Files
MeltMind/tools/run_s2_pilot_when_ready.py

74 lines
4.5 KiB
Python

"""Wait for the existing preparation attempt, then run the already bounded pilot.
Never launches/restarts preparation. Waiting uses no CUDA; all GPU work remains
inside run_s2_pilot's single-card probe-plus-four-run 7200-second guard.
"""
import argparse
import fcntl
import json
from pathlib import Path
import time
import traceback
from freeze_g1_components import sha
from s2_e1_supervisor import save
from prepare_s2_pilot_training import finalize
from run_s2_pilot import run as pilot
from s2_pilot_terminal import terminal_batch,continuation_allowed
def preparation_terminal(source):
path=source/'batch-complete.json'
if not path.exists():return False
batch,_=terminal_batch(source)
if not (source/'merged-terminal.json').exists() and continuation_allowed(source,batch):return False
if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']:
raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch')
if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)):
raise ValueError('Exactly 499 distinct terminal attempts required')
return True
def run(source,root,watch,gpu_uuid):
watch.mkdir(exist_ok=True)
with (watch/'execution.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (watch/'started.json').exists():raise FileExistsError('Continuation already attempted; inspect evidence, do not restart')
assert (source/'batch-started.json').exists(),'Existing preparation must already be running'
assert not root.exists(),'No overwrite of a prior pilot'
paths=[str(p) for p in Path('tools').glob('*s2*pilot*.py')]+[str(p) for p in Path('tools/s2_models').glob('*.py')]
paths+=['tools/s2_gpu_process_supervisor.py','tools/s2_e1_supervisor.py','tools/freeze_g1_components.py']
frozen={p:sha(p) for p in sorted(set(paths))}
expected=sha(source/'contract.json');start=time.monotonic()
save(watch/'started.json',dict(started_unix=time.time(),preparation_root=str(source),preparation_contract_sha256=expected,
pilot_root=str(root),gpu_uuid=gpu_uuid,code_sha256=frozen,waiting_uses_GPU=False,
authorizing_rulings=['1794681576.845789','1794681807.376669','1794683016.134929']))
try:
while not preparation_terminal(source):
progress=json.loads((source/'progress.json').read_text())
stage='WAITING_FOR_ACCEPTED_MERGED_CONTINUATION' if (source/'batch-complete.json').exists() else 'WAITING_FOR_EXISTING_PREPARATION'
save(watch/'progress.json',dict(stage=stage,completed=progress['completed'],
waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True))
time.sleep(10)
assert sha(source/'contract.json')==expected
for p,h in frozen.items():assert sha(p)==h,p
plan=finalize(source,root,gpu_uuid)
terminal,terminal_path=terminal_batch(source)
save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED',
batch_sha256=sha(terminal_path),terminal_manifest=str(terminal_path),
preparation_sources=terminal.get('source_provenance',[]),success_manifest_sha256=plan['success_manifest_sha256'],
successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),
pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start))
save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),
total_GPU_wall_seconds_limit=7200,preparation_wait_excluded_from_GPU_time=True))
pilot(root)
result=json.loads((root/'orchestrator-complete.json').read_text())
save(watch/'complete.json',dict(**result,preparation_wait_and_pilot_wall_seconds=time.monotonic()-start))
except Exception as exc:
save(watch/'failure.json',dict(status='STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),
wall_seconds=time.monotonic()-start,preparation_not_restarted=True));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('source',type=Path);p.add_argument('root',type=Path);p.add_argument('watch',type=Path);p.add_argument('gpu_uuid');a=p.parse_args()
run(a.source,a.root,a.watch,a.gpu_uuid)