"""Wait for the existing preparation attempt, then run the already bounded pilot. Never launches/restarts preparation. Waiting uses no CUDA; all GPU work remains inside run_s2_pilot's single-card probe-plus-four-run 7200-second guard. """ import argparse import fcntl import json from pathlib import Path import time import traceback from freeze_g1_components import sha from s2_e1_supervisor import save from prepare_s2_pilot_training import finalize from run_s2_pilot import run as pilot from s2_pilot_terminal import terminal_batch,continuation_allowed def preparation_terminal(source): path=source/'batch-complete.json' if not path.exists():return False batch,_=terminal_batch(source) if not (source/'merged-terminal.json').exists() and continuation_allowed(source,batch):return False if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']: raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch') if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)): raise ValueError('Exactly 499 distinct terminal attempts required') return True def run(source,root,watch,gpu_uuid): watch.mkdir(exist_ok=True) with (watch/'execution.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (watch/'started.json').exists():raise FileExistsError('Continuation already attempted; inspect evidence, do not restart') assert (source/'batch-started.json').exists(),'Existing preparation must already be running' assert not root.exists(),'No overwrite of a prior pilot' paths=[str(p) for p in Path('tools').glob('*s2*pilot*.py')]+[str(p) for p in Path('tools/s2_models').glob('*.py')] paths+=['tools/s2_gpu_process_supervisor.py','tools/s2_e1_supervisor.py','tools/freeze_g1_components.py'] frozen={p:sha(p) for p in sorted(set(paths))} expected=sha(source/'contract.json');start=time.monotonic() save(watch/'started.json',dict(started_unix=time.time(),preparation_root=str(source),preparation_contract_sha256=expected, pilot_root=str(root),gpu_uuid=gpu_uuid,code_sha256=frozen,waiting_uses_GPU=False, authorizing_rulings=['1794681576.845789','1794681807.376669','1794683016.134929'])) try: while not preparation_terminal(source): progress=json.loads((source/'progress.json').read_text()) stage='WAITING_FOR_ACCEPTED_MERGED_CONTINUATION' if (source/'batch-complete.json').exists() else 'WAITING_FOR_EXISTING_PREPARATION' save(watch/'progress.json',dict(stage=stage,completed=progress['completed'], waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True)) time.sleep(10) assert sha(source/'contract.json')==expected for p,h in frozen.items():assert sha(p)==h,p plan=finalize(source,root,gpu_uuid) terminal,terminal_path=terminal_batch(source) save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED', batch_sha256=sha(terminal_path),terminal_manifest=str(terminal_path), preparation_sources=terminal.get('source_provenance',[]),success_manifest_sha256=plan['success_manifest_sha256'], successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()), pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start)) save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root), total_GPU_wall_seconds_limit=7200,preparation_wait_excluded_from_GPU_time=True)) pilot(root) result=json.loads((root/'orchestrator-complete.json').read_text()) save(watch/'complete.json',dict(**result,preparation_wait_and_pilot_wall_seconds=time.monotonic()-start)) except Exception as exc: save(watch/'failure.json',dict(status='STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(), wall_seconds=time.monotonic()-start,preparation_not_restarted=True));raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('source',type=Path);p.add_argument('root',type=Path);p.add_argument('watch',type=Path);p.add_argument('gpu_uuid');a=p.parse_args() run(a.source,a.root,a.watch,a.gpu_uuid)