"""Wait only for the authorized recovery, validate three sources, then pilot.""" import argparse import fcntl import json from pathlib import Path import time import traceback from freeze_g1_components import sha from s2_e1_supervisor import save from s2_incident_recovery_manifest import create_manifest from prepare_s2_recovered_pilot_training import finalize from run_s2_recovered_pilot import run as pilot def run(original,incident,recovery,root,watch,gpu_uuid): watch.mkdir(exist_ok=True) with (watch/'execution.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) assert not (watch/'started.json').exists() and not root.exists(),'No repeated recovery-chain attempt' assert (recovery/'batch-started.json').exists() paths=[str(p) for p in Path('tools').glob('*s2*recover*.py')]+[str(p) for p in Path('tools').glob('*s2*pilot*.py')]+[str(p) for p in Path('tools/s2_models').glob('*.py')] paths+=['tools/s2_e1_supervisor.py','tools/freeze_g1_components.py'] frozen={p:sha(p) for p in sorted(set(paths))};expected=sha(recovery/'contract.json');start=time.monotonic() save(watch/'started.json',dict(started_unix=time.time(),pid=__import__('os').getpid(),recovery_root=str(recovery), recovery_contract_sha256=expected,incident_ledger_sha256=sha(incident/'incident-ledger.json'),code_sha256=frozen, pilot_root=str(root),gpu_uuid=gpu_uuid,ruling='1794689807.792819',waiting_uses_GPU=False)) try: while not (recovery/'batch-complete.json').exists(): p=json.loads((recovery/'progress.json').read_text()) save(watch/'progress.json',dict(stage='WAITING_FOR_AUTHORIZED_RECOVERY',completed=p['completed'], recovery_stop_reason=p['stop_reason'],waiting_seconds=time.monotonic()-start,GPU_started=False)) time.sleep(10) assert sha(recovery/'contract.json')==expected for p,h in frozen.items():assert sha(p)==h,p terminal=create_manifest(incident,recovery) plan=finalize(original,root,gpu_uuid,terminal) save(watch/'preparation-accepted.json',dict(status='THREE_SOURCES_499_TERMINALS_VERIFIED',merged_terminal=str(terminal), merged_terminal_sha256=sha(terminal),successful=plan['successful'],pilot_plan_sha256=sha(root/'pilot-plan.json'), failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),waiting_seconds=time.monotonic()-start)) save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),GPU_wall_limit_seconds=7200)) pilot(root);result=json.loads((root/'orchestrator-complete.json').read_text()) save(watch/'complete.json',dict(**result,wall_seconds_including_CPU_wait=time.monotonic()-start)) except Exception as exc: save(watch/'failure.json',dict(status='STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-start));raise if __name__=='__main__': p=argparse.ArgumentParser() for name in ['original','incident','recovery','root','watch']:p.add_argument(name,type=Path) p.add_argument('gpu_uuid');a=p.parse_args();run(a.original,a.incident,a.recovery,a.root,a.watch,a.gpu_uuid)