"""Engineering continuation of zero-run v2; reuse probe, preserve both stops.""" import argparse import fcntl import json from pathlib import Path import time import traceback from freeze_g1_components import sha from s2_e1_supervisor import save from s2_gpu_idle_gate import inventory, recorded_wait from s2_precision_continuation_guard import run as train_runs def run(root): with (root/'continuation.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) assert not (root/'started.json').exists(),'No automatic continuation retry' contract=json.loads((root/'continuation-contract.json').read_text()) for p,h in contract['code_sha256'].items():assert sha(p)==h,p for p,h in contract['source_sha256'].items():assert sha(p)==h,p source=Path(contract['source']);old=json.loads((source/'training-contract.json').read_text()) ended=json.loads((source/'pilot-complete.json').read_text()) assert ended['status']=='PILOT_INCOMPLETE' and ended['reason']=='SELECTED_GPU_BUSY_BEFORE_RUN' and ended['runs']==[] previous=json.loads((source/'orchestrator-complete.json').read_text())['total_wall_seconds'] assert previous==contract['prior_wall_seconds'] and old['E_max']==old['cosine_T_max']==42 assert sha(old['timing_probe_receipt'])==old['timing_probe_sha256'] started=time.monotonic();save(root/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'continuation-contract.json'),prior_wall_seconds=previous)) try: names=inventory();selected=old['gpu_uuid'];assert 'V100' in names[selected] gate=recorded_wait(root,'initial-selected',selected,min(600,7200-previous-(time.monotonic()-started)-5)) switched=False if gate['status']!='IDLE_CONFIRMED': if not gate['last'].get('occupants'):raise RuntimeError('Selected GPU failed idle gate without identified foreign occupant; no switch') for uuid,name in sorted(names.items()): if uuid==selected or name!=names[selected]:continue remaining=7200-previous-(time.monotonic()-started)-5 if remaining<=20:break alternative=recorded_wait(root,'alternative-'+uuid,uuid,min(600,remaining)) if alternative['status']=='IDLE_CONFIRMED':selected=uuid;switched=True;break if not switched:raise RuntimeError('No identical idle V100 after foreign-occupant timeout') c=dict(old);c['gpu_uuid']=selected;c['code_sha256']=dict(old['code_sha256'],**contract['code_sha256']) c['engineering_continuation']=dict(source=str(source),source_sha256=contract['source_sha256'], prior_wall_seconds=previous,probe_reused=True,probe_rerun=False,gpu_switched=switched, original_gpu_uuid=old['gpu_uuid'],selected_gpu_uuid=selected,ruling='1794695421.177269') save(root/'training-contract.json',c) train_runs(root,prior_wall_seconds=previous+time.monotonic()-started) result=json.loads((root/'pilot-complete.json').read_text()) save(root/'complete.json',dict(status=result['status'],selected_global_width=result['selected_global_width'], cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous, limit_seconds=7200,probe_reused=True,probe_rerun=False,test_labels_read=False)) except Exception as exc: save(root/'failure.json',dict(status='CONTINUATION_STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(), cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous,limit_seconds=7200));raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)