"""One irreversible attempt: bounded timing probe, frozen E_max, then four runs.""" import argparse import fcntl import json from pathlib import Path import sys import time import traceback from freeze_g1_components import sha from s2_e1_supervisor import save from s2_precision_pilot_gpu_guard import guarded_worker,run as train_runs from s2_pilot_train import common_epoch_limit def freeze_training_contract(root,plan,probe): assert probe['status']=='COMPLETE' and probe['labels_read'] is False and probe['metrics_evaluated'] is False assert [r['case'] for r in probe['rows']]==plan['probe_cases'] nodes=sum(r['nodes'] for r in probe['rows']);assert nodes==probe['probe_nodes'] timing=common_epoch_limit(sum(r['training_seconds'] for r in probe['rows'])/nodes, sum(r['inference_seconds'] for r in probe['rows'])/nodes,plan['learning_nodes'],plan['validation_nodes']) assert all(probe[k]==v for k,v in timing.items()) c=dict(plan,**timing,timing_probe_receipt=str(root/'timing-probe/result.json'),timing_probe_sha256=sha(root/'timing-probe/result.json'), timing_probe_guard_sha256=sha(root/'timing-probe-guard-receipt.json'), probe_guard_wall_seconds=json.loads((root/'timing-probe-guard-receipt.json').read_text())['total_run_wall_seconds'], standardizer_fingerprint=probe['standardizer_fingerprint']) with (root/'training-contract.json').open('x') as f:json.dump(c,f,indent=2);f.write('\n') return c def run(root): with (root/'orchestrator.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) assert not (root/'orchestrator-started.json').exists(),'No automatic probe/pilot retry' plan=json.loads((root/'pilot-plan.json').read_text()) for p,h in plan['code_sha256'].items():assert sha(p)==h,p start=time.monotonic();save(root/'orchestrator-started.json',dict(started_unix=time.time(),plan_sha256=sha(root/'pilot-plan.json'))) try: receipt,probe=guarded_worker(root,dict(name='timing-probe'),-1,plan['gpu_uuid'],300, [sys.executable,'tools/run_s2_pilot_probe_v2.py',str(root)]) if receipt['status']!='PASS' or not probe or probe['status']!='COMPLETE':raise RuntimeError('Timing probe failed; no training, no retry') coverage=dict(**plan['coverage_boundary'],actual_probe_max_nodes=max(r['nodes'] for r in probe['rows']),probe_receipt_sha256=sha(root/'timing-probe/result.json'),probe_guard_sha256=sha(root/'timing-probe-guard-receipt.json')) assert coverage['actual_probe_max_nodes']==coverage['max_successful_learning_nodes'] save(root/'timing-probe/coverage-addendum.json',coverage) freeze_training_contract(root,plan,probe) train_runs(root,prior_wall_seconds=time.monotonic()-start) result=json.loads((root/'pilot-complete.json').read_text()) save(root/'orchestrator-complete.json',dict(status=result['status'],total_wall_seconds=time.monotonic()-start, selected_global_width=result['selected_global_width'],test_labels_read=False,probe_included_in_7200_seconds=True)) except Exception as exc: save(root/'orchestrator-failure.json',dict(status='FAILED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),total_wall_seconds=time.monotonic()-start));raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)