55 lines
3.4 KiB
Python
55 lines
3.4 KiB
Python
"""One irreversible attempt: bounded timing probe, frozen E_max, then four runs."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from s2_recovered_pilot_gpu_guard import guarded_worker,run as train_runs
|
|
from s2_pilot_train import common_epoch_limit
|
|
|
|
|
|
def freeze_training_contract(root,plan,probe):
|
|
assert probe['status']=='COMPLETE' and probe['labels_read'] is False and probe['metrics_evaluated'] is False
|
|
assert [r['case'] for r in probe['rows']]==plan['probe_cases']
|
|
nodes=sum(r['nodes'] for r in probe['rows']);assert nodes==probe['probe_nodes']
|
|
timing=common_epoch_limit(sum(r['training_seconds'] for r in probe['rows'])/nodes,
|
|
sum(r['inference_seconds'] for r in probe['rows'])/nodes,plan['learning_nodes'],plan['validation_nodes'])
|
|
assert all(probe[k]==v for k,v in timing.items())
|
|
c=dict(plan,**timing,timing_probe_receipt=str(root/'timing-probe/result.json'),timing_probe_sha256=sha(root/'timing-probe/result.json'),
|
|
timing_probe_guard_sha256=sha(root/'timing-probe-guard-receipt.json'),
|
|
probe_guard_wall_seconds=json.loads((root/'timing-probe-guard-receipt.json').read_text())['total_run_wall_seconds'],
|
|
standardizer_fingerprint=probe['standardizer_fingerprint'])
|
|
with (root/'training-contract.json').open('x') as f:json.dump(c,f,indent=2);f.write('\n')
|
|
return c
|
|
|
|
|
|
def run(root):
|
|
with (root/'orchestrator.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
assert not (root/'orchestrator-started.json').exists(),'No automatic probe/pilot retry'
|
|
plan=json.loads((root/'pilot-plan.json').read_text())
|
|
for p,h in plan['code_sha256'].items():assert sha(p)==h,p
|
|
start=time.monotonic();save(root/'orchestrator-started.json',dict(started_unix=time.time(),plan_sha256=sha(root/'pilot-plan.json')))
|
|
try:
|
|
receipt,probe=guarded_worker(root,dict(name='timing-probe'),-1,plan['gpu_uuid'],300,
|
|
[sys.executable,'tools/run_s2_pilot_probe.py',str(root)])
|
|
if receipt['status']!='PASS' or not probe or probe['status']!='COMPLETE':raise RuntimeError('Timing probe failed; no training, no retry')
|
|
coverage=dict(**plan['coverage_boundary'],actual_probe_max_nodes=max(r['nodes'] for r in probe['rows']),probe_receipt_sha256=sha(root/'timing-probe/result.json'),probe_guard_sha256=sha(root/'timing-probe-guard-receipt.json'))
|
|
assert coverage['actual_probe_max_nodes']==coverage['max_successful_learning_nodes']
|
|
save(root/'timing-probe/coverage-addendum.json',coverage)
|
|
freeze_training_contract(root,plan,probe)
|
|
train_runs(root,prior_wall_seconds=time.monotonic()-start)
|
|
result=json.loads((root/'pilot-complete.json').read_text())
|
|
save(root/'orchestrator-complete.json',dict(status=result['status'],total_wall_seconds=time.monotonic()-start,
|
|
selected_global_width=result['selected_global_width'],test_labels_read=False,probe_included_in_7200_seconds=True))
|
|
except Exception as exc:
|
|
save(root/'orchestrator-failure.json',dict(status='FAILED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),total_wall_seconds=time.monotonic()-start));raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
|