Files
MeltMind/tools/run_s2_precision_continuation.py

58 lines
3.8 KiB
Python

"""Engineering continuation of zero-run v2; reuse probe, preserve both stops."""
import argparse
import fcntl
import json
from pathlib import Path
import time
import traceback
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_gpu_idle_gate import inventory, recorded_wait
from s2_precision_continuation_guard import run as train_runs
def run(root):
with (root/'continuation.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
assert not (root/'started.json').exists(),'No automatic continuation retry'
contract=json.loads((root/'continuation-contract.json').read_text())
for p,h in contract['code_sha256'].items():assert sha(p)==h,p
for p,h in contract['source_sha256'].items():assert sha(p)==h,p
source=Path(contract['source']);old=json.loads((source/'training-contract.json').read_text())
ended=json.loads((source/'pilot-complete.json').read_text())
assert ended['status']=='PILOT_INCOMPLETE' and ended['reason']=='SELECTED_GPU_BUSY_BEFORE_RUN' and ended['runs']==[]
previous=json.loads((source/'orchestrator-complete.json').read_text())['total_wall_seconds']
assert previous==contract['prior_wall_seconds'] and old['E_max']==old['cosine_T_max']==42
assert sha(old['timing_probe_receipt'])==old['timing_probe_sha256']
started=time.monotonic();save(root/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'continuation-contract.json'),prior_wall_seconds=previous))
try:
names=inventory();selected=old['gpu_uuid'];assert 'V100' in names[selected]
gate=recorded_wait(root,'initial-selected',selected,min(600,7200-previous-(time.monotonic()-started)-5))
switched=False
if gate['status']!='IDLE_CONFIRMED':
if not gate['last'].get('occupants'):raise RuntimeError('Selected GPU failed idle gate without identified foreign occupant; no switch')
for uuid,name in sorted(names.items()):
if uuid==selected or name!=names[selected]:continue
remaining=7200-previous-(time.monotonic()-started)-5
if remaining<=20:break
alternative=recorded_wait(root,'alternative-'+uuid,uuid,min(600,remaining))
if alternative['status']=='IDLE_CONFIRMED':selected=uuid;switched=True;break
if not switched:raise RuntimeError('No identical idle V100 after foreign-occupant timeout')
c=dict(old);c['gpu_uuid']=selected;c['code_sha256']=dict(old['code_sha256'],**contract['code_sha256'])
c['engineering_continuation']=dict(source=str(source),source_sha256=contract['source_sha256'],
prior_wall_seconds=previous,probe_reused=True,probe_rerun=False,gpu_switched=switched,
original_gpu_uuid=old['gpu_uuid'],selected_gpu_uuid=selected,ruling='1794695421.177269')
save(root/'training-contract.json',c)
train_runs(root,prior_wall_seconds=previous+time.monotonic()-started)
result=json.loads((root/'pilot-complete.json').read_text())
save(root/'complete.json',dict(status=result['status'],selected_global_width=result['selected_global_width'],
cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous,
limit_seconds=7200,probe_reused=True,probe_rerun=False,test_labels_read=False))
except Exception as exc:
save(root/'failure.json',dict(status='CONTINUATION_STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),
cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous,limit_seconds=7200));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)