58 lines
3.8 KiB
Python
58 lines
3.8 KiB
Python
"""Engineering continuation of zero-run v2; reuse probe, preserve both stops."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import traceback
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from s2_gpu_idle_gate import inventory, recorded_wait
|
|
from s2_precision_continuation_guard import run as train_runs
|
|
|
|
|
|
def run(root):
|
|
with (root/'continuation.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
assert not (root/'started.json').exists(),'No automatic continuation retry'
|
|
contract=json.loads((root/'continuation-contract.json').read_text())
|
|
for p,h in contract['code_sha256'].items():assert sha(p)==h,p
|
|
for p,h in contract['source_sha256'].items():assert sha(p)==h,p
|
|
source=Path(contract['source']);old=json.loads((source/'training-contract.json').read_text())
|
|
ended=json.loads((source/'pilot-complete.json').read_text())
|
|
assert ended['status']=='PILOT_INCOMPLETE' and ended['reason']=='SELECTED_GPU_BUSY_BEFORE_RUN' and ended['runs']==[]
|
|
previous=json.loads((source/'orchestrator-complete.json').read_text())['total_wall_seconds']
|
|
assert previous==contract['prior_wall_seconds'] and old['E_max']==old['cosine_T_max']==42
|
|
assert sha(old['timing_probe_receipt'])==old['timing_probe_sha256']
|
|
started=time.monotonic();save(root/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'continuation-contract.json'),prior_wall_seconds=previous))
|
|
try:
|
|
names=inventory();selected=old['gpu_uuid'];assert 'V100' in names[selected]
|
|
gate=recorded_wait(root,'initial-selected',selected,min(600,7200-previous-(time.monotonic()-started)-5))
|
|
switched=False
|
|
if gate['status']!='IDLE_CONFIRMED':
|
|
if not gate['last'].get('occupants'):raise RuntimeError('Selected GPU failed idle gate without identified foreign occupant; no switch')
|
|
for uuid,name in sorted(names.items()):
|
|
if uuid==selected or name!=names[selected]:continue
|
|
remaining=7200-previous-(time.monotonic()-started)-5
|
|
if remaining<=20:break
|
|
alternative=recorded_wait(root,'alternative-'+uuid,uuid,min(600,remaining))
|
|
if alternative['status']=='IDLE_CONFIRMED':selected=uuid;switched=True;break
|
|
if not switched:raise RuntimeError('No identical idle V100 after foreign-occupant timeout')
|
|
c=dict(old);c['gpu_uuid']=selected;c['code_sha256']=dict(old['code_sha256'],**contract['code_sha256'])
|
|
c['engineering_continuation']=dict(source=str(source),source_sha256=contract['source_sha256'],
|
|
prior_wall_seconds=previous,probe_reused=True,probe_rerun=False,gpu_switched=switched,
|
|
original_gpu_uuid=old['gpu_uuid'],selected_gpu_uuid=selected,ruling='1794695421.177269')
|
|
save(root/'training-contract.json',c)
|
|
train_runs(root,prior_wall_seconds=previous+time.monotonic()-started)
|
|
result=json.loads((root/'pilot-complete.json').read_text())
|
|
save(root/'complete.json',dict(status=result['status'],selected_global_width=result['selected_global_width'],
|
|
cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous,
|
|
limit_seconds=7200,probe_reused=True,probe_rerun=False,test_labels_read=False))
|
|
except Exception as exc:
|
|
save(root/'failure.json',dict(status='CONTINUATION_STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),
|
|
cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous,limit_seconds=7200));raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
|