51 lines
3.2 KiB
Python
51 lines
3.2 KiB
Python
"""Wait only for the authorized recovery, validate three sources, then pilot."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import traceback
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from s2_incident_recovery_manifest import create_manifest
|
|
from prepare_s2_recovered_pilot_training import finalize
|
|
from run_s2_recovered_pilot import run as pilot
|
|
|
|
|
|
def run(original,incident,recovery,root,watch,gpu_uuid):
|
|
watch.mkdir(exist_ok=True)
|
|
with (watch/'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
assert not (watch/'started.json').exists() and not root.exists(),'No repeated recovery-chain attempt'
|
|
assert (recovery/'batch-started.json').exists()
|
|
paths=[str(p) for p in Path('tools').glob('*s2*recover*.py')]+[str(p) for p in Path('tools').glob('*s2*pilot*.py')]+[str(p) for p in Path('tools/s2_models').glob('*.py')]
|
|
paths+=['tools/s2_e1_supervisor.py','tools/freeze_g1_components.py']
|
|
frozen={p:sha(p) for p in sorted(set(paths))};expected=sha(recovery/'contract.json');start=time.monotonic()
|
|
save(watch/'started.json',dict(started_unix=time.time(),pid=__import__('os').getpid(),recovery_root=str(recovery),
|
|
recovery_contract_sha256=expected,incident_ledger_sha256=sha(incident/'incident-ledger.json'),code_sha256=frozen,
|
|
pilot_root=str(root),gpu_uuid=gpu_uuid,ruling='1794689807.792819',waiting_uses_GPU=False))
|
|
try:
|
|
while not (recovery/'batch-complete.json').exists():
|
|
p=json.loads((recovery/'progress.json').read_text())
|
|
save(watch/'progress.json',dict(stage='WAITING_FOR_AUTHORIZED_RECOVERY',completed=p['completed'],
|
|
recovery_stop_reason=p['stop_reason'],waiting_seconds=time.monotonic()-start,GPU_started=False))
|
|
time.sleep(10)
|
|
assert sha(recovery/'contract.json')==expected
|
|
for p,h in frozen.items():assert sha(p)==h,p
|
|
terminal=create_manifest(incident,recovery)
|
|
plan=finalize(original,root,gpu_uuid,terminal)
|
|
save(watch/'preparation-accepted.json',dict(status='THREE_SOURCES_499_TERMINALS_VERIFIED',merged_terminal=str(terminal),
|
|
merged_terminal_sha256=sha(terminal),successful=plan['successful'],pilot_plan_sha256=sha(root/'pilot-plan.json'),
|
|
failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),waiting_seconds=time.monotonic()-start))
|
|
save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),GPU_wall_limit_seconds=7200))
|
|
pilot(root);result=json.loads((root/'orchestrator-complete.json').read_text())
|
|
save(watch/'complete.json',dict(**result,wall_seconds_including_CPU_wait=time.monotonic()-start))
|
|
except Exception as exc:
|
|
save(watch/'failure.json',dict(status='STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-start));raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser()
|
|
for name in ['original','incident','recovery','root','watch']:p.add_argument(name,type=Path)
|
|
p.add_argument('gpu_uuid');a=p.parse_args();run(a.original,a.incident,a.recovery,a.root,a.watch,a.gpu_uuid)
|