Files
MeltMind/tools/run_s2_recovery_chain.py
T

51 lines
3.2 KiB
Python

"""Wait only for the authorized recovery, validate three sources, then pilot."""
import argparse
import fcntl
import json
from pathlib import Path
import time
import traceback
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_incident_recovery_manifest import create_manifest
from prepare_s2_recovered_pilot_training import finalize
from run_s2_recovered_pilot import run as pilot
def run(original,incident,recovery,root,watch,gpu_uuid):
watch.mkdir(exist_ok=True)
with (watch/'execution.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
assert not (watch/'started.json').exists() and not root.exists(),'No repeated recovery-chain attempt'
assert (recovery/'batch-started.json').exists()
paths=[str(p) for p in Path('tools').glob('*s2*recover*.py')]+[str(p) for p in Path('tools').glob('*s2*pilot*.py')]+[str(p) for p in Path('tools/s2_models').glob('*.py')]
paths+=['tools/s2_e1_supervisor.py','tools/freeze_g1_components.py']
frozen={p:sha(p) for p in sorted(set(paths))};expected=sha(recovery/'contract.json');start=time.monotonic()
save(watch/'started.json',dict(started_unix=time.time(),pid=__import__('os').getpid(),recovery_root=str(recovery),
recovery_contract_sha256=expected,incident_ledger_sha256=sha(incident/'incident-ledger.json'),code_sha256=frozen,
pilot_root=str(root),gpu_uuid=gpu_uuid,ruling='1794689807.792819',waiting_uses_GPU=False))
try:
while not (recovery/'batch-complete.json').exists():
p=json.loads((recovery/'progress.json').read_text())
save(watch/'progress.json',dict(stage='WAITING_FOR_AUTHORIZED_RECOVERY',completed=p['completed'],
recovery_stop_reason=p['stop_reason'],waiting_seconds=time.monotonic()-start,GPU_started=False))
time.sleep(10)
assert sha(recovery/'contract.json')==expected
for p,h in frozen.items():assert sha(p)==h,p
terminal=create_manifest(incident,recovery)
plan=finalize(original,root,gpu_uuid,terminal)
save(watch/'preparation-accepted.json',dict(status='THREE_SOURCES_499_TERMINALS_VERIFIED',merged_terminal=str(terminal),
merged_terminal_sha256=sha(terminal),successful=plan['successful'],pilot_plan_sha256=sha(root/'pilot-plan.json'),
failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),waiting_seconds=time.monotonic()-start))
save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),GPU_wall_limit_seconds=7200))
pilot(root);result=json.loads((root/'orchestrator-complete.json').read_text())
save(watch/'complete.json',dict(**result,wall_seconds_including_CPU_wait=time.monotonic()-start))
except Exception as exc:
save(watch/'failure.json',dict(status='STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-start));raise
if __name__=='__main__':
p=argparse.ArgumentParser()
for name in ['original','incident','recovery','root','watch']:p.add_argument(name,type=Path)
p.add_argument('gpu_uuid');a=p.parse_args();run(a.original,a.incident,a.recovery,a.root,a.watch,a.gpu_uuid)