74 lines
4.5 KiB
Python
74 lines
4.5 KiB
Python
"""Wait for the existing preparation attempt, then run the already bounded pilot.
|
|
|
|
Never launches/restarts preparation. Waiting uses no CUDA; all GPU work remains
|
|
inside run_s2_pilot's single-card probe-plus-four-run 7200-second guard.
|
|
"""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import traceback
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from prepare_s2_pilot_training import finalize
|
|
from run_s2_pilot import run as pilot
|
|
from s2_pilot_terminal import terminal_batch,continuation_allowed
|
|
|
|
|
|
def preparation_terminal(source):
|
|
path=source/'batch-complete.json'
|
|
if not path.exists():return False
|
|
batch,_=terminal_batch(source)
|
|
if not (source/'merged-terminal.json').exists() and continuation_allowed(source,batch):return False
|
|
if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']:
|
|
raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch')
|
|
if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)):
|
|
raise ValueError('Exactly 499 distinct terminal attempts required')
|
|
return True
|
|
|
|
|
|
def run(source,root,watch,gpu_uuid):
|
|
watch.mkdir(exist_ok=True)
|
|
with (watch/'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (watch/'started.json').exists():raise FileExistsError('Continuation already attempted; inspect evidence, do not restart')
|
|
assert (source/'batch-started.json').exists(),'Existing preparation must already be running'
|
|
assert not root.exists(),'No overwrite of a prior pilot'
|
|
paths=[str(p) for p in Path('tools').glob('*s2*pilot*.py')]+[str(p) for p in Path('tools/s2_models').glob('*.py')]
|
|
paths+=['tools/s2_gpu_process_supervisor.py','tools/s2_e1_supervisor.py','tools/freeze_g1_components.py']
|
|
frozen={p:sha(p) for p in sorted(set(paths))}
|
|
expected=sha(source/'contract.json');start=time.monotonic()
|
|
save(watch/'started.json',dict(started_unix=time.time(),preparation_root=str(source),preparation_contract_sha256=expected,
|
|
pilot_root=str(root),gpu_uuid=gpu_uuid,code_sha256=frozen,waiting_uses_GPU=False,
|
|
authorizing_rulings=['1794681576.845789','1794681807.376669','1794683016.134929']))
|
|
try:
|
|
while not preparation_terminal(source):
|
|
progress=json.loads((source/'progress.json').read_text())
|
|
stage='WAITING_FOR_ACCEPTED_MERGED_CONTINUATION' if (source/'batch-complete.json').exists() else 'WAITING_FOR_EXISTING_PREPARATION'
|
|
save(watch/'progress.json',dict(stage=stage,completed=progress['completed'],
|
|
waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True))
|
|
time.sleep(10)
|
|
assert sha(source/'contract.json')==expected
|
|
for p,h in frozen.items():assert sha(p)==h,p
|
|
plan=finalize(source,root,gpu_uuid)
|
|
terminal,terminal_path=terminal_batch(source)
|
|
save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED',
|
|
batch_sha256=sha(terminal_path),terminal_manifest=str(terminal_path),
|
|
preparation_sources=terminal.get('source_provenance',[]),success_manifest_sha256=plan['success_manifest_sha256'],
|
|
successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),
|
|
pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start))
|
|
save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),
|
|
total_GPU_wall_seconds_limit=7200,preparation_wait_excluded_from_GPU_time=True))
|
|
pilot(root)
|
|
result=json.loads((root/'orchestrator-complete.json').read_text())
|
|
save(watch/'complete.json',dict(**result,preparation_wait_and_pilot_wall_seconds=time.monotonic()-start))
|
|
except Exception as exc:
|
|
save(watch/'failure.json',dict(status='STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),
|
|
wall_seconds=time.monotonic()-start,preparation_not_restarted=True));raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('source',type=Path);p.add_argument('root',type=Path);p.add_argument('watch',type=Path);p.add_argument('gpu_uuid');a=p.parse_args()
|
|
run(a.source,a.root,a.watch,a.gpu_uuid)
|