"""Wait for the sole ten terminal, then supervised reference and cost gates.""" import argparse import fcntl import os from pathlib import Path import signal import sys import time import psutil from s2_recovery_supervisor import save,supervise,stop_group from s3b_runtime import read,sha from s3b_v04_gates import verify_gate_config def run(root): lock=(root/'execution.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) c=verify_gate_config(root);ten=Path(c['ten_root']);external=Path(c['stage_guard_root']) if (root/'started.json').exists():raise FileExistsError('Never restart gate chain') os.sched_setaffinity(0,[30,31]) save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),code_freeze=read(root/'code-freeze.json'))) stopped=False def stop(*_): nonlocal stopped stopped=True for p in psutil.Process(os.getpid()).children():stop_group(p.pid) signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop) guards=[];reason=None;terminal=None;active=None extra=c['additional_synthetic_CPU_s'] def cumulative():return terminal['cumulative_worker_CPU_s']+extra+sum(g['cpu_seconds'] for g in guards) try: while not (ten/'complete.json').exists() or not (external/'complete.json').exists(): if stopped:raise RuntimeError('Gate chain interrupted') if (external/'failure.json').exists():raise RuntimeError('External stage guard failed') save(root/'progress.json',dict(stage='WAITING_ORIGINAL_TEN',unix=time.time(),new_scientific_workers=0)) time.sleep(5) terminal=read(ten/'complete.json');ext=read(external/'complete.json') if terminal['status']!='TEN_PASS_AWAITING_REFERENCE_AND_COST' or ext['status']!='OBSERVED_QUEUE_TERMINAL': reason='TEN_OR_STAGE_CAP_GATE_FAILED';return 1 for kind in ['reference','census']: verify_gate_config(root) if stopped:reason='GATE_CHAIN_INTERRUPTED';return 1 if cumulative()+1800+7200>72000:reason='GATE_FULL_JOB_RESERVATION';return 1 dest=root/kind;dest.mkdir(exist_ok=False);active=kind save(root/'progress.json',dict(stage=kind.upper(),cumulative_settled_CPU_s=cumulative())) w=supervise(['taskset','-c','16',sys.executable,'tools/s3b_v04_gates.py',str(root),kind],dest,'worker', dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800, tree_rss_bytes=4*2**30,disk_bytes=2**30,max_processes=1),poll=.2) guards.append(w);active=None if w['status']!='PASS' or w['monitor_errors']:reason=kind.upper()+'_GUARD_FAILED';return 1 result=read(dest/'output/result.json') if kind=='reference' and result['status']!='PASS':reason='EXACT_REFERENCE_GATE_FAILED';return 1 if kind=='census': terms=result['projection_terms'] projected=terms['variable_projection_CPU_s']+terms['fixed_projection_CPU_s']+cumulative()+7200 gate=dict(status='PASS' if projected<=72000 else 'FAIL',projected_total_worker_CPU_s=projected, cap_CPU_s=72000,postfit_reserve_CPU_s=7200,cumulative_actual_CPU_s=cumulative(), projection_terms=terms,reference_gate_sha256=sha(root/'reference/output/result.json'), census_sha256=sha(dest/'output/result.json'),full617_launched=False) save(root/'cost-gate.json',gate) if gate['status']!='PASS':reason='COST_PROJECTION_GATE_FAILED';return 1 return 0 except Exception as exc: reason='GATE_CHAIN_EXCEPTION' save(root/'failure.json',dict(type=type(exc).__name__,reason=str(exc),active=active)) raise finally: save(root/'complete.json',dict(status='GATES_PASS_READY_FOR_FULL617' if reason is None else 'STOPPED', reason=reason,guards=guards,ten_complete_sha256=sha(ten/'complete.json') if (ten/'complete.json').exists() else None, additional_synthetic_CPU_s=extra,cumulative_worker_CPU_s=cumulative() if terminal else None, account_complete=active is None,full617_launched=False,no_solver_switch=True,labels_read=False, chain_CPU_s=time.process_time())) if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))