Files
MeltMind/tools/run_s3b_v04_gates.py
T

76 lines
4.3 KiB
Python

"""Wait for the sole ten terminal, then supervised reference and cost gates."""
import argparse
import fcntl
import os
from pathlib import Path
import signal
import sys
import time
import psutil
from s2_recovery_supervisor import save,supervise,stop_group
from s3b_runtime import read,sha
from s3b_v04_gates import verify_gate_config
def run(root):
lock=(root/'execution.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
c=verify_gate_config(root);ten=Path(c['ten_root']);external=Path(c['stage_guard_root'])
if (root/'started.json').exists():raise FileExistsError('Never restart gate chain')
os.sched_setaffinity(0,[30,31])
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),code_freeze=read(root/'code-freeze.json')))
stopped=False
def stop(*_):
nonlocal stopped
stopped=True
for p in psutil.Process(os.getpid()).children():stop_group(p.pid)
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
guards=[];reason=None;terminal=None;active=None
extra=c['additional_synthetic_CPU_s']
def cumulative():return terminal['cumulative_worker_CPU_s']+extra+sum(g['cpu_seconds'] for g in guards)
try:
while not (ten/'complete.json').exists() or not (external/'complete.json').exists():
if stopped:raise RuntimeError('Gate chain interrupted')
if (external/'failure.json').exists():raise RuntimeError('External stage guard failed')
save(root/'progress.json',dict(stage='WAITING_ORIGINAL_TEN',unix=time.time(),new_scientific_workers=0))
time.sleep(5)
terminal=read(ten/'complete.json');ext=read(external/'complete.json')
if terminal['status']!='TEN_PASS_AWAITING_REFERENCE_AND_COST' or ext['status']!='OBSERVED_QUEUE_TERMINAL':
reason='TEN_OR_STAGE_CAP_GATE_FAILED';return 1
for kind in ['reference','census']:
verify_gate_config(root)
if stopped:reason='GATE_CHAIN_INTERRUPTED';return 1
if cumulative()+1800+7200>72000:reason='GATE_FULL_JOB_RESERVATION';return 1
dest=root/kind;dest.mkdir(exist_ok=False);active=kind
save(root/'progress.json',dict(stage=kind.upper(),cumulative_settled_CPU_s=cumulative()))
w=supervise(['taskset','-c','16',sys.executable,'tools/s3b_v04_gates.py',str(root),kind],dest,'worker',
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,
tree_rss_bytes=4*2**30,disk_bytes=2**30,max_processes=1),poll=.2)
guards.append(w);active=None
if w['status']!='PASS' or w['monitor_errors']:reason=kind.upper()+'_GUARD_FAILED';return 1
result=read(dest/'output/result.json')
if kind=='reference' and result['status']!='PASS':reason='EXACT_REFERENCE_GATE_FAILED';return 1
if kind=='census':
terms=result['projection_terms']
projected=terms['variable_projection_CPU_s']+terms['fixed_projection_CPU_s']+cumulative()+7200
gate=dict(status='PASS' if projected<=72000 else 'FAIL',projected_total_worker_CPU_s=projected,
cap_CPU_s=72000,postfit_reserve_CPU_s=7200,cumulative_actual_CPU_s=cumulative(),
projection_terms=terms,reference_gate_sha256=sha(root/'reference/output/result.json'),
census_sha256=sha(dest/'output/result.json'),full617_launched=False)
save(root/'cost-gate.json',gate)
if gate['status']!='PASS':reason='COST_PROJECTION_GATE_FAILED';return 1
return 0
except Exception as exc:
reason='GATE_CHAIN_EXCEPTION'
save(root/'failure.json',dict(type=type(exc).__name__,reason=str(exc),active=active))
raise
finally:
save(root/'complete.json',dict(status='GATES_PASS_READY_FOR_FULL617' if reason is None else 'STOPPED',
reason=reason,guards=guards,ten_complete_sha256=sha(ten/'complete.json') if (ten/'complete.json').exists() else None,
additional_synthetic_CPU_s=extra,cumulative_worker_CPU_s=cumulative() if terminal else None,
account_complete=active is None,full617_launched=False,no_solver_switch=True,labels_read=False,
chain_CPU_s=time.process_time()))
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))