76 lines
4.3 KiB
Python
76 lines
4.3 KiB
Python
"""Wait for the sole ten terminal, then supervised reference and cost gates."""
|
|
import argparse
|
|
import fcntl
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import sys
|
|
import time
|
|
import psutil
|
|
from s2_recovery_supervisor import save,supervise,stop_group
|
|
from s3b_runtime import read,sha
|
|
from s3b_v04_gates import verify_gate_config
|
|
|
|
|
|
def run(root):
|
|
lock=(root/'execution.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
c=verify_gate_config(root);ten=Path(c['ten_root']);external=Path(c['stage_guard_root'])
|
|
if (root/'started.json').exists():raise FileExistsError('Never restart gate chain')
|
|
os.sched_setaffinity(0,[30,31])
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),code_freeze=read(root/'code-freeze.json')))
|
|
stopped=False
|
|
def stop(*_):
|
|
nonlocal stopped
|
|
stopped=True
|
|
for p in psutil.Process(os.getpid()).children():stop_group(p.pid)
|
|
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
|
|
guards=[];reason=None;terminal=None;active=None
|
|
extra=c['additional_synthetic_CPU_s']
|
|
def cumulative():return terminal['cumulative_worker_CPU_s']+extra+sum(g['cpu_seconds'] for g in guards)
|
|
try:
|
|
while not (ten/'complete.json').exists() or not (external/'complete.json').exists():
|
|
if stopped:raise RuntimeError('Gate chain interrupted')
|
|
if (external/'failure.json').exists():raise RuntimeError('External stage guard failed')
|
|
save(root/'progress.json',dict(stage='WAITING_ORIGINAL_TEN',unix=time.time(),new_scientific_workers=0))
|
|
time.sleep(5)
|
|
terminal=read(ten/'complete.json');ext=read(external/'complete.json')
|
|
if terminal['status']!='TEN_PASS_AWAITING_REFERENCE_AND_COST' or ext['status']!='OBSERVED_QUEUE_TERMINAL':
|
|
reason='TEN_OR_STAGE_CAP_GATE_FAILED';return 1
|
|
for kind in ['reference','census']:
|
|
verify_gate_config(root)
|
|
if stopped:reason='GATE_CHAIN_INTERRUPTED';return 1
|
|
if cumulative()+1800+7200>72000:reason='GATE_FULL_JOB_RESERVATION';return 1
|
|
dest=root/kind;dest.mkdir(exist_ok=False);active=kind
|
|
save(root/'progress.json',dict(stage=kind.upper(),cumulative_settled_CPU_s=cumulative()))
|
|
w=supervise(['taskset','-c','16',sys.executable,'tools/s3b_v04_gates.py',str(root),kind],dest,'worker',
|
|
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,
|
|
tree_rss_bytes=4*2**30,disk_bytes=2**30,max_processes=1),poll=.2)
|
|
guards.append(w);active=None
|
|
if w['status']!='PASS' or w['monitor_errors']:reason=kind.upper()+'_GUARD_FAILED';return 1
|
|
result=read(dest/'output/result.json')
|
|
if kind=='reference' and result['status']!='PASS':reason='EXACT_REFERENCE_GATE_FAILED';return 1
|
|
if kind=='census':
|
|
terms=result['projection_terms']
|
|
projected=terms['variable_projection_CPU_s']+terms['fixed_projection_CPU_s']+cumulative()+7200
|
|
gate=dict(status='PASS' if projected<=72000 else 'FAIL',projected_total_worker_CPU_s=projected,
|
|
cap_CPU_s=72000,postfit_reserve_CPU_s=7200,cumulative_actual_CPU_s=cumulative(),
|
|
projection_terms=terms,reference_gate_sha256=sha(root/'reference/output/result.json'),
|
|
census_sha256=sha(dest/'output/result.json'),full617_launched=False)
|
|
save(root/'cost-gate.json',gate)
|
|
if gate['status']!='PASS':reason='COST_PROJECTION_GATE_FAILED';return 1
|
|
return 0
|
|
except Exception as exc:
|
|
reason='GATE_CHAIN_EXCEPTION'
|
|
save(root/'failure.json',dict(type=type(exc).__name__,reason=str(exc),active=active))
|
|
raise
|
|
finally:
|
|
save(root/'complete.json',dict(status='GATES_PASS_READY_FOR_FULL617' if reason is None else 'STOPPED',
|
|
reason=reason,guards=guards,ten_complete_sha256=sha(ten/'complete.json') if (ten/'complete.json').exists() else None,
|
|
additional_synthetic_CPU_s=extra,cumulative_worker_CPU_s=cumulative() if terminal else None,
|
|
account_complete=active is None,full617_launched=False,no_solver_switch=True,labels_read=False,
|
|
chain_CPU_s=time.process_time()))
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))
|