118 lines
7.7 KiB
Python
118 lines
7.7 KiB
Python
"""Frozen min-rank candidate registration; bounded workers, CPU, disk and failure checkpoints."""
|
|
from pathlib import Path
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import psutil
|
|
from g1_register_candidates import ROOT, check_contract
|
|
from s2_e1_supervisor import supervise, save, disk_usage, stop_group
|
|
from g1_registration_checkpoint import first_block_projection
|
|
|
|
def guard(index):
|
|
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text());n=1
|
|
job=ROOT/'jobs'/f'job-{index:02d}';job.mkdir(parents=True,exist_ok=True)
|
|
limit=n*c['per_pair_wall_seconds']
|
|
r=supervise([sys.executable,'tools/g1_register_candidates.py',str(index)],job,'worker',
|
|
dict(total_cpu_seconds=limit+1,job_cpu_seconds=limit,job_wall_seconds=limit,
|
|
tree_rss_bytes=c['per_worker_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=2),poll=.25)
|
|
save(job/'guard-result.json',{'worker':r,'supervisor_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2])})
|
|
return 0 if r['status']=='PASS' else 1
|
|
|
|
def terminate_workers(active):
|
|
# Guards stay alive to reap killed workers and write failure receipts.
|
|
for p,_ in active.values():
|
|
try:
|
|
for child in psutil.Process(p.pid).children():stop_group(child.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
|
|
def run():
|
|
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text())
|
|
lock=(ROOT/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (ROOT/'batch-started.json').exists():raise FileExistsError('Batch already attempted; no automatic restart')
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
save(ROOT/'batch-started.json',{'pid':os.getpid(),'started_unix':time.time(),'jobs':len(plan['jobs']),'pairs':len(plan['jobs']),'workers':c['workers'],'contract_sha256':__import__('hashlib').sha256((ROOT/'contract.json').read_bytes()).hexdigest()})
|
|
started=time.monotonic();active={};done=[];next_index=0;reason=None;peak_rss=0;first_checkpoint_done=False
|
|
def interrupted(*_):
|
|
nonlocal reason
|
|
reason='coordinator_interrupted'
|
|
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
|
try:
|
|
while active or (next_index<len(plan['jobs']) and reason is None):
|
|
while reason is None and len(active)<c['workers'] and next_index<len(plan['jobs']) and (first_checkpoint_done or next_index<100):
|
|
i=next_index;next_index+=1;path=ROOT/'jobs'/f'job-{i:02d}';path.mkdir(parents=True,exist_ok=True)
|
|
log=(path/'guard.log').open('x')
|
|
p=subprocess.Popen([sys.executable,__file__,'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
|
active[i]=(p,log)
|
|
print(json.dumps({'event':'STARTED','job':i,'guard_pid':p.pid}),flush=True)
|
|
for i,(p,log) in list(active.items()):
|
|
code=p.poll()
|
|
if code is not None:
|
|
log.close();path=ROOT/'jobs'/f'job-{i:02d}'/'guard-result.json'
|
|
result=json.loads(path.read_text()) if path.exists() else {'worker':{'status':'GUARD_FAILED','cpu_seconds':0},'supervisor_cpu_seconds':0}
|
|
done.append({'index':i,'returncode':code,**result});del active[i]
|
|
if len(done)%100==0 and sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:
|
|
reason='failure_rate_over_2_percent'
|
|
print(json.dumps({'event':'FINISHED','job':i,'status':result['worker']['status'],'completed':len(done)}),flush=True)
|
|
if not first_checkpoint_done and len(done)==100:
|
|
checkpoint_rows=[]
|
|
for record in sorted(done,key=lambda r:r['index']):
|
|
row={'status':record['worker']['status'],'cpu_seconds':record['worker']['cpu_seconds']}
|
|
path=ROOT/'custodian/results'/f"pair-{record['index']:04d}.json"
|
|
if path.exists():
|
|
data=json.loads(path.read_text());row.update(max_starts_per_branch=data['max_starts_per_branch'],geometry_load_wall_seconds=data['geometry_load_wall_seconds'])
|
|
checkpoint_rows.append(row)
|
|
checkpoint=first_block_projection(checkpoint_rows,len(plan['jobs']))
|
|
save(ROOT/'public/first-100-checkpoint.json',checkpoint);first_checkpoint_done=True
|
|
if checkpoint['decision']=='STOP_AND_REPORT':reason='first_100_projection_stop'
|
|
print(json.dumps({'event':'FIRST_100_CHECKPOINT',**checkpoint}),flush=True)
|
|
# Evaluate failures at every 100 terminal jobs and at the final partial batch.
|
|
if done and len(done)==len(plan['jobs']):
|
|
if sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:reason='failure_rate_over_2_percent'
|
|
if time.monotonic()-started>=c['wall_seconds']:reason='global_wall_limit'
|
|
if disk_usage(ROOT)>=c['disk_bytes']:reason='global_disk_limit'
|
|
live_cpu=live_rss=0
|
|
for p,_ in active.values():
|
|
try:
|
|
for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True):
|
|
try:
|
|
t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss
|
|
except psutil.NoSuchProcess:pass
|
|
except psutil.NoSuchProcess:pass
|
|
peak_rss=max(peak_rss,live_rss)
|
|
completed_cpu=sum(r['worker']['cpu_seconds']+r['supervisor_cpu_seconds'] for r in done)
|
|
if completed_cpu+live_cpu>=c['worker_cpu_budget_seconds']:reason='global_cpu_limit'
|
|
if live_rss>=c['aggregate_rss_bytes']:reason='aggregate_rss_limit'
|
|
if reason:terminate_workers(active)
|
|
save(ROOT/'progress.json',{'completed_jobs':len(done),'active_jobs':list(active),'next_job':next_index,
|
|
'completed_worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
|
|
'live_tree_cpu_seconds':live_cpu,'active_tree_rss_bytes':live_rss,'stop_reason':reason,'elapsed_seconds':time.monotonic()-started})
|
|
if active:time.sleep(.5)
|
|
finally:
|
|
terminate_workers(active)
|
|
for p,log in active.values():
|
|
try:p.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:stop_group(p.pid);p.wait()
|
|
log.close()
|
|
failures=sum(r['worker']['status']!='PASS' for r in done)
|
|
passed=reason is None and len(done)==len(plan['jobs']) and failures==0
|
|
summary={'status':'ALL_REGISTRATIONS_COMPLETE' if passed else 'REGISTRATION_INCOMPLETE',
|
|
'jobs':done,'failed_jobs':failures,'failure_rate_at_final_checkpoint':failures/len(done) if done else None,
|
|
'stop_reason':reason,'worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
|
|
'guard_cpu_seconds':sum(r['supervisor_cpu_seconds'] for r in done),
|
|
'coordinator_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
'wall_seconds':time.monotonic()-started,'peak_aggregate_tree_rss_bytes':peak_rss,
|
|
'disk_bytes':disk_usage(ROOT),'thresholds_frozen_before_registration':True,'candidate_generation_completed':True,'no_retries':True}
|
|
save(ROOT/'batch-complete.json' if passed else ROOT/'batch-incomplete.json',summary)
|
|
print(json.dumps({k:v for k,v in summary.items() if k!='jobs'}),flush=True)
|
|
return 0 if passed else 1
|
|
|
|
if __name__=='__main__':
|
|
ap=argparse.ArgumentParser();ap.add_argument('--guard',type=int);a=ap.parse_args()
|
|
sys.exit(guard(a.guard) if a.guard is not None else run())
|