Files
MeltMind/tools/run_g1_registered_candidates.py
T

118 lines
7.7 KiB
Python

"""Frozen min-rank candidate registration; bounded workers, CPU, disk and failure checkpoints."""
from pathlib import Path
import argparse
import fcntl
import json
import os
import resource
import signal
import subprocess
import sys
import time
import psutil
from g1_register_candidates import ROOT, check_contract
from s2_e1_supervisor import supervise, save, disk_usage, stop_group
from g1_registration_checkpoint import first_block_projection
def guard(index):
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text());n=1
job=ROOT/'jobs'/f'job-{index:02d}';job.mkdir(parents=True,exist_ok=True)
limit=n*c['per_pair_wall_seconds']
r=supervise([sys.executable,'tools/g1_register_candidates.py',str(index)],job,'worker',
dict(total_cpu_seconds=limit+1,job_cpu_seconds=limit,job_wall_seconds=limit,
tree_rss_bytes=c['per_worker_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=2),poll=.25)
save(job/'guard-result.json',{'worker':r,'supervisor_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2])})
return 0 if r['status']=='PASS' else 1
def terminate_workers(active):
# Guards stay alive to reap killed workers and write failure receipts.
for p,_ in active.values():
try:
for child in psutil.Process(p.pid).children():stop_group(child.pid)
except psutil.NoSuchProcess:pass
def run():
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text())
lock=(ROOT/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (ROOT/'batch-started.json').exists():raise FileExistsError('Batch already attempted; no automatic restart')
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
save(ROOT/'batch-started.json',{'pid':os.getpid(),'started_unix':time.time(),'jobs':len(plan['jobs']),'pairs':len(plan['jobs']),'workers':c['workers'],'contract_sha256':__import__('hashlib').sha256((ROOT/'contract.json').read_bytes()).hexdigest()})
started=time.monotonic();active={};done=[];next_index=0;reason=None;peak_rss=0;first_checkpoint_done=False
def interrupted(*_):
nonlocal reason
reason='coordinator_interrupted'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
try:
while active or (next_index<len(plan['jobs']) and reason is None):
while reason is None and len(active)<c['workers'] and next_index<len(plan['jobs']) and (first_checkpoint_done or next_index<100):
i=next_index;next_index+=1;path=ROOT/'jobs'/f'job-{i:02d}';path.mkdir(parents=True,exist_ok=True)
log=(path/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[i]=(p,log)
print(json.dumps({'event':'STARTED','job':i,'guard_pid':p.pid}),flush=True)
for i,(p,log) in list(active.items()):
code=p.poll()
if code is not None:
log.close();path=ROOT/'jobs'/f'job-{i:02d}'/'guard-result.json'
result=json.loads(path.read_text()) if path.exists() else {'worker':{'status':'GUARD_FAILED','cpu_seconds':0},'supervisor_cpu_seconds':0}
done.append({'index':i,'returncode':code,**result});del active[i]
if len(done)%100==0 and sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:
reason='failure_rate_over_2_percent'
print(json.dumps({'event':'FINISHED','job':i,'status':result['worker']['status'],'completed':len(done)}),flush=True)
if not first_checkpoint_done and len(done)==100:
checkpoint_rows=[]
for record in sorted(done,key=lambda r:r['index']):
row={'status':record['worker']['status'],'cpu_seconds':record['worker']['cpu_seconds']}
path=ROOT/'custodian/results'/f"pair-{record['index']:04d}.json"
if path.exists():
data=json.loads(path.read_text());row.update(max_starts_per_branch=data['max_starts_per_branch'],geometry_load_wall_seconds=data['geometry_load_wall_seconds'])
checkpoint_rows.append(row)
checkpoint=first_block_projection(checkpoint_rows,len(plan['jobs']))
save(ROOT/'public/first-100-checkpoint.json',checkpoint);first_checkpoint_done=True
if checkpoint['decision']=='STOP_AND_REPORT':reason='first_100_projection_stop'
print(json.dumps({'event':'FIRST_100_CHECKPOINT',**checkpoint}),flush=True)
# Evaluate failures at every 100 terminal jobs and at the final partial batch.
if done and len(done)==len(plan['jobs']):
if sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:reason='failure_rate_over_2_percent'
if time.monotonic()-started>=c['wall_seconds']:reason='global_wall_limit'
if disk_usage(ROOT)>=c['disk_bytes']:reason='global_disk_limit'
live_cpu=live_rss=0
for p,_ in active.values():
try:
for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True):
try:
t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss
except psutil.NoSuchProcess:pass
except psutil.NoSuchProcess:pass
peak_rss=max(peak_rss,live_rss)
completed_cpu=sum(r['worker']['cpu_seconds']+r['supervisor_cpu_seconds'] for r in done)
if completed_cpu+live_cpu>=c['worker_cpu_budget_seconds']:reason='global_cpu_limit'
if live_rss>=c['aggregate_rss_bytes']:reason='aggregate_rss_limit'
if reason:terminate_workers(active)
save(ROOT/'progress.json',{'completed_jobs':len(done),'active_jobs':list(active),'next_job':next_index,
'completed_worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
'live_tree_cpu_seconds':live_cpu,'active_tree_rss_bytes':live_rss,'stop_reason':reason,'elapsed_seconds':time.monotonic()-started})
if active:time.sleep(.5)
finally:
terminate_workers(active)
for p,log in active.values():
try:p.wait(timeout=10)
except subprocess.TimeoutExpired:stop_group(p.pid);p.wait()
log.close()
failures=sum(r['worker']['status']!='PASS' for r in done)
passed=reason is None and len(done)==len(plan['jobs']) and failures==0
summary={'status':'ALL_REGISTRATIONS_COMPLETE' if passed else 'REGISTRATION_INCOMPLETE',
'jobs':done,'failed_jobs':failures,'failure_rate_at_final_checkpoint':failures/len(done) if done else None,
'stop_reason':reason,'worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
'guard_cpu_seconds':sum(r['supervisor_cpu_seconds'] for r in done),
'coordinator_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
'wall_seconds':time.monotonic()-started,'peak_aggregate_tree_rss_bytes':peak_rss,
'disk_bytes':disk_usage(ROOT),'thresholds_frozen_before_registration':True,'candidate_generation_completed':True,'no_retries':True}
save(ROOT/'batch-complete.json' if passed else ROOT/'batch-incomplete.json',summary)
print(json.dumps({k:v for k,v in summary.items() if k!='jobs'}),flush=True)
return 0 if passed else 1
if __name__=='__main__':
ap=argparse.ArgumentParser();ap.add_argument('--guard',type=int);a=ap.parse_args()
sys.exit(guard(a.guard) if a.guard is not None else run())