"""Frozen min-rank candidate registration; bounded workers, CPU, disk and failure checkpoints.""" from pathlib import Path import argparse import fcntl import json import os import resource import signal import subprocess import sys import time import psutil from g1_register_candidates import ROOT, check_contract from s2_e1_supervisor import supervise, save, disk_usage, stop_group from g1_registration_checkpoint import first_block_projection def guard(index): c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text());n=1 job=ROOT/'jobs'/f'job-{index:02d}';job.mkdir(parents=True,exist_ok=True) limit=n*c['per_pair_wall_seconds'] r=supervise([sys.executable,'tools/g1_register_candidates.py',str(index)],job,'worker', dict(total_cpu_seconds=limit+1,job_cpu_seconds=limit,job_wall_seconds=limit, tree_rss_bytes=c['per_worker_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=2),poll=.25) save(job/'guard-result.json',{'worker':r,'supervisor_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2])}) return 0 if r['status']=='PASS' else 1 def terminate_workers(active): # Guards stay alive to reap killed workers and write failure receipts. for p,_ in active.values(): try: for child in psutil.Process(p.pid).children():stop_group(child.pid) except psutil.NoSuchProcess:pass def run(): c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text()) lock=(ROOT/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (ROOT/'batch-started.json').exists():raise FileExistsError('Batch already attempted; no automatic restart') env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') save(ROOT/'batch-started.json',{'pid':os.getpid(),'started_unix':time.time(),'jobs':len(plan['jobs']),'pairs':len(plan['jobs']),'workers':c['workers'],'contract_sha256':__import__('hashlib').sha256((ROOT/'contract.json').read_bytes()).hexdigest()}) started=time.monotonic();active={};done=[];next_index=0;reason=None;peak_rss=0;first_checkpoint_done=False def interrupted(*_): nonlocal reason reason='coordinator_interrupted' signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) try: while active or (next_index.02: reason='failure_rate_over_2_percent' print(json.dumps({'event':'FINISHED','job':i,'status':result['worker']['status'],'completed':len(done)}),flush=True) if not first_checkpoint_done and len(done)==100: checkpoint_rows=[] for record in sorted(done,key=lambda r:r['index']): row={'status':record['worker']['status'],'cpu_seconds':record['worker']['cpu_seconds']} path=ROOT/'custodian/results'/f"pair-{record['index']:04d}.json" if path.exists(): data=json.loads(path.read_text());row.update(max_starts_per_branch=data['max_starts_per_branch'],geometry_load_wall_seconds=data['geometry_load_wall_seconds']) checkpoint_rows.append(row) checkpoint=first_block_projection(checkpoint_rows,len(plan['jobs'])) save(ROOT/'public/first-100-checkpoint.json',checkpoint);first_checkpoint_done=True if checkpoint['decision']=='STOP_AND_REPORT':reason='first_100_projection_stop' print(json.dumps({'event':'FIRST_100_CHECKPOINT',**checkpoint}),flush=True) # Evaluate failures at every 100 terminal jobs and at the final partial batch. if done and len(done)==len(plan['jobs']): if sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:reason='failure_rate_over_2_percent' if time.monotonic()-started>=c['wall_seconds']:reason='global_wall_limit' if disk_usage(ROOT)>=c['disk_bytes']:reason='global_disk_limit' live_cpu=live_rss=0 for p,_ in active.values(): try: for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True): try: t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss except psutil.NoSuchProcess:pass except psutil.NoSuchProcess:pass peak_rss=max(peak_rss,live_rss) completed_cpu=sum(r['worker']['cpu_seconds']+r['supervisor_cpu_seconds'] for r in done) if completed_cpu+live_cpu>=c['worker_cpu_budget_seconds']:reason='global_cpu_limit' if live_rss>=c['aggregate_rss_bytes']:reason='aggregate_rss_limit' if reason:terminate_workers(active) save(ROOT/'progress.json',{'completed_jobs':len(done),'active_jobs':list(active),'next_job':next_index, 'completed_worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done), 'live_tree_cpu_seconds':live_cpu,'active_tree_rss_bytes':live_rss,'stop_reason':reason,'elapsed_seconds':time.monotonic()-started}) if active:time.sleep(.5) finally: terminate_workers(active) for p,log in active.values(): try:p.wait(timeout=10) except subprocess.TimeoutExpired:stop_group(p.pid);p.wait() log.close() failures=sum(r['worker']['status']!='PASS' for r in done) passed=reason is None and len(done)==len(plan['jobs']) and failures==0 summary={'status':'ALL_REGISTRATIONS_COMPLETE' if passed else 'REGISTRATION_INCOMPLETE', 'jobs':done,'failed_jobs':failures,'failure_rate_at_final_checkpoint':failures/len(done) if done else None, 'stop_reason':reason,'worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done), 'guard_cpu_seconds':sum(r['supervisor_cpu_seconds'] for r in done), 'coordinator_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), 'wall_seconds':time.monotonic()-started,'peak_aggregate_tree_rss_bytes':peak_rss, 'disk_bytes':disk_usage(ROOT),'thresholds_frozen_before_registration':True,'candidate_generation_completed':True,'no_retries':True} save(ROOT/'batch-complete.json' if passed else ROOT/'batch-incomplete.json',summary) print(json.dumps({k:v for k,v in summary.items() if k!='jobs'}),flush=True) return 0 if passed else 1 if __name__=='__main__': ap=argparse.ArgumentParser();ap.add_argument('--guard',type=int);a=ap.parse_args() sys.exit(guard(a.guard) if a.guard is not None else run())