"""Four guarded calibration workers; all CPU/failures retained, no retries.""" from pathlib import Path import argparse import fcntl import json import os import resource import signal import subprocess import sys import time import psutil from g1_copy_calibration import ROOT, check_contract from s2_e1_supervisor import supervise, save, disk_usage, stop_group def guard(index): c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text());n=len(plan['jobs'][index]['pairs']) job=ROOT/'jobs'/f'job-{index:02d}';job.mkdir(parents=True,exist_ok=True) limit=n*c['per_pair_wall_seconds'] r=supervise([sys.executable,'tools/g1_copy_calibration.py',str(index)],job,'worker', dict(total_cpu_seconds=limit+1,job_cpu_seconds=limit,job_wall_seconds=limit, tree_rss_bytes=c['per_worker_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=2),poll=.25) save(job/'guard-result.json',{'worker':r,'supervisor_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2])}) return 0 if r['status']=='PASS' else 1 def terminate_workers(active): # Guards stay alive to reap killed workers and write failure receipts. for p,_ in active.values(): try: for child in psutil.Process(p.pid).children():stop_group(child.pid) except psutil.NoSuchProcess:pass def run(): c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text()) lock=(ROOT/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (ROOT/'batch-started.json').exists():raise FileExistsError('Batch already attempted; no automatic restart') env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') save(ROOT/'batch-started.json',{'pid':os.getpid(),'started_unix':time.time(),'jobs':len(plan['jobs']),'pairs':sum(len(j['pairs']) for j in plan['jobs']),'workers':c['workers'],'contract_sha256':__import__('hashlib').sha256((ROOT/'contract.json').read_bytes()).hexdigest()}) started=time.monotonic();active={};done=[];next_index=0;reason=None;peak_rss=0 def interrupted(*_): nonlocal reason reason='coordinator_interrupted' signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) try: while active or (next_index=c['wall_seconds']:reason='global_wall_limit' if disk_usage(ROOT)>=c['disk_bytes']:reason='global_disk_limit' live_cpu=live_rss=0 for p,_ in active.values(): try: for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True): try: t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss except psutil.NoSuchProcess:pass except psutil.NoSuchProcess:pass peak_rss=max(peak_rss,live_rss) completed_cpu=sum(r['worker']['cpu_seconds']+r['supervisor_cpu_seconds'] for r in done) if completed_cpu+live_cpu>=c['worker_cpu_budget_seconds']:reason='global_cpu_limit' if live_rss>=c['aggregate_rss_bytes']:reason='aggregate_rss_limit' if reason:terminate_workers(active) save(ROOT/'progress.json',{'completed_jobs':len(done),'active_jobs':list(active),'next_job':next_index, 'completed_worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done), 'live_tree_cpu_seconds':live_cpu,'active_tree_rss_bytes':live_rss,'stop_reason':reason,'elapsed_seconds':time.monotonic()-started}) if active:time.sleep(.5) finally: terminate_workers(active) for p,log in active.values(): try:p.wait(timeout=10) except subprocess.TimeoutExpired:stop_group(p.pid);p.wait() log.close() failures=sum(r['worker']['status']!='PASS' for r in done) passed=reason is None and len(done)==len(plan['jobs']) and failures==0 summary={'status':'CALIBRATION_RESIDUALS_COMPLETE' if passed else 'CALIBRATION_INCOMPLETE', 'jobs':done,'failed_jobs':failures,'failure_rate_at_final_checkpoint':failures/len(done) if done else None, 'stop_reason':reason,'worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done), 'guard_cpu_seconds':sum(r['supervisor_cpu_seconds'] for r in done), 'coordinator_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), 'wall_seconds':time.monotonic()-started,'peak_aggregate_tree_rss_bytes':peak_rss, 'disk_bytes':disk_usage(ROOT),'thresholds_frozen':False,'candidate_generation':False,'no_retries':True} save(ROOT/'batch-complete.json' if passed else ROOT/'batch-incomplete.json',summary) print(json.dumps({k:v for k,v in summary.items() if k!='jobs'}),flush=True) return 0 if passed else 1 if __name__=='__main__': ap=argparse.ArgumentParser();ap.add_argument('--guard',type=int);a=ap.parse_args() sys.exit(guard(a.guard) if a.guard is not None else run())