"""Approved 131 geometry-only attempts; independent stage CPU and cumulative disk.""" import argparse import fcntl import json import math import os from pathlib import Path import resource import signal import subprocess import sys import time from freeze_g1_components import sha from s2_recovery_supervisor import supervise,save,tree_usage from s2_recovery_monitoring import disk_usage,Monitor,budget_reason from run_s2_pilot_preparation import terminate def contract(root): c=json.loads((root/'contract.json').read_text()) for p,h in c['code_sha256'].items():assert sha(p)==h,p for p,h in c['source_sha256'].items():assert sha(p)==h,p assert len(c['cases'])==131 and len({s['case'] for s in c['cases']})==131 for spec in c['cases']: a=spec['attempt'] assert a['decode_labels'] is False and a['retries']==0 expected=3600 if a['kind']=='labelled_timeout_repreparation' else 1800 assert a['wall_limit_seconds']==expected and a['tree_rss_bytes']==1342177280 roots=[Path(p).resolve() for p in c['budget_roots']] assert all(a!=b and a not in b.parents and b not in a.parents for i,a in enumerate(roots) for b in roots[i+1:]) return c def guard(root,index): c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True) wall=c['cases'][index]['attempt']['wall_limit_seconds'] r=supervise([sys.executable,'tools/s2_main_prepare.py',str(root),str(index)],job,'worker', dict(total_cpu_seconds=wall+1,job_cpu_seconds=wall,job_wall_seconds=wall,tree_rss_bytes=1342177280, disk_bytes=100*2**30,max_processes=1),poll=.25) monitor=Monitor(job/'accounting-errors.jsonl') monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), attempt=c['cases'][index]['attempt'],label_arrays_decoded=[])),.25) return 0 if r['status']=='PASS' else 1 def run(root): with (root/'batch.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root) def run_locked(root): c=contract(root) assert not (root/'batch-started.json').exists(),'One main preparation batch only; never restart' assert c['limits']['workers']==16 assert c['limits']['worker_cpu_hours']==20 and c['limits']['disk_bytes']==100*2**30 monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic() save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0)), limits=c['limits'],labels_decoded=False)) active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') def interrupted(*_): nonlocal reason reason='MAIN_PREPARATION_COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) def read_account(path): r=json.loads(path.read_text());w=r['worker'] assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0 assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0 assert isinstance(w['returncode'],int) and isinstance(w['status'],str) assert 'stop_reason' in w return r try: while active or (next_indexc['limits']['worker_cpu_hours']*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT' if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT' result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) and reason is None else 'STOPPED',stop_reason=reason,jobs=done, not_run_indices=list(range(next_index,len(c['cases']))),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent, cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done), coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start, peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done), monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False) monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5) print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True) if result['status']=='ATTEMPTS_COMPLETE': from freeze_s2_main_roster import freeze freeze(root) return 0 if result['status']=='ATTEMPTS_COMPLETE' else 1 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args() sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))