"""16 guarded workers, shared CPU/disk caps, exact first-50 terminal barrier.""" import argparse import fcntl import json import os from pathlib import Path import resource import signal import subprocess import sys import time import psutil from freeze_g1_components import sha from s2_e1_supervisor import supervise,save,disk_usage,stop_group def contract(root): c=json.loads((root/'contract.json').read_text()) for p,h in c['code_sha256'].items():assert sha(p)==h,p return c def guard(root,index): c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True) r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker', dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280, disk_bytes=100*2**30,max_processes=1),poll=.25) save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]))) return 0 if r['status']=='PASS' else 1 def projection(c,done): by_index={r['index']:r for r in done};observed={False:[0.,0],True:[0.,0]} for i,r in by_index.items(): cached=bool(c['cases'][i].get('reuse_preparation'));observed[cached][0]+=r['worker']['cpu_seconds'];observed[cached][1]+=c['cases'][i]['tet_nodes'] rate={key:(cpu/nodes if nodes else c['initial_cpu_seconds_per_node']) for key,(cpu,nodes) in observed.items()} spent=sum(r['worker']['cpu_seconds'] for r in done) remaining=sum(rate[bool(s.get('reuse_preparation'))]*s['tet_nodes'] for i,s in enumerate(c['cases']) if i not in by_index) return dict(projected_worker_cpu_seconds=spent+remaining,spent_worker_cpu_seconds=spent, observed_rate_seconds_per_node={str(k):v for k,v in rate.items()},method='Separate reused-complete and newly prepared observed node rates; initial rate only for unseen category') def terminate(active): for process,_ in active.values(): try: for child in psutil.Process(process.pid).children():stop_group(child.pid) except psutil.NoSuchProcess:pass def run(root): with (root/'batch.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) return run_locked(root) def run_locked(root): c=contract(root) assert not (root/'batch-started.json').exists(),'Batch already attempted; inspect receipts, do not restart' assert c['initial_projection_cpu_hours']<=60 and c['initial_projection_gib']<=100 env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') start=time.monotonic();save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'))) active={};done=[];next_index=0;stop_reason=None;checkpoint=False;peak_rss=0 def interrupted(*_): nonlocal stop_reason stop_reason='COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) try: while active or (next_index=60*3600:stop_reason='GLOBAL_CPU_LIMIT' if used>=100*2**30:stop_reason='GLOBAL_DISK_LIMIT' if len(done)==min(50,len(c['cases'])) and not active and not checkpoint and stop_reason is None: proj=projection(c,done);finished_nodes=sum(c['cases'][r['index']]['tet_nodes'] for r in done) projected_disk=used/finished_nodes*sum(s['tet_nodes'] for s in c['cases']) checkpoint=True if proj['projected_worker_cpu_seconds']>60*3600 or projected_disk>100*2**30:stop_reason='FIRST_50_PROJECTION_OVER_CAP' save(root/'first-50-checkpoint.json',dict(**proj,projected_disk_bytes=projected_disk,continue_within_cap=stop_reason is None,completed=len(done),failed=sum(r['worker']['status']!='PASS' for r in done))) print(json.dumps(dict(event='FIRST_50_CHECKPOINT',**proj,projected_disk_bytes=projected_disk,stop_reason=stop_reason)),flush=True) if stop_reason:terminate(active) save(root/'progress.json',dict(completed=len(done),active=list(active),next_index=next_index,first50_complete=checkpoint,worker_cpu_seconds=cpu, live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,elapsed_seconds=time.monotonic()-start,stop_reason=stop_reason)) if active:time.sleep(.5) finally: terminate(active) for p,log in active.values(): try:p.wait(timeout=10) except subprocess.TimeoutExpired:stop_group(p.pid);p.wait() log.close() result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) else 'STOPPED',stop_reason=stop_reason,jobs=done, not_run_indices=list(range(next_index,len(c['cases']))),worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done), guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), wall_seconds=time.monotonic()-start,peak_aggregate_rss_bytes=peak_rss,disk_bytes=disk_usage(root), failed=sum(r['worker']['status']!='PASS' for r in done),retry=False,replacement=False,test_labels_read=False) save(root/'batch-complete.json',result);print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True) if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args() sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))