"""Sequential sampled O2 jobs, external 20ms whole-process budget supervision.""" import argparse import fcntl import hashlib import json import os from pathlib import Path import signal import subprocess import sys import time def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest() def read(p):return json.loads(Path(p).read_text()) def save(p,v):Path(p).write_text(json.dumps(v,indent=2)+'\n') def memory_tree(pid): pending=[pid];seen=set();total_rss=0;total_hwm=0 while pending: p=pending.pop() if p in seen:continue seen.add(p) try: lines=Path(f'/proc/{p}/status').read_text().splitlines() values={line.split(':',1)[0]:int(line.split()[1])*1024 for line in lines if line.startswith(('VmRSS:','VmHWM:'))} total_rss+=values.get('VmRSS',0);total_hwm+=values.get('VmHWM',0) pending.extend(map(int,Path(f'/proc/{p}/task/{p}/children').read_text().split())) except (FileNotFoundError,ProcessLookupError):pass return total_rss,total_hwm,len(seen) def job(out,phase,index,s): base=out/phase;receipt=base/(s['job']+'.supervisor.json') if receipt.exists() or (base/s['job']).exists():raise FileExistsError(s['job']) start=time.monotonic();peak=0;reason=None;trigger=None;samples=0;next_trace=0.;max_gap=0.;last=start with (base/(s['job']+'.log')).open('x') as log,(base/(s['job']+'.monitor.jsonl')).open('x') as trace: proc=subprocess.Popen([sys.executable,'tools/run_sampled_o2.py',str(out),phase,str(index)],stdout=log,stderr=subprocess.STDOUT, env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'),start_new_session=True) while proc.poll() is None: tick=time.monotonic();elapsed=tick-start;max_gap=max(max_gap,tick-last);last=tick rss,hwm,processes=memory_tree(proc.pid);oldpeak=peak;peak=max(peak,rss,hwm);samples+=1 if elapsed>=next_trace or peak>oldpeak: trace.write(json.dumps(dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm,processes=processes))+'\n');trace.flush();next_trace=elapsed+1 if elapsed>1800:reason='wall_budget_exceeded' elif max(rss,hwm)>1342177280:reason='memory_budget_exceeded' if reason: trigger=dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm) os.killpg(proc.pid,signal.SIGTERM) try:proc.wait(timeout=2) except subprocess.TimeoutExpired:os.killpg(proc.pid,signal.SIGKILL);proc.wait() break time.sleep(.02) wall=time.monotonic()-start result_path=base/s['job']/'result.json';r=read(result_path) if result_path.exists() else None if r:peak=max(peak,r['peak_rss_bytes']) if reason is None and wall>1800:reason='wall_budget_exceeded_after_exit' if reason is None and peak>1342177280:reason='memory_budget_exceeded_after_exit' status='PASS' if reason is None and proc.returncode==0 and r and r['status']=='COMPLETE' else 'FAIL' record=dict(job=s['job'],case=s['case'],pose=s['pose'],phase=phase,pid=proc.pid,status=status, returncode=proc.returncode,stop_reason=reason,trigger=trigger,wall_seconds=wall,peak_rss_bytes=peak, wall_budget_seconds=1800,memory_budget_bytes=1342177280,poll_interval_seconds=.02, monitor_samples=samples,max_observed_poll_gap_seconds=max_gap,whole_process_tree_supervised=True, producer_final_peak_available=r is not None,result_sha256=sha(result_path) if r else None, n_ref=r['n_ref'] if status=='PASS' else None,eligible_draw_count=r['eligible_draw_count'] if status=='PASS' else None, partial_result_available=r is not None) save(receipt,record);print(json.dumps(record),flush=True) return record def main(): p=argparse.ArgumentParser();p.add_argument('output',type=Path);p.add_argument('phase',choices=['equivalence','panel']);a=p.parse_args() out=a.output;c=read(out/'execution_contract.json') with (out/'execution.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) started=out/(a.phase+'.started.json') with started.open('x') as f:json.dump(dict(contract_sha256=sha(out/'execution_contract.json'),phase=a.phase,pid=os.getpid(),started_unix=time.time()),f) if a.phase=='panel':assert read(out/'equivalence_gate.json')['status']=='PASS' rows=[] for i,s in enumerate(c[a.phase]): r=job(out,a.phase,i,s);rows.append(r) if r['status']!='PASS' and (a.phase=='equivalence' or r['stop_reason'] is None): save(out/(a.phase+'.stopped.json'),dict(reason='Equivalence or unexpected producer failure; no further jobs launched',failed_job=s['job'],records=rows)) if a.phase=='equivalence':save(out/'equivalence_gate.json',dict(status='FAIL',failed_job=s['job'],records=rows)) raise SystemExit(1) save(out/(a.phase+'.completion.json'),dict(status='COMPLETE',records=rows)) if a.phase=='equivalence': save(out/'equivalence_gate.json',dict(status='PASS',fields=len(rows),result_sha256={r['job']:r['result_sha256'] for r in rows},code_sha256=c['code_sha256'])) if __name__=='__main__':main()