93 lines
5.2 KiB
Python
93 lines
5.2 KiB
Python
"""Sequential sampled O2 jobs, external 20ms whole-process budget supervision."""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
|
|
def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
def save(p,v):Path(p).write_text(json.dumps(v,indent=2)+'\n')
|
|
|
|
|
|
def memory_tree(pid):
|
|
pending=[pid];seen=set();total_rss=0;total_hwm=0
|
|
while pending:
|
|
p=pending.pop()
|
|
if p in seen:continue
|
|
seen.add(p)
|
|
try:
|
|
lines=Path(f'/proc/{p}/status').read_text().splitlines()
|
|
values={line.split(':',1)[0]:int(line.split()[1])*1024 for line in lines if line.startswith(('VmRSS:','VmHWM:'))}
|
|
total_rss+=values.get('VmRSS',0);total_hwm+=values.get('VmHWM',0)
|
|
pending.extend(map(int,Path(f'/proc/{p}/task/{p}/children').read_text().split()))
|
|
except (FileNotFoundError,ProcessLookupError):pass
|
|
return total_rss,total_hwm,len(seen)
|
|
|
|
|
|
def job(out,phase,index,s):
|
|
base=out/phase;receipt=base/(s['job']+'.supervisor.json')
|
|
if receipt.exists() or (base/s['job']).exists():raise FileExistsError(s['job'])
|
|
start=time.monotonic();peak=0;reason=None;trigger=None;samples=0;next_trace=0.;max_gap=0.;last=start
|
|
with (base/(s['job']+'.log')).open('x') as log,(base/(s['job']+'.monitor.jsonl')).open('x') as trace:
|
|
proc=subprocess.Popen([sys.executable,'tools/run_sampled_o2.py',str(out),phase,str(index)],stdout=log,stderr=subprocess.STDOUT,
|
|
env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'),start_new_session=True)
|
|
while proc.poll() is None:
|
|
tick=time.monotonic();elapsed=tick-start;max_gap=max(max_gap,tick-last);last=tick
|
|
rss,hwm,processes=memory_tree(proc.pid);oldpeak=peak;peak=max(peak,rss,hwm);samples+=1
|
|
if elapsed>=next_trace or peak>oldpeak:
|
|
trace.write(json.dumps(dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm,processes=processes))+'\n');trace.flush();next_trace=elapsed+1
|
|
if elapsed>1800:reason='wall_budget_exceeded'
|
|
elif max(rss,hwm)>1342177280:reason='memory_budget_exceeded'
|
|
if reason:
|
|
trigger=dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm)
|
|
os.killpg(proc.pid,signal.SIGTERM)
|
|
try:proc.wait(timeout=2)
|
|
except subprocess.TimeoutExpired:os.killpg(proc.pid,signal.SIGKILL);proc.wait()
|
|
break
|
|
time.sleep(.02)
|
|
wall=time.monotonic()-start
|
|
result_path=base/s['job']/'result.json';r=read(result_path) if result_path.exists() else None
|
|
if r:peak=max(peak,r['peak_rss_bytes'])
|
|
if reason is None and wall>1800:reason='wall_budget_exceeded_after_exit'
|
|
if reason is None and peak>1342177280:reason='memory_budget_exceeded_after_exit'
|
|
status='PASS' if reason is None and proc.returncode==0 and r and r['status']=='COMPLETE' else 'FAIL'
|
|
record=dict(job=s['job'],case=s['case'],pose=s['pose'],phase=phase,pid=proc.pid,status=status,
|
|
returncode=proc.returncode,stop_reason=reason,trigger=trigger,wall_seconds=wall,peak_rss_bytes=peak,
|
|
wall_budget_seconds=1800,memory_budget_bytes=1342177280,poll_interval_seconds=.02,
|
|
monitor_samples=samples,max_observed_poll_gap_seconds=max_gap,whole_process_tree_supervised=True,
|
|
producer_final_peak_available=r is not None,result_sha256=sha(result_path) if r else None,
|
|
n_ref=r['n_ref'] if status=='PASS' else None,eligible_draw_count=r['eligible_draw_count'] if status=='PASS' else None,
|
|
partial_result_available=r is not None)
|
|
save(receipt,record);print(json.dumps(record),flush=True)
|
|
return record
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('output',type=Path);p.add_argument('phase',choices=['equivalence','panel']);a=p.parse_args()
|
|
out=a.output;c=read(out/'execution_contract.json')
|
|
with (out/'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
started=out/(a.phase+'.started.json')
|
|
with started.open('x') as f:json.dump(dict(contract_sha256=sha(out/'execution_contract.json'),phase=a.phase,pid=os.getpid(),started_unix=time.time()),f)
|
|
if a.phase=='panel':assert read(out/'equivalence_gate.json')['status']=='PASS'
|
|
rows=[]
|
|
for i,s in enumerate(c[a.phase]):
|
|
r=job(out,a.phase,i,s);rows.append(r)
|
|
if r['status']!='PASS' and (a.phase=='equivalence' or r['stop_reason'] is None):
|
|
save(out/(a.phase+'.stopped.json'),dict(reason='Equivalence or unexpected producer failure; no further jobs launched',failed_job=s['job'],records=rows))
|
|
if a.phase=='equivalence':save(out/'equivalence_gate.json',dict(status='FAIL',failed_job=s['job'],records=rows))
|
|
raise SystemExit(1)
|
|
save(out/(a.phase+'.completion.json'),dict(status='COMPLETE',records=rows))
|
|
if a.phase=='equivalence':
|
|
save(out/'equivalence_gate.json',dict(status='PASS',fields=len(rows),result_sha256={r['job']:r['result_sha256'] for r in rows},code_sha256=c['code_sha256']))
|
|
|
|
|
|
if __name__=='__main__':main()
|