Files
MeltMind/tools/supervise_sampled_o2.py

93 lines
5.2 KiB
Python

"""Sequential sampled O2 jobs, external 20ms whole-process budget supervision."""
import argparse
import fcntl
import hashlib
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
def read(p):return json.loads(Path(p).read_text())
def save(p,v):Path(p).write_text(json.dumps(v,indent=2)+'\n')
def memory_tree(pid):
pending=[pid];seen=set();total_rss=0;total_hwm=0
while pending:
p=pending.pop()
if p in seen:continue
seen.add(p)
try:
lines=Path(f'/proc/{p}/status').read_text().splitlines()
values={line.split(':',1)[0]:int(line.split()[1])*1024 for line in lines if line.startswith(('VmRSS:','VmHWM:'))}
total_rss+=values.get('VmRSS',0);total_hwm+=values.get('VmHWM',0)
pending.extend(map(int,Path(f'/proc/{p}/task/{p}/children').read_text().split()))
except (FileNotFoundError,ProcessLookupError):pass
return total_rss,total_hwm,len(seen)
def job(out,phase,index,s):
base=out/phase;receipt=base/(s['job']+'.supervisor.json')
if receipt.exists() or (base/s['job']).exists():raise FileExistsError(s['job'])
start=time.monotonic();peak=0;reason=None;trigger=None;samples=0;next_trace=0.;max_gap=0.;last=start
with (base/(s['job']+'.log')).open('x') as log,(base/(s['job']+'.monitor.jsonl')).open('x') as trace:
proc=subprocess.Popen([sys.executable,'tools/run_sampled_o2.py',str(out),phase,str(index)],stdout=log,stderr=subprocess.STDOUT,
env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'),start_new_session=True)
while proc.poll() is None:
tick=time.monotonic();elapsed=tick-start;max_gap=max(max_gap,tick-last);last=tick
rss,hwm,processes=memory_tree(proc.pid);oldpeak=peak;peak=max(peak,rss,hwm);samples+=1
if elapsed>=next_trace or peak>oldpeak:
trace.write(json.dumps(dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm,processes=processes))+'\n');trace.flush();next_trace=elapsed+1
if elapsed>1800:reason='wall_budget_exceeded'
elif max(rss,hwm)>1342177280:reason='memory_budget_exceeded'
if reason:
trigger=dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm)
os.killpg(proc.pid,signal.SIGTERM)
try:proc.wait(timeout=2)
except subprocess.TimeoutExpired:os.killpg(proc.pid,signal.SIGKILL);proc.wait()
break
time.sleep(.02)
wall=time.monotonic()-start
result_path=base/s['job']/'result.json';r=read(result_path) if result_path.exists() else None
if r:peak=max(peak,r['peak_rss_bytes'])
if reason is None and wall>1800:reason='wall_budget_exceeded_after_exit'
if reason is None and peak>1342177280:reason='memory_budget_exceeded_after_exit'
status='PASS' if reason is None and proc.returncode==0 and r and r['status']=='COMPLETE' else 'FAIL'
record=dict(job=s['job'],case=s['case'],pose=s['pose'],phase=phase,pid=proc.pid,status=status,
returncode=proc.returncode,stop_reason=reason,trigger=trigger,wall_seconds=wall,peak_rss_bytes=peak,
wall_budget_seconds=1800,memory_budget_bytes=1342177280,poll_interval_seconds=.02,
monitor_samples=samples,max_observed_poll_gap_seconds=max_gap,whole_process_tree_supervised=True,
producer_final_peak_available=r is not None,result_sha256=sha(result_path) if r else None,
n_ref=r['n_ref'] if status=='PASS' else None,eligible_draw_count=r['eligible_draw_count'] if status=='PASS' else None,
partial_result_available=r is not None)
save(receipt,record);print(json.dumps(record),flush=True)
return record
def main():
p=argparse.ArgumentParser();p.add_argument('output',type=Path);p.add_argument('phase',choices=['equivalence','panel']);a=p.parse_args()
out=a.output;c=read(out/'execution_contract.json')
with (out/'execution.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
started=out/(a.phase+'.started.json')
with started.open('x') as f:json.dump(dict(contract_sha256=sha(out/'execution_contract.json'),phase=a.phase,pid=os.getpid(),started_unix=time.time()),f)
if a.phase=='panel':assert read(out/'equivalence_gate.json')['status']=='PASS'
rows=[]
for i,s in enumerate(c[a.phase]):
r=job(out,a.phase,i,s);rows.append(r)
if r['status']!='PASS' and (a.phase=='equivalence' or r['stop_reason'] is None):
save(out/(a.phase+'.stopped.json'),dict(reason='Equivalence or unexpected producer failure; no further jobs launched',failed_job=s['job'],records=rows))
if a.phase=='equivalence':save(out/'equivalence_gate.json',dict(status='FAIL',failed_job=s['job'],records=rows))
raise SystemExit(1)
save(out/(a.phase+'.completion.json'),dict(status='COMPLETE',records=rows))
if a.phase=='equivalence':
save(out/'equivalence_gate.json',dict(status='PASS',fields=len(rows),result_sha256={r['job']:r['result_sha256'] for r in rows},code_sha256=c['code_sha256']))
if __name__=='__main__':main()