Files
MeltMind/tools/supervise_s1_scoring.py

60 lines
4.0 KiB
Python

"""S1 command/receipt adapter for the existing whole-process O2 supervisor."""
import fcntl,json,os,signal,subprocess,sys,time
from pathlib import Path
from supervise_sampled_o2 import memory_tree
from run_s1_fold import sha,read,save
WALL_SECONDS=1800
MEMORY_BYTES=1342177280
def supervise(command,base,result_path,wall_limit=WALL_SECONDS,memory_limit=MEMORY_BYTES):
start=time.monotonic();peak=0;reason=None;trigger=None;samples=0;next_trace=0.;max_gap=0.;last=start;max_processes=0
with Path(str(base)+'.log').open('x') as log,Path(str(base)+'.monitor.jsonl').open('x') as trace:
proc=subprocess.Popen(command,stdout=log,stderr=subprocess.STDOUT,env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'),start_new_session=True)
while proc.poll() is None:
tick=time.monotonic();elapsed=tick-start;max_gap=max(max_gap,tick-last);last=tick
rss,hwm,processes=memory_tree(proc.pid);max_processes=max(max_processes,processes);oldpeak=peak;peak=max(peak,rss,hwm);samples+=1
if elapsed>=next_trace or peak>oldpeak:
trace.write(json.dumps(dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm,processes=processes))+'\n');trace.flush();next_trace=elapsed+1
if elapsed>wall_limit:reason='wall_budget_exceeded'
elif max(rss,hwm)>memory_limit:reason='memory_budget_exceeded'
if reason:
trigger=dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm)
os.killpg(proc.pid,signal.SIGTERM)
try:proc.wait(timeout=2)
except subprocess.TimeoutExpired:os.killpg(proc.pid,signal.SIGKILL);proc.wait()
break
time.sleep(.02)
wall=time.monotonic()-start
r=read(result_path) if result_path.exists() else None
if r:peak=max(peak,r['peak_rss_bytes'])
if reason is None and wall>wall_limit:reason='wall_budget_exceeded_after_exit'
if reason is None and peak>memory_limit:reason='memory_budget_exceeded_after_exit'
status='PASS' if reason is None and proc.returncode==0 and r and r['status']=='COMPLETE' else 'FAIL'
record=dict(status=status,pid=proc.pid,returncode=proc.returncode,stop_reason=reason,trigger=trigger,wall_seconds=wall,peak_rss_bytes=peak,
wall_budget_seconds=wall_limit,memory_budget_bytes=memory_limit,poll_interval_seconds=.02,monitor_samples=samples,max_observed_poll_gap_seconds=max_gap,
max_observed_processes=max_processes,whole_process_tree_supervised=True,producer_final_peak_available=r is not None,
result_sha256=sha(result_path) if r else None,logical_knn_queries=r['logical_knn_queries'] if r else None,tie_radius_queries=r['tie_radius_queries'] if r else None)
save(Path(str(base)+'.supervisor.json'),record);return record
def main():
out=Path(sys.argv[1]);c=read(out/'execution_contract.json')
if c['wall_seconds_per_fold']!=WALL_SECONDS or c['memory_bytes_per_fold']!=MEMORY_BYTES:raise ValueError('Budget is not the approved one')
with (out/'execution.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
with (out/'started.json').open('x') as f:json.dump(dict(contract_sha256=sha(out/'execution_contract.json'),pid=os.getpid(),started_unix=time.time()),f)
rows=[]
for fold in c['folds']:
i=fold['fold_index'];base=out/'folds'/f'{i:02d}'
if base.exists() or Path(str(base)+'.supervisor.json').exists():raise FileExistsError(base)
r=supervise([sys.executable,'tools/run_s1_fold.py',str(out),str(i)],base,base/'result.json');r['fold_index']=i;rows.append(r)
print(json.dumps(r),flush=True)
if r['status']!='PASS' and r['stop_reason'] is None:
save(out/'stopped.json',dict(status='UNEXPECTED_FAILURE',failed_fold=i,records=rows));raise SystemExit(1)
save(out/'completion.json',dict(status='COMPLETE' if all(r['status']=='PASS' for r in rows) else 'COMPLETE_WITH_BUDGET_FAILURES',records=rows))
if __name__=='__main__':main()