60 lines
4.0 KiB
Python
60 lines
4.0 KiB
Python
"""S1 command/receipt adapter for the existing whole-process O2 supervisor."""
|
|
import fcntl,json,os,signal,subprocess,sys,time
|
|
from pathlib import Path
|
|
from supervise_sampled_o2 import memory_tree
|
|
from run_s1_fold import sha,read,save
|
|
|
|
WALL_SECONDS=1800
|
|
MEMORY_BYTES=1342177280
|
|
|
|
|
|
def supervise(command,base,result_path,wall_limit=WALL_SECONDS,memory_limit=MEMORY_BYTES):
|
|
start=time.monotonic();peak=0;reason=None;trigger=None;samples=0;next_trace=0.;max_gap=0.;last=start;max_processes=0
|
|
with Path(str(base)+'.log').open('x') as log,Path(str(base)+'.monitor.jsonl').open('x') as trace:
|
|
proc=subprocess.Popen(command,stdout=log,stderr=subprocess.STDOUT,env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'),start_new_session=True)
|
|
while proc.poll() is None:
|
|
tick=time.monotonic();elapsed=tick-start;max_gap=max(max_gap,tick-last);last=tick
|
|
rss,hwm,processes=memory_tree(proc.pid);max_processes=max(max_processes,processes);oldpeak=peak;peak=max(peak,rss,hwm);samples+=1
|
|
if elapsed>=next_trace or peak>oldpeak:
|
|
trace.write(json.dumps(dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm,processes=processes))+'\n');trace.flush();next_trace=elapsed+1
|
|
if elapsed>wall_limit:reason='wall_budget_exceeded'
|
|
elif max(rss,hwm)>memory_limit:reason='memory_budget_exceeded'
|
|
if reason:
|
|
trigger=dict(elapsed_seconds=elapsed,rss_bytes=rss,hwm_sum_bytes=hwm)
|
|
os.killpg(proc.pid,signal.SIGTERM)
|
|
try:proc.wait(timeout=2)
|
|
except subprocess.TimeoutExpired:os.killpg(proc.pid,signal.SIGKILL);proc.wait()
|
|
break
|
|
time.sleep(.02)
|
|
wall=time.monotonic()-start
|
|
r=read(result_path) if result_path.exists() else None
|
|
if r:peak=max(peak,r['peak_rss_bytes'])
|
|
if reason is None and wall>wall_limit:reason='wall_budget_exceeded_after_exit'
|
|
if reason is None and peak>memory_limit:reason='memory_budget_exceeded_after_exit'
|
|
status='PASS' if reason is None and proc.returncode==0 and r and r['status']=='COMPLETE' else 'FAIL'
|
|
record=dict(status=status,pid=proc.pid,returncode=proc.returncode,stop_reason=reason,trigger=trigger,wall_seconds=wall,peak_rss_bytes=peak,
|
|
wall_budget_seconds=wall_limit,memory_budget_bytes=memory_limit,poll_interval_seconds=.02,monitor_samples=samples,max_observed_poll_gap_seconds=max_gap,
|
|
max_observed_processes=max_processes,whole_process_tree_supervised=True,producer_final_peak_available=r is not None,
|
|
result_sha256=sha(result_path) if r else None,logical_knn_queries=r['logical_knn_queries'] if r else None,tie_radius_queries=r['tie_radius_queries'] if r else None)
|
|
save(Path(str(base)+'.supervisor.json'),record);return record
|
|
|
|
|
|
def main():
|
|
out=Path(sys.argv[1]);c=read(out/'execution_contract.json')
|
|
if c['wall_seconds_per_fold']!=WALL_SECONDS or c['memory_bytes_per_fold']!=MEMORY_BYTES:raise ValueError('Budget is not the approved one')
|
|
with (out/'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
with (out/'started.json').open('x') as f:json.dump(dict(contract_sha256=sha(out/'execution_contract.json'),pid=os.getpid(),started_unix=time.time()),f)
|
|
rows=[]
|
|
for fold in c['folds']:
|
|
i=fold['fold_index'];base=out/'folds'/f'{i:02d}'
|
|
if base.exists() or Path(str(base)+'.supervisor.json').exists():raise FileExistsError(base)
|
|
r=supervise([sys.executable,'tools/run_s1_fold.py',str(out),str(i)],base,base/'result.json');r['fold_index']=i;rows.append(r)
|
|
print(json.dumps(r),flush=True)
|
|
if r['status']!='PASS' and r['stop_reason'] is None:
|
|
save(out/'stopped.json',dict(status='UNEXPECTED_FAILURE',failed_fold=i,records=rows));raise SystemExit(1)
|
|
save(out/'completion.json',dict(status='COMPLETE' if all(r['status']=='PASS' for r in rows) else 'COMPLETE_WITH_BUDGET_FAILURES',records=rows))
|
|
|
|
|
|
if __name__=='__main__':main()
|