Files
MeltMind/tools/supervise_o2_rep_safe.py
T

49 lines
2.5 KiB
Python

"""Post-run fix for empty /proc memory fields at process exit. Original frozen supervisor is preserved separately."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import subprocess
import sys
import time
def status_peak_bytes(lines):
return max((int(x.split()[1])*1024 for x in lines if x.startswith(('VmRSS:','VmHWM:'))),default=0)
def main():
p=argparse.ArgumentParser();p.add_argument('output',type=Path);p.add_argument('case_index',type=int);p.add_argument('pose');a=p.parse_args()
c=json.loads((a.output/'execution_contract.json').read_text());s=c['cases'][a.case_index]
lock=(a.output/'execution.lock').open('w');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
receipt=a.output/(s['case']+'_'+a.pose+'.supervisor.json')
if receipt.exists():raise FileExistsError(receipt)
log=(a.output/(s['case']+'_'+a.pose+'.log')).open('x')
start=time.monotonic();peak=0;reason=None
proc=subprocess.Popen([sys.executable,'tools/run_o2_rep.py',str(a.output),str(a.case_index),a.pose],stdout=log,stderr=subprocess.STDOUT,env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'))
while proc.poll() is None:
elapsed=time.monotonic()-start
try:
lines=Path(f'/proc/{proc.pid}/status').read_text().splitlines()
rss=status_peak_bytes(lines);peak=max(peak,rss)
except (FileNotFoundError,StopIteration):rss=0
if elapsed>s['wall_seconds']:reason='wall_reservation_exceeded'
elif rss>s['memory_bytes']:reason='RSS_reservation_exceeded'
if reason:
proc.terminate()
try:proc.wait(timeout=2)
except subprocess.TimeoutExpired:proc.kill();proc.wait()
break
time.sleep(.02)
result=a.output/(s['case']+'_'+a.pose)/'result.json'
if result.exists():peak=max(peak,json.loads(result.read_text())['peak_rss_kib']*1024)
if reason is None and time.monotonic()-start>s['wall_seconds']:reason='wall_reservation_exceeded'
if reason is None and peak>s['memory_bytes']:reason='RSS_reservation_exceeded'
log.close();record=dict(case=s['case'],pose=a.pose,pid=proc.pid,wall_seconds=time.monotonic()-start,observed_peak_RSS_bytes=peak,returncode=proc.returncode,stop_reason=reason,reservation_seconds=s['wall_seconds'],reservation_memory_bytes=s['memory_bytes'])
receipt.write_text(json.dumps(record,indent=2)+'\n');print(json.dumps(record,indent=2),flush=True)
if reason or proc.returncode:sys.exit(1)
if __name__=='__main__':main()