Files
MeltMind/tools/supervise_o2_rep.py
T

45 lines
2.3 KiB
Python

"""External wall-clock/RSS guard; each Type A pose is its own field."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import subprocess
import sys
import time
def main():
p=argparse.ArgumentParser();p.add_argument('output',type=Path);p.add_argument('case_index',type=int);p.add_argument('pose');a=p.parse_args()
c=json.loads((a.output/'execution_contract.json').read_text());s=c['cases'][a.case_index]
lock=(a.output/'execution.lock').open('w');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
receipt=a.output/(s['case']+'_'+a.pose+'.supervisor.json')
if receipt.exists():raise FileExistsError(receipt)
log=(a.output/(s['case']+'_'+a.pose+'.log')).open('x')
start=time.monotonic();peak=0;reason=None
proc=subprocess.Popen([sys.executable,'tools/run_o2_rep.py',str(a.output),str(a.case_index),a.pose],stdout=log,stderr=subprocess.STDOUT,env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'))
while proc.poll() is None:
elapsed=time.monotonic()-start
try:
lines=Path(f'/proc/{proc.pid}/status').read_text().splitlines()
rss=max(int(x.split()[1])*1024 for x in lines if x.startswith(('VmRSS:','VmHWM:')));peak=max(peak,rss)
except (FileNotFoundError,StopIteration):rss=0
if elapsed>s['wall_seconds']:reason='wall_reservation_exceeded'
elif rss>s['memory_bytes']:reason='RSS_reservation_exceeded'
if reason:
proc.terminate()
try:proc.wait(timeout=2)
except subprocess.TimeoutExpired:proc.kill();proc.wait()
break
time.sleep(.02)
result=a.output/(s['case']+'_'+a.pose)/'result.json'
if result.exists():peak=max(peak,json.loads(result.read_text())['peak_rss_kib']*1024)
if reason is None and time.monotonic()-start>s['wall_seconds']:reason='wall_reservation_exceeded'
if reason is None and peak>s['memory_bytes']:reason='RSS_reservation_exceeded'
log.close();record=dict(case=s['case'],pose=a.pose,pid=proc.pid,wall_seconds=time.monotonic()-start,observed_peak_RSS_bytes=peak,returncode=proc.returncode,stop_reason=reason,reservation_seconds=s['wall_seconds'],reservation_memory_bytes=s['memory_bytes'])
receipt.write_text(json.dumps(record,indent=2)+'\n');print(json.dumps(record,indent=2),flush=True)
if reason or proc.returncode:sys.exit(1)
if __name__=='__main__':main()