49 lines
2.5 KiB
Python
49 lines
2.5 KiB
Python
"""Post-run fix for empty /proc memory fields at process exit. Original frozen supervisor is preserved separately."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
|
|
def status_peak_bytes(lines):
|
|
return max((int(x.split()[1])*1024 for x in lines if x.startswith(('VmRSS:','VmHWM:'))),default=0)
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('output',type=Path);p.add_argument('case_index',type=int);p.add_argument('pose');a=p.parse_args()
|
|
c=json.loads((a.output/'execution_contract.json').read_text());s=c['cases'][a.case_index]
|
|
lock=(a.output/'execution.lock').open('w');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
receipt=a.output/(s['case']+'_'+a.pose+'.supervisor.json')
|
|
if receipt.exists():raise FileExistsError(receipt)
|
|
log=(a.output/(s['case']+'_'+a.pose+'.log')).open('x')
|
|
start=time.monotonic();peak=0;reason=None
|
|
proc=subprocess.Popen([sys.executable,'tools/run_o2_rep.py',str(a.output),str(a.case_index),a.pose],stdout=log,stderr=subprocess.STDOUT,env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'))
|
|
while proc.poll() is None:
|
|
elapsed=time.monotonic()-start
|
|
try:
|
|
lines=Path(f'/proc/{proc.pid}/status').read_text().splitlines()
|
|
rss=status_peak_bytes(lines);peak=max(peak,rss)
|
|
except (FileNotFoundError,StopIteration):rss=0
|
|
if elapsed>s['wall_seconds']:reason='wall_reservation_exceeded'
|
|
elif rss>s['memory_bytes']:reason='RSS_reservation_exceeded'
|
|
if reason:
|
|
proc.terminate()
|
|
try:proc.wait(timeout=2)
|
|
except subprocess.TimeoutExpired:proc.kill();proc.wait()
|
|
break
|
|
time.sleep(.02)
|
|
result=a.output/(s['case']+'_'+a.pose)/'result.json'
|
|
if result.exists():peak=max(peak,json.loads(result.read_text())['peak_rss_kib']*1024)
|
|
if reason is None and time.monotonic()-start>s['wall_seconds']:reason='wall_reservation_exceeded'
|
|
if reason is None and peak>s['memory_bytes']:reason='RSS_reservation_exceeded'
|
|
log.close();record=dict(case=s['case'],pose=a.pose,pid=proc.pid,wall_seconds=time.monotonic()-start,observed_peak_RSS_bytes=peak,returncode=proc.returncode,stop_reason=reason,reservation_seconds=s['wall_seconds'],reservation_memory_bytes=s['memory_bytes'])
|
|
receipt.write_text(json.dumps(record,indent=2)+'\n');print(json.dumps(record,indent=2),flush=True)
|
|
if reason or proc.returncode:sys.exit(1)
|
|
|
|
|
|
if __name__=='__main__':main()
|