Files
MeltMind/tools/supervise_s1_panel.py

47 lines
2.7 KiB
Python

"""Sequential, locked, per-part external limits; no retries or replacement draws."""
import fcntl
import json
import os
from pathlib import Path
import subprocess
import sys
import time
def main():
out=Path(sys.argv[1]);c=json.loads((out/'count_contract.json').read_text())
lock=(out/'execution.lock').open('w');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
with (out/'batch.started.json').open('x') as f:json.dump(dict(started_epoch=time.time()),f)
for index,spec in enumerate(c['cases']):
case=spec['case'];receipt=out/(case+'.supervisor.json');assert not receipt.exists() and not (out/case).exists()
with (out/(case+'.log')).open('x') as log:
start=time.monotonic();peak=0;reason=None
proc=subprocess.Popen([sys.executable,'tools/count_s1_panel.py',str(out),str(index)],stdout=log,stderr=subprocess.STDOUT,env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'))
while proc.poll() is None:
try:
lines=Path(f'/proc/{proc.pid}/status').read_text().splitlines()
rss=max((int(x.split()[1])*1024 for x in lines if x.startswith(('VmRSS:','VmHWM:'))),default=0);peak=max(peak,rss)
except FileNotFoundError:pass
if time.monotonic()-start>c['wall_seconds_per_part']:reason='wall_budget_exceeded'
elif peak>c['memory_bytes_per_part']:reason='memory_budget_exceeded'
if reason:
proc.terminate()
try:proc.wait(timeout=2)
except subprocess.TimeoutExpired:proc.kill();proc.wait()
break
time.sleep(.02)
result=out/case/'result.json'
if result.exists():peak=max(peak,json.loads(result.read_text())['peak_rss_bytes'])
elapsed=time.monotonic()-start
if reason is None and elapsed>c['wall_seconds_per_part']:reason='wall_budget_exceeded'
if reason is None and peak>c['memory_bytes_per_part']:reason='memory_budget_exceeded'
record=dict(case=case,panel_index=index,returncode=proc.returncode,stop_reason=reason,wall_seconds=elapsed,peak_rss_bytes=peak,
wall_budget_seconds=c['wall_seconds_per_part'],memory_budget_bytes=c['memory_bytes_per_part'],status='PASS' if proc.returncode==0 and reason is None else 'FAILED_OR_STOPPED')
receipt.write_text(json.dumps(record,indent=2)+'\n');print(json.dumps(record),flush=True)
if proc.returncode and reason is None:
print('STOP_BATCH_UNEXPECTED_ERROR; preserve evidence and diagnose before remaining parts',flush=True);sys.exit(1)
print('BATCH_COMPLETE',flush=True)
if __name__=='__main__':main()