47 lines
2.7 KiB
Python
47 lines
2.7 KiB
Python
"""Sequential, locked, per-part external limits; no retries or replacement draws."""
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
|
|
def main():
|
|
out=Path(sys.argv[1]);c=json.loads((out/'count_contract.json').read_text())
|
|
lock=(out/'execution.lock').open('w');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
with (out/'batch.started.json').open('x') as f:json.dump(dict(started_epoch=time.time()),f)
|
|
for index,spec in enumerate(c['cases']):
|
|
case=spec['case'];receipt=out/(case+'.supervisor.json');assert not receipt.exists() and not (out/case).exists()
|
|
with (out/(case+'.log')).open('x') as log:
|
|
start=time.monotonic();peak=0;reason=None
|
|
proc=subprocess.Popen([sys.executable,'tools/count_s1_panel.py',str(out),str(index)],stdout=log,stderr=subprocess.STDOUT,env=dict(os.environ,OMP_NUM_THREADS='2',OPENBLAS_NUM_THREADS='2'))
|
|
while proc.poll() is None:
|
|
try:
|
|
lines=Path(f'/proc/{proc.pid}/status').read_text().splitlines()
|
|
rss=max((int(x.split()[1])*1024 for x in lines if x.startswith(('VmRSS:','VmHWM:'))),default=0);peak=max(peak,rss)
|
|
except FileNotFoundError:pass
|
|
if time.monotonic()-start>c['wall_seconds_per_part']:reason='wall_budget_exceeded'
|
|
elif peak>c['memory_bytes_per_part']:reason='memory_budget_exceeded'
|
|
if reason:
|
|
proc.terminate()
|
|
try:proc.wait(timeout=2)
|
|
except subprocess.TimeoutExpired:proc.kill();proc.wait()
|
|
break
|
|
time.sleep(.02)
|
|
result=out/case/'result.json'
|
|
if result.exists():peak=max(peak,json.loads(result.read_text())['peak_rss_bytes'])
|
|
elapsed=time.monotonic()-start
|
|
if reason is None and elapsed>c['wall_seconds_per_part']:reason='wall_budget_exceeded'
|
|
if reason is None and peak>c['memory_bytes_per_part']:reason='memory_budget_exceeded'
|
|
record=dict(case=case,panel_index=index,returncode=proc.returncode,stop_reason=reason,wall_seconds=elapsed,peak_rss_bytes=peak,
|
|
wall_budget_seconds=c['wall_seconds_per_part'],memory_budget_bytes=c['memory_bytes_per_part'],status='PASS' if proc.returncode==0 and reason is None else 'FAILED_OR_STOPPED')
|
|
receipt.write_text(json.dumps(record,indent=2)+'\n');print(json.dumps(record),flush=True)
|
|
if proc.returncode and reason is None:
|
|
print('STOP_BATCH_UNEXPECTED_ERROR; preserve evidence and diagnose before remaining parts',flush=True);sys.exit(1)
|
|
print('BATCH_COMPLETE',flush=True)
|
|
|
|
|
|
if __name__=='__main__':main()
|