Files
MeltMind/tools/run_s3b_ten.py
T

74 lines
4.4 KiB
Python

"""Sole sequential fixed-ten cost measurement, never retries or batches events."""
import argparse
import fcntl
import os
from pathlib import Path
import resource
import signal
import sys
import time
import psutil
from s2_recovery_supervisor import save, supervise, stop_group
from s2_recovery_monitoring import disk_usage
from s3b_runtime import contract, read, sha, reserve_ok
def run(root):
lock=(root/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
c=contract(root);lim=c['limits']
if (root/'started.json').exists(): raise FileExistsError('Never restart this ten-part attempt')
os.sched_setaffinity(0,c['coordinator_cpus'])
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
code_freeze=read(root/'code-freeze.json'),stage='FIXED_TEN',label_reads=0))
interrupted=False
def stop(*_):
nonlocal interrupted
interrupted=True
for child in psutil.Process(os.getpid()).children(): stop_group(child.pid)
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
start=time.monotonic();done=[];reason=None;active=None
try:
for index,spec in enumerate(c['ten_parts']):
contract(root)
spent=c['prior_synthetic_worker_CPU_s']+sum(r['worker']['cpu_seconds'] for r in done)
if interrupted: reason='COORDINATOR_INTERRUPTED';break
if not reserve_ok(spent,lim['part_CPU_s'],lim['ten_CPU_s']):reason='TEN_FULL_JOB_RESERVATION';break
if disk_usage(root)>=lim['ten_disk_bytes']:reason='TEN_DISK_LIMIT';break
dest=root/'parts'/f'part-{index:02d}';dest.mkdir(parents=True,exist_ok=False)
active=index
save(root/'progress.json',dict(stage='FIXED_TEN_RUNNING',active_index=index,case=spec['case'],
completed=len(done),settled_worker_CPU_s=spent,elapsed_s=time.monotonic()-start))
w=supervise(['taskset','-c',str(c['worker_cpu']),sys.executable,'tools/run_s3b_part.py',str(root),str(index)],
dest,'worker',dict(total_cpu_seconds=lim['part_CPU_s']+1,job_cpu_seconds=lim['part_CPU_s'],
job_wall_seconds=lim['part_wall_s'],tree_rss_bytes=lim['part_RSS_bytes'],
disk_bytes=lim['ten_disk_bytes']-disk_usage(root),max_processes=1),poll=.2)
result=dest/'output/result.json';failure=dest/'output/failure.json'
receipt=dict(index=index,case=spec['case'],worker=w,
result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None,
failure_path=str(failure) if failure.exists() else None,failure_sha256=sha(failure) if failure.exists() else None)
save(dest/'guard-result.json',receipt);done.append(receipt);active=None
print(f'S3B_TEN_TERMINAL index={index} case={spec["case"]} status={w["status"]}',flush=True)
if w['monitor_errors']:reason='MONITOR_ERRORS';break
if w['status']!='PASS':reason=w['stop_reason'] or 'WORKER_NUMERICAL_OR_INPUT_FAILURE';break
if not result.exists() or read(result)['status'] not in ('COMPLETE','BLOCKED'):
reason='MISSING_TERMINAL_SCIENTIFIC_RESULT';break
if interrupted:reason='COORDINATOR_INTERRUPTED'
except Exception as exc:
reason='COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),active_index=active))
raise
finally:
report=dict(status='TEN_TERMINAL_AWAITING_COST_CENSUS' if reason is None and len(done)==10 else 'TEN_STOPPED',
stop_reason=reason,receipts=done,attempted=len(done),
not_run_cases=[r['case'] for i,r in enumerate(c['ten_parts']) if i not in {v['index'] for v in done}],
worker_CPU_s=c['prior_synthetic_worker_CPU_s']+sum(r['worker']['cpu_seconds'] for r in done),
coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_s=time.monotonic()-start,
account_complete=active is None,no_retries=True,labels_read=False,event_batching=False,
note='No full queue launched. A complete ten requires full-count cost census before continuation; no repeat user-go.')
save(root/'complete.json',report);save(root/'progress.json',report)
return 0 if reason is None else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))