74 lines
4.4 KiB
Python
74 lines
4.4 KiB
Python
"""Sole sequential fixed-ten cost measurement, never retries or batches events."""
|
|
import argparse
|
|
import fcntl
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import sys
|
|
import time
|
|
import psutil
|
|
from s2_recovery_supervisor import save, supervise, stop_group
|
|
from s2_recovery_monitoring import disk_usage
|
|
from s3b_runtime import contract, read, sha, reserve_ok
|
|
|
|
|
|
def run(root):
|
|
lock=(root/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
c=contract(root);lim=c['limits']
|
|
if (root/'started.json').exists(): raise FileExistsError('Never restart this ten-part attempt')
|
|
os.sched_setaffinity(0,c['coordinator_cpus'])
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
|
|
code_freeze=read(root/'code-freeze.json'),stage='FIXED_TEN',label_reads=0))
|
|
interrupted=False
|
|
def stop(*_):
|
|
nonlocal interrupted
|
|
interrupted=True
|
|
for child in psutil.Process(os.getpid()).children(): stop_group(child.pid)
|
|
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
|
|
start=time.monotonic();done=[];reason=None;active=None
|
|
try:
|
|
for index,spec in enumerate(c['ten_parts']):
|
|
contract(root)
|
|
spent=c['prior_synthetic_worker_CPU_s']+sum(r['worker']['cpu_seconds'] for r in done)
|
|
if interrupted: reason='COORDINATOR_INTERRUPTED';break
|
|
if not reserve_ok(spent,lim['part_CPU_s'],lim['ten_CPU_s']):reason='TEN_FULL_JOB_RESERVATION';break
|
|
if disk_usage(root)>=lim['ten_disk_bytes']:reason='TEN_DISK_LIMIT';break
|
|
dest=root/'parts'/f'part-{index:02d}';dest.mkdir(parents=True,exist_ok=False)
|
|
active=index
|
|
save(root/'progress.json',dict(stage='FIXED_TEN_RUNNING',active_index=index,case=spec['case'],
|
|
completed=len(done),settled_worker_CPU_s=spent,elapsed_s=time.monotonic()-start))
|
|
w=supervise(['taskset','-c',str(c['worker_cpu']),sys.executable,'tools/run_s3b_part.py',str(root),str(index)],
|
|
dest,'worker',dict(total_cpu_seconds=lim['part_CPU_s']+1,job_cpu_seconds=lim['part_CPU_s'],
|
|
job_wall_seconds=lim['part_wall_s'],tree_rss_bytes=lim['part_RSS_bytes'],
|
|
disk_bytes=lim['ten_disk_bytes']-disk_usage(root),max_processes=1),poll=.2)
|
|
result=dest/'output/result.json';failure=dest/'output/failure.json'
|
|
receipt=dict(index=index,case=spec['case'],worker=w,
|
|
result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None,
|
|
failure_path=str(failure) if failure.exists() else None,failure_sha256=sha(failure) if failure.exists() else None)
|
|
save(dest/'guard-result.json',receipt);done.append(receipt);active=None
|
|
print(f'S3B_TEN_TERMINAL index={index} case={spec["case"]} status={w["status"]}',flush=True)
|
|
if w['monitor_errors']:reason='MONITOR_ERRORS';break
|
|
if w['status']!='PASS':reason=w['stop_reason'] or 'WORKER_NUMERICAL_OR_INPUT_FAILURE';break
|
|
if not result.exists() or read(result)['status'] not in ('COMPLETE','BLOCKED'):
|
|
reason='MISSING_TERMINAL_SCIENTIFIC_RESULT';break
|
|
if interrupted:reason='COORDINATOR_INTERRUPTED'
|
|
except Exception as exc:
|
|
reason='COORDINATOR_EXCEPTION'
|
|
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),active_index=active))
|
|
raise
|
|
finally:
|
|
report=dict(status='TEN_TERMINAL_AWAITING_COST_CENSUS' if reason is None and len(done)==10 else 'TEN_STOPPED',
|
|
stop_reason=reason,receipts=done,attempted=len(done),
|
|
not_run_cases=[r['case'] for i,r in enumerate(c['ten_parts']) if i not in {v['index'] for v in done}],
|
|
worker_CPU_s=c['prior_synthetic_worker_CPU_s']+sum(r['worker']['cpu_seconds'] for r in done),
|
|
coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_s=time.monotonic()-start,
|
|
account_complete=active is None,no_retries=True,labels_read=False,event_batching=False,
|
|
note='No full queue launched. A complete ten requires full-count cost census before continuation; no repeat user-go.')
|
|
save(root/'complete.json',report);save(root/'progress.json',report)
|
|
return 0 if reason is None else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))
|