Files
MeltMind/tools/run_s3b_v04_ten.py

94 lines
6.2 KiB
Python

"""Sole sequential two-candidate measurement; one corpus solver, shared ledger."""
import argparse
import fcntl
import os
from pathlib import Path
import resource
import signal
import sys
import time
import psutil
from s2_recovery_supervisor import save, supervise, stop_group
from s2_recovery_monitoring import disk_usage
from s3b_v04_runtime import contract, read, sha, reserve_job, failure_classification, may_switch
def run(root):
lock=(root/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
c=contract(root);lim=c['limits']
if (root/'started.json').exists():raise FileExistsError('Never restart a candidate measurement')
os.sched_setaffinity(0,c['coordinator_cpus'])
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
code_freeze=read(root/'code-freeze.json'),stage='V04_FIXED_TEN',labels_read=False))
interrupted=False
def stop(*_):
nonlocal interrupted
interrupted=True
for child in psutil.Process(os.getpid()).children():stop_group(child.pid)
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
start=time.monotonic();done=[];candidates=[];reason=None;active=None;selected=None
def spent():return c['new_synthetic_worker_CPU_s']+sum(r['worker']['cpu_seconds'] for r in done)
try:
for method in c['candidate_order']:
candidate=[];classification=None
for index,spec in enumerate(c['ten_parts']):
contract(root)
if interrupted:reason='COORDINATOR_INTERRUPTED';break
if not reserve_job(spent(),lim['part_CPU_s']):reason='SHARED_FULL_JOB_RESERVATION';break
if disk_usage(root)>=lim['ten_disk_bytes']:reason='TEN_DISK_LIMIT';break
dest=root/'candidates'/method/'parts'/f'part-{index:02d}'
dest.mkdir(parents=True,exist_ok=False);active=dict(method=method,index=index)
save(root/'progress.json',dict(stage='V04_FIXED_TEN_RUNNING',**active,case=spec['case'],
completed_this_candidate=len(candidate),all_attempts=len(done),
new_settled_worker_CPU_s=spent(),old_worker_CPU_s=c['old_worker_CPU_s'],
remaining_compute_CPU_s=lim['total_worker_CPU_s']-lim['postfit_reserve_CPU_s']-c['old_worker_CPU_s']-spent()))
w=supervise(['taskset','-c',str(c['worker_cpu']),sys.executable,'tools/run_s3b_v04_part.py',str(root),str(index),method],
dest,'worker',dict(total_cpu_seconds=lim['part_CPU_s']+1,job_cpu_seconds=lim['part_CPU_s'],
job_wall_seconds=lim['part_wall_s'],tree_rss_bytes=lim['part_RSS_bytes'],
disk_bytes=lim['ten_disk_bytes']-disk_usage(root),max_processes=1),poll=.2)
paths={n:dest/'output'/f'{n}.json' for n in ['result','failure','progress']}
data={n:read(p) if p.exists() else None for n,p in paths.items()}
classification=failure_classification(w,data['failure'],data['progress'],data['result'])
receipt=dict(method=method,index=index,case=spec['case'],worker=w,classification=classification,
files={n:dict(path=str(p),sha256=sha(p)) for n,p in paths.items() if p.exists()})
save(dest/'guard-result.json',receipt);done.append(receipt);candidate.append(receipt);active=None
print(f'V04_TERMINAL method={method} index={index} status={classification["category"]}',flush=True)
if classification['category']!='PASS':reason=classification['category'];break
candidate_pass=len(candidate)==10 and all(r['classification']['category']=='PASS' for r in candidate)
summary=dict(method=method,status='PASS' if candidate_pass else 'STOPPED',reason=reason,
classification=classification,attempts=candidate,
not_run_cases=[r['case'] for i,r in enumerate(c['ten_parts']) if i not in {v['index'] for v in candidate}])
candidates.append(summary);save(root/f'candidate-{method}.json',summary)
if candidate_pass:
selected=method;reason=None
save(root/'selected-solver.json',dict(method=selected,contract_sha256=sha(root/'contract.json'),
candidate_summary_sha256=sha(root/f'candidate-{method}.json'),
selection_basis='all ten complete within resource and numerical checks; no accuracy/labels read',
reuse_ten=True,reference_gate_not_yet_run=True))
break
if interrupted or reason in {'SHARED_FULL_JOB_RESERVATION','TEN_DISK_LIMIT'}:break
if classification and may_switch(method,classification):
save(root/'fallback-decision.json',dict(from_method=method,to_method='gmres_ilu',
classification=classification,new_worker_CPU_s=spent(),restart_same_fixed_ten_from_index=0))
reason=None
else:break
except Exception as exc:
reason='COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),active=active))
raise
finally:
report=dict(status='TEN_PASS_AWAITING_REFERENCE_AND_COST' if selected else 'TEN_STOPPED',
stop_reason=reason,selected_solver=selected,candidates=candidates,attempts=len(done),
new_worker_CPU_s=spent(),old_worker_CPU_s=c['old_worker_CPU_s'],
cumulative_worker_CPU_s=c['old_worker_CPU_s']+spent(),postfit_reserve_CPU_s=lim['postfit_reserve_CPU_s'],
remaining_compute_CPU_s=lim['total_worker_CPU_s']-lim['postfit_reserve_CPU_s']-c['old_worker_CPU_s']-spent(),
coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_s=time.monotonic()-start,
account_complete=active is None,no_silent_retries=True,labels_read=False,
note='Reference and full-count cost gates still required; no full617 launch by this coordinator.')
save(root/'complete.json',report);save(root/'progress.json',report)
return 0 if selected else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))