"""Authorized seven-part infrastructure continuation; original three PASS reused.""" import argparse import fcntl import os from pathlib import Path import resource import signal import sys import time import psutil from s2_recovery_supervisor import save, supervise, stop_group from s2_recovery_monitoring import disk_usage from s3b_v04_runtime import contract, read, sha, reserve_job, failure_classification, may_switch def run(root): lock=(root/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) c=contract(root);lim=c['limits'] if (root/'started.json').exists():raise FileExistsError('Never restart a candidate measurement') os.sched_setaffinity(0,c['coordinator_cpus']) save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'), code_freeze=read(root/'code-freeze.json'),stage='V04_FIXED_TEN',labels_read=False)) interrupted=False def stop(*_): nonlocal interrupted interrupted=True for child in psutil.Process(os.getpid()).children():stop_group(child.pid) signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop) start=time.monotonic();done=list(c['reused_receipts']);candidates=[];reason=None;active=None;selected=None def spent():return c['new_synthetic_worker_CPU_s']+sum(r['worker']['cpu_seconds'] for r in done) try: for method in c['candidate_order']: candidate=[r for r in done if r['method']==method];classification=None for index,spec in enumerate(c['ten_parts']): if index in {r['index'] for r in candidate}:continue contract(root) if interrupted:reason='COORDINATOR_INTERRUPTED';break if not reserve_job(spent(),lim['part_CPU_s']):reason='SHARED_FULL_JOB_RESERVATION';break stage_spent=(c['new_synthetic_worker_CPU_s'] if method=='splu' else 0.)+sum(r['worker']['cpu_seconds'] for r in done if r['method']==method) if stage_spent+lim['part_CPU_s']>10800:reason='CANDIDATE_STAGE_FULL_JOB_RESERVATION';break if disk_usage(root)>=lim['ten_disk_bytes']:reason='TEN_DISK_LIMIT';break dest=root/'candidates'/method/'parts'/f'part-{index:02d}' dest.mkdir(parents=True,exist_ok=False);active=dict(method=method,index=index) save(root/'progress.json',dict(stage='V04_FIXED_TEN_RUNNING',**active,case=spec['case'], completed_this_candidate=len(candidate),all_attempts=len(done), new_settled_worker_CPU_s=spent(),old_worker_CPU_s=c['old_worker_CPU_s'], remaining_compute_CPU_s=lim['total_worker_CPU_s']-lim['postfit_reserve_CPU_s']-c['old_worker_CPU_s']-spent())) w=supervise(['taskset','-c',str(c['worker_cpu']),sys.executable,'tools/run_s3b_v04_part.py',str(root),str(index),method], dest,'worker',dict(total_cpu_seconds=lim['part_CPU_s']+1,job_cpu_seconds=lim['part_CPU_s'], job_wall_seconds=lim['part_wall_s'],tree_rss_bytes=lim['part_RSS_bytes'], disk_bytes=lim['ten_disk_bytes']-disk_usage(root),max_processes=1),poll=.2) paths={n:dest/'output'/f'{n}.json' for n in ['result','failure','progress']} data={n:read(p) if p.exists() else None for n,p in paths.items()} classification=failure_classification(w,data['failure'],data['progress'],data['result']) result_data=data['result'] or {} receipt=dict(method=method,index=index,case=spec['case'],worker=w,classification=classification, pressure_calls=result_data.get('pressure_calls'),early_resolves=result_data.get('early_resolves'), dead_end_share=(result_data['early_resolves']/result_data['pressure_calls']) if result_data.get('pressure_calls') else None, files={n:dict(path=str(p),sha256=sha(p)) for n,p in paths.items() if p.exists()}) save(dest/'guard-result.json',receipt);done.append(receipt);candidate.append(receipt);active=None print(f'V04_TERMINAL method={method} index={index} status={classification["category"]}',flush=True) if classification['category']!='PASS':reason=classification['category'];break candidate_pass=len(candidate)==10 and all(r['classification']['category']=='PASS' for r in candidate) summary=dict(method=method,status='PASS' if candidate_pass else 'STOPPED',reason=reason, classification=classification,attempts=candidate, not_run_cases=[r['case'] for i,r in enumerate(c['ten_parts']) if i not in {v['index'] for v in candidate}]) candidates.append(summary);save(root/f'candidate-{method}.json',summary) if candidate_pass: selected=method;reason=None save(root/'selected-solver.json',dict(method=selected,contract_sha256=sha(root/'contract.json'), candidate_summary_sha256=sha(root/f'candidate-{method}.json'), selection_basis='all ten complete within resource and numerical checks; no accuracy/labels read', reuse_ten=True,reference_gate_not_yet_run=True)) break if interrupted or reason in {'SHARED_FULL_JOB_RESERVATION','TEN_DISK_LIMIT','CANDIDATE_STAGE_FULL_JOB_RESERVATION'}:break if classification and may_switch(method,classification): save(root/'fallback-decision.json',dict(from_method=method,to_method='gmres_ilu', classification=classification,new_worker_CPU_s=spent(),restart_same_fixed_ten_from_index=0)) reason=None else:break except Exception as exc: reason='COORDINATOR_EXCEPTION' save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),active=active)) raise finally: report=dict(status='TEN_PASS_AWAITING_REFERENCE_AND_COST' if selected else 'TEN_STOPPED', stop_reason=reason,selected_solver=selected,candidates=candidates,attempts=len(done), new_worker_CPU_s=spent(),old_worker_CPU_s=c['old_worker_CPU_s'], cumulative_worker_CPU_s=c['old_worker_CPU_s']+spent(),postfit_reserve_CPU_s=lim['postfit_reserve_CPU_s'], remaining_compute_CPU_s=lim['total_worker_CPU_s']-lim['postfit_reserve_CPU_s']-c['old_worker_CPU_s']-spent(), coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_s=time.monotonic()-start, account_complete=active is None,no_silent_retries=True,labels_read=False,reused_original_PASS=3,new_attempts=len(done)-3, candidate_CPU_s={method:(c['new_synthetic_worker_CPU_s'] if method=='splu' else 0.)+sum(r['worker']['cpu_seconds'] for r in done if r['method']==method) for method in c['candidate_order']}, note='Reference and full-count cost gates still required; no full617 launch by this coordinator.') save(root/'complete.json',report);save(root/'progress.json',report) lock.close() return 0 if selected else 1 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))