102 lines
7.2 KiB
Python
102 lines
7.2 KiB
Python
"""Authorized seven-part infrastructure continuation; original three PASS reused."""
|
|
import argparse
|
|
import fcntl
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import sys
|
|
import time
|
|
import psutil
|
|
from s2_recovery_supervisor import save, supervise, stop_group
|
|
from s2_recovery_monitoring import disk_usage
|
|
from s3b_v04_runtime import contract, read, sha, reserve_job, failure_classification, may_switch
|
|
|
|
|
|
def run(root):
|
|
lock=(root/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
c=contract(root);lim=c['limits']
|
|
if (root/'started.json').exists():raise FileExistsError('Never restart a candidate measurement')
|
|
os.sched_setaffinity(0,c['coordinator_cpus'])
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
|
|
code_freeze=read(root/'code-freeze.json'),stage='V04_FIXED_TEN',labels_read=False))
|
|
interrupted=False
|
|
def stop(*_):
|
|
nonlocal interrupted
|
|
interrupted=True
|
|
for child in psutil.Process(os.getpid()).children():stop_group(child.pid)
|
|
signal.signal(signal.SIGTERM,stop);signal.signal(signal.SIGINT,stop)
|
|
start=time.monotonic();done=list(c['reused_receipts']);candidates=[];reason=None;active=None;selected=None
|
|
def spent():return c['new_synthetic_worker_CPU_s']+sum(r['worker']['cpu_seconds'] for r in done)
|
|
try:
|
|
for method in c['candidate_order']:
|
|
candidate=[r for r in done if r['method']==method];classification=None
|
|
for index,spec in enumerate(c['ten_parts']):
|
|
if index in {r['index'] for r in candidate}:continue
|
|
contract(root)
|
|
if interrupted:reason='COORDINATOR_INTERRUPTED';break
|
|
if not reserve_job(spent(),lim['part_CPU_s']):reason='SHARED_FULL_JOB_RESERVATION';break
|
|
stage_spent=(c['new_synthetic_worker_CPU_s'] if method=='splu' else 0.)+sum(r['worker']['cpu_seconds'] for r in done if r['method']==method)
|
|
if stage_spent+lim['part_CPU_s']>10800:reason='CANDIDATE_STAGE_FULL_JOB_RESERVATION';break
|
|
if disk_usage(root)>=lim['ten_disk_bytes']:reason='TEN_DISK_LIMIT';break
|
|
dest=root/'candidates'/method/'parts'/f'part-{index:02d}'
|
|
dest.mkdir(parents=True,exist_ok=False);active=dict(method=method,index=index)
|
|
save(root/'progress.json',dict(stage='V04_FIXED_TEN_RUNNING',**active,case=spec['case'],
|
|
completed_this_candidate=len(candidate),all_attempts=len(done),
|
|
new_settled_worker_CPU_s=spent(),old_worker_CPU_s=c['old_worker_CPU_s'],
|
|
remaining_compute_CPU_s=lim['total_worker_CPU_s']-lim['postfit_reserve_CPU_s']-c['old_worker_CPU_s']-spent()))
|
|
w=supervise(['taskset','-c',str(c['worker_cpu']),sys.executable,'tools/run_s3b_v04_part.py',str(root),str(index),method],
|
|
dest,'worker',dict(total_cpu_seconds=lim['part_CPU_s']+1,job_cpu_seconds=lim['part_CPU_s'],
|
|
job_wall_seconds=lim['part_wall_s'],tree_rss_bytes=lim['part_RSS_bytes'],
|
|
disk_bytes=lim['ten_disk_bytes']-disk_usage(root),max_processes=1),poll=.2)
|
|
paths={n:dest/'output'/f'{n}.json' for n in ['result','failure','progress']}
|
|
data={n:read(p) if p.exists() else None for n,p in paths.items()}
|
|
classification=failure_classification(w,data['failure'],data['progress'],data['result'])
|
|
result_data=data['result'] or {}
|
|
receipt=dict(method=method,index=index,case=spec['case'],worker=w,classification=classification,
|
|
pressure_calls=result_data.get('pressure_calls'),early_resolves=result_data.get('early_resolves'),
|
|
dead_end_share=(result_data['early_resolves']/result_data['pressure_calls']) if result_data.get('pressure_calls') else None,
|
|
files={n:dict(path=str(p),sha256=sha(p)) for n,p in paths.items() if p.exists()})
|
|
save(dest/'guard-result.json',receipt);done.append(receipt);candidate.append(receipt);active=None
|
|
print(f'V04_TERMINAL method={method} index={index} status={classification["category"]}',flush=True)
|
|
if classification['category']!='PASS':reason=classification['category'];break
|
|
candidate_pass=len(candidate)==10 and all(r['classification']['category']=='PASS' for r in candidate)
|
|
summary=dict(method=method,status='PASS' if candidate_pass else 'STOPPED',reason=reason,
|
|
classification=classification,attempts=candidate,
|
|
not_run_cases=[r['case'] for i,r in enumerate(c['ten_parts']) if i not in {v['index'] for v in candidate}])
|
|
candidates.append(summary);save(root/f'candidate-{method}.json',summary)
|
|
if candidate_pass:
|
|
selected=method;reason=None
|
|
save(root/'selected-solver.json',dict(method=selected,contract_sha256=sha(root/'contract.json'),
|
|
candidate_summary_sha256=sha(root/f'candidate-{method}.json'),
|
|
selection_basis='all ten complete within resource and numerical checks; no accuracy/labels read',
|
|
reuse_ten=True,reference_gate_not_yet_run=True))
|
|
break
|
|
if interrupted or reason in {'SHARED_FULL_JOB_RESERVATION','TEN_DISK_LIMIT','CANDIDATE_STAGE_FULL_JOB_RESERVATION'}:break
|
|
if classification and may_switch(method,classification):
|
|
save(root/'fallback-decision.json',dict(from_method=method,to_method='gmres_ilu',
|
|
classification=classification,new_worker_CPU_s=spent(),restart_same_fixed_ten_from_index=0))
|
|
reason=None
|
|
else:break
|
|
except Exception as exc:
|
|
reason='COORDINATOR_EXCEPTION'
|
|
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),active=active))
|
|
raise
|
|
finally:
|
|
report=dict(status='TEN_PASS_AWAITING_REFERENCE_AND_COST' if selected else 'TEN_STOPPED',
|
|
stop_reason=reason,selected_solver=selected,candidates=candidates,attempts=len(done),
|
|
new_worker_CPU_s=spent(),old_worker_CPU_s=c['old_worker_CPU_s'],
|
|
cumulative_worker_CPU_s=c['old_worker_CPU_s']+spent(),postfit_reserve_CPU_s=lim['postfit_reserve_CPU_s'],
|
|
remaining_compute_CPU_s=lim['total_worker_CPU_s']-lim['postfit_reserve_CPU_s']-c['old_worker_CPU_s']-spent(),
|
|
coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_s=time.monotonic()-start,
|
|
account_complete=active is None,no_silent_retries=True,labels_read=False,reused_original_PASS=3,new_attempts=len(done)-3,
|
|
candidate_CPU_s={method:(c['new_synthetic_worker_CPU_s'] if method=='splu' else 0.)+sum(r['worker']['cpu_seconds'] for r in done if r['method']==method) for method in c['candidate_order']},
|
|
note='Reference and full-count cost gates still required; no full617 launch by this coordinator.')
|
|
save(root/'complete.json',report);save(root/'progress.json',report)
|
|
lock.close()
|
|
return 0 if selected else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))
|