Files
MeltMind/tools/phase2_b15_recovery.py
T

107 lines
8.4 KiB
Python

"""5932 single B15 infrastructure recovery; original scientific worker pinned."""
import argparse,fcntl,json,os,resource,sys,time,subprocess,signal,shutil
from pathlib import Path
from phase2_b1213_gate import config
from phase2_h1_worker_v2 import checked,save,sha
from phase2_fixed_io import BASE,freeze
from r4bc_guard import supervise
from s2_recovery_supervisor import tree_usage
from s2_recovery_monitoring import disk_usage
def read(p):return json.loads(Path(p).read_text())
def validated(root):
c=config(root)
if c['continuation_approval']!='1794905932.019429' or c['explicit_recovery_indices']!=[288,290,291]:raise PermissionError('One approved recovery')
if len(c['reused_indices'])!=289 or len(c['new_indices'])!=142 or set(c['new_indices'])!=set(range(431))-set(c['reused_indices']):raise PermissionError('Exact289+3+139 scope')
if c['total_cpu_cap']!=7200 or c['disk_bytes']!=12*2**30:raise PermissionError('Unchanged total; disk12GiB')
return c
def phase_projection(charged,observed_cost,completed_weight,remaining_weight):
if completed_weight<=0 or min(charged,observed_cost,remaining_weight)<0:raise ValueError('Projection inputs')
return charged+1.25*observed_cost*remaining_weight/completed_weight
def guard(root,mode,index,slot,cap):
c=validated(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);d=root/'guards'/f'{mode}-{index:04d}';start=time.process_time()
cmd=[sys.executable,c['scientific_entry'],mode,str(root)]+([str(index)] if mode=='prep' else [])
g=supervise(cmd,d,dict(cpu_seconds=cap,wall_seconds=60 if mode=='prep' else cap+120,rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes']),global_stop=root/'global-stop.json')
save(d/'guard-result.json',dict(mode=mode,index=index,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1
def run(root):
c=validated(root);os.sched_setaffinity(0,[29]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single recovery')
free=shutil.disk_usage(root).free
if free<2*c['disk_bytes']:raise RuntimeError('Insufficient free space before launch')
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[];prepared=set(c['reused_indices']);prep_cost=c['projection']['observed_prep_cpu_seconds'];prep_nodes=c['projection']['completed_nodes']
def total():
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['prior_cpu_seconds']+60+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_,_ in active.values())
def projected(mode):
charge=total()
if mode=='prep':return phase_projection(charge,prep_cost,prep_nodes,sum(r['nodes'] for i,r in enumerate(c['parts']) if i not in prepared))
p=root/'phases'/mode/'progress.json'
if mode=='predict' and p.exists():
a=read(p);n=a['parts'];rows=c['rows']['validation'];elapsed=sum(tree_usage(p.pid)[0] for p,_,_ in active.values())
if n:return phase_projection(charge,elapsed,sum(r['nodes'] for r in rows[:n]),sum(r['nodes'] for r in rows[n:]))
if mode=='train' and p.exists() and not (root/'phases/train/result.json').exists():
a=read(p);n=a.get('parts',0);elapsed=sum(tree_usage(p.pid)[0] for p,_,_ in active.values())
if n:return phase_projection(charge,elapsed,n,max(0,320-n))+c['phase_estimates']['train']
return charge
def batch(mode,ids,jobcap=None):
nonlocal settled,prep_cost,prep_nodes
pending=list(ids)
while pending or active:
sample=dict(cpu_seconds=total(),disk_bytes=disk_usage(root),rss_bytes=sum(tree_usage(p.pid)[1] for p,_,_ in active.values()),signals=list(requested));reasons=[]
if requested:reasons.append('COORDINATOR_SIGNAL')
if sample['cpu_seconds']>7200-120:reasons.append('TOTAL_CPU_RESERVE')
if sample['disk_bytes']>c['disk_bytes']:reasons.append('DIRECTORY_DISK_LIMIT')
if sample['rss_bytes']>32*2**30:reasons.append('TREE_RSS_LIMIT')
estimate=projected(mode)
if estimate>7200:reasons.append('PHASE_PROJECTION_EXCEEDS_TOTAL')
if reasons:
save(root/'stop-trigger.json',dict(reasons=reasons,sample=sample,phase=mode,projected_cumulative_cpu=estimate,unix=time.time()));raise RuntimeError('RESOURCE_OR_SIGNAL_STOP')
for slot,(p,index,cap) in list(active.items()):
if p.poll() is not None:
rr=read(root/'guards'/f'{mode}-{index:04d}'/'guard-result.json');cost=rr['guard']['cpu_seconds']+rr['guard_cpu_seconds'];settled+=cost;records.append(rr);del active[slot]
if p.returncode or rr['guard']['status']!='PASS':raise RuntimeError('WORKER_NOT_PASS')
if mode=='prep':prepared.add(index);prep_cost+=cost;prep_nodes+=c['parts'][index]['nodes']
for slot in (range(4) if mode=='prep' else range(1)):
if pending and slot not in active:
remaining=7200-total()-sum(z[2] for z in active.values())-120;cap=min(jobcap,remaining) if jobcap else remaining
if cap<1 or (mode=='prep' and cap<jobcap):
if not active:raise RuntimeError('TOTAL_RESERVATION_EXHAUSTED')
break
i=pending.pop(0)
if mode=='prep':(root/'parts'/f'{i:04d}').mkdir(parents=True,exist_ok=False)
p=subprocess.Popen([sys.executable,'tools/phase2_b15_recovery.py','guard',str(root),mode,str(i),str(slot),str(cap)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);active[slot]=(p,i,cap)
save(root/'live-progress.json',dict(stage=mode,pending=len(pending),active=[dict(pid=p.pid,index=i,slot=s) for s,(p,i,_) in active.items()],new_terminal=len(records),cumulative_cpu_seconds=total(),phase_projected_total=estimate,unix=time.time()));time.sleep(.2)
def phase_gate(mode):
projected_total=total()+c['phase_estimates'][mode];save(root/(mode+'-cost-check.json'),dict(projected_cumulative_cpu_seconds=projected_total,phase_planning_estimate=c['phase_estimates'][mode],soft_estimate_not_job_cap=True,passed=projected_total<=7200))
if projected_total>7200:raise RuntimeError('PHASE_PROJECTION_EXCEEDS_TOTAL')
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),prior_cpu_seconds=c['prior_cpu_seconds'],reused=289,explicit_recoveries=3,never_attempted=139,free_bytes=free,disk_required_free=2*c['disk_bytes']))
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
try:
mode='prep';batch(mode,c['new_indices'],30);entries=[]
for i,row in enumerate(c['parts']):
p=root/'parts'/f'{i:04d}'/'result.json';r=read(p);checked(p.parent/'geometry.npz',r['geometry_sha256']);entries.append(dict(case=row['case'],index=i,result_sha256=sha(p)))
save(root/'features-manifest.json',dict(entries=entries,all431_terminal=True,labels_read=False,reused289=True));freeze(root,'features',[root/'contract.json',root/'features-manifest.json',*[BASE/p for p in c['code_sha256']]])
for mode in ['train','predict','score']:
phase_gate(mode)
if mode=='predict':freeze(root,'model',[root/'contract.json',root/'phases/train/model.ubj',root/'phases/train/result.json',*[BASE/p for p in c['code_sha256']]])
if mode=='score':freeze(root,'predictions',[root/'contract.json',root/'phases/predict/manifest.json',*[BASE/p for p in c['code_sha256']]])
(root/'phases'/mode).mkdir(parents=True,exist_ok=False);batch(mode,[0])
save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'phases/score/result.json')['decision'],cumulative_cpu_seconds=total(),records=records,models_fit=1,old_stop_preserved=True,accounting_complete=True,retry=False))
except BaseException as exc:
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
for p,_,_ in active.values():
try:os.kill(p.pid,signal.SIGTERM)
except ProcessLookupError:pass
for p,index,_ in active.values():
p.wait();receipt=root/'guards'/f'{mode}-{index:04d}'/'guard-result.json'
if receipt.exists():records.append(read(receipt))
active.clear();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,old_stop_preserved=True,accounting_complete=True,retry=False));raise
finally:
for s,h in old.items():signal.signal(s,h)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','guard']);p.add_argument('root',type=Path);p.add_argument('mode',nargs='?');p.add_argument('index',nargs='?',type=int);p.add_argument('slot',nargs='?',type=int);p.add_argument('cap',nargs='?',type=float);a=p.parse_args()
if a.action=='run':run(a.root)
else:raise SystemExit(guard(a.root,a.mode,a.index,a.slot,a.cap))