Files
MeltMind/tools/phase2_e2_geometry_queue.py
T

74 lines
5.6 KiB
Python

"""E2 first-ten cost gate, then bounded label-free geometry; no fit or scoring."""
import argparse,fcntl,json,os,resource,signal,subprocess,sys,time
from pathlib import Path
from phase2_e2_geometry import config
from phase2_h1_worker_v2 import sha,save
from r4bc_guard import supervise
from s2_recovery_supervisor import tree_usage
from s2_recovery_monitoring import disk_usage
def read(p):return json.loads(Path(p).read_text())
def cost_projection(cpu,done_nodes,total_nodes,postfit,setup=60):
if done_nodes<=0 or total_nodes<done_nodes or cpu<setup:raise ValueError('Invalid cost checkpoint')
return setup+1.25*(cpu-setup)*total_nodes/done_nodes+postfit
def guard(root,index,slot):
c=config(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);start=time.process_time();d=root/'guards'/f'{index:04d}'
g=supervise([sys.executable,'tools/phase2_e2_geometry.py',str(root),str(index)],d,dict(cpu_seconds=c['part_cpu_seconds'],wall_seconds=c['part_wall_seconds'],rss_bytes=c['part_rss_bytes'],disk_bytes=c['disk_bytes']),global_stop=root/'global-stop.json')
save(d/'guard-result.json',dict(index=index,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1
def run(root):
c=config(root);os.sched_setaffinity(0,[30,31]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('E2 geometry single attempt')
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[]
def total():
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 60+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_ in active.values())
def batch(indices):
nonlocal settled
pending=list(indices)
while pending or active:
sample=dict(cpu_seconds=total(),disk_bytes=disk_usage(root),rss_bytes=sum(tree_usage(p.pid)[1] for p,_ in active.values()),signals=list(requested));reasons=[]
if requested:reasons.append('COORDINATOR_SIGNAL')
if sample['cpu_seconds']>14400-120:reasons.append('TOTAL_CPU_RESERVE')
if sample['disk_bytes']>c['disk_bytes']:reasons.append('DISK_LIMIT')
if sample['rss_bytes']>32*2**30:reasons.append('RSS_LIMIT')
if reasons:save(root/'stop-trigger.json',dict(reasons=reasons,sample=sample));raise RuntimeError('RESOURCE_OR_SIGNAL_STOP')
for slot,(p,i) in list(active.items()):
if p.poll() is not None:
rr=read(root/'guards'/f'{i:04d}'/'guard-result.json');settled+=rr['guard']['cpu_seconds']+rr['guard_cpu_seconds'];records.append(rr);del active[slot]
if p.returncode or rr['guard']['status']!='PASS':raise RuntimeError('WORKER_NOT_PASS')
for slot in range(4):
if pending and slot not in active:
if total()+(len(active)+1)*c['part_cpu_seconds']+120>14400:
if not active:raise RuntimeError('TOTAL_ADMISSION_RESERVE')
break
i=pending.pop(0);(root/'parts'/f'{i:04d}').mkdir(parents=True,exist_ok=False)
p=subprocess.Popen([sys.executable,'tools/phase2_e2_geometry_queue.py','guard',str(root),str(i),str(slot)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);active[slot]=(p,i)
save(root/'live-progress.json',dict(stage='FIRST_TEN' if max(indices)<10 else 'GEOMETRY',terminal=len(records),pending=len(pending),active=[dict(slot=k,pid=p.pid,index=i) for k,(p,i) in active.items()],cumulative_cpu_seconds=total(),unix=time.time()));time.sleep(.2)
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),first_ten=list(range(10)),new_fits=0,new_labels=0))
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
try:
batch(list(range(10)));projected=cost_projection(total(),sum(r['nodes'] for r in c['parts'][:10]),sum(r['nodes'] for r in c['parts']),c['postfit_projection']['total'])
save(root/'first-ten-cost-check.json',dict(projected_total_cpu_seconds=projected,total_cap=14400,postfit_projection=c['postfit_projection'],actual_cpu_seconds=total(),passed=projected<=14400,denominator_all_ten_including_missing=True))
if projected>14400:raise RuntimeError('FIRST_TEN_COST_PROJECTION_EXCEEDS_TOTAL')
batch(list(range(10,552)));entries=[]
for i,row in enumerate(c['parts']):
p=root/'parts'/f'{i:04d}'/'result.json';v=read(p);entries.append(dict(case=row['case'],index=i,result_sha256=sha(p),valid={a:v['arms'][a]['valid'] for a in ['G','T','C']}))
save(root/'geometry-manifest.json',dict(entries=entries,all552_terminal=True,labels_read=False,cross_arm_filter=False))
save(root/'complete.json',dict(status='GEOMETRY_COMPLETE_POSTFIT_PENDING',cumulative_cpu_seconds=total(),records=records,new_fits=0,new_predictions=0,scored=False,accounting_complete=True))
except BaseException as exc:
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
for p,_ in active.values():
try:os.kill(p.pid,signal.SIGTERM)
except ProcessLookupError:pass
for p,i in active.values():
p.wait();f=root/'guards'/f'{i:04d}'/'guard-result.json'
if f.exists():records.append(read(f))
active.clear();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,accounting_complete=True,retry=False));raise
finally:
for s,h in old.items():signal.signal(s,h)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','guard']);p.add_argument('root',type=Path);p.add_argument('index',nargs='?',type=int);p.add_argument('slot',nargs='?',type=int);a=p.parse_args()
if a.action=='run':run(a.root)
else:raise SystemExit(guard(a.root,a.index,a.slot))