94 lines
7.0 KiB
Python
94 lines
7.0 KiB
Python
"""Cal-v2 preparation only:493 recomputations,52 hash reuse,7 ineligible."""
|
|
import argparse,fcntl,json,os,resource,signal,subprocess,sys,time
|
|
from pathlib import Path
|
|
from phase2_fixed_io import BASE
|
|
from phase2_h1_worker_v2 import checked,save,sha
|
|
from phase2_home_freeze_io import verify,freeze
|
|
from phase2_gate_outcome_features import prepare
|
|
from r4bc_guard import supervise
|
|
from s2_recovery_supervisor import tree_usage
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
def validate_scope(c):
|
|
if c['approval']!='1794916421.489089' or c['total_cpu_seconds']!=72000 or len(c['parts'])!=493 or len(c['exact_reuse'])!=52 or len(c['invalid_cases'])!=7:raise PermissionError('Fixed full calibration v2 scope')
|
|
names=[p['row']['case'] for p in c['parts']]+[p['case'] for p in c['exact_reuse']]+c['invalid_cases']
|
|
if len(set(names))!=552 or set(names)!=set(c['original_cases']):raise ValueError('Disjoint complete552 roster required')
|
|
cpus=c['worker_cpus']
|
|
if len(cpus)!=12 or len(set(cpus))!=12 or set(cpus)&{8,10,12,14,30,31}:raise PermissionError('B17 and coordinator cores reserved')
|
|
if c['geometry_cpu_cap']+c['postfit_reserve']!=72000 or c['postfit_reserve']<3600:raise PermissionError('Shared calibration budget/reserve')
|
|
for p in c['parts']:
|
|
ids=p['gate_raw_ids']
|
|
if not ids or ids!=sorted(set(ids)):raise ValueError('Nonempty unique sorted sealed gate IDs required')
|
|
def projection(used,parts,setup,reserve):
|
|
return used*sum(p['row']['nodes'] for p in parts)/sum(p['row']['nodes'] for p in parts[:10])+setup+reserve
|
|
def config(root):
|
|
c=read(root/'contract.json')
|
|
validate_scope(c)
|
|
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(BASE/p,h)
|
|
s=Path(c['archive']).stat();a=c['archive_stat']
|
|
if (s.st_size,s.st_ino,s.st_mtime_ns)!=(a['bytes'],a['inode'],a['mtime_ns']):raise ValueError('Archive identity')
|
|
return c
|
|
|
|
def guard(root,index,slot):
|
|
c=config(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);start=time.process_time();d=root/'parts'/f'{index:04d}'
|
|
g=supervise([sys.executable,'tools/phase2_gate_outcome_queue.py','worker',str(root),str(index)],d,dict(cpu_seconds=1800,wall_seconds=1800,rss_bytes=4*2**30,disk_bytes=2*2**30),global_stop=root/'global-stop.json')
|
|
save(d/'guard-result.json',dict(index=index,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1
|
|
|
|
def run(root):
|
|
c=config(root);verify(root,'launch');lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('Single calibration v2 queue')
|
|
os.sched_setaffinity(0,[30,31]);start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[]
|
|
def total():
|
|
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['setup_seconds']+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_ in active.values())
|
|
def batch(indices,slots):
|
|
nonlocal settled
|
|
pending=list(indices)
|
|
while pending or active:
|
|
if requested or total()>c['geometry_cpu_cap']-120 or disk_usage(root)>c['disk_bytes'] or sum(tree_usage(p.pid)[1] for p,_ in active.values())>48*2**30:raise RuntimeError('Resource/signal stop')
|
|
for slot,(p,i) in list(active.items()):
|
|
if p.poll() is not None:
|
|
r=read(root/'parts'/f'{i:04d}'/'guard-result.json');records.append(r);settled+=r['guard']['cpu_seconds']+r['guard_cpu_seconds'];del active[slot]
|
|
if p.returncode or r['guard']['status']!='PASS':raise RuntimeError('FEATURE_WORKER_STOP_NO_RETRY')
|
|
for slot in range(slots):
|
|
if pending and slot not in active:
|
|
if total()+(len(active)+1)*1800+120>c['geometry_cpu_cap']:
|
|
if not active:raise RuntimeError('Full per-part reservation exceeds remaining budget')
|
|
break
|
|
i=pending.pop(0);p=subprocess.Popen([sys.executable,'tools/phase2_gate_outcome_queue.py','guard',str(root),str(i),str(slot)],start_new_session=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL);active[slot]=(p,i)
|
|
save(root/'live-progress.json',dict(stage='GATE_DEPENDENT_FEATURES_ONLY',pending=len(pending),active=[dict(slot=s,pid=p.pid,index=i) for s,(p,i) in active.items()],terminal=len(records),cumulative_cpu_seconds=total(),unix=time.time()));time.sleep(.5)
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),new_feature_parts=493,exact_reused=52,ineligible=7,new_labels_read=False))
|
|
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
|
|
try:
|
|
batch(range(10),4)
|
|
used=sum(r['guard']['cpu_seconds']+r['guard_cpu_seconds'] for r in records)
|
|
projected=projection(used,c['parts'],c['setup_seconds'],c['postfit_reserve'])
|
|
save(root/'first-ten-cost-check.json',dict(first10_cpu_seconds=used,projected_total_cpu_seconds=projected,factor=1.,projection_kind='node weighted mean, fixed total cap supplies margin',postfit_reserve=c['postfit_reserve'],cap=72000,passed=projected<=72000))
|
|
if projected>72000:raise RuntimeError('FIRST_TEN_COST_STOP_NO_EXTENSION')
|
|
batch(range(10,493),12)
|
|
entries=[]
|
|
for i,item in enumerate(c['parts']):
|
|
p=root/'parts'/f'{i:04d}'/'result.json';d=read(p)
|
|
for name,h in d['output_sha256'].items():checked(p.parent/name,h)
|
|
entries.append(dict(case=item['row']['case'],index=i,path=str(p.parent),result_sha256=sha(p),output_sha256=d['output_sha256']))
|
|
save(root/'features-manifest.json',dict(recomputed=entries,exact_reuse=c['exact_reuse'],invalid_cases=c['invalid_cases'],parts_total=552,labels_read=False,new_fits=0,new_predictions=0))
|
|
save(root/'complete.json',dict(status='FEATURES_COMPLETE_PREDICTIONS_AND_COMPARISON_PENDING',records=records,cumulative_cpu_seconds=total(),total_cpu_seconds=72000,accounting_complete=True,new_fits=0,new_predictions=0,new_v2_labels_read=0,retry=False))
|
|
freeze(root,'features',[root/'contract.json',root/'features-manifest.json',root/'complete.json',*[BASE/p for p in c['code_sha256']]])
|
|
save(root/'post-freeze-cost.json',dict(cumulative_cpu_seconds=total(),within_cap=total()<=72000))
|
|
except BaseException as exc:
|
|
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
|
|
for p,_ in active.values():
|
|
try:os.kill(p.pid,signal.SIGTERM)
|
|
except ProcessLookupError:pass
|
|
for p,i in active.values():
|
|
p.wait();f=root/'parts'/f'{i:04d}'/'guard-result.json'
|
|
if f.exists():records.append(read(f))
|
|
active.clear();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,cumulative_cpu_seconds=total(),accounting_complete=True,retry=False,new_v2_labels_read=0));raise
|
|
finally:
|
|
for s,h in old.items():signal.signal(s,h)
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','guard','worker']);p.add_argument('root',type=Path);p.add_argument('index',nargs='?',type=int);p.add_argument('slot',nargs='?',type=int);a=p.parse_args()
|
|
if a.action=='run':run(a.root)
|
|
elif a.action=='guard':raise SystemExit(guard(a.root,a.index,a.slot))
|
|
else:prepare(a.root,config(a.root),a.index)
|