163 lines
13 KiB
Python
163 lines
13 KiB
Python
"""Five-fold held-out-test confirmation; immutable G/model reuse and bounded queue."""
|
|
import argparse,fcntl,json,os,resource,sys,time,subprocess,signal,traceback
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import xgboost as xgb
|
|
from phase2_h1_worker_v2 import checked,sha,save
|
|
from phase2_fixed_io import BASE,freeze,verify
|
|
from phase2_b1213_features import b13,cached,GeometryFailure
|
|
from phase2_b1213_gate import train as fit,coverage
|
|
from r4bc_guard import supervise
|
|
from s2_recovery_supervisor import tree_usage
|
|
from s2_recovery_monitoring import disk_usage
|
|
from s2_custody_common import prediction_inputs
|
|
from s2_custody_statistics import part_metrics
|
|
from r4bc_common import paired_difference
|
|
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
def config(root):
|
|
c=read(root/'contract.json')
|
|
if c['approval']!='1794904215.545139' or c['evaluation_role']!='test':raise PermissionError('Final test ruling required')
|
|
if xgb.__version__!='2.1.4' or np.__version__!='1.26.4':raise ValueError('Library identity')
|
|
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(BASE/p,h)
|
|
st=Path(c['archive']).stat();prior=c['archive_stat']
|
|
if (st.st_size,st.st_mtime_ns,st.st_ino)!=(prior['bytes'],prior['mtime_ns'],prior['inode']):raise ValueError('Original archive identity changed')
|
|
alltest=[]
|
|
for f in c['folds']:
|
|
for role in ['learning','test']:
|
|
for row in f['rows'][role]:
|
|
a=next(a for a in row['assignments'] if a['fold']==f['fold'])
|
|
if a['role']!=role or (role=='learning' and '100' not in a['learning_pools']):raise ValueError('Frozen fold role mismatch')
|
|
if {r['component_id'] for r in f['rows']['learning']}&{r['component_id'] for r in f['rows']['test']}:raise ValueError('Component leakage')
|
|
alltest.extend(r['case'] for r in f['rows']['test'])
|
|
if len(alltest)!=552 or len(set(alltest))!=552:raise ValueError('Test must partition552')
|
|
return c
|
|
|
|
def field(root,c,index):
|
|
row=c['parts'][index];dest=root/'parts'/f'{index:04d}';start=time.process_time()
|
|
try:e,r=b13(row,c,dest)
|
|
except GeometryFailure as exc:e=np.full((row['nodes'],2),np.nan);r=dict(status='SCIENTIFIC_MISSING',reason=str(exc),labels_read=False)
|
|
if e.shape!=(row['nodes'],2) or np.isinf(e).any():raise ValueError('Invalid G readout')
|
|
np.save(dest/'extra.npy',e,allow_pickle=False);save(dest/'result.json',dict(case=row['case'],index=index,component_id=row['component_id'],extra_sha256=sha(dest/'extra.npy'),**r,cpu_seconds=time.process_time()-start))
|
|
|
|
def predict(root,c,fold):
|
|
d=root/'folds'/str(fold);verify(d,'model');f=c['folds'][fold];model=xgb.XGBRegressor();model.load_model(d/'phases/train/model.ubj');dest=d/'phases/predict';dest.mkdir(parents=True,exist_ok=False)
|
|
by={r['case']:i for i,r in enumerate(c['parts'])};out=[]
|
|
for row in f['rows']['test']:
|
|
q=prediction_inputs(row);raw=cached(row,'raw_ids');ix=np.searchsorted(raw,q['raw_ids'])
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Query IDs')
|
|
p=root/'parts'/f"{by[row['case']]:04d}";r=read(p/'result.json');e=np.load(checked(p/'extra.npy',r['extra_sha256']),allow_pickle=False);features=cached(row,'features');X=np.column_stack([features[ix],e[ix]]).astype(np.float32)
|
|
pred=np.asarray(model.predict(X),np.float64)*row['preparation']['t_set'];base=c['A_predictions'][row['case']]
|
|
if base['fold']!=fold or base['pool']!=100:raise ValueError('A prediction role mismatch')
|
|
with np.load(checked(base['path'],base['sha256']),allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'],raw):raise ValueError('A original raw IDs differ')
|
|
A=z['A_seconds'][ix].copy()
|
|
if not np.isfinite(pred).all() or not np.isfinite(A).all():raise ValueError('Nonfinite prediction')
|
|
p=dest/(row['case']+'.npz');np.savez_compressed(p,pred=pred,A=A,raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
|
|
out.append(dict(case=row['case'],component_id=row['component_id'],fold=fold,path=str(p),sha256=sha(p),G_valid=r['status']=='GEOMETRY_COMPLETE',A_source_sha256=base['sha256']))
|
|
save(dest/'progress.json',dict(parts=len(out),test_labels_read=False))
|
|
save(dest/'manifest.json',dict(entries=out,test_labels_read=False))
|
|
|
|
def pooled(rows):
|
|
groups={}
|
|
for r in rows:groups.setdefault(r['component_id'],[]).append(r)
|
|
result={}
|
|
for metric in ['relative_l2','last_location','kendall_tau_b']:
|
|
values={arm:{g:float(np.mean([r[arm][metric]['value'] for r in rs])) for g,rs in groups.items() if all(r[arm][metric]['value'] is not None for r in rs)} for arm in ['new','A']}
|
|
result[metric]=dict(means={a:float(np.mean(list(v.values()))) if v else None for a,v in values.items()},components={a:len(v) for a,v in values.items()},new_minus_A=paired_difference(values['new'],values['A']))
|
|
return result
|
|
|
|
def score(root,c):
|
|
verify(root,'predictions');dest=root/'score';dest.mkdir(exist_ok=False);by={r['case']:r for r in c['parts']};rows=[]
|
|
manifest=read(root/'predictions-manifest.json')['entries']
|
|
if len(manifest)!=552 or {e['case'] for e in manifest}!=set(by):raise ValueError('Full test roster incomplete')
|
|
for e in manifest:
|
|
row=by[e['case']];save(dest/'next-target-access.json',dict(case=e['case'],fold=e['fold'],role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy']))
|
|
truth=cached(row,'seconds');raw=cached(row,'raw_ids')
|
|
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
|
|
ix=np.searchsorted(raw,z['raw_ids'])
|
|
if not np.array_equal(raw[ix],z['raw_ids']):raise ValueError('Score raw IDs')
|
|
rr=dict(case=e['case'],component_id=e['component_id'],fold=e['fold'],G_valid=e['G_valid'])
|
|
for arm in ['new','A']:rr[arm]=part_metrics(z['pred' if arm=='new' else 'A'],truth[ix],z['counts'],z['near_wall'],z['raw_ids'],z['coordinates_over_Rg'],int(e['case'].split('_')[0]))
|
|
rows.append(rr);save(dest/'progress.json',dict(parts=len(rows)))
|
|
result=pooled(rows);primary=result['relative_l2'];complete=primary['components']==dict(new=204,A=204);high=primary['new_minus_A']['high']
|
|
decision='CONFIRMED_FIXED_GENERIC_CONTEXT_INCREMENT' if complete and high is not None and high<0 else 'NOT_CONFIRMED_FIXED_RECIPE' if complete else 'PARK_UNDEFINED'
|
|
save(dest/'result.json',dict(status='COMPLETE',decision=decision,parts=552,components=204,metrics=result,per_fold={str(f):pooled([r for r in rows if r['fold']==f]) for f in range(5)},rows=rows,invalid_G_parts=[r['case'] for r in rows if not r['G_valid']],gate='pooled field CI high<0 only',validation_used=False,E2_started=False))
|
|
|
|
def worker(root,mode,index):
|
|
c=config(root)
|
|
if mode=='field':field(root,c,index)
|
|
elif mode=='train':
|
|
f=c['folds'][index];d=root/'folds'/str(index);(d/'phases/train').mkdir(parents=True,exist_ok=False);fit(d,dict(c,rows=f['rows'],extra_columns=2))
|
|
elif mode=='predict':predict(root,c,index)
|
|
else:score(root,c)
|
|
|
|
def guard(root,mode,index,slot):
|
|
c=config(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);d=root/'guards'/f'{mode}-{index:04d}';start=time.process_time();cap=c['job_caps'][mode]
|
|
result=supervise([sys.executable,'tools/phase2_b13_e1.py','worker',str(root),mode,str(index)],d,dict(cpu_seconds=cap,wall_seconds=cap+120,rss_bytes=8*2**30,disk_bytes=5*2**30),global_stop=root/'global-stop.json')
|
|
save(d/'guard-result.json',dict(mode=mode,index=index,guard=result,guard_cpu_seconds=time.process_time()-start));return 0 if result['status']=='PASS' else 1
|
|
|
|
def run(root):
|
|
c=config(root);os.sched_setaffinity(0,[30,31]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('Single E1 launch')
|
|
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[]
|
|
def total():
|
|
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 60+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_,_ in active.values())
|
|
def batch(mode,ids,phase_cap=None):
|
|
nonlocal settled
|
|
pending=list(ids);phase0=total()
|
|
while pending or active:
|
|
if requested or total()>10800-120 or (phase_cap and total()-phase0>phase_cap) or disk_usage(root)>20*2**30 or sum(tree_usage(p.pid)[1] for p,_,_ in active.values())>32*2**30:raise RuntimeError('GLOBAL_RESOURCE_OR_SIGNAL_STOP')
|
|
for slot,(p,index,_) in list(active.items()):
|
|
if p.poll() is not None:
|
|
r=read(root/'guards'/f'{mode}-{index:04d}'/'guard-result.json');settled+=r['guard']['cpu_seconds']+r['guard_cpu_seconds'];records.append(r);del active[slot]
|
|
if p.returncode!=0 or r['guard']['status']!='PASS':raise RuntimeError('WORKER_NOT_PASS')
|
|
for slot in range(4):
|
|
if pending and slot not in active:
|
|
reserve=sum(c['job_caps'][mode] for _ in active)+c['job_caps'][mode]+300
|
|
if total()+reserve>10800:
|
|
if not active:raise RuntimeError('BUDGET_RESERVATION_EXHAUSTED')
|
|
break
|
|
index=pending.pop(0)
|
|
if mode=='field':(root/'parts'/f'{index:04d}').mkdir(parents=True,exist_ok=False)
|
|
p=subprocess.Popen([sys.executable,'tools/phase2_b13_e1.py','guard',str(root),mode,str(index),str(slot)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);active[slot]=(p,index,time.time())
|
|
save(root/'live-progress.json',dict(stage=mode,pending=len(pending),active=[dict(slot=k,pid=p.pid,index=i) for k,(p,i,_) in active.items()],settled_jobs=len(records),cumulative_cpu_seconds=total(),unix=time.time()));time.sleep(.2)
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
|
|
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
|
|
try:
|
|
batch('field',c['new_geometry_indices'],2200)
|
|
entries=[];results={}
|
|
for i,row in enumerate(c['parts']):
|
|
p=root/'parts'/f'{i:04d}'/'result.json';r=read(p);checked(p.parent/'extra.npy',r['extra_sha256']);results[row['case']]=r;entries.append(dict(case=row['case'],index=i,result_sha256=sha(p)))
|
|
cov={str(f['fold']):{role:coverage(rows,results) for role,rows in f['rows'].items()} for f in c['folds']};save(root/'coverage.json',cov);save(root/'features-manifest.json',dict(entries=entries,all552_terminal=True,labels_read=False))
|
|
if any(v['fraction']<.8 for f in cov.values() for v in f.values()):
|
|
save(root/'complete.json',dict(status='COMPLETE',decision='PARK_G_COVERAGE',coverage=cov,cumulative_cpu_seconds=total(),records=records,new_fits=0));return
|
|
for f in c['folds']:
|
|
d=root/'folds'/str(f['fold']);d.mkdir(parents=True,exist_ok=False);save(d/'contract.json',c);save(d/'features-manifest.json',read(root/'features-manifest.json'));(d/'parts').symlink_to(root/'parts',target_is_directory=True)
|
|
freeze(d,'features',[d/'contract.json',d/'features-manifest.json',root/'coverage.json',*[BASE/p for p in c['code_sha256']]])
|
|
d=root/'folds/0/phases/train';d.mkdir(parents=True,exist_ok=False)
|
|
import shutil
|
|
shutil.copyfile(checked(c['fold0_model']['path'],c['fold0_model']['sha256']),d/'model.ubj');save(d/'result.json',dict(status='FIT_REUSED_BY_HASH',checkpoint_sha256=sha(d/'model.ubj'),original=c['fold0_model']))
|
|
batch('train',[1,2,3,4])
|
|
for f in range(5):
|
|
d=root/'folds'/str(f);freeze(d,'model',[d/'contract.json',d/'phases/train/model.ubj',d/'phases/train/result.json',*[BASE/p for p in c['code_sha256']]])
|
|
batch('predict',range(5));entries=[]
|
|
for f in range(5):entries.extend(read(root/'folds'/str(f)/'phases/predict/manifest.json')['entries'])
|
|
save(root/'predictions-manifest.json',dict(entries=entries,all_predictions_terminal=True));freeze(root,'predictions',[root/'contract.json',root/'predictions-manifest.json',*[BASE/p for p in c['code_sha256']]])
|
|
batch('score',[0]);save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'score/result.json')['decision'],cumulative_cpu_seconds=total(),records=records,new_fits=4,reused_fold0=True,E2_started=False))
|
|
except BaseException as exc:
|
|
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
|
|
for p,_,_ in active.values():
|
|
try:os.kill(p.pid,signal.SIGTERM)
|
|
except ProcessLookupError:pass
|
|
for p,index,_ in active.values():p.wait()
|
|
# Reaped guard/worker child CPU is included once by total().
|
|
active.clear();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,accounting_complete=True,retry=False));raise
|
|
finally:
|
|
for s,h in old.items():signal.signal(s,h)
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','worker','guard']);p.add_argument('root',type=Path);p.add_argument('mode',nargs='?');p.add_argument('index',nargs='?',type=int);p.add_argument('slot',nargs='?',type=int);a=p.parse_args()
|
|
if a.action=='run':run(a.root)
|
|
elif a.action=='worker':worker(a.root,a.mode,a.index)
|
|
else:raise SystemExit(guard(a.root,a.mode,a.index,a.slot))
|