Freeze B13 E1 five-fold test confirmation and exact baseline reuse
This commit is contained in:
1 parent
76dea96bd2
commit
783608f73b
5 files changed
+363631
No files matched your search
@@ -0,0 +1,162 @@
|
||||
"""Five-fold held-out-test confirmation; immutable G/model reuse and bounded queue."""
|
||||
import argparse,fcntl,json,os,resource,sys,time,subprocess,signal,traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import xgboost as xgb
|
||||
from phase2_h1_worker_v2 import checked,sha,save
|
||||
from phase2_fixed_io import BASE,freeze,verify
|
||||
from phase2_b1213_features import b13,cached,GeometryFailure
|
||||
from phase2_b1213_gate import train as fit,coverage
|
||||
from r4bc_guard import supervise
|
||||
from s2_recovery_supervisor import tree_usage
|
||||
from s2_recovery_monitoring import disk_usage
|
||||
from s2_custody_common import prediction_inputs
|
||||
from s2_custody_statistics import part_metrics
|
||||
from r4bc_common import paired_difference
|
||||
|
||||
def read(p):return json.loads(Path(p).read_text())
|
||||
def config(root):
|
||||
c=read(root/'contract.json')
|
||||
if c['approval']!='1794904215.545139' or c['evaluation_role']!='test':raise PermissionError('Final test ruling required')
|
||||
if xgb.__version__!='2.1.4' or np.__version__!='1.26.4':raise ValueError('Library identity')
|
||||
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(BASE/p,h)
|
||||
st=Path(c['archive']).stat();prior=c['archive_stat']
|
||||
if (st.st_size,st.st_mtime_ns,st.st_ino)!=(prior['bytes'],prior['mtime_ns'],prior['inode']):raise ValueError('Original archive identity changed')
|
||||
alltest=[]
|
||||
for f in c['folds']:
|
||||
for role in ['learning','test']:
|
||||
for row in f['rows'][role]:
|
||||
a=next(a for a in row['assignments'] if a['fold']==f['fold'])
|
||||
if a['role']!=role or (role=='learning' and '100' not in a['learning_pools']):raise ValueError('Frozen fold role mismatch')
|
||||
if {r['component_id'] for r in f['rows']['learning']}&{r['component_id'] for r in f['rows']['test']}:raise ValueError('Component leakage')
|
||||
alltest.extend(r['case'] for r in f['rows']['test'])
|
||||
if len(alltest)!=552 or len(set(alltest))!=552:raise ValueError('Test must partition552')
|
||||
return c
|
||||
|
||||
def field(root,c,index):
|
||||
row=c['parts'][index];dest=root/'parts'/f'{index:04d}';start=time.process_time()
|
||||
try:e,r=b13(row,c,dest)
|
||||
except GeometryFailure as exc:e=np.full((row['nodes'],2),np.nan);r=dict(status='SCIENTIFIC_MISSING',reason=str(exc),labels_read=False)
|
||||
if e.shape!=(row['nodes'],2) or np.isinf(e).any():raise ValueError('Invalid G readout')
|
||||
np.save(dest/'extra.npy',e,allow_pickle=False);save(dest/'result.json',dict(case=row['case'],index=index,component_id=row['component_id'],extra_sha256=sha(dest/'extra.npy'),**r,cpu_seconds=time.process_time()-start))
|
||||
|
||||
def predict(root,c,fold):
|
||||
d=root/'folds'/str(fold);verify(d,'model');f=c['folds'][fold];model=xgb.XGBRegressor();model.load_model(d/'phases/train/model.ubj');dest=d/'phases/predict';dest.mkdir(parents=True,exist_ok=False)
|
||||
by={r['case']:i for i,r in enumerate(c['parts'])};out=[]
|
||||
for row in f['rows']['test']:
|
||||
q=prediction_inputs(row);raw=cached(row,'raw_ids');ix=np.searchsorted(raw,q['raw_ids'])
|
||||
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Query IDs')
|
||||
p=root/'parts'/f"{by[row['case']]:04d}";r=read(p/'result.json');e=np.load(checked(p/'extra.npy',r['extra_sha256']),allow_pickle=False);features=cached(row,'features');X=np.column_stack([features[ix],e[ix]]).astype(np.float32)
|
||||
pred=np.asarray(model.predict(X),np.float64)*row['preparation']['t_set'];base=c['A_predictions'][row['case']]
|
||||
if base['fold']!=fold or base['pool']!=100:raise ValueError('A prediction role mismatch')
|
||||
with np.load(checked(base['path'],base['sha256']),allow_pickle=False) as z:
|
||||
if not np.array_equal(z['raw_ids'],raw):raise ValueError('A original raw IDs differ')
|
||||
A=z['A_seconds'][ix].copy()
|
||||
if not np.isfinite(pred).all() or not np.isfinite(A).all():raise ValueError('Nonfinite prediction')
|
||||
p=dest/(row['case']+'.npz');np.savez_compressed(p,pred=pred,A=A,raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
|
||||
out.append(dict(case=row['case'],component_id=row['component_id'],fold=fold,path=str(p),sha256=sha(p),G_valid=r['status']=='GEOMETRY_COMPLETE',A_source_sha256=base['sha256']))
|
||||
save(dest/'progress.json',dict(parts=len(out),test_labels_read=False))
|
||||
save(dest/'manifest.json',dict(entries=out,test_labels_read=False))
|
||||
|
||||
def pooled(rows):
|
||||
groups={}
|
||||
for r in rows:groups.setdefault(r['component_id'],[]).append(r)
|
||||
result={}
|
||||
for metric in ['relative_l2','last_location','kendall_tau_b']:
|
||||
values={arm:{g:float(np.mean([r[arm][metric]['value'] for r in rs])) for g,rs in groups.items() if all(r[arm][metric]['value'] is not None for r in rs)} for arm in ['new','A']}
|
||||
result[metric]=dict(means={a:float(np.mean(list(v.values()))) if v else None for a,v in values.items()},components={a:len(v) for a,v in values.items()},new_minus_A=paired_difference(values['new'],values['A']))
|
||||
return result
|
||||
|
||||
def score(root,c):
|
||||
verify(root,'predictions');dest=root/'score';dest.mkdir(exist_ok=False);by={r['case']:r for r in c['parts']};rows=[]
|
||||
manifest=read(root/'predictions-manifest.json')['entries']
|
||||
if len(manifest)!=552 or {e['case'] for e in manifest}!=set(by):raise ValueError('Full test roster incomplete')
|
||||
for e in manifest:
|
||||
row=by[e['case']];save(dest/'next-target-access.json',dict(case=e['case'],fold=e['fold'],role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy']))
|
||||
truth=cached(row,'seconds');raw=cached(row,'raw_ids')
|
||||
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
|
||||
ix=np.searchsorted(raw,z['raw_ids'])
|
||||
if not np.array_equal(raw[ix],z['raw_ids']):raise ValueError('Score raw IDs')
|
||||
rr=dict(case=e['case'],component_id=e['component_id'],fold=e['fold'],G_valid=e['G_valid'])
|
||||
for arm in ['new','A']:rr[arm]=part_metrics(z['pred' if arm=='new' else 'A'],truth[ix],z['counts'],z['near_wall'],z['raw_ids'],z['coordinates_over_Rg'],int(e['case'].split('_')[0]))
|
||||
rows.append(rr);save(dest/'progress.json',dict(parts=len(rows)))
|
||||
result=pooled(rows);primary=result['relative_l2'];complete=primary['components']==dict(new=204,A=204);high=primary['new_minus_A']['high']
|
||||
decision='CONFIRMED_FIXED_GENERIC_CONTEXT_INCREMENT' if complete and high is not None and high<0 else 'NOT_CONFIRMED_FIXED_RECIPE' if complete else 'PARK_UNDEFINED'
|
||||
save(dest/'result.json',dict(status='COMPLETE',decision=decision,parts=552,components=204,metrics=result,per_fold={str(f):pooled([r for r in rows if r['fold']==f]) for f in range(5)},rows=rows,invalid_G_parts=[r['case'] for r in rows if not r['G_valid']],gate='pooled field CI high<0 only',validation_used=False,E2_started=False))
|
||||
|
||||
def worker(root,mode,index):
|
||||
c=config(root)
|
||||
if mode=='field':field(root,c,index)
|
||||
elif mode=='train':
|
||||
f=c['folds'][index];d=root/'folds'/str(index);(d/'phases/train').mkdir(parents=True,exist_ok=False);fit(d,dict(c,rows=f['rows'],extra_columns=2))
|
||||
elif mode=='predict':predict(root,c,index)
|
||||
else:score(root,c)
|
||||
|
||||
def guard(root,mode,index,slot):
|
||||
c=config(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);d=root/'guards'/f'{mode}-{index:04d}';start=time.process_time();cap=c['job_caps'][mode]
|
||||
result=supervise([sys.executable,'tools/phase2_b13_e1.py','worker',str(root),mode,str(index)],d,dict(cpu_seconds=cap,wall_seconds=cap+120,rss_bytes=8*2**30,disk_bytes=5*2**30),global_stop=root/'global-stop.json')
|
||||
save(d/'guard-result.json',dict(mode=mode,index=index,guard=result,guard_cpu_seconds=time.process_time()-start));return 0 if result['status']=='PASS' else 1
|
||||
|
||||
def run(root):
|
||||
c=config(root);os.sched_setaffinity(0,[30,31]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Single E1 launch')
|
||||
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[]
|
||||
def total():
|
||||
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 60+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_,_ in active.values())
|
||||
def batch(mode,ids,phase_cap=None):
|
||||
nonlocal settled
|
||||
pending=list(ids);phase0=total()
|
||||
while pending or active:
|
||||
if requested or total()>10800-120 or (phase_cap and total()-phase0>phase_cap) or disk_usage(root)>20*2**30 or sum(tree_usage(p.pid)[1] for p,_,_ in active.values())>32*2**30:raise RuntimeError('GLOBAL_RESOURCE_OR_SIGNAL_STOP')
|
||||
for slot,(p,index,_) in list(active.items()):
|
||||
if p.poll() is not None:
|
||||
r=read(root/'guards'/f'{mode}-{index:04d}'/'guard-result.json');settled+=r['guard']['cpu_seconds']+r['guard_cpu_seconds'];records.append(r);del active[slot]
|
||||
if p.returncode!=0 or r['guard']['status']!='PASS':raise RuntimeError('WORKER_NOT_PASS')
|
||||
for slot in range(4):
|
||||
if pending and slot not in active:
|
||||
reserve=sum(c['job_caps'][mode] for _ in active)+c['job_caps'][mode]+300
|
||||
if total()+reserve>10800:
|
||||
if not active:raise RuntimeError('BUDGET_RESERVATION_EXHAUSTED')
|
||||
break
|
||||
index=pending.pop(0)
|
||||
if mode=='field':(root/'parts'/f'{index:04d}').mkdir(parents=True,exist_ok=False)
|
||||
p=subprocess.Popen([sys.executable,'tools/phase2_b13_e1.py','guard',str(root),mode,str(index),str(slot)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);active[slot]=(p,index,time.time())
|
||||
save(root/'live-progress.json',dict(stage=mode,pending=len(pending),active=[dict(slot=k,pid=p.pid,index=i) for k,(p,i,_) in active.items()],settled_jobs=len(records),cumulative_cpu_seconds=total(),unix=time.time()));time.sleep(.2)
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
|
||||
try:
|
||||
batch('field',c['new_geometry_indices'],2200)
|
||||
entries=[];results={}
|
||||
for i,row in enumerate(c['parts']):
|
||||
p=root/'parts'/f'{i:04d}'/'result.json';r=read(p);checked(p.parent/'extra.npy',r['extra_sha256']);results[row['case']]=r;entries.append(dict(case=row['case'],index=i,result_sha256=sha(p)))
|
||||
cov={str(f['fold']):{role:coverage(rows,results) for role,rows in f['rows'].items()} for f in c['folds']};save(root/'coverage.json',cov);save(root/'features-manifest.json',dict(entries=entries,all552_terminal=True,labels_read=False))
|
||||
if any(v['fraction']<.8 for f in cov.values() for v in f.values()):
|
||||
save(root/'complete.json',dict(status='COMPLETE',decision='PARK_G_COVERAGE',coverage=cov,cumulative_cpu_seconds=total(),records=records,new_fits=0));return
|
||||
for f in c['folds']:
|
||||
d=root/'folds'/str(f['fold']);d.mkdir(parents=True,exist_ok=False);save(d/'contract.json',c);save(d/'features-manifest.json',read(root/'features-manifest.json'));(d/'parts').symlink_to(root/'parts',target_is_directory=True)
|
||||
freeze(d,'features',[d/'contract.json',d/'features-manifest.json',root/'coverage.json',*[BASE/p for p in c['code_sha256']]])
|
||||
d=root/'folds/0/phases/train';d.mkdir(parents=True,exist_ok=False)
|
||||
import shutil
|
||||
shutil.copyfile(checked(c['fold0_model']['path'],c['fold0_model']['sha256']),d/'model.ubj');save(d/'result.json',dict(status='FIT_REUSED_BY_HASH',checkpoint_sha256=sha(d/'model.ubj'),original=c['fold0_model']))
|
||||
batch('train',[1,2,3,4])
|
||||
for f in range(5):
|
||||
d=root/'folds'/str(f);freeze(d,'model',[d/'contract.json',d/'phases/train/model.ubj',d/'phases/train/result.json',*[BASE/p for p in c['code_sha256']]])
|
||||
batch('predict',range(5));entries=[]
|
||||
for f in range(5):entries.extend(read(root/'folds'/str(f)/'phases/predict/manifest.json')['entries'])
|
||||
save(root/'predictions-manifest.json',dict(entries=entries,all_predictions_terminal=True));freeze(root,'predictions',[root/'contract.json',root/'predictions-manifest.json',*[BASE/p for p in c['code_sha256']]])
|
||||
batch('score',[0]);save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'score/result.json')['decision'],cumulative_cpu_seconds=total(),records=records,new_fits=4,reused_fold0=True,E2_started=False))
|
||||
except BaseException as exc:
|
||||
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
|
||||
for p,_,_ in active.values():
|
||||
try:os.kill(p.pid,signal.SIGTERM)
|
||||
except ProcessLookupError:pass
|
||||
for p,index,_ in active.values():p.wait()
|
||||
# Reaped guard/worker child CPU is included once by total().
|
||||
active.clear();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,accounting_complete=True,retry=False));raise
|
||||
finally:
|
||||
for s,h in old.items():signal.signal(s,h)
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','worker','guard']);p.add_argument('root',type=Path);p.add_argument('mode',nargs='?');p.add_argument('index',nargs='?',type=int);p.add_argument('slot',nargs='?',type=int);a=p.parse_args()
|
||||
if a.action=='run':run(a.root)
|
||||
elif a.action=='worker':worker(a.root,a.mode,a.index)
|
||||
else:raise SystemExit(guard(a.root,a.mode,a.index,a.slot))
|
||||
@@ -0,0 +1,26 @@
|
||||
import ast,unittest
|
||||
import numpy as np
|
||||
from phase2_b13_e1 import pooled
|
||||
from phase2_b1213_gate import coverage
|
||||
from s2_custody_statistics import kendall_pairs
|
||||
class E1Tests(unittest.TestCase):
|
||||
def metric(v):return {m:dict(value=v,reason=None) for m in ['relative_l2','last_location','kendall_tau_b']}
|
||||
def test_component_not_part_weight(self):
|
||||
rows=[dict(component_id='a',new=self.metric(0),A=self.metric(0))]+[dict(component_id='b',new=self.metric(2),A=self.metric(0)) for i in range(10)]
|
||||
self.assertEqual(pooled(rows)['relative_l2']['new_minus_A']['value'],1)
|
||||
def test_coverage_requires_all_members(self):
|
||||
rows=[dict(component_id='a',case='1'),dict(component_id='a',case='2'),dict(component_id='b',case='3')]
|
||||
result={'1':dict(status='GEOMETRY_COMPLETE'),'2':dict(status='SCIENTIFIC_MISSING'),'3':dict(status='GEOMETRY_COMPLETE')}
|
||||
self.assertEqual(coverage(rows,result)['fraction'],.5)
|
||||
def test_undefined_component_retained_as_missing(self):
|
||||
rows=[dict(component_id='a',new=self.metric(None),A=self.metric(1))]
|
||||
self.assertIsNone(pooled(rows)['relative_l2']['new_minus_A']['high'])
|
||||
def test_frozen_fit_reads_learning_only(self):
|
||||
from pathlib import Path
|
||||
tree=ast.parse(Path('tools/phase2_b1213_gate.py').read_text());fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='train')
|
||||
constants=[n.value for n in ast.walk(fn) if isinstance(n,ast.Constant)]
|
||||
self.assertIn('learning',constants);self.assertNotIn('validation',constants);self.assertNotIn('test',constants)
|
||||
def test_tau_ties_undefined(self):
|
||||
self.assertIsNone(kendall_pairs([1,1,1],[1,2,3],7)['value']);self.assertEqual(kendall_pairs([1,2,3],[1,2,3],7)['value'],1.)
|
||||
metric=E1Tests.metric;E1Tests.metric=staticmethod(metric)
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user