"""E2: immutable T/C geometry -> ten fits -> all predictions -> raw/adjusted score.""" import argparse,fcntl,json,os,resource,signal,subprocess,sys,time from pathlib import Path from collections import Counter import numpy as np import xgboost as xgb from phase2_h1_worker_v2 import checked,sha,save from phase2_fixed_io import BASE,verify from phase2_local_freeze_io import freeze from phase2_b1213_features import cached from gre24_models import xgb_model from s2_custody_common import prediction_inputs from s2_custody_statistics import part_metrics from phase2_e2_statistics import analyse from r4bc_guard import supervise from s2_recovery_supervisor import tree_usage from s2_recovery_monitoring import disk_usage def read(p):return json.loads(Path(p).read_text()) def validate_roles(c): parts={r['case']:r for r in c['parts']};tests=[];groups={} if len(parts)!=552 or len(c['folds'])!=5:raise ValueError('Frozen552/five folds required') for f in c['folds']: if f['fold'] not in range(5):raise ValueError('Fold identity') for role in ['learning','test']: rows=f['rows'][role] if len({r['case'] for r in rows})!=len(rows):raise ValueError('Duplicate fold role') for r in rows: if r!=parts[r['case']]:raise ValueError('Fold/part data changed') a=next(a for a in r['assignments'] if a['fold']==f['fold']) if a['role']!=role or (role=='learning' and '100' not in a['learning_pools']):raise ValueError('Learning/test assignment mismatch') if {r['component_id'] for r in f['rows']['learning']}&{r['component_id'] for r in f['rows']['test']}:raise ValueError('Component leakage') for r in f['rows']['test']: tests.append(r['case']);groups.setdefault(r['component_id'],set()).add(f['fold']) if len(tests)!=552 or set(tests)!=set(parts) or len(groups)!=204 or any(len(v)!=1 for v in groups.values()):raise ValueError('Test roster must partition552/204 once') def config(root): c=read(root/'contract.json') if c['approval']!='1794907827.062989' or c['total_cpu_seconds']!=14400 or c['new_fits']!=10 or c['new_geometry']!=0:raise PermissionError('Original E2 scope required') if np.__version__!='1.26.4' or xgb.__version__!='2.1.4':raise ValueError('Pinned libraries') for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(BASE/p,h) st=Path(c['archive']).stat();s=c['archive_stat'] if (st.st_size,st.st_mtime_ns,st.st_ino)!=(s['bytes'],s['mtime_ns'],s['inode']):raise ValueError('Archive identity') validate_roles(c);return c def job(index): if index not in range(10):raise ValueError('Only ten fit/predict cells') return index//2,('T','C')[index%2] def jobroot(root,index):return root/'cells'/f'{index:02d}' def extras(row,arm,entries): e=entries[row['case']]['arms'][arm];v=np.load(checked(e['path'],e['sha256']),allow_pickle=False) if v.shape!=(row['nodes'],2) or np.isinf(v).any():raise ValueError('Geometry feature shape/infinity') if e['valid'] and not np.isfinite(v).all():raise ValueError('Valid arm readout nonfinite') if not e['valid'] and not np.isnan(v).all():raise ValueError('Missing arm must retain two nativeNaN columns') return v def train(root,c,index): verify(root,'features');fold,arm=job(index);rows=c['folds'][fold]['rows']['learning'];dest=jobroot(root,index)/'train';dest.mkdir(parents=True,exist_ok=False) entries={e['case']:e for e in read(root/'features-manifest.json')['entries']};n=sum(r['nodes'] for r in rows);X=np.empty((n,21),np.float32);Y=np.empty(n,np.float32);offset=0;access=[] for row in rows: f=cached(row,'features');e=extras(row,arm,entries);raw=cached(row,'raw_ids') if f.shape!=(row['nodes'],19) or not np.isfinite(f).all() or len(raw)!=row['nodes']:raise ValueError('Frozen learning feature identity') access.append(dict(case=row['case'],fold=fold,arm=arm,role='learning',learning_pool=100,target_sha256=row['v5_cache']['output_sha256']['target.npy'],raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'],nodes=row['nodes']));save(dest/'target-access.json',access) y=cached(row,'target');end=offset+len(y) if y.shape!=(row['nodes'],) or not np.isfinite(y).all():raise ValueError('Learning target shape/nonfinite') X[offset:end,:19]=f;X[offset:end,19:]=e;Y[offset:end]=y;offset=end save(dest/'progress.json',dict(stage='LOAD_LEARNING',parts=len(access),nodes=offset)) if offset!=n:raise ValueError('Learning count') model=xgb_model(0,1);save(dest/'progress.json',dict(stage='FIT',nodes=n,parts=len(access),columns=21)) model.fit(X,Y,sample_weight=np.ones(n,np.float64));model.save_model(dest/'model.ubj');save(dest/'xgboost-config.json',json.loads(model.get_booster().save_config())) save(dest/'result.json',dict(status='FIT_COMPLETE',fold=fold,arm=arm,checkpoint_sha256=sha(dest/'model.ubj'),nodes=n,parts=len(rows),columns=21,unit_node_weights=True,test_labels_read=False)) def query_identity(q,z): for key in ['raw_ids','counts','coordinates_over_Rg','near_wall']: if not np.array_equal(q[key],z[key]):raise ValueError('Frozen G query mismatch: '+key) def predict(root,c,index): verify(root,'models');fold,arm=job(index);d=jobroot(root,index);model=xgb.XGBRegressor();model.load_model(d/'train/model.ubj');dest=d/'predict';dest.mkdir(exist_ok=False) entries={e['case']:e for e in read(root/'features-manifest.json')['entries']};G={e['case']:e for e in read(c['G_prediction_manifest'])['entries']};out=[] for row in c['folds'][fold]['rows']['test']: q=prediction_inputs(row);raw=cached(row,'raw_ids');ix=np.searchsorted(raw,q['raw_ids']) if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Test query IDs') g=G[row['case']] if g['fold']!=fold or g['component_id']!=row['component_id']:raise ValueError('G fold/component mismatch') with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z:query_identity(q,z) f=cached(row,'features');e=extras(row,arm,entries);X=np.column_stack([f[ix],e[ix]]).astype(np.float32);pred=np.asarray(model.predict(X),np.float64)*row['preparation']['t_set'] if not np.isfinite(pred).all():raise ValueError('Nonfinite prediction') p=dest/(row['case']+'.npz');np.savez_compressed(p,pred=pred,**{k:q[k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']}) out.append(dict(case=row['case'],component_id=row['component_id'],fold=fold,arm=arm,path=str(p),sha256=sha(p),model_sha256=sha(d/'train/model.ubj'),G_prediction_sha256=g['sha256'])) save(dest/'progress.json',dict(parts=len(out),test_labels_read=False)) save(dest/'manifest.json',dict(entries=out,test_labels_read=False,units='seconds')) def score(root,c): verify(root,'predictions');dest=root/'score';dest.mkdir(exist_ok=False);m=read(root/'predictions-manifest.json')['entries'];by={r['case']:r for r in c['parts']};entries={e['case']:e for e in read(root/'features-manifest.json')['entries']} if len(m)!=1104 or len({(e['case'],e['arm']) for e in m})!=1104 or {(e['case'],e['arm']) for e in m}!={(s,a) for s in by for a in ['T','C']}:raise ValueError('All1104 T/C predictions required') pred={(e['case'],e['arm']):e for e in m};gm={e['case']:e for e in read(c['G_prediction_manifest'])['entries']};gr={r['case']:r for r in read(c['G_score'])['rows']};rows=[];access=[] if set(gm)!=set(by) or set(gr)!=set(by):raise ValueError('Frozen G full roster mismatch') for case,row in by.items(): geometry=read(checked(entries[case]['geometry_result'],entries[case]['geometry_sha256']));g=gm[case];old=gr[case] if old['fold']!=g['fold'] or old['component_id']!=row['component_id']:raise ValueError('G scoring identity') access.append(dict(case=case,fold=g['fold'],role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy']));save(dest/'target-access.json',access) truth=cached(row,'seconds');raw=cached(row,'raw_ids');rr=dict(case=case,component_id=row['component_id'],fold=g['fold'],metrics={'G':old['new']},geometry=geometry) with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z: q={k:z[k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']};ix=np.searchsorted(raw,q['raw_ids']) if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Score raw IDs') for arm in ['T','C']: e=pred[case,arm] if e['fold']!=g['fold'] or e['G_prediction_sha256']!=g['sha256']:raise ValueError('Prediction role/source mismatch') with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as v: query_identity(q,v);rr['metrics'][arm]=part_metrics(v['pred'],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0])) rows.append(rr);save(dest/'progress.json',dict(stage='PART_METRICS',parts=len(rows))) save(dest/'part-metrics.json',dict(rows=rows,original_G_metrics_reused=True));save(dest/'progress.json',dict(stage='PAIRED_COMPONENT_BOOTSTRAP_REFIT',parts=552,replicates=2000)) result,arrays=analyse(rows);np.savez_compressed(dest/'bootstrap.npz',**arrays) result.update(status='COMPLETE',rows=rows,coverage=read(root/'coverage.json'),geometry_cost=c['prior_geometry_cpu_seconds'],new_models=10,reused_G_models=5,new_geometry=0,validation_used=False,new_decodes=0,bootstrap_sha256=sha(dest/'bootstrap.npz')) save(dest/'result.json',result) def worker(root,mode,index): c=config(root) if mode=='score':score(root,c) else:{'train':train,'predict':predict}[mode](root,c,index) def guard(root,mode,index,slot): c=config(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);start=time.process_time();cap=c['job_caps'][mode];d=root/'guards'/f'{mode}-{index:02d}' g=supervise([sys.executable,'tools/phase2_e2_postfit.py','worker',str(root),mode,str(index)],d,dict(cpu_seconds=cap,wall_seconds=cap+180,rss_bytes=8*2**30,disk_bytes=c['disk_bytes']),global_stop=root/'global-stop.json') save(d/'guard-result.json',dict(mode=mode,index=index,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1 def admission(total,active_count,job_cap,limit=14400):return total+(active_count+1)*job_cap+120<=limit def run(root): c=config(root);os.sched_setaffinity(0,[30,31]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('Single attempt E2 postfit') verify(root,'launch');verify(root,'features');start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[] def total(): u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['prior_geometry_cpu_seconds']+c['setup_seconds']+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_,_ in active.values()) def limits(): sample=dict(cpu_seconds=total(),rss_bytes=sum(tree_usage(p.pid)[1] for p,_,_ in active.values()),disk_bytes=disk_usage(root)+disk_usage(Path(c['geometry_root'])),signals=list(requested));reasons=[] if sample['cpu_seconds']>14400-120:reasons.append('TOTAL_CPU_RESERVE') if sample['rss_bytes']>32*2**30:reasons.append('RSS_LIMIT') if sample['disk_bytes']>c['disk_bytes']:reasons.append('COMBINED_GEOMETRY_POSTFIT_DISK_LIMIT') if requested:reasons.append('COORDINATOR_SIGNAL') if reasons:save(root/'stop-trigger.json',dict(reasons=reasons,sample=sample));raise RuntimeError('RESOURCE_OR_SIGNAL_STOP') def batch(mode,indices): nonlocal settled pending=list(indices) while pending or active: limits() for slot,(p,i,m) in list(active.items()): if p.poll() is not None: r=read(root/'guards'/f'{m}-{i:02d}'/'guard-result.json');settled+=r['guard']['cpu_seconds']+r['guard_cpu_seconds'];records.append(r);del active[slot] if p.returncode or r['guard']['status']!='PASS':raise RuntimeError('WORKER_NOT_PASS') for slot in range(4): if pending and slot not in active: if not admission(total(),len(active),c['job_caps'][mode]): if not active:raise RuntimeError('TOTAL_ADMISSION_RESERVE') break i=pending.pop(0);p=subprocess.Popen([sys.executable,'tools/phase2_e2_postfit.py','guard',str(root),mode,str(i),str(slot)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);active[slot]=(p,i,mode) save(root/'live-progress.json',dict(stage=mode,pending=len(pending),active=[dict(slot=s,pid=p.pid,index=i,mode=m) for s,(p,i,m) in active.items()],terminal=len(records),cumulative_cpu_seconds=total(),unix=time.time()));time.sleep(.5) save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),prior_geometry_cpu_seconds=c['prior_geometry_cpu_seconds'],new_geometry=0)) old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]} try: batch('train',range(10));freeze(root,'models',[root/'contract.json',*[jobroot(root,i)/'train'/n for i in range(10) for n in ['model.ubj','result.json','target-access.json']],*[BASE/p for p in c['code_sha256']]]) batch('predict',range(10));entries=[] for i in range(10):entries.extend(read(jobroot(root,i)/'predict/manifest.json')['entries']) save(root/'predictions-manifest.json',dict(entries=entries,all1104_terminal=True,reused_G_manifest=c['G_prediction_manifest'],reused_G_manifest_sha256=c['source_sha256'][c['G_prediction_manifest']],labels_read=False)) freeze(root,'predictions',[root/'contract.json',root/'predictions-manifest.json',root/'features-manifest.json',Path(c['G_prediction_manifest']),Path(c['G_score']),*[BASE/p for p in c['code_sha256']]]) batch('score',[0]);limits();save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'score/result.json')['decision'],cumulative_cpu_seconds=total(),prior_geometry_cpu_seconds=c['prior_geometry_cpu_seconds'],records=records,new_fits=10,new_predictions=1104,reused_G_predictions=552,accounting_complete=True,retry=False)) freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'score/bootstrap.npz']) save(root/'post-terminal-freeze-cost.json',dict(cumulative_cpu_seconds=total(),includes_terminal_freeze=True,within_cap=total()<=14400)) except BaseException as exc: save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc))) for p,_,_ in active.values(): try:os.kill(p.pid,signal.SIGTERM) except ProcessLookupError:pass for p,i,m in active.values(): p.wait();f=root/'guards'/f'{m}-{i:02d}'/'guard-result.json' if f.exists():records.append(read(f)) active.clear();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,accounting_complete=True,retry=False));raise finally: for s,h in old.items():signal.signal(s,h) if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('action',choices=['run','worker','guard']);p.add_argument('root',type=Path);p.add_argument('mode',nargs='?');p.add_argument('index',nargs='?',type=int);p.add_argument('slot',nargs='?',type=int);a=p.parse_args() if a.action=='run':run(a.root) elif a.action=='worker':worker(a.root,a.mode,a.index) else:raise SystemExit(guard(a.root,a.mode,a.index,a.slot))