Files
MeltMind/tools/phase2_e2_postfit.py
T

186 lines
15 KiB
Python

"""E2: immutable T/C geometry -> ten fits -> all predictions -> raw/adjusted score."""
import argparse,fcntl,json,os,resource,signal,subprocess,sys,time
from pathlib import Path
from collections import Counter
import numpy as np
import xgboost as xgb
from phase2_h1_worker_v2 import checked,sha,save
from phase2_fixed_io import BASE,verify
from phase2_local_freeze_io import freeze
from phase2_b1213_features import cached
from gre24_models import xgb_model
from s2_custody_common import prediction_inputs
from s2_custody_statistics import part_metrics
from phase2_e2_statistics import analyse
from r4bc_guard import supervise
from s2_recovery_supervisor import tree_usage
from s2_recovery_monitoring import disk_usage
def read(p):return json.loads(Path(p).read_text())
def validate_roles(c):
parts={r['case']:r for r in c['parts']};tests=[];groups={}
if len(parts)!=552 or len(c['folds'])!=5:raise ValueError('Frozen552/five folds required')
for f in c['folds']:
if f['fold'] not in range(5):raise ValueError('Fold identity')
for role in ['learning','test']:
rows=f['rows'][role]
if len({r['case'] for r in rows})!=len(rows):raise ValueError('Duplicate fold role')
for r in rows:
if r!=parts[r['case']]:raise ValueError('Fold/part data changed')
a=next(a for a in r['assignments'] if a['fold']==f['fold'])
if a['role']!=role or (role=='learning' and '100' not in a['learning_pools']):raise ValueError('Learning/test assignment mismatch')
if {r['component_id'] for r in f['rows']['learning']}&{r['component_id'] for r in f['rows']['test']}:raise ValueError('Component leakage')
for r in f['rows']['test']:
tests.append(r['case']);groups.setdefault(r['component_id'],set()).add(f['fold'])
if len(tests)!=552 or set(tests)!=set(parts) or len(groups)!=204 or any(len(v)!=1 for v in groups.values()):raise ValueError('Test roster must partition552/204 once')
def config(root):
c=read(root/'contract.json')
if c['approval']!='1794907827.062989' or c['total_cpu_seconds']!=14400 or c['new_fits']!=10 or c['new_geometry']!=0:raise PermissionError('Original E2 scope required')
if np.__version__!='1.26.4' or xgb.__version__!='2.1.4':raise ValueError('Pinned libraries')
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(BASE/p,h)
st=Path(c['archive']).stat();s=c['archive_stat']
if (st.st_size,st.st_mtime_ns,st.st_ino)!=(s['bytes'],s['mtime_ns'],s['inode']):raise ValueError('Archive identity')
validate_roles(c);return c
def job(index):
if index not in range(10):raise ValueError('Only ten fit/predict cells')
return index//2,('T','C')[index%2]
def jobroot(root,index):return root/'cells'/f'{index:02d}'
def extras(row,arm,entries):
e=entries[row['case']]['arms'][arm];v=np.load(checked(e['path'],e['sha256']),allow_pickle=False)
if v.shape!=(row['nodes'],2) or np.isinf(v).any():raise ValueError('Geometry feature shape/infinity')
if e['valid'] and not np.isfinite(v).all():raise ValueError('Valid arm readout nonfinite')
if not e['valid'] and not np.isnan(v).all():raise ValueError('Missing arm must retain two nativeNaN columns')
return v
def train(root,c,index):
verify(root,'features');fold,arm=job(index);rows=c['folds'][fold]['rows']['learning'];dest=jobroot(root,index)/'train';dest.mkdir(parents=True,exist_ok=False)
entries={e['case']:e for e in read(root/'features-manifest.json')['entries']};n=sum(r['nodes'] for r in rows);X=np.empty((n,21),np.float32);Y=np.empty(n,np.float32);offset=0;access=[]
for row in rows:
f=cached(row,'features');e=extras(row,arm,entries);raw=cached(row,'raw_ids')
if f.shape!=(row['nodes'],19) or not np.isfinite(f).all() or len(raw)!=row['nodes']:raise ValueError('Frozen learning feature identity')
access.append(dict(case=row['case'],fold=fold,arm=arm,role='learning',learning_pool=100,target_sha256=row['v5_cache']['output_sha256']['target.npy'],raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'],nodes=row['nodes']));save(dest/'target-access.json',access)
y=cached(row,'target');end=offset+len(y)
if y.shape!=(row['nodes'],) or not np.isfinite(y).all():raise ValueError('Learning target shape/nonfinite')
X[offset:end,:19]=f;X[offset:end,19:]=e;Y[offset:end]=y;offset=end
save(dest/'progress.json',dict(stage='LOAD_LEARNING',parts=len(access),nodes=offset))
if offset!=n:raise ValueError('Learning count')
model=xgb_model(0,1);save(dest/'progress.json',dict(stage='FIT',nodes=n,parts=len(access),columns=21))
model.fit(X,Y,sample_weight=np.ones(n,np.float64));model.save_model(dest/'model.ubj');save(dest/'xgboost-config.json',json.loads(model.get_booster().save_config()))
save(dest/'result.json',dict(status='FIT_COMPLETE',fold=fold,arm=arm,checkpoint_sha256=sha(dest/'model.ubj'),nodes=n,parts=len(rows),columns=21,unit_node_weights=True,test_labels_read=False))
def query_identity(q,z):
for key in ['raw_ids','counts','coordinates_over_Rg','near_wall']:
if not np.array_equal(q[key],z[key]):raise ValueError('Frozen G query mismatch: '+key)
def predict(root,c,index):
verify(root,'models');fold,arm=job(index);d=jobroot(root,index);model=xgb.XGBRegressor();model.load_model(d/'train/model.ubj');dest=d/'predict';dest.mkdir(exist_ok=False)
entries={e['case']:e for e in read(root/'features-manifest.json')['entries']};G={e['case']:e for e in read(c['G_prediction_manifest'])['entries']};out=[]
for row in c['folds'][fold]['rows']['test']:
q=prediction_inputs(row);raw=cached(row,'raw_ids');ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Test query IDs')
g=G[row['case']]
if g['fold']!=fold or g['component_id']!=row['component_id']:raise ValueError('G fold/component mismatch')
with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z:query_identity(q,z)
f=cached(row,'features');e=extras(row,arm,entries);X=np.column_stack([f[ix],e[ix]]).astype(np.float32);pred=np.asarray(model.predict(X),np.float64)*row['preparation']['t_set']
if not np.isfinite(pred).all():raise ValueError('Nonfinite prediction')
p=dest/(row['case']+'.npz');np.savez_compressed(p,pred=pred,**{k:q[k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']})
out.append(dict(case=row['case'],component_id=row['component_id'],fold=fold,arm=arm,path=str(p),sha256=sha(p),model_sha256=sha(d/'train/model.ubj'),G_prediction_sha256=g['sha256']))
save(dest/'progress.json',dict(parts=len(out),test_labels_read=False))
save(dest/'manifest.json',dict(entries=out,test_labels_read=False,units='seconds'))
def score(root,c):
verify(root,'predictions');dest=root/'score';dest.mkdir(exist_ok=False);m=read(root/'predictions-manifest.json')['entries'];by={r['case']:r for r in c['parts']};entries={e['case']:e for e in read(root/'features-manifest.json')['entries']}
if len(m)!=1104 or len({(e['case'],e['arm']) for e in m})!=1104 or {(e['case'],e['arm']) for e in m}!={(s,a) for s in by for a in ['T','C']}:raise ValueError('All1104 T/C predictions required')
pred={(e['case'],e['arm']):e for e in m};gm={e['case']:e for e in read(c['G_prediction_manifest'])['entries']};gr={r['case']:r for r in read(c['G_score'])['rows']};rows=[];access=[]
if set(gm)!=set(by) or set(gr)!=set(by):raise ValueError('Frozen G full roster mismatch')
for case,row in by.items():
geometry=read(checked(entries[case]['geometry_result'],entries[case]['geometry_sha256']));g=gm[case];old=gr[case]
if old['fold']!=g['fold'] or old['component_id']!=row['component_id']:raise ValueError('G scoring identity')
access.append(dict(case=case,fold=g['fold'],role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy']));save(dest/'target-access.json',access)
truth=cached(row,'seconds');raw=cached(row,'raw_ids');rr=dict(case=case,component_id=row['component_id'],fold=g['fold'],metrics={'G':old['new']},geometry=geometry)
with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z:
q={k:z[k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']};ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Score raw IDs')
for arm in ['T','C']:
e=pred[case,arm]
if e['fold']!=g['fold'] or e['G_prediction_sha256']!=g['sha256']:raise ValueError('Prediction role/source mismatch')
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as v:
query_identity(q,v);rr['metrics'][arm]=part_metrics(v['pred'],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
rows.append(rr);save(dest/'progress.json',dict(stage='PART_METRICS',parts=len(rows)))
save(dest/'part-metrics.json',dict(rows=rows,original_G_metrics_reused=True));save(dest/'progress.json',dict(stage='PAIRED_COMPONENT_BOOTSTRAP_REFIT',parts=552,replicates=2000))
result,arrays=analyse(rows);np.savez_compressed(dest/'bootstrap.npz',**arrays)
result.update(status='COMPLETE',rows=rows,coverage=read(root/'coverage.json'),geometry_cost=c['prior_geometry_cpu_seconds'],new_models=10,reused_G_models=5,new_geometry=0,validation_used=False,new_decodes=0,bootstrap_sha256=sha(dest/'bootstrap.npz'))
save(dest/'result.json',result)
def worker(root,mode,index):
c=config(root)
if mode=='score':score(root,c)
else:{'train':train,'predict':predict}[mode](root,c,index)
def guard(root,mode,index,slot):
c=config(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);start=time.process_time();cap=c['job_caps'][mode];d=root/'guards'/f'{mode}-{index:02d}'
g=supervise([sys.executable,'tools/phase2_e2_postfit.py','worker',str(root),mode,str(index)],d,dict(cpu_seconds=cap,wall_seconds=cap+180,rss_bytes=8*2**30,disk_bytes=c['disk_bytes']),global_stop=root/'global-stop.json')
save(d/'guard-result.json',dict(mode=mode,index=index,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1
def admission(total,active_count,job_cap,limit=14400):return total+(active_count+1)*job_cap+120<=limit
def run(root):
c=config(root);os.sched_setaffinity(0,[30,31]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single attempt E2 postfit')
verify(root,'launch');verify(root,'features');start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[]
def total():
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['prior_geometry_cpu_seconds']+c['setup_seconds']+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_,_ in active.values())
def limits():
sample=dict(cpu_seconds=total(),rss_bytes=sum(tree_usage(p.pid)[1] for p,_,_ in active.values()),disk_bytes=disk_usage(root)+disk_usage(Path(c['geometry_root'])),signals=list(requested));reasons=[]
if sample['cpu_seconds']>14400-120:reasons.append('TOTAL_CPU_RESERVE')
if sample['rss_bytes']>32*2**30:reasons.append('RSS_LIMIT')
if sample['disk_bytes']>c['disk_bytes']:reasons.append('COMBINED_GEOMETRY_POSTFIT_DISK_LIMIT')
if requested:reasons.append('COORDINATOR_SIGNAL')
if reasons:save(root/'stop-trigger.json',dict(reasons=reasons,sample=sample));raise RuntimeError('RESOURCE_OR_SIGNAL_STOP')
def batch(mode,indices):
nonlocal settled
pending=list(indices)
while pending or active:
limits()
for slot,(p,i,m) in list(active.items()):
if p.poll() is not None:
r=read(root/'guards'/f'{m}-{i:02d}'/'guard-result.json');settled+=r['guard']['cpu_seconds']+r['guard_cpu_seconds'];records.append(r);del active[slot]
if p.returncode or r['guard']['status']!='PASS':raise RuntimeError('WORKER_NOT_PASS')
for slot in range(4):
if pending and slot not in active:
if not admission(total(),len(active),c['job_caps'][mode]):
if not active:raise RuntimeError('TOTAL_ADMISSION_RESERVE')
break
i=pending.pop(0);p=subprocess.Popen([sys.executable,'tools/phase2_e2_postfit.py','guard',str(root),mode,str(i),str(slot)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);active[slot]=(p,i,mode)
save(root/'live-progress.json',dict(stage=mode,pending=len(pending),active=[dict(slot=s,pid=p.pid,index=i,mode=m) for s,(p,i,m) in active.items()],terminal=len(records),cumulative_cpu_seconds=total(),unix=time.time()));time.sleep(.5)
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),prior_geometry_cpu_seconds=c['prior_geometry_cpu_seconds'],new_geometry=0))
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
try:
batch('train',range(10));freeze(root,'models',[root/'contract.json',*[jobroot(root,i)/'train'/n for i in range(10) for n in ['model.ubj','result.json','target-access.json']],*[BASE/p for p in c['code_sha256']]])
batch('predict',range(10));entries=[]
for i in range(10):entries.extend(read(jobroot(root,i)/'predict/manifest.json')['entries'])
save(root/'predictions-manifest.json',dict(entries=entries,all1104_terminal=True,reused_G_manifest=c['G_prediction_manifest'],reused_G_manifest_sha256=c['source_sha256'][c['G_prediction_manifest']],labels_read=False))
freeze(root,'predictions',[root/'contract.json',root/'predictions-manifest.json',root/'features-manifest.json',Path(c['G_prediction_manifest']),Path(c['G_score']),*[BASE/p for p in c['code_sha256']]])
batch('score',[0]);limits();save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'score/result.json')['decision'],cumulative_cpu_seconds=total(),prior_geometry_cpu_seconds=c['prior_geometry_cpu_seconds'],records=records,new_fits=10,new_predictions=1104,reused_G_predictions=552,accounting_complete=True,retry=False))
freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'score/bootstrap.npz'])
save(root/'post-terminal-freeze-cost.json',dict(cumulative_cpu_seconds=total(),includes_terminal_freeze=True,within_cap=total()<=14400))
except BaseException as exc:
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
for p,_,_ in active.values():
try:os.kill(p.pid,signal.SIGTERM)
except ProcessLookupError:pass
for p,i,m in active.values():
p.wait();f=root/'guards'/f'{m}-{i:02d}'/'guard-result.json'
if f.exists():records.append(read(f))
active.clear();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,accounting_complete=True,retry=False));raise
finally:
for s,h in old.items():signal.signal(s,h)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','worker','guard']);p.add_argument('root',type=Path);p.add_argument('mode',nargs='?');p.add_argument('index',nargs='?',type=int);p.add_argument('slot',nargs='?',type=int);a=p.parse_args()
if a.action=='run':run(a.root)
elif a.action=='worker':worker(a.root,a.mode,a.index)
else:raise SystemExit(guard(a.root,a.mode,a.index,a.slot))