Files
MeltMind/tools/phase2_gate_outcome_postfit.py
T

163 lines
14 KiB
Python

"""Single continuation of fixed calibration: reuse models, predict, freeze, compare."""
import argparse,fcntl,json,os,resource,signal,sys,time
from pathlib import Path
import numpy as np
from phase2_fixed_io import BASE
from phase2_h1_worker_v2 import checked,sha,save
from phase2_home_freeze_io import verify,freeze
from phase2_gate_outcome_review import validate_review
from phase2_gate_outcome_statistics import two_metrics,summarize
from phase2_b1213_features import cached
from s2_custody_common import prediction_inputs
from r4bc_guard import supervise
from s2_recovery_monitoring import disk_usage
def read(p):return json.loads(Path(p).read_text())
def config(root):
c=read(root/'contract.json');verify(root,'implementation')
if c['total_cpu_seconds']!=72000 or c['postfit_cpu_seconds']!=3600 or c['new_fits']!=0 or c['candidate_label_reads']!=0:raise PermissionError('Fixed scope')
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
prep=read(c['preparation_contract']);comparison=read(c['comparison_contract'])
if comparison['rule']!=c['comparison_rule'] or comparison['bands']!=c['bands']:raise ValueError('Fixed7650 statistics changed')
if len(c['parts'])!=552 or len({r['case'] for r in c['parts']})!=552 or {r['case'] for r in c['parts']}!=set(prep['original_cases']):raise ValueError('Original552 roster')
old=read(checked(comparison['original_prediction_manifest'],comparison['original_prediction_manifest_sha256']))['entries'];by={r['case']:r for r in c['parts']}
if len(old)!=552 or set(by)!={e['case'] for e in old}:raise ValueError('Original prediction roster')
for e in old:
row=by[e['case']];test=[a['fold'] for a in row['assignments'] if a['role']=='test']
if test!=[e['fold']] or row['component_id']!=e['component_id']:raise ValueError('Frozen fold/component identity')
return c,prep,comparison,{e['case']:e for e in old}
def terminal_inputs(root,c,prep):
p=Path(c['preparation_root']);verify(p,'features');done=read(p/'complete.json');cost=read(p/'post-freeze-cost.json')
if done['status']!='FEATURES_COMPLETE_PREDICTIONS_AND_COMPARISON_PENDING' or not done['accounting_complete'] or len(done['records'])!=493 or any(r['guard']['status']!='PASS' for r in done['records']):raise PermissionError('All493 original PASS required')
if {r['index'] for r in done['records']}!=set(range(493)):raise ValueError('Distinct493 terminal jobs')
if not cost['within_cap'] or cost['cumulative_cpu_seconds']<done['cumulative_cpu_seconds']:raise ValueError('Complete original CPU ledger required')
import psutil
for process in psutil.process_iter(['pid','cmdline']):
args=process.info['cmdline'] or []
if any(x.endswith('phase2_gate_outcome_queue.py') for x in args):raise PermissionError('Original feature processes still active')
m=read(p/'features-manifest.json');new=m['recomputed'];exact=m['exact_reuse'];invalid=m['invalid_cases']
if len(new)!=493 or exact!=prep['exact_reuse'] or invalid!=prep['invalid_cases']:raise ValueError('Fixed493/52/7 partition')
expected={item['row']['case']:i for i,item in enumerate(prep['parts'])}
if len({e['case'] for e in new})!=493 or set(expected)!={e['case'] for e in new}:raise ValueError('Recomputation roster')
for e in new:
d=Path(e['path']);r=read(checked(d/'result.json',e['result_sha256']))
if e['index']!=expected[e['case']] or r['case']!=e['case'] or r['output_sha256']!=e['output_sha256']:raise ValueError('Feature result identity')
for n,h in e['output_sha256'].items():checked(d/n,h)
for e in exact:
for path,h in e['source_sha256'].items():checked(path,h)
return dict(prior_cpu_seconds=cost['cumulative_cpu_seconds'],original_complete_sha256=sha(p/'complete.json'),original_cost_sha256=sha(p/'post-freeze-cost.json'),features_manifest_sha256=sha(p/'features-manifest.json'),recomputed=new,exact_reuse=exact,invalid_cases=invalid,terminal_pid_absent=True)
def adopted(root,c):
verify(root,'inputs');a=read(root/'adopted-inputs.json');p=Path(c['preparation_root'])
checked(p/'complete.json',a['original_complete_sha256']);checked(p/'post-freeze-cost.json',a['original_cost_sha256']);checked(p/'features-manifest.json',a['features_manifest_sha256'])
if a['prior_cpu_seconds']!=read(p/'post-freeze-cost.json')['cumulative_cpu_seconds']:raise ValueError('Original cost changed')
return a
def query_identity(q,z):
for name in ['raw_ids','counts','coordinates_over_Rg','near_wall']:
if not np.array_equal(q[name],z[name]):raise ValueError('Frozen query identity: '+name)
def predict_arrays(A,G,features,extra,raw,q,t_set):
ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Query IDs outside full-node array')
if features.shape!=(len(raw),19) or extra.shape!=(len(raw),2) or not np.isfinite(features).all() or np.isinf(extra).any():raise ValueError('Invalid19+2 features')
if not np.isfinite(t_set) or t_set<=0:raise ValueError('Original clock')
x=features[ix].astype(np.float32);g=np.column_stack([features[ix],extra[ix]]).astype(np.float32)
out=dict(A=np.asarray(A.predict(x),np.float64)*t_set,G=np.asarray(G.predict(g),np.float64)*t_set)
if any(v.shape!=(len(ix),) or not np.isfinite(v).all() for v in out.values()):raise ValueError('Nonfinite prediction; no auto exclusion')
return out
def predict(root,fold):
import joblib,xgboost as xgb
from threadpoolctl import threadpool_limits
c,prep,cmp,old=config(root);a=adopted(root,c);by={e['case']:e for e in a['recomputed']};spec=cmp['models'][fold]
if spec['fold']!=fold or xgb.__version__!='2.1.4' or np.__version__!='1.26.4':raise ValueError('Model/library identity')
A=joblib.load(checked(spec['A']['path'],spec['A']['sha256']));G=xgb.XGBRegressor();G.load_model(checked(spec['G']['path'],spec['G']['sha256']));A.set_params(n_jobs=1);G.set_params(n_jobs=1)
if A.get_booster().num_features()!=19 or G.get_booster().num_features()!=21:raise ValueError('Model feature widths')
dest=root/'predict'/str(fold);dest.mkdir(parents=True,exist_ok=False);entries=[]
with threadpool_limits(1):
for row in c['parts']:
case=row['case'];base=old[case]
if base['fold']!=fold or case not in by:continue
e=by[case];path=Path(e['path']);q=prediction_inputs(row);raw=cached(row,'raw_ids')
with np.load(checked(base['path'],base['sha256']),allow_pickle=False) as z:query_identity(q,z)
f=np.load(checked(path/'features.npy',e['output_sha256']['features.npy']),allow_pickle=False);g=np.load(checked(path/'G_extra.npy',e['output_sha256']['G_extra.npy']),allow_pickle=False)
values=predict_arrays(A,G,f,g,raw,q,row['preparation']['t_set']);out=dest/(case+'.npz');np.savez_compressed(out,**values,**{n:q[n] for n in ['raw_ids','counts','coordinates_over_Rg','near_wall']})
entries.append(dict(case=case,component_id=row['component_id'],fold=fold,path=str(out),sha256=sha(out),original=base,exact_reuse=False,negative_predictions={k:int(np.sum(v<0)) for k,v in values.items()}));save(dest/'progress.json',dict(parts=len(entries),labels_read=False))
save(dest/'manifest.json',dict(entries=entries,labels_read=False,new_fits=0))
def freeze_predictions(root,c,cmp,old,a):
entries=[]
for f in range(5):entries.extend(read(root/'predict'/str(f)/'manifest.json')['entries'])
if len(entries)!=493 or {e['case'] for e in entries}!={e['case'] for e in a['recomputed']}:raise ValueError('All493 new predictions required')
for e in a['exact_reuse']:
base=old[e['case']]
if e['prediction']!=base:raise ValueError('Exact52 original prediction identity')
entries.append(dict(case=e['case'],component_id=base['component_id'],fold=base['fold'],path=base['path'],sha256=base['sha256'],original=base,exact_reuse=True))
if len({e['case'] for e in entries})!=545:raise ValueError('Distinct545 predictions')
for e in entries:checked(e['path'],e['sha256'])
save(root/'predictions-manifest.json',dict(entries=entries,invalid_cases=a['invalid_cases'],eligible_denominator=552,new_parts=493,reused_parts=52,labels_read=False))
freeze(root,'predictions',[root/'contract.json',root/'adopted-inputs.json',root/'predictions-manifest.json',*[Path(p) for p in c['code_sha256']]])
def score(root):
c,prep,cmp,old=config(root);adopted(root,c);verify(root,'predictions');validate_review(root) # Before ANY target values or comparison.
dest=root/'score';dest.mkdir(exist_ok=False);manifest=read(root/'predictions-manifest.json');entries={e['case']:e for e in manifest['entries']};invalid=set(manifest['invalid_cases']);prior={r['case']:r for r in read(checked(c['original_score'],c['source_sha256'][c['original_score']]))['rows']}
if len(entries)!=545 or invalid!=set(prep['invalid_cases']) or set(entries)&invalid or set(entries)|invalid!={r['case'] for r in c['parts']}:raise ValueError('Complete eligibility roster')
rows=[];access=[];save(dest/'review-consumed.json',dict(verdict='PASS',review_sha256=sha(root/'review.json'),unix=time.time(),before_target_access=True))
for row in c['parts']:
case=row['case'];base=old[case];prior_row=prior[case]
if prior_row['fold']!=base['fold'] or prior_row['component_id']!=row['component_id']:raise ValueError('Original score roles')
reference=dict(A={k:prior_row['A'][k] for k in ['relative_l2','last_location']},G={k:prior_row['new'][k] for k in ['relative_l2','last_location']})
out=dict(case=case,component_id=row['component_id'],fold=base['fold'],LOG_gate=reference,label_gate=None,eligibility='INVALID_GATE' if case in invalid else 'VALID')
if case not in invalid:
e=entries[case]
if e['original']!=base or e['component_id']!=row['component_id'] or e['fold']!=base['fold']:raise ValueError('Prediction fold/component identity')
if e['exact_reuse']:
checked(e['path'],e['sha256']);out['label_gate']=reference;out['reuse']='Exact seed set, original prediction and metrics by hash'
else:
access.append(dict(case=case,target_sha256=row['v5_cache']['output_sha256']['seconds.npy'],existing_old552_cache=True));save(dest/'target-access.json',access)
truth=cached(row,'seconds');raw=cached(row,'raw_ids')
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z,np.load(checked(base['path'],base['sha256']),allow_pickle=False) as zz:
query_identity(z,zz);ix=np.searchsorted(raw,z['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],z['raw_ids']):raise ValueError('Score query IDs')
out['label_gate']={arm:two_metrics(z[arm],truth[ix],z) for arm in ['A','G']}
rows.append(out);save(dest/'progress.json',dict(parts=len(rows)))
result=summarize(rows,{r['component_id'] for r in c['parts']});save(dest/'result.json',dict(status='COMPLETE',**result,rows=rows,eligibility_denominator=552,invalid_gate_parts=sorted(invalid),eligibility_loss=7/552,exact_seed_prediction_reuse=52,new_prediction_parts=493,original_exact_set_FAIL_preserved=True,candidate_label_reads=0,new_decodes=0,new_fits=0,comparison_rule=cmp['rule'],review_sha256=sha(root/'review.json')))
def remaining_cap(prior,post_spent,requested):
remaining=min(3600-post_spent,72000-prior-post_spent)-120
if requested>remaining:raise RuntimeError('Insufficient shared budget for fixed job; no retry/extension')
return requested
def run(root):
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,prep,cmp,old=config(root);validate_review(root);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single postfit launch; no restart')
os.sched_setaffinity(0,[8]);save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),phase='VERIFY_ORIGINAL_TERMINAL',new_fits=0,candidate_label_reads=0));prior=None;records=[];charged=0.;requested=[]
def post_cost():
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['setup_cpu_seconds']+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
def job(mode,index,cap):
nonlocal charged
if requested or disk_usage(root)>c['disk_bytes']:raise RuntimeError('Signal/disk stop')
cap=remaining_cap(prior,post_cost(),cap);d=root/'guards'/f'{mode}-{index}'
g=supervise([sys.executable,'tools/phase2_gate_outcome_postfit.py',mode,str(root),str(index)],d,dict(cpu_seconds=cap,wall_seconds=cap+180,rss_bytes=4*2**30,disk_bytes=c['disk_bytes']))
charged+=g['cpu_seconds'];records.append(dict(mode=mode,index=index,guard=g));save(root/'live-progress.json',dict(stage=mode,records=records,prior_cpu_once=prior,postfit_cpu_seconds=post_cost(),cumulative_cpu_seconds=prior+post_cost()))
if g['status']!='PASS' or post_cost()>3600 or prior+post_cost()>72000:raise RuntimeError('Postfit guard/budget stop')
old_handlers={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
try:
a=terminal_inputs(root,c,prep);prior=a['prior_cpu_seconds'];save(root/'adopted-inputs.json',a);freeze(root,'inputs',[root/'contract.json',root/'adopted-inputs.json',Path(c['preparation_root'])/'complete.json',Path(c['preparation_root'])/'post-freeze-cost.json',Path(c['preparation_root'])/'features-manifest.json'])
for f in range(5):job('predict',f,300)
freeze_predictions(root,c,cmp,old,a);job('score',0,600)
save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'score/result.json')['decision'],records=records,prior_cpu_once=prior,postfit_cpu_seconds=post_cost(),cumulative_cpu_seconds=prior+post_cost(),accounting_complete=True,new_fits=0,new_decodes=0,retry=False))
freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json',root/'predictions-manifest.json'])
save(root/'post-terminal-cost.json',dict(cumulative_cpu_seconds=prior+post_cost(),postfit_cpu_seconds=post_cost(),within_cap=prior+post_cost()<=72000 and post_cost()<=3600))
except BaseException as exc:
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,prior_cpu_once=prior,postfit_cpu_seconds=post_cost(),cumulative_cpu_seconds=prior+post_cost() if prior is not None else None,accounting_complete=prior is not None,retry=False));raise
finally:
for s,h in old_handlers.items():signal.signal(s,h)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','predict','score']);p.add_argument('root',type=Path);p.add_argument('index',nargs='?',type=int,default=0);a=p.parse_args()
if a.action=='run':run(a.root)
elif a.action=='predict':predict(a.root,a.index)
else:score(a.root)