124 lines
11 KiB
Python
124 lines
11 KiB
Python
"""Reviewed continued gate postfit; original models/statistics, explicit333+160 custody."""
|
|
import argparse,fcntl,os,resource,signal,sys,time
|
|
from pathlib import Path
|
|
import psutil
|
|
import numpy as np
|
|
from phase2_h1_worker_v2 import checked,sha,save
|
|
from phase2_home_freeze_io import freeze,verify
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_gate_outcome_statistics import two_metrics,summarize
|
|
from phase2_gate_outcome_postfit import predict_arrays,query_identity
|
|
from phase2_gate_continued_postfit_common import read,config,terminal_inputs,adopted,require_budget
|
|
from phase2_b1213_features import cached
|
|
from s2_custody_common import prediction_inputs
|
|
from r4bc_guard import supervise
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
def prediction_identity(entries,c,old,a):
|
|
exact={e['case']:e for e in a['exact_reuse']};new={e['case'] for e in a['recomputed']}
|
|
if len(entries)!=545 or len({e['case'] for e in entries})!=545 or {e['case'] for e in entries}!=set(exact)|new:raise ValueError('Exact545 eligible prediction roster')
|
|
for e in entries:
|
|
case=e['case'];base=old[case]
|
|
if e['original']!=base or e['fold']!=base['fold'] or e['component_id']!=base['component_id'] or e['exact_reuse'] is not (case in exact):raise ValueError('Frozen prediction role/reuse identity')
|
|
if case in exact:
|
|
if exact[case]['prediction']!=base or e['path']!=base['path'] or e['sha256']!=base['sha256']:raise ValueError('Exact52 immutable prediction identity')
|
|
elif Path(e['path']).resolve()!=(Path(c['postfit_root'])/'predict'/str(e['fold'])/(case+'.npz')).resolve():raise ValueError('New493 prediction path')
|
|
|
|
def predict(root,fold):
|
|
import joblib,xgboost as xgb
|
|
from threadpoolctl import threadpool_limits
|
|
c,prep,cmp,old=config(root);validate_review(root);a=adopted(root,c);by={e['case']:e for e in a['recomputed']};spec=cmp['models'][fold]
|
|
if spec['fold']!=fold or xgb.__version__!='2.1.4' or np.__version__!='1.26.4':raise ValueError('Model/library identity')
|
|
A=joblib.load(checked(spec['A']['path'],spec['A']['sha256']));G=xgb.XGBRegressor();G.load_model(checked(spec['G']['path'],spec['G']['sha256']));A.set_params(n_jobs=1);G.set_params(n_jobs=1)
|
|
if A.get_booster().num_features()!=19 or G.get_booster().num_features()!=21:raise ValueError('Model feature widths')
|
|
dest=root/'predict'/str(fold);dest.mkdir(parents=True,exist_ok=False);entries=[]
|
|
with threadpool_limits(1):
|
|
for row in c['parts']:
|
|
case=row['case'];base=old[case]
|
|
if base['fold']!=fold or case not in by:continue
|
|
e=by[case];path=Path(e['path']);q=prediction_inputs(row);raw=cached(row,'raw_ids')
|
|
with np.load(checked(base['path'],base['sha256']),allow_pickle=False) as z:query_identity(q,z)
|
|
f=np.load(checked(path/'features.npy',e['output_sha256']['features.npy']),allow_pickle=False);g=np.load(checked(path/'G_extra.npy',e['output_sha256']['G_extra.npy']),allow_pickle=False)
|
|
values=predict_arrays(A,G,f,g,raw,q,row['preparation']['t_set']);out=dest/(case+'.npz');np.savez_compressed(out,**values,**{n:q[n] for n in ['raw_ids','counts','coordinates_over_Rg','near_wall']})
|
|
entries.append(dict(case=case,component_id=row['component_id'],fold=fold,path=str(out),sha256=sha(out),original=base,exact_reuse=False,negative_predictions={k:int(np.sum(v<0)) for k,v in values.items()}));save(dest/'progress.json',dict(parts=len(entries),labels_read=False))
|
|
save(dest/'manifest.json',dict(entries=entries,labels_read=False,new_fits=0))
|
|
|
|
def freeze_predictions(root,c,cmp,old,a):
|
|
entries=[]
|
|
for f in range(5):entries.extend(read(root/'predict'/str(f)/'manifest.json')['entries'])
|
|
if len(entries)!=493 or {e['case'] for e in entries}!={e['case'] for e in a['recomputed']}:raise ValueError('All493 new predictions required')
|
|
for e in a['exact_reuse']:
|
|
base=old[e['case']]
|
|
if e['prediction']!=base:raise ValueError('Exact52 original prediction identity')
|
|
entries.append(dict(case=e['case'],component_id=base['component_id'],fold=base['fold'],path=base['path'],sha256=base['sha256'],original=base,exact_reuse=True))
|
|
if len({e['case'] for e in entries})!=545:raise ValueError('Distinct545 predictions')
|
|
prediction_identity(entries,c,old,a)
|
|
for e in entries:checked(e['path'],e['sha256'])
|
|
save(root/'predictions-manifest.json',dict(entries=entries,invalid_cases=a['invalid_cases'],eligible_denominator=552,new_parts=493,reused_parts=52,labels_read=False))
|
|
freeze(root,'predictions',[root/'contract.json',root/'adopted-inputs.json',root/'predictions-manifest.json',*[Path(p) for p in c['code_sha256']]])
|
|
|
|
def score(root):
|
|
c,prep,cmp,old=config(root);validate_review(root);a=adopted(root,c);verify(root,'predictions') # Before ANY target values or comparison.
|
|
manifest=read(root/'predictions-manifest.json');prediction_identity(manifest['entries'],c,old,a)
|
|
dest=root/'score';dest.mkdir(exist_ok=False);entries={e['case']:e for e in manifest['entries']};invalid=set(manifest['invalid_cases']);prior={r['case']:r for r in read(checked(c['original_score'],c['source_sha256'][c['original_score']]))['rows']}
|
|
if len(entries)!=545 or invalid!=set(prep['invalid_cases']) or set(entries)&invalid or set(entries)|invalid!={r['case'] for r in c['parts']}:raise ValueError('Complete eligibility roster')
|
|
rows=[];access=[];save(dest/'review-consumed.json',dict(verdict='PASS',review_sha256=sha(root/'review.json'),unix=time.time(),before_target_access=True))
|
|
for row in c['parts']:
|
|
case=row['case'];base=old[case];prior_row=prior[case]
|
|
if prior_row['fold']!=base['fold'] or prior_row['component_id']!=row['component_id']:raise ValueError('Original score roles')
|
|
reference=dict(A={k:prior_row['A'][k] for k in ['relative_l2','last_location']},G={k:prior_row['new'][k] for k in ['relative_l2','last_location']})
|
|
out=dict(case=case,component_id=row['component_id'],fold=base['fold'],LOG_gate=reference,label_gate=None,eligibility='INVALID_GATE' if case in invalid else 'VALID')
|
|
if case not in invalid:
|
|
e=entries[case]
|
|
if e['original']!=base or e['component_id']!=row['component_id'] or e['fold']!=base['fold']:raise ValueError('Prediction fold/component identity')
|
|
if e['exact_reuse']:
|
|
checked(e['path'],e['sha256']);out['label_gate']=reference;out['reuse']='Exact seed set, original prediction and metrics by hash'
|
|
else:
|
|
access.append(dict(case=case,target_sha256=row['v5_cache']['output_sha256']['seconds.npy'],existing_old552_cache=True));save(dest/'target-access.json',access)
|
|
truth=cached(row,'seconds');raw=cached(row,'raw_ids')
|
|
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z,np.load(checked(base['path'],base['sha256']),allow_pickle=False) as zz:
|
|
query_identity(z,zz);ix=np.searchsorted(raw,z['raw_ids'])
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],z['raw_ids']):raise ValueError('Score query IDs')
|
|
out['label_gate']={arm:two_metrics(z[arm],truth[ix],z) for arm in ['A','G']}
|
|
rows.append(out);save(dest/'progress.json',dict(parts=len(rows)))
|
|
if len(access)!=493 or len({e['case'] for e in access})!=493:raise ValueError('Exactly493 existing target reads;52 exact and7 invalid untouched')
|
|
result=summarize(rows,{r['component_id'] for r in c['parts']});save(dest/'result.json',dict(status='COMPLETE',**result,rows=rows,eligibility_denominator=552,invalid_gate_parts=sorted(invalid),eligibility_loss=7/552,exact_seed_prediction_reuse=52,new_prediction_parts=493,original_exact_set_FAIL_preserved=True,candidate_label_reads=0,new_decodes=0,new_fits=0,comparison_rule=cmp['rule'],review_sha256=sha(root/'review.json')))
|
|
|
|
def run(root):
|
|
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,prep,cmp,old=config(root);validate_review(root);a=terminal_inputs(root,c,prep)
|
|
lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('One continued postfit; no retries')
|
|
for p in psutil.process_iter(['pid','cmdline']):
|
|
if p.pid!=os.getpid() and any(Path(x).name in ['phase2_gate_outcome_postfit.py','phase2_gate_continued_postfit.py'] for x in p.info['cmdline'] or []):raise PermissionError('Existing gate postfit process')
|
|
os.sched_setaffinity(0,[8]);prior=a['prior_cpu_seconds'];records=[];charged=0.;requested=[]
|
|
def post_cost():
|
|
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 120+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
|
def job(mode,index,cap,remaining):
|
|
nonlocal charged
|
|
require_budget(c,prior,post_cost(),remaining)
|
|
if requested or disk_usage(root)>c['disk_bytes']:raise RuntimeError('Signal/disk stop')
|
|
g=supervise([sys.executable,'tools/phase2_gate_continued_postfit.py',mode,str(root),str(index)],root/'guards'/f'{mode}-{index}',dict(cpu_seconds=cap,wall_seconds=cap+180,rss_bytes=4*2**30,disk_bytes=c['disk_bytes']))
|
|
charged+=g['cpu_seconds'];records.append(dict(mode=mode,index=index,guard=g));save(root/'live-progress.json',dict(stage=mode,records=records,prior_cpu_once=prior,original_cpu_already_in_prior=a['original_cpu_already_in_prior'],postfit_cpu_seconds=post_cost(),cumulative_cpu_seconds=prior+post_cost()))
|
|
if g['status']!='PASS' or post_cost()>3600 or prior+post_cost()>c['total_cpu_seconds'] or disk_usage(root)>c['disk_bytes']:raise RuntimeError('Postfit guard/budget/disk stop')
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),phase='CANONICAL_TERMINAL_ADOPTED',prior_cpu_once=prior,original_cpu_already_in_prior=a['original_cpu_already_in_prior'],review_sha256=sha(root/'review.json'),new_fits=0,candidate_label_reads=0))
|
|
handlers={s:signal.signal(s,lambda n,f:requested.append(n)) for s in [signal.SIGTERM,signal.SIGINT]}
|
|
try:
|
|
require_budget(c,prior,post_cost(),5);save(root/'adopted-inputs.json',a);p=Path(c['preparation_root']);oldroot=Path(c['original_preparation_root'])
|
|
freeze(root,'inputs',[root/'contract.json',root/'adopted-inputs.json',oldroot/'complete.json',p/'complete.json',p/'post-freeze-cost.json',p/'features-manifest.json',p/'features-git-freeze.json',p/'reused-manifest.json'])
|
|
for fold in range(5):job('predict',fold,300,5-fold)
|
|
freeze_predictions(root,c,cmp,old,a);job('score',0,600,0)
|
|
require_budget(c,prior,post_cost(),0,False)
|
|
save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'score/result.json')['decision'],records=records,prior_cpu_once=prior,original_cpu_already_in_prior=a['original_cpu_already_in_prior'],postfit_cpu_seconds=post_cost(),cumulative_cpu_seconds=prior+post_cost(),total_cpu_seconds=c['total_cpu_seconds'],accounting_complete=True,new_fits=0,new_decodes=0,retry=False))
|
|
freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json',root/'predictions-manifest.json'])
|
|
cost=post_cost();save(root/'post-terminal-cost.json',dict(cumulative_cpu_seconds=prior+cost,postfit_cpu_seconds=cost,total_cpu_seconds=c['total_cpu_seconds'],within_cap=prior+cost<=c['total_cpu_seconds'] and cost<=3600))
|
|
if prior+cost>c['total_cpu_seconds'] or cost>3600:raise RuntimeError('Terminal work exceeded cap')
|
|
except BaseException as exc:
|
|
cost=post_cost();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,prior_cpu_once=prior,postfit_cpu_seconds=cost,cumulative_cpu_seconds=prior+cost,accounting_complete=True,retry=False));raise
|
|
finally:
|
|
for s,h in handlers.items():signal.signal(s,h)
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','predict','score']);p.add_argument('root',type=Path);p.add_argument('index',nargs='?',type=int,default=0);a=p.parse_args()
|
|
if a.action=='run':run(a.root)
|
|
elif a.action=='predict':predict(a.root,a.index)
|
|
else:score(a.root)
|