157 lines
9.5 KiB
Python
157 lines
9.5 KiB
Python
"""Postfit stages require actual Git-frozen manifests; never launch fits or decode VTU."""
|
|
import argparse
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import hashlib
|
|
|
|
import joblib
|
|
import numpy as np
|
|
from threadpoolctl import threadpool_limits
|
|
from r4bc_common import (contract, read, checked, sha, save, role_rows, require_role, features,
|
|
query_inputs, exact, InputViolation, paired_difference, ARMS)
|
|
from s2_custody_common import prediction_inputs
|
|
from s2_custody_statistics import part_metrics
|
|
|
|
METRICS=('relative_l2','kendall_tau_b','last_location','near_wall')
|
|
|
|
|
|
def git_gate(root, name, required):
|
|
proof=read(root/name)
|
|
if not proof.get('commit') or not proof.get('checkout'):raise InputViolation('No Git freeze evidence')
|
|
for rel in required:
|
|
path=root/rel;expected=proof['sha256'][rel];checked(path,expected)
|
|
repository_path=str(path.relative_to(Path.cwd())) if path.is_absolute() else str(path)
|
|
raw=subprocess.check_output(['git','show',proof['commit']+':'+repository_path],cwd=proof['checkout'])
|
|
if hashlib.sha256(raw).hexdigest()!=expected:raise InputViolation('Git object differs from local freeze')
|
|
return proof
|
|
|
|
|
|
def training_gate(root):
|
|
c=contract(root);git_gate(root,'training-git-freeze.json',['contract.json','batch-complete.json','models-terminal.json'])
|
|
done=read(root/'batch-complete.json')
|
|
if done['global_stop'] or not done['accounting_complete'] or done['postfit_state']!='READY_FOR_GIT_CHECKPOINT_FREEZE':
|
|
raise InputViolation('Training terminal gate not satisfied')
|
|
checked(root/'models-terminal.json',done['model_manifest_sha256'])
|
|
return c,read(root/'models-terminal.json')['models']
|
|
|
|
|
|
def predict(root,cell,arm):
|
|
c,models=training_gate(root)
|
|
m=next(m for m in models if m['cell']==cell and m['arm']==arm)
|
|
if m['status']!='PASS':raise InputViolation('No valid final checkpoint')
|
|
dest=root/'predictions'/f'cell-{cell:02d}-{arm}';dest.mkdir(parents=True,exist_ok=False)
|
|
start=time.process_time();index={}
|
|
with threadpool_limits(1):
|
|
model=joblib.load(checked(m['model_path'],m['model_sha256']))
|
|
for row in role_rows(c,m['fold'],m['pool'],'test'):
|
|
require_role(row,m['fold'],'test',m['pool'])
|
|
x,q=features(row,arm,full=False)
|
|
pred=np.asarray(model.predict(x),dtype=np.float64)*row['preparation']['t_set']
|
|
if pred.shape!=q['raw_ids'].shape or not np.isfinite(pred).all():raise ValueError('Invalid test prediction')
|
|
path=dest/(row['case']+'.npz')
|
|
np.savez_compressed(path,seconds=pred,raw_ids=q['raw_ids'],counts=q['counts'])
|
|
index[row['case']]=dict(path=str(path),sha256=sha(path),array='seconds')
|
|
save(dest/'progress.json',dict(parts=len(index),cpu_seconds=time.process_time()-start))
|
|
save(dest/'index.json',index)
|
|
save(dest/'result.json',dict(status='COMPLETE',cell=cell,arm=arm,index_sha256=sha(dest/'index.json'),
|
|
model_sha256=m['model_sha256'],cpu_seconds=time.process_time()-start,
|
|
target_values_read=False,new_decodes=0,model_fit=False))
|
|
|
|
|
|
def freeze_predictions(root):
|
|
c,models=training_gate(root);entries=[];missing=[]
|
|
for m in models:
|
|
dest=root/'predictions'/f"cell-{m['cell']:02d}-{m['arm']}"
|
|
guard=root/'prediction-guards'/f"cell-{m['cell']:02d}-{m['arm']}"/'worker.receipt.json'
|
|
if m['status']!='PASS' or not guard.exists() or read(guard)['status']!='PASS':
|
|
missing.append(dict(cell=m['cell'],arm=m['arm'],reason=m['reason'] or 'PREDICTION_NOT_PASS'));continue
|
|
result=read(dest/'result.json');checked(dest/'index.json',result['index_sha256'])
|
|
index=read(dest/'index.json')
|
|
expected={r['case'] for r in role_rows(c,m['fold'],m['pool'],'test')}
|
|
if set(index)!=expected:raise InputViolation('Prediction domain incomplete')
|
|
for case,v in index.items():
|
|
checked(v['path'],v['sha256']);entries.append(dict(pool=m['pool'],fold=m['fold'],arm=m['arm'],case=case,**v))
|
|
for row in c['parts']:
|
|
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
|
for pool in (100,50,25):
|
|
h=row['historical_predictions'][str(pool)];checked(h['path'],h['sha256'])
|
|
entries.append(dict(pool=pool,fold=fold,arm='H0',case=row['case'],array='seconds',**h))
|
|
e=row['r4a'];checked(e['predictions'],e['predictions_sha256'])
|
|
entries.append(dict(pool=pool,fold=fold,arm='EIK',case=row['case'],array='F',path=e['predictions'],sha256=e['predictions_sha256']))
|
|
save(root/'predictions-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
|
|
models_terminal_sha256=sha(root/'models-terminal.json'),entries=entries,missing=missing,
|
|
target_values_read=False,new_decodes=0,common_parts=552,common_components=204))
|
|
|
|
|
|
def component_table(rows, roster, arms, metric):
|
|
by={(r['case'],r['arm']):r for r in rows};groups={}
|
|
for r in roster:groups.setdefault(r['component_id'],[]).append(r['case'])
|
|
values={a:{} for a in arms};excluded={}
|
|
for group,cases in groups.items():
|
|
missing=[(case,a) for case in cases for a in arms
|
|
if by.get((case,a),{}).get(metric,{}).get('value') is None
|
|
or not np.isfinite(by.get((case,a),{}).get(metric,{}).get('value',np.nan))]
|
|
if missing:excluded[group]=missing;continue
|
|
for arm in arms:values[arm][group]=float(np.mean([by[case,arm][metric]['value'] for case in cases]))
|
|
ids=sorted(next(iter(values.values())))
|
|
return dict(components=len(ids),parts=sum(len(groups[g]) for g in ids),component_ids=ids,
|
|
means={a:float(np.mean(list(v.values()))) if v else None for a,v in values.items()},
|
|
values=values,excluded_components=excluded)
|
|
|
|
|
|
def score(root):
|
|
c,_=training_gate(root)
|
|
git_gate(root,'scoring-git-freeze.json',['contract.json','predictions-freeze.json'])
|
|
frozen=read(root/'predictions-freeze.json')
|
|
if frozen['contract_sha256']!=sha(root/'contract.json'):raise InputViolation('Prediction contract mismatch')
|
|
for e in frozen['entries']:checked(e['path'],e['sha256'])
|
|
# Every prediction is verified before the first cached test target value.
|
|
entries={}
|
|
for e in frozen['entries']:entries.setdefault(e['case'],[]).append(e)
|
|
out=root/'scores';out.mkdir(exist_ok=False);start=time.process_time();rows=[];access=[]
|
|
for row in c['parts']:
|
|
case=row['case'];q=prediction_inputs(row);label=row['label']
|
|
with np.load(checked(label['path'],label['sha256']),allow_pickle=False) as z:
|
|
exact(z['raw_ids'],q['raw_ids'],'scoring target IDs');exact(z['counts'],q['counts'],'scoring target counts')
|
|
truth=z['seconds'].copy()
|
|
access.append(dict(case=case,path=label['path'],sha256=label['sha256']))
|
|
for entry in entries[case]:
|
|
with np.load(entry['path'],allow_pickle=False) as z:
|
|
exact(z['raw_ids'],q['raw_ids'],'scoring prediction IDs');exact(z['counts'],q['counts'],'scoring prediction counts')
|
|
metrics=part_metrics(z[entry['array']],truth,q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
|
|
rows.append(dict(case=case,component_id=row['component_id'],pool=entry['pool'],fold=entry['fold'],arm=entry['arm'],**metrics))
|
|
save(out/'progress.json',dict(parts=len(access),cpu_seconds=time.process_time()-start))
|
|
summary={}
|
|
edges=[('H0','H0p'),('H0p','H1'),('H1','H2'),('H2','H3'),('H3','H4'),('H4','A')]
|
|
for pool in (100,50,25):
|
|
selected=[r for r in rows if r['pool']==pool];info={}
|
|
for metric in METRICS:
|
|
b=component_table(selected,c['parts'],('EIK','A','Q'),metric)
|
|
chain=component_table(selected,c['parts'],('H0p','H1','H2','H3','H4','A'),metric)
|
|
edge_reports={}
|
|
for before,after in edges:
|
|
item=component_table(selected,c['parts'],(before,after),metric)
|
|
if metric=='relative_l2':item['difference']=paired_difference(item['values'][before],item['values'][after])
|
|
edge_reports[before+'->'+after]=item
|
|
if metric=='relative_l2':
|
|
b['differences']={a+'->'+z:paired_difference(b['values'][a],b['values'][z]) for a,z in [('EIK','A'),('EIK','Q'),('Q','A')]}
|
|
info[metric]=dict(primary=b,chain_common=chain,edges=edge_reports)
|
|
folds={a:sorted({r['fold'] for r in selected if r['arm']==a}) for a in ARMS}
|
|
summary[str(pool)]=dict(metrics=info,valid_prediction_folds=folds,
|
|
complete_five_fold_all_arms=all(len(v)==5 for v in folds.values()))
|
|
save(out/'part-scores.json',rows);save(out/'target-access.json',access)
|
|
save(out/'summary.json',dict(status='SCORED_DESCRIPTIVE',pools=summary,
|
|
predictions_freeze_sha256=sha(root/'predictions-freeze.json'),new_decodes=0,new_fits=0,
|
|
part_scores_sha256=sha(out/'part-scores.json'),cpu_seconds=time.process_time()-start,
|
|
limitations=['Fixed grouped v5 domain; not fresh blind evaluation','Failure domains reported per comparison',
|
|
'H0 reference only; H0 to H0p includes roster/HPO response','Final learner step includes float32 and training policy']))
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('mode',choices=['predict','freeze','score'])
|
|
p.add_argument('--cell',type=int);p.add_argument('--arm',choices=ARMS);a=p.parse_args()
|
|
if a.mode=='predict':predict(a.root,a.cell,a.arm)
|
|
elif a.mode=='freeze':freeze_predictions(a.root)
|
|
else:score(a.root)
|