105 lines
6.8 KiB
Python
105 lines
6.8 KiB
Python
"""Approved post-stop scoring: all arms share complete canonical components."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import numpy as np
|
|
from gre24_prepare_v2 import sha
|
|
from r4a_part import contract, exact
|
|
from s2_custody_common import checked, prediction_inputs
|
|
from s2_custody_statistics import part_metrics, contrast
|
|
from s2_recovery_supervisor import save
|
|
|
|
ARMS=('EIK','GRAPH','B0_S2')
|
|
METRICS=('relative_l2','kendall_tau_b','last_location','near_wall')
|
|
|
|
|
|
def summaries(rows, roster, pass_ids):
|
|
by={r['case']:r for r in rows};groups={}
|
|
for r in roster:groups.setdefault(r['component_id'],[]).append(r)
|
|
failed_components={k:dict(parts=[{'case':r['case'],'nodes':r['nodes'],'prediction_pass':r['case'] in pass_ids} for r in members],
|
|
nodes=sum(r['nodes'] for r in members))
|
|
for k,members in groups.items() if any(r['case'] not in pass_ids for r in members)}
|
|
report={}
|
|
for metric in METRICS:
|
|
values={a:{} for a in ARMS};undefined={}
|
|
for group,members in sorted(groups.items()):
|
|
if group in failed_components:continue
|
|
absent=[]
|
|
for member in members:
|
|
for arm in ARMS:
|
|
v=by.get(member['case'],{}).get('metrics',{}).get(arm,{}).get(metric,{}).get('value')
|
|
if v is None or not np.isfinite(v):absent.append(dict(case=member['case'],arm=arm,metric=metric))
|
|
if absent:undefined[group]=absent;continue
|
|
for arm in ARMS:
|
|
values[arm][group]=float(np.mean([by[r['case']]['metrics'][arm][metric]['value'] for r in members]))
|
|
component_ids=sorted(values['EIK'])
|
|
item=dict(component_ids=component_ids,components=len(component_ids),
|
|
parts=sum(len(groups[g]) for g in component_ids),nodes=sum(sum(r['nodes'] for r in groups[g]) for g in component_ids),
|
|
means={a:float(np.mean(list(v.values()))) if v else None for a,v in values.items()},
|
|
component_values=values,undefined_components_excluded_from_all_arms=undefined)
|
|
if metric=='relative_l2':
|
|
item['contrasts']={}
|
|
for candidate in ['GRAPH','B0_S2']:
|
|
delta=contrast(values['EIK'],values[candidate],replicates=2000,seed=2026111408)
|
|
delta.pop('region',None)
|
|
delta.update(formula='1 - E_'+candidate+'/E_EIK',reference='EIK',candidate=candidate,
|
|
interpretation='Positive means candidate has lower error than EIK; descriptive percentile interval, no gate')
|
|
item['contrasts'][candidate]=delta
|
|
report[metric]=item
|
|
return report,failed_components
|
|
|
|
|
|
def score(root):
|
|
c=contract(root);eng=json.loads((root/'continuation-contract.json').read_text())
|
|
for p,h in eng['code_sha256'].items():checked(p,h)
|
|
frozen=root/'pass-predictions-freeze.json';freeze=json.loads(frozen.read_text())
|
|
if freeze['contract_sha256']!=sha(root/'contract.json') or freeze['continuation_contract_sha256']!=sha(root/'continuation-contract.json'):
|
|
raise ValueError('Freeze/contract mismatch')
|
|
requested={r['case'] for r in c['parts']};passed=set(freeze['parts']);failed={r['case'] for r in freeze['canonical_failures']}
|
|
if passed&failed or passed|failed!=requested:raise ValueError('Canonical coverage mismatch')
|
|
for entry in freeze['parts'].values():
|
|
checked(entry['result'],entry['result_sha256']);checked(entry['predictions'],entry['predictions_sha256'])
|
|
# Labels are opened only after every PASS prediction hash has been verified.
|
|
labels={r['case']:r for r in json.loads(Path(c['label_index']).read_text())}
|
|
historical={r['case']:r for r in json.loads(Path(c['historical_scores']).read_text()) if r['arm']=='B0'}
|
|
out=root/'postfit';out.mkdir(exist_ok=True);start=time.process_time();rows=[]
|
|
for row in c['parts']:
|
|
case=row['case']
|
|
if case not in passed:continue
|
|
q=prediction_inputs(row);label=labels[case]
|
|
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
|
if label['fold']!=fold:raise ValueError('Cached fold mismatch')
|
|
with np.load(checked(label['path'],label['sha256']),allow_pickle=False) as z:
|
|
exact(z['raw_ids'],q['raw_ids'],'target/query IDs');exact(z['counts'],q['counts'],'target/query counts');truth=z['seconds'].copy()
|
|
if not np.isfinite(truth).all() or np.any(truth<0):raise ValueError('Invalid cached truth')
|
|
entry=freeze['parts'][case]
|
|
with np.load(checked(entry['predictions'],entry['predictions_sha256']),allow_pickle=False) as z:
|
|
exact(z['raw_ids'],q['raw_ids'],'prediction IDs');exact(z['counts'],q['counts'],'prediction multiplicity')
|
|
metrics={arm:part_metrics(z[array],truth,q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
|
|
for arm,array in [('EIK','F'),('GRAPH','G')]}
|
|
b0=historical.get(case)
|
|
if b0 is None:raise ValueError('Missing historical B0 row')
|
|
metrics['B0_S2']={metric:b0[metric] for metric in METRICS}
|
|
rows.append(dict(case=case,component_id=row['component_id'],fold=fold,metrics=metrics,
|
|
cached_label_sha256=label['sha256'],prediction_sha256=entry['predictions_sha256']))
|
|
save(out/'progress.json',dict(stage='CACHE_ONLY_SCORING',parts=len(rows),worker_cpu_seconds=time.process_time()-start))
|
|
report,excluded=summaries(rows,c['parts'],passed)
|
|
save(out/'part-scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen)))
|
|
save(out/'score-summary.json',dict(status='SCORED_DESCRIPTIVE_COMPLETE_COMPONENTS',authorizing_ruling='1794830693.515879',
|
|
requested_parts=len(requested),requested_components=len({r['component_id'] for r in c['parts']}),
|
|
pass_prediction_parts=len(passed),canonical_failures=freeze['canonical_failures'],excluded_components=excluded,
|
|
summary=report,delta_direction='EIK reference; 1 - GRAPH/EIK and 1 - B0_S2/EIK',
|
|
superseded_draft_direction='1 - EIK/GRAPH was not scored; changed by Claude1794830693',
|
|
historical_B0_interpretation='whole distance construction; not pure propagation attribution',
|
|
new_decodes=0,new_fits=0,worker_cpu_seconds=time.process_time()-start,
|
|
predictions_freeze_sha256=sha(frozen),part_scores_sha256=sha(out/'part-scores.json'),
|
|
limitations=['Fixed v5 geometry and S2 query domain, not fresh blind testing',
|
|
'Failed components excluded identically from every arm; no trimmed membership',
|
|
'Component-only descriptive bootstrap; no CI or 10% gate',
|
|
'Historical B0 differs in full distance construction, not just propagation']))
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();score(a.root)
|