Files
MeltMind/tools/r4a_score.py

77 lines
5.1 KiB
Python

"""Score already frozen R4 predictions against the original S2 custody cache."""
import argparse
import json
from pathlib import Path
import time
import numpy as np
from gre24_prepare_v2 import sha
from r4a_part import contract, exact
from r4a_core import paired_summary, scoring_allowed
from s2_custody_common import checked, prediction_inputs
from s2_custody_statistics import part_metrics
from s2_recovery_supervisor import save
def score(root):
c=contract(root); start=time.process_time();out=root/'postfit';out.mkdir(exist_ok=True)
terminal=json.loads((root/'preparation-complete.json').read_text())
if not scoring_allowed(terminal['records'],len(c['parts']),terminal['stop_reason']):
raise ValueError('Any failure, missing attempt or stop forbids automatic scoring')
freeze=json.loads((root/'predictions-freeze.json').read_text())
if freeze['contract_sha256']!=sha(root/'contract.json') or freeze['preparation_complete_sha256']!=sha(root/'preparation-complete.json'):
raise ValueError('Prediction freeze identity mismatch')
if set(freeze['parts'])!={r['case'] for r in c['parts']}: raise ValueError('Incomplete prediction roster')
# Verify every paired prediction before opening any target cache.
for entry in freeze['parts'].values():
checked(entry['result'],entry['result_sha256'])
checked(entry['predictions'],entry['predictions_sha256'])
labels={x['case']:x for x in json.loads(Path(c['label_index']).read_text())}
rows=[]
for row in c['parts']:
case=row['case']; q=prediction_inputs(row); item=labels[case]
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
if item['fold']!=fold:raise ValueError('Custody test-fold mismatch')
with np.load(checked(item['path'],item['sha256']),allow_pickle=False) as z:
exact(z['raw_ids'],q['raw_ids'],'cached truth/query IDs')
exact(z['counts'],q['counts'],'cached truth multiplicity')
truth=z['seconds'].copy()
if not np.isfinite(truth).all() or np.any(truth<0):raise ValueError('Invalid cached truth')
entry=freeze['parts'][case]
with np.load(checked(entry['predictions'],entry['predictions_sha256']),allow_pickle=False) as z:
exact(z['raw_ids'],q['raw_ids'],'prediction IDs');exact(z['counts'],q['counts'],'prediction counts')
metrics={arm:part_metrics(z[arm],truth,q['counts'],q['near_wall'],q['raw_ids'],
q['coordinates_over_Rg'],int(case.split('_')[0])) for arm in ['F','G']}
rows.append(dict(case=case,fold=fold,component_id=row['component_id'],metrics=metrics,label_sha256=item['sha256']))
save(out/'score-progress.json',dict(parts=len(rows),cpu_seconds=time.process_time()-start))
summary={metric:paired_summary(rows,c['parts'],metric) for metric in ['relative_l2','kendall_tau_b','last_location','near_wall']}
historical=json.loads(Path(c['historical_scores']).read_text())
b0={r['case']:r for r in historical if r['arm']=='B0'}
groups={}
for row in c['parts']:groups.setdefault(row['component_id'],[]).append(row['case'])
historical_summary={}
for metric in summary:
values={};missing={}
for component,cases in sorted(groups.items()):
a=[b0.get(case,{}).get(metric,{}).get('value') for case in cases]
if any(v is None or not np.isfinite(v) for v in a):missing[component]='missing or undefined historical B0 metric'
else:values[component]=float(np.mean(a))
paired=summary['relative_l2']['component_values']['F']
shared={k:v for k,v in values.items() if k in paired}
historical_summary[metric]=dict(original_complete_components=len(values),original_mean=float(np.mean(list(values.values()))) if values else None,
on_main_paired_components=len(shared),on_main_paired_mean=float(np.mean(list(shared.values()))) if shared else None,
excluded=missing,interpretation='historical whole distance construction, not the matched graph arm')
save(out/'part-scores.json',dict(rows=rows,predictions_freeze_sha256=sha(root/'predictions-freeze.json')))
results=[json.loads(Path(x['result']).read_text()) for x in freeze['parts'].values()]
save(out/'score-summary.json',dict(status='SCORED_DESCRIPTIVE',summary=summary,historical_B0=historical_summary,
requested_parts=617,requested_components=209,paired_prediction_parts=len(rows),
full_tet_nodes=sum(x['nodes'] for x in results),full_tet_volume=sum(x['volume'] for x in results),
requested_node_coverage=1.,requested_volume_coverage=1.,query_nodes=sum(x['query_nodes'] for x in results),
new_decodes=0,new_fits=0,predictions_freeze_sha256=sha(root/'predictions-freeze.json'),
part_scores_sha256=sha(out/'part-scores.json'),worker_cpu_seconds=time.process_time()-start,
limitations=['Fixed v5/S2 query domain; not fresh blinded test','Shared cell speed and fixed clock only',
'Component-only descriptive percentile intervals; no gate','No full-node paper score or R4b/c']))
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();score(a.root)