77 lines
5.1 KiB
Python
77 lines
5.1 KiB
Python
"""Score already frozen R4 predictions against the original S2 custody cache."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import numpy as np
|
|
from gre24_prepare_v2 import sha
|
|
from r4a_part import contract, exact
|
|
from r4a_core import paired_summary, scoring_allowed
|
|
from s2_custody_common import checked, prediction_inputs
|
|
from s2_custody_statistics import part_metrics
|
|
from s2_recovery_supervisor import save
|
|
|
|
|
|
def score(root):
|
|
c=contract(root); start=time.process_time();out=root/'postfit';out.mkdir(exist_ok=True)
|
|
terminal=json.loads((root/'preparation-complete.json').read_text())
|
|
if not scoring_allowed(terminal['records'],len(c['parts']),terminal['stop_reason']):
|
|
raise ValueError('Any failure, missing attempt or stop forbids automatic scoring')
|
|
freeze=json.loads((root/'predictions-freeze.json').read_text())
|
|
if freeze['contract_sha256']!=sha(root/'contract.json') or freeze['preparation_complete_sha256']!=sha(root/'preparation-complete.json'):
|
|
raise ValueError('Prediction freeze identity mismatch')
|
|
if set(freeze['parts'])!={r['case'] for r in c['parts']}: raise ValueError('Incomplete prediction roster')
|
|
# Verify every paired prediction before opening any target cache.
|
|
for entry in freeze['parts'].values():
|
|
checked(entry['result'],entry['result_sha256'])
|
|
checked(entry['predictions'],entry['predictions_sha256'])
|
|
labels={x['case']:x for x in json.loads(Path(c['label_index']).read_text())}
|
|
rows=[]
|
|
for row in c['parts']:
|
|
case=row['case']; q=prediction_inputs(row); item=labels[case]
|
|
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
|
if item['fold']!=fold:raise ValueError('Custody test-fold mismatch')
|
|
with np.load(checked(item['path'],item['sha256']),allow_pickle=False) as z:
|
|
exact(z['raw_ids'],q['raw_ids'],'cached truth/query IDs')
|
|
exact(z['counts'],q['counts'],'cached truth multiplicity')
|
|
truth=z['seconds'].copy()
|
|
if not np.isfinite(truth).all() or np.any(truth<0):raise ValueError('Invalid cached truth')
|
|
entry=freeze['parts'][case]
|
|
with np.load(checked(entry['predictions'],entry['predictions_sha256']),allow_pickle=False) as z:
|
|
exact(z['raw_ids'],q['raw_ids'],'prediction IDs');exact(z['counts'],q['counts'],'prediction counts')
|
|
metrics={arm:part_metrics(z[arm],truth,q['counts'],q['near_wall'],q['raw_ids'],
|
|
q['coordinates_over_Rg'],int(case.split('_')[0])) for arm in ['F','G']}
|
|
rows.append(dict(case=case,fold=fold,component_id=row['component_id'],metrics=metrics,label_sha256=item['sha256']))
|
|
save(out/'score-progress.json',dict(parts=len(rows),cpu_seconds=time.process_time()-start))
|
|
summary={metric:paired_summary(rows,c['parts'],metric) for metric in ['relative_l2','kendall_tau_b','last_location','near_wall']}
|
|
historical=json.loads(Path(c['historical_scores']).read_text())
|
|
b0={r['case']:r for r in historical if r['arm']=='B0'}
|
|
groups={}
|
|
for row in c['parts']:groups.setdefault(row['component_id'],[]).append(row['case'])
|
|
historical_summary={}
|
|
for metric in summary:
|
|
values={};missing={}
|
|
for component,cases in sorted(groups.items()):
|
|
a=[b0.get(case,{}).get(metric,{}).get('value') for case in cases]
|
|
if any(v is None or not np.isfinite(v) for v in a):missing[component]='missing or undefined historical B0 metric'
|
|
else:values[component]=float(np.mean(a))
|
|
paired=summary['relative_l2']['component_values']['F']
|
|
shared={k:v for k,v in values.items() if k in paired}
|
|
historical_summary[metric]=dict(original_complete_components=len(values),original_mean=float(np.mean(list(values.values()))) if values else None,
|
|
on_main_paired_components=len(shared),on_main_paired_mean=float(np.mean(list(shared.values()))) if shared else None,
|
|
excluded=missing,interpretation='historical whole distance construction, not the matched graph arm')
|
|
save(out/'part-scores.json',dict(rows=rows,predictions_freeze_sha256=sha(root/'predictions-freeze.json')))
|
|
results=[json.loads(Path(x['result']).read_text()) for x in freeze['parts'].values()]
|
|
save(out/'score-summary.json',dict(status='SCORED_DESCRIPTIVE',summary=summary,historical_B0=historical_summary,
|
|
requested_parts=617,requested_components=209,paired_prediction_parts=len(rows),
|
|
full_tet_nodes=sum(x['nodes'] for x in results),full_tet_volume=sum(x['volume'] for x in results),
|
|
requested_node_coverage=1.,requested_volume_coverage=1.,query_nodes=sum(x['query_nodes'] for x in results),
|
|
new_decodes=0,new_fits=0,predictions_freeze_sha256=sha(root/'predictions-freeze.json'),
|
|
part_scores_sha256=sha(out/'part-scores.json'),worker_cpu_seconds=time.process_time()-start,
|
|
limitations=['Fixed v5/S2 query domain; not fresh blinded test','Shared cell speed and fixed clock only',
|
|
'Component-only descriptive percentile intervals; no gate','No full-node paper score or R4b/c']))
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();score(a.root)
|