140 lines
9.0 KiB
Python
140 lines
9.0 KiB
Python
"""Frozen predictions -> per-part fields/topology -> frozen descriptive report."""
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
import numpy as np
|
|
from scipy.sparse import csr_matrix
|
|
from phase2_h1_worker_v2 import contract,sha,checked,save
|
|
from phase2_h1_topology import events,match
|
|
from phase2_h1_decision import branch_decision
|
|
from s2_custody_common import prediction_inputs
|
|
from s2_custody_statistics import part_metrics,kendall_pairs,last_location
|
|
from gre24_features_v2 import per_geometry_metrics,appendix_summary
|
|
from r4bc_common import paired_difference
|
|
|
|
ARMS=('A','H1','cF','EIK')
|
|
ARRAYS=dict(A='A_seconds',H1='seconds',cF='cF_seconds',EIK='EIK_seconds')
|
|
|
|
|
|
def scoring_gate(root):
|
|
from phase2_h1_recovery_gate import scoring_gate as verified
|
|
return verified(root)
|
|
|
|
|
|
def score_part(root,protocol,case):
|
|
c,frozen=scoring_gate(root);row=c['parts'][protocol][case]
|
|
dest=root/'scores'/protocol/case;dest.mkdir(parents=True,exist_ok=False);start=time.process_time()
|
|
entries=[e for e in frozen['entries'] if e['protocol']==protocol and e['case']==case]
|
|
# Verify every field for this part before opening its target; global freeze
|
|
# already verified every prediction file before permitting this worker.
|
|
for e in entries:checked(e['path'],e['sha256'])
|
|
gm=json.loads((root/'geometry/manifest.json').read_text())
|
|
ge=next(e for e in gm['records'] if e['protocol']==protocol and e['case']==case)
|
|
with np.load(checked(ge['path'],ge['sha256']),allow_pickle=False) as z:
|
|
raw=z['raw_ids'].copy();x=z['points'].copy();mass=z['mass'].copy();rg=float(z['rg'])
|
|
adjacency=csr_matrix((z['data'],z['indices'],z['indptr']),shape=(len(raw),len(raw)))
|
|
source=Path(row['output']);save(dest/'target-access.json',dict(protocol=protocol,case=case,role='test_scoring',
|
|
path=str(source/'seconds.npy'),sha256=row['output_sha256']['seconds.npy'],predictions_freeze_sha256=sha(root/'predictions-freeze.json')))
|
|
truth=np.load(checked(source/'seconds.npy',row['output_sha256']['seconds.npy']),allow_pickle=False)
|
|
cacheids=np.load(checked(source/'raw_ids.npy',row['output_sha256']['raw_ids.npy']),allow_pickle=False)
|
|
if not np.array_equal(raw,cacheids) or truth.shape!=raw.shape or not np.isfinite(truth).all():raise ValueError('Score domain mismatch')
|
|
if protocol=='grouped':
|
|
q=prediction_inputs(c['grouped_rows'][case]);ix=np.searchsorted(raw,q['raw_ids'])
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Query raw ID mismatch')
|
|
label=c['grouped_rows'][case]['label']
|
|
with np.load(checked(label['path'],label['sha256']),allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'],q['raw_ids']) or not np.array_equal(z['counts'],q['counts']) or not np.array_equal(z['seconds'],truth[ix]):raise ValueError('Frozen query truth differs from tet truth')
|
|
else:ix=np.arange(len(raw));q=None
|
|
rows=[]
|
|
for e in entries:
|
|
with np.load(e['path'],allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'],raw):raise ValueError('Prediction raw ID mismatch')
|
|
for arm in ARMS:
|
|
pred=z[ARRAYS[arm]]
|
|
if protocol=='grouped':metrics=part_metrics(pred[ix],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
|
|
else:
|
|
met=per_geometry_metrics(truth,pred)
|
|
metrics=dict(relative_l2=dict(value=met['rrmse_rms'],reason=None if met['rrmse_rms'] is not None else 'zero truth denominator'),
|
|
kendall_tau_b=kendall_pairs(truth,pred,int(case.split('_')[0])),
|
|
last_location=dict(value=last_location(truth,pred,raw,x/rg),reason=None),paper=met)
|
|
rows.append(dict(case=case,protocol=protocol,fold=e['fold'],pool=e['pool'],arm=arm,**metrics))
|
|
# Field scores survive a later topology resource stop; their provenance is
|
|
# independent and the guard must report topology as incomplete, never PASS.
|
|
save(dest/'core.json',dict(status='FIELD_SCORES_COMPLETE',rows=rows,cpu_seconds=time.process_time()-start))
|
|
ref=events(truth,mass,raw,x,adjacency);save(dest/'truth-events.json',ref);topo=[]
|
|
for e in entries:
|
|
with np.load(e['path'],allow_pickle=False) as z:
|
|
for arm in ARMS:
|
|
pred=events(z[ARRAYS[arm]],mass,raw,x,adjacency)
|
|
detail=dict(case=case,protocol=protocol,fold=e['fold'],pool=e['pool'],arm=arm,predicted=pred,
|
|
match=match(ref,pred,adjacency,rg))
|
|
save(dest/f"events-{e['index']:02d}-{arm}.json",detail);topo.append(detail)
|
|
save(dest/'topology.json',dict(status='TOPOLOGY_COMPLETE',rows=topo,cpu_seconds=time.process_time()-start))
|
|
|
|
|
|
def intervals(rows,c,pool,metric):
|
|
lookup={(r['case'],r['arm']):r for r in rows if r['protocol']=='grouped' and r['pool']==pool}
|
|
groups={}
|
|
for case,row in c['parts']['grouped'].items():groups.setdefault(row['clock_group'],[]).append(case)
|
|
values={arm:{} for arm in ARMS};missing={arm:{} for arm in ARMS}
|
|
for arm in ARMS:
|
|
for g,cases in sorted(groups.items()):
|
|
block=[];bad=[]
|
|
for case in cases:
|
|
r=lookup.get((case,arm));v=None if r is None else r.get(metric,{}).get('value')
|
|
if v is None:bad.append(case)
|
|
else:block.append(v)
|
|
if bad:missing[arm][g]=bad
|
|
else:values[arm][g]=float(np.mean(block))
|
|
# Existing helper is before-after: pass H1 first for the approved H1-A sign.
|
|
delta=paired_difference(values['H1'],values['A'])
|
|
return dict(means={a:float(np.mean(list(v.values()))) if v else None for a,v in values.items()},
|
|
components={a:len(v) for a,v in values.items()},values=values,missing=missing,H1_minus_A=delta)
|
|
|
|
|
|
def summarize(root):
|
|
c,frozen=scoring_gate(root);rows=[];topo=[];coverage=[]
|
|
guard_by_part={}
|
|
for path in (root/'tasks').glob('score-*.json'):
|
|
desc=json.loads(path.read_text());g=root/'guards'/desc['name']/'guard-result.json'
|
|
guard_by_part[(desc['protocol'],desc['case'])]=json.loads(g.read_text()) if g.exists() else None
|
|
for protocol in ('grouped','v4_random'):
|
|
for case in c['parts'][protocol]:
|
|
d=root/'scores'/protocol/case;core=d/'core.json';tp=d/'topology.json'
|
|
if core.exists():rows.extend(json.loads(core.read_text())['rows'])
|
|
if tp.exists():topo.extend(json.loads(tp.read_text())['rows'])
|
|
g=guard_by_part.get((protocol,case));reason=None if tp.exists() else ('not requested/no valid test prediction' if (protocol,case) not in guard_by_part else 'missing guard receipt' if g is None else g.get('stop_reason') or 'worker failure before topology completed')
|
|
coverage.append(dict(protocol=protocol,case=case,field_complete=core.exists(),topology_complete=tp.exists(),
|
|
missing_topology_reason=reason,guard_status=None if g is None else g['status']))
|
|
grouped={str(p):{metric:intervals(rows,c,p,metric) for metric in ('relative_l2','last_location','kendall_tau_b')} for p in (100,50,25)}
|
|
decision=branch_decision(*(grouped['100'][m]['H1_minus_A'] for m in ('relative_l2','last_location','kendall_tau_b')))
|
|
v4={}
|
|
for arm in ARMS:
|
|
v4[arm]={}
|
|
for denominator in ('rms','maximum','mean'):
|
|
key='rrmse_'+denominator;blocks=[[r['paper'][key] for r in rows if r['protocol']=='v4_random' and r['arm']==arm and r['fold']==f and r['paper'][key] is not None] for f in range(5)]
|
|
v4[arm][key]=dict(summary=appendix_summary(blocks) if all(blocks) else None,
|
|
defined=[len(v) for v in blocks],fraction_below_point1=[float(np.mean(np.asarray(v)<.1)) if v else None for v in blocks])
|
|
totals={}
|
|
for r in topo:
|
|
key=f"{r['protocol']}/{r['pool']}/{r['fold']}/{r['arm']}";m=r['match']
|
|
t=totals.setdefault(key,dict(TP=0,FP=0,FN=0,distances=[],empty_classes={},parts=0))
|
|
for k in ('TP','FP','FN'):t[k]+=m[k]
|
|
t['distances'].extend(m['matched_distance_over_Rg']);t['parts']+=1
|
|
category=m['empty_class'];t['empty_classes'][category]=t['empty_classes'].get(category,0)+1
|
|
for t in totals.values():
|
|
t['precision']=t['TP']/(t['TP']+t['FP']) if t['TP']+t['FP'] else None
|
|
t['recall']=t['TP']/(t['TP']+t['FN']) if t['TP']+t['FN'] else None
|
|
t['median_distance_over_Rg']=float(np.median(t['distances'])) if t['distances'] else None
|
|
save(root/'summary.json',dict(status='SCORED_DESCRIPTIVE',grouped=grouped,v4=v4,tree_decision=decision,
|
|
topology=totals,coverage=coverage,field_rows=len(rows),topology_rows=len(topo),
|
|
topology_interpretation='auxiliary; B3 stability gate must pass before treating as usable diagnostic',
|
|
new_decodes=0,full_training_matrix=False))
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('mode',choices=['part','summary']);p.add_argument('protocol',nargs='?');p.add_argument('case',nargs='?');a=p.parse_args()
|
|
(score_part(a.root,a.protocol,a.case) if a.mode=='part' else summarize(a.root))
|