38 lines
2.4 KiB
Python
38 lines
2.4 KiB
Python
"""Frozen full-domain RMSE/max(truth) summaries on identical paired support."""
|
|
import numpy as np
|
|
|
|
from track_a_postfit_rebind_common import OMITTED, SPLITS
|
|
from track_a_postfit_statistics import paired_uncertainty
|
|
|
|
|
|
def summarize(rows, jobs):
|
|
if tuple(j['id'] for j in jobs) != SPLITS or {r['split'] for r in rows} != set(SPLITS):
|
|
raise ValueError('Exactly the four approved original splits required')
|
|
per = []
|
|
for j in jobs:
|
|
chosen = [r for r in rows if r['split'] == j['id']]
|
|
if {r['case'] for r in chosen} != set(j['test']) or len(chosen) != len(j['test']):
|
|
raise ValueError('Complete paired prediction support required')
|
|
valid = [r for r in chosen if r['A'] is not None and r['H2'] is not None]
|
|
summaries = {}
|
|
for method in ['A', 'H2']:
|
|
values = np.asarray([r[method] for r in valid], np.float64)
|
|
if not np.isfinite(values).all() or np.any(values < 0):
|
|
raise ValueError('Invalid paired metric')
|
|
summaries[method] = {name: float(fn(values)) if len(values) else None
|
|
for name, fn in [('min',np.min),('median',np.median),('max',np.max)]}
|
|
delta = np.asarray([r['H2']-r['A'] for r in valid],np.float64)
|
|
per.append(dict(split=j['id'],requested_test_parts=315,paired_predictions=len(chosen),
|
|
scored_pairs=len(valid),missing_original=list(j['missing_test']),
|
|
undefined=[r['case'] for r in chosen if r['A'] is None or r['H2'] is None],
|
|
summaries=summaries,paired_mean_H2_minus_A=float(delta.mean()) if len(delta) else None,
|
|
paired_median_H2_minus_A=float(np.median(delta)) if len(delta) else None))
|
|
means = {method: {name: float(np.mean([r['summaries'][method][name] for r in per]))
|
|
if all(r['summaries'][method][name] is not None for r in per) else None
|
|
for name in ['min','median','max']} for method in ['A','H2']}
|
|
return dict(per_split=per,mean_of_four_split_summaries=means,
|
|
evaluated_splits=list(SPLITS), excluded_split=OMITTED,
|
|
direction='Negative paired H2-minus-A favours H2; descriptive only',
|
|
descriptive_paired_uncertainty=paired_uncertainty(rows),significance_gate=False,metric='RMSE / max(full truth)',
|
|
full_requested_315_coverage=all(not r['missing_original'] for r in per))
|