Deliver read-only S2 D1-D4 posthoc diagnostics with oracle boundaries

This commit is contained in:
Frank14f committed 2026-11-15 16:38:35 +08:00
1 parent eeeca7c9ea
commit daed8e873a
14 files changed
+295682

No files matched your search

+113
View File
@@ -0,0 +1,113 @@
"""Post-hoc D1/D2/D4 from immutable existing outputs; no model or label loading."""
from collections import defaultdict
import json
from pathlib import Path
import time
import numpy as np
from freeze_g1_components import sha
from s2_recovery_supervisor import save
ROOT=Path('artifacts/s2-posthoc-diagnostics-2026-11-15')
EVAL=Path('artifacts/s2-custody-evaluation-2026-11-15')
def read(path):return json.loads(Path(path).read_text())
def grouped(values,ids):
groups=defaultdict(list)
for value,key in zip(values,ids):groups[key].append(float(value))
return float(np.mean([np.mean(v) for v in groups.values()]))
def main():
start=time.process_time();assert not ROOT.exists(),'Do not overwrite diagnostic evidence'
result=read(EVAL/'custody/result.json')
for name in ['component-scores.json','part-scores.json','tables.json']:
assert sha(EVAL/'custody'/name)==result['output_sha256'][name]
scores=read(EVAL/'custody/component-scores.json');curves={}
arms=['N','E','R2','B0'];pools=[25,50,100]
for pool in pools:
for arm in arms:
curves[pool,arm]={r['component_id']:r['value'] for r in scores if r['subset']=='all' and r['metric']=='relative_l2' and r['arm']==arm and r['pool']==pool}
common=set.intersection(*(set(v) for v in curves.values()))
all_comparison=set(curves[25,'N'])&set(curves[100,'N'])
d1=dict(posthoc=True,common_components=len(common),common_component_ids=sorted(common),common_folds=[0,2,3,4],
rows=[dict(pool_component_fraction=pool/100,arm=arm,original_components=len(curves[pool,arm]),
original_mean=float(np.mean(list(curves[pool,arm].values()))),
common_mean=float(np.mean([curves[pool,arm][k] for k in sorted(common)]))) for pool in pools for arm in arms],
N_25_to100_all5_relative_error_reduction=float(1-np.mean([curves[100,'N'][k] for k in all_comparison])/np.mean([curves[25,'N'][k] for k in all_comparison])),
note='Original50pctN denominator differs; common fourfold curve is a separate descriptive diagnostic, not replacement primary analysis')
records=[];histories={}
for fold in range(5):
for arm in ['N','E']:
for seed in [0,1,2]:
name=f'fold{fold}-pool100-{arm}128-seed{seed}';base=Path('artifacts/s2-main-matrix-2026-11-15/cells')/name/name
rr=read(base/'result.json');h=read(base/'history.json');histories[name]=h
y=np.array([v['validation_component_equal_relative_l2'] for v in h]);train=np.array([v['training_global_weighted_mse'] for v in h])
n=min(5,len(y));slope=float(np.polyfit(np.arange(n),y[-n:],1)[0])
records.append(dict(name=name,fold=fold,arm=arm,seed=seed,epochs=len(h),best_epoch_1based=rr['best_epoch']+1,
best_validation=rr['best_validation_score'],final_validation=float(y[-1]),
best_in_last5=rr['best_epoch']>=len(h)-5,best_at_final=rr['best_epoch']==len(h)-1,
final5_slope=slope,final5_endpoint_relative_change=float(y[-1]/y[-n]-1),
first_validation=float(y[0]),final_training_loss=float(train[-1]),final5_training_relative_change=float(train[-1]/train[-n]-1)))
summary=[]
for arm in ['N','E']:
rr=[r for r in records if r['arm']==arm]
summary.append(dict(arm=arm,runs=len(rr),all_reached42=all(r['epochs']==42 for r in rr),
median_best_epoch=float(np.median([r['best_epoch_1based'] for r in rr])),
best_in_last5=sum(r['best_in_last5'] for r in rr),best_at_final=sum(r['best_at_final'] for r in rr),
negative_final5_validation_slope=sum(r['final5_slope']<0 for r in rr),
median_final5_validation_change=float(np.median([r['final5_endpoint_relative_change'] for r in rr])),
median_final5_training_change=float(np.median([r['final5_training_relative_change'] for r in rr]))))
d2=dict(posthoc=True,epoch_display='one based',records=records,summary=summary,
limitation='Late best epoch/negative local slope does not prove a longer schedule will beat R2; cosine endpoint and fitting choices are confounded')
roster=read('artifacts/s2-main-preparation-2026-11-15/final-roster.json')['parts']
partrows=read(EVAL/'custody/part-scores.json');by=defaultdict(list)
for r in partrows:
if r['arm']=='B0' or r['pool']==100:by[r['case'],r['arm']].append(r['relative_l2']['value'])
geom=[]
for r in roster:
if not r['included']:continue
p=Path(r['preparation']['preparation_root'])/'result.json';assert sha(p)==r['preparation']['result_sha256'];s=read(p)['scales']
means={arm:float(np.mean(by[r['case'],arm])) for arm in arms}
assert len(by[r['case'],'N'])==len(by[r['case'],'E'])==3
geom.append(dict(case=r['case'],component_id=r['component_id'],nodes=r['nodes'],Rg=s['Rg'],t_part=s['t_part'],
errors=means,N_minus_B0=means['N']-means['B0'],N_loses_to_B0=means['N']>means['B0']))
bins=[]
for feature in ['Rg','t_part','nodes']:
x=np.array([r[feature] for r in geom]);cuts=np.quantile(x,[1/3,2/3],method='linear');assignment=np.searchsorted(cuts,x,side='right')
for b in range(3):
rr=[r for i,r in enumerate(geom) if assignment[i]==b];ids=[r['component_id'] for r in rr]
bins.append(dict(feature=feature,bin=b,tertile_cutpoints=cuts.tolist(),min=min(r[feature] for r in rr),max=max(r[feature] for r in rr),
parts=len(rr),components=len(set(ids)),N_loses_parts=sum(r['N_loses_to_B0'] for r in rr),
component_equal_loss_fraction=grouped([r['N_loses_to_B0'] for r in rr],ids),
component_equal_difference=grouped([r['N_minus_B0'] for r in rr],ids),
component_equal_N=grouped([r['errors']['N'] for r in rr],ids),
component_equal_B0=grouped([r['errors']['B0'] for r in rr],ids)))
d4=dict(posthoc=True,parts=617,geometry_units='original prepared geometry units; nodes count',
bins=bins,part_records=geom,N_loses_parts=sum(r['N_loses_to_B0'] for r in geom),
component_equal_loss_fraction=grouped([r['N_loses_to_B0'] for r in geom],[r['component_id'] for r in geom]),
limitation='Geometry-only empirical tertiles on617parts, posthoc descriptive; within-bin parts/component equal; no causal claim or multiplicity test')
ROOT.mkdir();save(ROOT/'D1-learning-curves.json',d1);save(ROOT/'D2-epoch-history.json',d2);save(ROOT/'D4-geometry-errors.json',d4)
# Standalone scientific figure from existing tabulations, no new fitted model.
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
fig,axes=plt.subplots(1,3,figsize=(13,3.8))
for arm in arms:
axes[0].plot([.25,.5,1],[100*next(r['common_mean'] for r in d1['rows'] if r['arm']==arm and r['pool_component_fraction']==p) for p in [.25,.5,1]],'-o',label=arm)
axes[0].set(xlabel='Learning component fraction',ylabel='Component-equal relative L2 (%)',title='D1: common four-fold diagnostic');axes[0].legend()
for ax,arm in zip(axes[1:],['N','E']):
hs=[h for name,h in histories.items() if f'-{arm}128-' in name]
for h in hs:ax.plot([r['epoch']+1 for r in h],[r['validation_component_equal_relative_l2'] for r in h],alpha=.3,lw=.8)
ax.set(xlabel='Epoch',ylabel='Validation relative L2',title=f'D2: {arm}, 100% pool, 15 runs')
fig.tight_layout();fig.savefig(ROOT/'D1-D2-curves.png',dpi=200);plt.close(fig)
save(ROOT/'receipt.json',dict(status='D1_D2_D4_COMPLETE',cpu_seconds=time.process_time()-start,
new_labels_decoded=False,models_loaded=False,new_training=False,original_statistics_modified=False,
D3='WAITING_MATHEMATICAL_BOUNDARIES',ruling='1794731320.994789',code_sha256=sha(__file__),
output_sha256={p.name:sha(p) for p in ROOT.iterdir() if p.is_file()}))
print(json.dumps(dict(D1_reduction=d1['N_25_to100_all5_relative_error_reduction'],D2=summary,D4_loses=d4['N_loses_parts'],CPU=time.process_time()-start)))
if __name__=='__main__':main()
+119
View File
@@ -0,0 +1,119 @@
"""D3 oracle diagnostic, not a calibration; only frozen prediction/label caches."""
import json
from pathlib import Path
import time
import numpy as np
from freeze_g1_components import sha
from s2_recovery_supervisor import save
from s2_custody_common import read_json,checked,test_rows,expected_models,prediction_path
from s2_custody_statistics import component_values
ROOT=Path('artifacts/s2-posthoc-diagnostics-2026-11-15')
EVAL=Path('artifacts/s2-custody-evaluation-2026-11-15')
def decompose(prediction,truth,counts):
p=np.asarray(prediction,dtype=np.float64);t=np.asarray(truth,dtype=np.float64);c=np.asarray(counts,dtype=np.float64)
if p.ndim!=1 or p.shape!=t.shape or p.shape!=c.shape or not len(p):return dict(valid=False,reason='empty or invalid shape')
if not all(np.isfinite(v).all() for v in [p,t,c]) or np.any(c<=0) or np.any(c!=np.floor(c)):
return dict(valid=False,reason='nonfinite eligible item or invalid counts')
ps=float(np.max(np.abs(p)));ts=float(np.max(np.abs(t)))
if ts==0:return dict(valid=False,reason='zero_truth',e2=None,H=None,S=None,a=None)
if ps==0:return dict(valid=False,reason='zero_pred',e2=1.,H=None,S=None,a=None)
# Unit-vector projection avoids squaring tiny P or subtracting close energies.
wp=np.sqrt(c)*(p/ps);wt=np.sqrt(c)*(t/ts)
pn=np.linalg.norm(wp);tn=np.linalg.norm(wt);u=wp/pn;v=wt/tn
cosine=float(u@v)
beta_ld=np.longdouble(ps)/np.longdouble(ts)*np.longdouble(pn)/np.longdouble(tn)
beta=float(beta_ld)
a_ld=np.longdouble(cosine)/beta_ld
a=float(a_ld) if abs(a_ld)<=np.finfo(float).max else None
e2=float(np.sum((beta*u-v)**2));H=float(np.sum((cosine*u-v)**2));S=float((beta-cosine)**2)
if not np.isfinite([e2,H,S]).all():return dict(valid=False,reason='unrepresentable normalized energy',a=a)
if not np.isclose(e2,H+S,rtol=1e-10,atol=1e-12):raise ArithmeticError('Orthogonal decomposition failed')
return dict(valid=True,reason=None,a=a,a_scientific=np.format_float_scientific(a_ld,precision=17),
e2=e2,H=H,S=S,identity_absolute_error=abs(e2-H-S),
sign_reversal=cosine<0,nonpositive_a=cosine<=0,zero_cross=cosine==0,
perfect_prediction=e2==0,negative_prediction_items=int(np.sum(p<0)))
def synthetic_checks():
for p,t,want in [([2,4],[1,2],(.5,0.,1.,1.)),([1,2],[1,2],(1.,0.,0.,0.)),
([-1,-2],[1,2],(-1.,0.,4.,4.)),([1,0],[0,1],(0.,1.,1.,2.)),
([1e-200,2e-200],[1,2],(1e200,0.,1.,1.))]:
d=decompose(p,t,[1,1]);assert d['valid']
np.testing.assert_allclose([d['a'],d['H'],d['S'],d['e2']],want,rtol=1e-12,atol=1e-12)
z=decompose([0,0],[1,2],[1,1]);assert not z['valid'] and z['reason']=='zero_pred' and z['e2']==1 and z['H'] is None
for p,t,c in [([1,2],[0,0],[1,1]),([],[],[]),([1,np.nan],[1,2],[1,1])]:assert not decompose(p,t,c)['valid']
p=np.array([.1,.4,.9]);t=np.array([.2,.3,1.]);c=np.array([3,1,7]);d=decompose(p,t,c)
np.testing.assert_allclose(d['a'],np.sum(c*p*t)/np.sum(c*p*p),rtol=1e-12)
np.testing.assert_allclose(d['e2'],np.sum(c*(p-t)**2)/np.sum(c*t*t),rtol=1e-12)
return 10
def main():
start=time.process_time();assert not (ROOT/'D3-started.json').exists(),'No duplicate diagnostic pass'
tests=synthetic_checks();c=read_json(EVAL/'contract.json');original=read_json(EVAL/'custody/result.json')
for name in ['label-index.json','part-scores.json']:checked(EVAL/'custody'/name,original['output_sha256'][name])
predindex=read_json(checked(EVAL/'prediction-index.json',read_json(EVAL/'predictions-complete.json')['prediction_index_sha256']))
labels={r['case']:r for r in read_json(EVAL/'custody/label-index.json')}
original_scores={(r['case'],r['arm'],r['pool'],r['seed']):r for r in read_json(EVAL/'custody/part-scores.json')}
save(ROOT/'D3-started.json',dict(rulings=['1794731518.379529','1794731544.355059','1794731547.266819'],
description='oracle diagnostic, not a calibration',synthetic_checks=tests,code_sha256=sha(__file__),
zero_pred='e2=1 retained as original error; a/H/S undefined and excluded from valid decomposition',
tolerance=dict(rtol=1e-10,atol=1e-12),new_labels_decoded=False))
rows=[];metric_rows=[];maximum_identity_error=maximum_frozen_error=0.
for fold in range(5):
for row in test_rows(c,fold):
label=labels[row['case']];checked(label['path'],label['sha256'])
with np.load(label['path'],allow_pickle=False) as z:truth=z['seconds'];ids=z['raw_ids'];counts=z['counts']
for model in expected_models(c,fold):
p=prediction_path(EVAL,model,row['case']);checked(p,predindex[str(p)])
with np.load(p,allow_pickle=False) as z:
assert np.array_equal(z['raw_ids'],ids) and np.array_equal(z['counts'],counts)
d=decompose(z['seconds'],truth,counts)
old=original_scores[row['case'],model['arm'],model['pool'],model['seed']]['relative_l2']['value']
if d['valid']:
error=abs(d['e2']-old**2);maximum_frozen_error=max(maximum_frozen_error,error)
assert np.isclose(d['e2'],old**2,rtol=1e-10,atol=1e-12)
maximum_identity_error=max(maximum_identity_error,d['identity_absolute_error'])
record=dict(case=row['case'],fold=fold,component_id=row['component_id'],arm=model['arm'],pool=model['pool'],seed=model['seed'],**d)
rows.append(record)
metric_rows.append(dict(case=row['case'],arm=model['arm'],pool=model['pool'],seed=model['seed'],
**{key:dict(value=d[key] if d['valid'] else None,reason=d['reason']) for key in ['e2','H','S']},
nonpositive=dict(value=float(d['nonpositive_a']) if d['valid'] else None,reason=d['reason']),
reversal=dict(value=float(d['sign_reversal']) if d['valid'] else None,reason=d['reason'])))
roster=read_json(c['roster'])['parts'];members={p for f in read_json(checked(c['exposure_membership'],c['exposure_membership_sha256'])) for p in f['parts']}
tables=[];comparisons=[]
for subset in ['all','not_known_exposed']:
for pool in [25,50,100]:
byarm={}
for arm in ['N','E','R2','B0']:
values={};details={};excluded={}
for metric in ['e2','H','S','nonpositive','reversal']:
values[metric],excluded,details=component_values(metric_rows,roster,arm,pool,metric,subset!='all',members)
keys=set.intersection(*(set(v) for v in values.values()));means={k:float(np.mean([v[x] for x in sorted(keys)])) if keys else None for k,v in values.items()}
if keys:assert np.isclose(means['e2'],means['H']+means['S'],rtol=1e-10,atol=1e-12)
tables.append(dict(subset=subset,pool=pool,arm=arm,components=len(keys),parts=sum(details[k]['parts'] for k in keys),
excluded_components=excluded,mean=means,oracle_removable_squared_error_share=means['S']/means['e2'] if keys and means['e2']>0 else None,
warning='oracle diagnostic, not a calibration; mean squared error is not square of primary mean L2'))
byarm[arm]=(values,details)
for ref,cand in [('N','E'),('R2','N'),('B0','N')]:
aa,details=byarm[ref];bb,_=byarm[cand];keys=set(aa['e2'])&set(bb['e2'])
am={k:float(np.mean([aa[k][x] for x in sorted(keys)])) for k in ['e2','H','S']}
bm={k:float(np.mean([bb[k][x] for x in sorted(keys)])) for k in ['e2','H','S']}
comparisons.append(dict(subset=subset,pool=pool,reference=ref,candidate=cand,components=len(keys),
parts=sum(details[x]['parts'] for x in keys),reference_means=am,candidate_means=bm,
candidate_minus_reference={k:bm[k]-am[k] for k in am},warning='squared-error diagnostic difference, not primary Delta'))
save(ROOT/'D3-part-decomposition.json',rows);save(ROOT/'D3-tables.json',tables);save(ROOT/'D3-comparisons.json',comparisons)
summary=dict(status='D3_COMPLETE',rows=len(rows),valid=sum(r['valid'] for r in rows),undefined=[r for r in rows if not r['valid']],
nonpositive_a_rows=sum(r.get('nonpositive_a',False) for r in rows),sign_reversal_rows=sum(r.get('sign_reversal',False) for r in rows),
zero_cross_rows=sum(r.get('zero_cross',False) for r in rows),maximum_identity_absolute_error=maximum_identity_error,
maximum_frozen_squared_error_difference=maximum_frozen_error,cpu_seconds=time.process_time()-start,
new_labels_decoded=False,predictions_modified=False,calibration=False,synthetic_checks=tests,
output_sha256={p.name:sha(p) for p in ROOT.glob('D3-*.json')})
save(ROOT/'D3-receipt.json',summary)
print(json.dumps(summary));print(json.dumps([x for x in tables if x['subset']=='all' and x['pool']==100]))
if __name__=='__main__':main()