Files
MeltMind/tools/report_s1_scoring.py
T

124 lines
12 KiB
Python

"""Validate saved folds and aggregate fixed scores. No kNN or rank refit."""
import csv,json,resource,sys,time
from pathlib import Path
from types import SimpleNamespace
import numpy as np
from run_s1_fold import read,save,sha,unpack_tree
from s1_scoring import ARMS,PERMUTATION_SEEDS,summarize_folds,digest
from s1_scoring_io import json_ready
def csv_write(path,rows):
with Path(path).open('x') as f:
w=csv.DictWriter(f,fieldnames=list(rows[0]),lineterminator='\n');w.writeheader();w.writerows(rows)
def endpoints(record):return record['interval95'] if record['interval95'] is not None else [None,None]
def direction(interval):
ci=interval['interval95']
if ci is None:return 'NA'
return 'ahead' if ci[1]<0 else ('behind' if ci[0]>0 else 'not_separated')
def decision_rows(aggregate):
rows=[];seedrows=[]
for contrast,slices in aggregate['contrasts'].items():
for slice_name,record in slices.items():
mean=record['five_seed_mean'];kind=mean['category'];d=direction(mean['source_level'])
singles=list(record['per_seed'].values());points=[x['source_equal_point'] for x in singles]
lo,hi=endpoints(mean['source_level']);ql,qh=endpoints(mean['query_level'])
rows.append(dict(contrast=contrast,category=kind,slice=slice_name,estimator='five_seed_mean',source_equal_difference=mean['source_equal_point'],part_equal_difference=mean['part_equal_point'],
fixed_LOPO_source_reweight_lower95=lo,fixed_LOPO_source_reweight_upper95=hi,source_reweight_SE=mean['source_level']['SE'],source_valid_replicates=mean['source_level']['valid_replicates'],
query_lower95=ql,query_upper95=qh,query_SE=mean['query_level']['SE'],query_valid_replicates=mean['query_level']['valid_replicates'],
primary_ranking=d if kind=='primary' else 'not_a_primary_decision',secondary_direction=d if kind=='secondary' else '',
seed_points_below_zero=sum(p<0 for p in points),seed_points_above_zero=sum(p>0 for p in points),seed_points_equal_zero=sum(p==0 for p in points),
seed_interval_classifications_agreeing_with_mean=sum(direction(x['source_level'])==d for x in singles),seed_count=5))
for seed,x in record['per_seed'].items():
sl,sh=endpoints(x['source_level']);qlo,qhi=endpoints(x['query_level'])
seedrows.append(dict(contrast=contrast,category=kind,slice=slice_name,seed=seed,source_equal_difference=x['source_equal_point'],source_lower95=sl,source_upper95=sh,query_lower95=qlo,query_upper95=qhi,use='stability_only_not_primary_decision'))
return rows,seedrows
def main():
start=time.monotonic();out=Path(sys.argv[1]);c=read(out/'execution_contract.json');completion=read(out/'completion.json')
if completion['status']!='COMPLETE' or len(completion['records'])!=24 or any(r['status']!='PASS' for r in completion['records']):raise ValueError('Required fold missing; do not silently drop sources')
for p,h in {**c['code_sha256'],**c['input_sha256']}.items():
if sha(p)!=h:raise ValueError('Frozen input/source changed: '+p)
foldrows=[];partrows=[];cliprows=[];neighborrows=[];controlrows=[];parts=[];compact=[];shuffle={};total_candidates=0
for i in range(24):
folder=out/'folds'/f'{i:02d}';result=read(folder/'result.json');supervisor=read(out/'folds'/f'{i:02d}.supervisor.json');plan=c['query_plan'][i]
if sha(folder/'result.json')!=supervisor['result_sha256']:raise ValueError('Result receipt mismatch')
for p,h in result['output_sha256'].items():
if sha(folder/p)!=h:raise ValueError('Saved array/structure changed')
with np.load(folder/'arrays.npz',allow_pickle=False) as z:r=unpack_tree(read(folder/'structure.json'),z)
node=r['test_nodes'];counts=node['counts'];eligible=node['eligible'];near=eligible&node['near'];boot=node['bootstrap_counts'];qcounts=counts[eligible]
if digest(boot)!=r['query_bootstrap_counts_sha256']:raise ValueError('Bootstrap counts digest differs')
if not np.array_equal(boot.sum(1),np.full(200,2000)):raise ValueError('Draw-slot conservation failure')
if result['logical_knn_queries']!=plan['logical_8NN_queries'] or result['tie_radius_queries']!=plan['tie_radius_queries']:raise ValueError('Actual query cost differs')
if len(set(map(tuple,r['physical_reference_ids'])))!=len(r['physical_reference_ids']):raise ValueError('Duplicate physical references')
if node['source'] in {key[0] for key in r['physical_reference_ids']}:raise ValueError('Source leakage')
for weights in r['rank_weights']:
np.testing.assert_allclose(np.sum(weights),1/23,rtol=0,atol=5e-16)
for m in r['rank_maps']:
if not (np.all(np.diff(m['values'])>0) and np.all(np.diff(m['cumulative'])>0) and m['cumulative'][0]==0 and m['cumulative'][-1]==1):raise ValueError('Invalid rank map')
for (arm,si),neighbors in r['nearest'].items():
idx=neighbors['indices'];dist=neighbors['distances']
if np.any(idx<0) or np.any(idx>=len(r['physical_reference_ids'])) or np.any(np.diff(np.sort(idx,axis=1),axis=1)==0):raise ValueError('Invalid eight distinct neighbors')
np.testing.assert_array_equal(dist.mean(1),r['scores'][arm][si,eligible])
if np.any(dist<0) or np.any(dist>1) or np.any(np.diff(dist,axis=1)<0):raise ValueError('Invalid RMS distances')
neighborrows.append(dict(fold=i,case=node['case'],arm=arm,seed=PERMUTATION_SEEDS[si] if arm not in ['A','B','C'] else 'all',
eligible_unique_nodes=len(qcounts),eligible_draws=int(qcounts.sum()),
draw_weighted_zero_distance_neighbor_fraction=float(np.dot(qcounts,neighbors['zero_distance_neighbors'])/(8*qcounts.sum())),
draw_weighted_distinct_neighbor_sources=float(np.dot(qcounts,neighbors['distinct_neighbor_sources'])/qcounts.sum()),
draw_weighted_largest_source_neighbor_fraction=float(np.dot(qcounts,neighbors['largest_source_neighbor_fraction'])/qcounts.sum()),
tie_candidates=neighbors['tie_candidate_evaluations']))
for arm in ARMS:
if arm in ['A','B','C']:
for si in range(1,5):
if r['scores'][arm][si].tobytes()!=r['scores'][arm][0].tobytes():raise ValueError('Cached baseline differs across seeds')
for si,seed in enumerate(PERMUTATION_SEEDS):
for slice_name,mask in [('whole',eligible),('near',near)]:
values=r['scores'][arm][si,mask];cc=counts[mask];bb=boot[:,mask]
point=float(np.dot(cc,values)/cc.sum());reps=bb@values/bb.sum(1)
stored=r['estimates'][arm][si][slice_name]
np.testing.assert_allclose(point,stored['point'],rtol=0,atol=1e-15)
np.testing.assert_allclose(reps,stored['replicates'],rtol=0,atol=1e-15)
qlo,qhi=endpoints(stored['query_level'])
partrows.append(dict(fold=i,case=node['case'],source=node['source'],arm=arm,seed=seed,slice=slice_name,score=point,query_lower95=qlo,query_upper95=qhi,query_SE=stored['query_level']['SE'],valid_query_replicates=stored['query_level']['valid_replicates'],accepted_draws=stored['accepted_draws'],positive_mass_share=stored['conditional_positive_mass_share'],full_mass_share=stored['full_mass_share'],boundary_unscored_mass_share=stored['boundary_unscored_mass_share']))
for slice_name,columns in r['query_clipping'].items():
for col,v in enumerate(columns):cliprows.append(dict(fold=i,case=node['case'],slice=slice_name,column=col,**v))
for (arm,si),by_part in r['control_reference_clipping'].items():
for col in range(10):
lo=[row[col]['below_fraction'] for row in by_part.values()];hi=[row[col]['above_fraction'] for row in by_part.values()]
controlrows.append(dict(fold=i,arm=arm,seed=PERMUTATION_SEEDS[si],column=col,equal_part_below_fraction=float(np.mean(lo)),equal_part_above_fraction=float(np.mean(hi)),max_part_below_fraction=max(lo),max_part_above_fraction=max(hi),parts_with_any_tail=sum(a>0 or b>0 for a,b in zip(lo,hi))))
for seed,by_part in r['shuffle_audits'].items():
for case,v in by_part.items():
key=case+'|'+seed
if key in shuffle and shuffle[key]!=v:raise ValueError('Part permutation/audit not reused across folds')
shuffle[key]=v
foldrows.append(dict(fold=i,case=node['case'],source=node['source'],status=supervisor['status'],wall_seconds=supervisor['wall_seconds'],peak_RSS_MiB=supervisor['peak_rss_bytes']/2**20,max_monitor_gap_seconds=supervisor['max_observed_poll_gap_seconds'],monitor_samples=supervisor['monitor_samples'],max_processes=supervisor['max_observed_processes'],eligible_unique_queries=int(eligible.sum()),eligible_draws=int(qcounts.sum()),eligible_near_draws=int(counts[near].sum()),training_references=len(r['physical_reference_ids']),logical_8NN_queries=result['logical_knn_queries'],tie_radius_queries=result['tie_radius_queries'],expected_logical_8NN_queries=plan['logical_8NN_queries'],tree_builds=result['tree_builds'],tie_candidate_evaluations=result['tie_candidate_evaluations']))
total_candidates+=result['tie_candidate_evaluations'];parts.append(SimpleNamespace(source=node['source']));compact.append(dict(status=r['status'],estimates=r['estimates']))
del r
aggregate=summarize_folds(parts,compact)
aggregate['source_interval_label']=c['source_interval_label'];aggregate['source_interval_scope']=c['source_interval_scope'];aggregate['primary_ranking_estimator']='five_seed_mean_only';aggregate['per_seed_decisions_in_raw_core_output']='diagnostic classifications only, not primary ranking';aggregate['zero_interpretation']='not separated, never equivalent'
decisions,stability=decision_rows(aggregate)
armrows=[]
for arm,slices in aggregate['arms'].items():
for slice_name,value in slices.items():
for seed,v in [('five_seed_mean',value['five_seed_mean']),*value['per_seed'].items()]:
lo,hi=endpoints(v['source_level']);ql,qh=endpoints(v['query_level'])
armrows.append(dict(arm=arm,slice=slice_name,seed=seed,source_equal_score=v['source_equal_point'],part_equal_score=v['part_equal_point'],source_reweight_lower95=lo,source_reweight_upper95=hi,query_lower95=ql,query_upper95=qh))
for name,rows in [('fold_summary.csv',foldrows),('part_scores.csv',partrows),('query_clipping.csv',cliprows),('control_clipping_summary.csv',controlrows),('neighbor_diagnostics.csv',neighborrows),('decision_table.csv',decisions),('seed_stability.csv',stability),('arm_summary.csv',armrows)]:csv_write(out/name,rows)
save(out/'aggregate.json',json_ready(aggregate));save(out/'shuffle_audits.json',shuffle)
verification=dict(status='PASS',folds=24,actual_logical_8NN_queries=sum(r['logical_8NN_queries'] for r in foldrows),actual_tie_radius_queries=sum(r['tie_radius_queries'] for r in foldrows),actual_tie_candidate_evaluations=total_candidates,
tree_builds=sum(r['tree_builds'] for r in foldrows),max_fold_wall_seconds=max(r['wall_seconds'] for r in foldrows),sum_fold_wall_seconds=sum(r['wall_seconds'] for r in foldrows),max_fold_RSS_MiB=max(r['peak_RSS_MiB'] for r in foldrows),max_monitor_gap_seconds=max(r['max_monitor_gap_seconds'] for r in foldrows),max_observed_processes=max(r['max_processes'] for r in foldrows),
core_source_unchanged=True,query_counts_match_all_folds=True,source_leakage=False,distinct_eight_neighbors_checked=True,all_saved_point_and_query_replica_ratios_recomputed=True,rank_map_and_weight_invariants_checked=True,fold_permutation_reuse_checked=True,
aggregation_performed_no_new_knn_queries=True,no_rank_refit=True,training=False,report_wall_seconds=time.monotonic()-start,report_peak_RSS_MiB=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024,
source_interval_label=c['source_interval_label'],source_interval_scope=c['source_interval_scope'],primary_decision=c['primary_decision'],report_code_sha256=sha(__file__))
save(out/'verification.json',verification);print(json.dumps(verification,indent=2));print(json.dumps([r for r in decisions if r['category']=='primary'],indent=2))
if __name__=='__main__':main()