124 lines
12 KiB
Python
124 lines
12 KiB
Python
"""Validate saved folds and aggregate fixed scores. No kNN or rank refit."""
|
|
import csv,json,resource,sys,time
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
import numpy as np
|
|
from run_s1_fold import read,save,sha,unpack_tree
|
|
from s1_scoring import ARMS,PERMUTATION_SEEDS,summarize_folds,digest
|
|
from s1_scoring_io import json_ready
|
|
|
|
|
|
def csv_write(path,rows):
|
|
with Path(path).open('x') as f:
|
|
w=csv.DictWriter(f,fieldnames=list(rows[0]),lineterminator='\n');w.writeheader();w.writerows(rows)
|
|
|
|
|
|
def endpoints(record):return record['interval95'] if record['interval95'] is not None else [None,None]
|
|
|
|
|
|
def direction(interval):
|
|
ci=interval['interval95']
|
|
if ci is None:return 'NA'
|
|
return 'ahead' if ci[1]<0 else ('behind' if ci[0]>0 else 'not_separated')
|
|
|
|
|
|
def decision_rows(aggregate):
|
|
rows=[];seedrows=[]
|
|
for contrast,slices in aggregate['contrasts'].items():
|
|
for slice_name,record in slices.items():
|
|
mean=record['five_seed_mean'];kind=mean['category'];d=direction(mean['source_level'])
|
|
singles=list(record['per_seed'].values());points=[x['source_equal_point'] for x in singles]
|
|
lo,hi=endpoints(mean['source_level']);ql,qh=endpoints(mean['query_level'])
|
|
rows.append(dict(contrast=contrast,category=kind,slice=slice_name,estimator='five_seed_mean',source_equal_difference=mean['source_equal_point'],part_equal_difference=mean['part_equal_point'],
|
|
fixed_LOPO_source_reweight_lower95=lo,fixed_LOPO_source_reweight_upper95=hi,source_reweight_SE=mean['source_level']['SE'],source_valid_replicates=mean['source_level']['valid_replicates'],
|
|
query_lower95=ql,query_upper95=qh,query_SE=mean['query_level']['SE'],query_valid_replicates=mean['query_level']['valid_replicates'],
|
|
primary_ranking=d if kind=='primary' else 'not_a_primary_decision',secondary_direction=d if kind=='secondary' else '',
|
|
seed_points_below_zero=sum(p<0 for p in points),seed_points_above_zero=sum(p>0 for p in points),seed_points_equal_zero=sum(p==0 for p in points),
|
|
seed_interval_classifications_agreeing_with_mean=sum(direction(x['source_level'])==d for x in singles),seed_count=5))
|
|
for seed,x in record['per_seed'].items():
|
|
sl,sh=endpoints(x['source_level']);qlo,qhi=endpoints(x['query_level'])
|
|
seedrows.append(dict(contrast=contrast,category=kind,slice=slice_name,seed=seed,source_equal_difference=x['source_equal_point'],source_lower95=sl,source_upper95=sh,query_lower95=qlo,query_upper95=qhi,use='stability_only_not_primary_decision'))
|
|
return rows,seedrows
|
|
|
|
|
|
def main():
|
|
start=time.monotonic();out=Path(sys.argv[1]);c=read(out/'execution_contract.json');completion=read(out/'completion.json')
|
|
if completion['status']!='COMPLETE' or len(completion['records'])!=24 or any(r['status']!='PASS' for r in completion['records']):raise ValueError('Required fold missing; do not silently drop sources')
|
|
for p,h in {**c['code_sha256'],**c['input_sha256']}.items():
|
|
if sha(p)!=h:raise ValueError('Frozen input/source changed: '+p)
|
|
foldrows=[];partrows=[];cliprows=[];neighborrows=[];controlrows=[];parts=[];compact=[];shuffle={};total_candidates=0
|
|
for i in range(24):
|
|
folder=out/'folds'/f'{i:02d}';result=read(folder/'result.json');supervisor=read(out/'folds'/f'{i:02d}.supervisor.json');plan=c['query_plan'][i]
|
|
if sha(folder/'result.json')!=supervisor['result_sha256']:raise ValueError('Result receipt mismatch')
|
|
for p,h in result['output_sha256'].items():
|
|
if sha(folder/p)!=h:raise ValueError('Saved array/structure changed')
|
|
with np.load(folder/'arrays.npz',allow_pickle=False) as z:r=unpack_tree(read(folder/'structure.json'),z)
|
|
node=r['test_nodes'];counts=node['counts'];eligible=node['eligible'];near=eligible&node['near'];boot=node['bootstrap_counts'];qcounts=counts[eligible]
|
|
if digest(boot)!=r['query_bootstrap_counts_sha256']:raise ValueError('Bootstrap counts digest differs')
|
|
if not np.array_equal(boot.sum(1),np.full(200,2000)):raise ValueError('Draw-slot conservation failure')
|
|
if result['logical_knn_queries']!=plan['logical_8NN_queries'] or result['tie_radius_queries']!=plan['tie_radius_queries']:raise ValueError('Actual query cost differs')
|
|
if len(set(map(tuple,r['physical_reference_ids'])))!=len(r['physical_reference_ids']):raise ValueError('Duplicate physical references')
|
|
if node['source'] in {key[0] for key in r['physical_reference_ids']}:raise ValueError('Source leakage')
|
|
for weights in r['rank_weights']:
|
|
np.testing.assert_allclose(np.sum(weights),1/23,rtol=0,atol=5e-16)
|
|
for m in r['rank_maps']:
|
|
if not (np.all(np.diff(m['values'])>0) and np.all(np.diff(m['cumulative'])>0) and m['cumulative'][0]==0 and m['cumulative'][-1]==1):raise ValueError('Invalid rank map')
|
|
for (arm,si),neighbors in r['nearest'].items():
|
|
idx=neighbors['indices'];dist=neighbors['distances']
|
|
if np.any(idx<0) or np.any(idx>=len(r['physical_reference_ids'])) or np.any(np.diff(np.sort(idx,axis=1),axis=1)==0):raise ValueError('Invalid eight distinct neighbors')
|
|
np.testing.assert_array_equal(dist.mean(1),r['scores'][arm][si,eligible])
|
|
if np.any(dist<0) or np.any(dist>1) or np.any(np.diff(dist,axis=1)<0):raise ValueError('Invalid RMS distances')
|
|
neighborrows.append(dict(fold=i,case=node['case'],arm=arm,seed=PERMUTATION_SEEDS[si] if arm not in ['A','B','C'] else 'all',
|
|
eligible_unique_nodes=len(qcounts),eligible_draws=int(qcounts.sum()),
|
|
draw_weighted_zero_distance_neighbor_fraction=float(np.dot(qcounts,neighbors['zero_distance_neighbors'])/(8*qcounts.sum())),
|
|
draw_weighted_distinct_neighbor_sources=float(np.dot(qcounts,neighbors['distinct_neighbor_sources'])/qcounts.sum()),
|
|
draw_weighted_largest_source_neighbor_fraction=float(np.dot(qcounts,neighbors['largest_source_neighbor_fraction'])/qcounts.sum()),
|
|
tie_candidates=neighbors['tie_candidate_evaluations']))
|
|
for arm in ARMS:
|
|
if arm in ['A','B','C']:
|
|
for si in range(1,5):
|
|
if r['scores'][arm][si].tobytes()!=r['scores'][arm][0].tobytes():raise ValueError('Cached baseline differs across seeds')
|
|
for si,seed in enumerate(PERMUTATION_SEEDS):
|
|
for slice_name,mask in [('whole',eligible),('near',near)]:
|
|
values=r['scores'][arm][si,mask];cc=counts[mask];bb=boot[:,mask]
|
|
point=float(np.dot(cc,values)/cc.sum());reps=bb@values/bb.sum(1)
|
|
stored=r['estimates'][arm][si][slice_name]
|
|
np.testing.assert_allclose(point,stored['point'],rtol=0,atol=1e-15)
|
|
np.testing.assert_allclose(reps,stored['replicates'],rtol=0,atol=1e-15)
|
|
qlo,qhi=endpoints(stored['query_level'])
|
|
partrows.append(dict(fold=i,case=node['case'],source=node['source'],arm=arm,seed=seed,slice=slice_name,score=point,query_lower95=qlo,query_upper95=qhi,query_SE=stored['query_level']['SE'],valid_query_replicates=stored['query_level']['valid_replicates'],accepted_draws=stored['accepted_draws'],positive_mass_share=stored['conditional_positive_mass_share'],full_mass_share=stored['full_mass_share'],boundary_unscored_mass_share=stored['boundary_unscored_mass_share']))
|
|
for slice_name,columns in r['query_clipping'].items():
|
|
for col,v in enumerate(columns):cliprows.append(dict(fold=i,case=node['case'],slice=slice_name,column=col,**v))
|
|
for (arm,si),by_part in r['control_reference_clipping'].items():
|
|
for col in range(10):
|
|
lo=[row[col]['below_fraction'] for row in by_part.values()];hi=[row[col]['above_fraction'] for row in by_part.values()]
|
|
controlrows.append(dict(fold=i,arm=arm,seed=PERMUTATION_SEEDS[si],column=col,equal_part_below_fraction=float(np.mean(lo)),equal_part_above_fraction=float(np.mean(hi)),max_part_below_fraction=max(lo),max_part_above_fraction=max(hi),parts_with_any_tail=sum(a>0 or b>0 for a,b in zip(lo,hi))))
|
|
for seed,by_part in r['shuffle_audits'].items():
|
|
for case,v in by_part.items():
|
|
key=case+'|'+seed
|
|
if key in shuffle and shuffle[key]!=v:raise ValueError('Part permutation/audit not reused across folds')
|
|
shuffle[key]=v
|
|
foldrows.append(dict(fold=i,case=node['case'],source=node['source'],status=supervisor['status'],wall_seconds=supervisor['wall_seconds'],peak_RSS_MiB=supervisor['peak_rss_bytes']/2**20,max_monitor_gap_seconds=supervisor['max_observed_poll_gap_seconds'],monitor_samples=supervisor['monitor_samples'],max_processes=supervisor['max_observed_processes'],eligible_unique_queries=int(eligible.sum()),eligible_draws=int(qcounts.sum()),eligible_near_draws=int(counts[near].sum()),training_references=len(r['physical_reference_ids']),logical_8NN_queries=result['logical_knn_queries'],tie_radius_queries=result['tie_radius_queries'],expected_logical_8NN_queries=plan['logical_8NN_queries'],tree_builds=result['tree_builds'],tie_candidate_evaluations=result['tie_candidate_evaluations']))
|
|
total_candidates+=result['tie_candidate_evaluations'];parts.append(SimpleNamespace(source=node['source']));compact.append(dict(status=r['status'],estimates=r['estimates']))
|
|
del r
|
|
aggregate=summarize_folds(parts,compact)
|
|
aggregate['source_interval_label']=c['source_interval_label'];aggregate['source_interval_scope']=c['source_interval_scope'];aggregate['primary_ranking_estimator']='five_seed_mean_only';aggregate['per_seed_decisions_in_raw_core_output']='diagnostic classifications only, not primary ranking';aggregate['zero_interpretation']='not separated, never equivalent'
|
|
decisions,stability=decision_rows(aggregate)
|
|
armrows=[]
|
|
for arm,slices in aggregate['arms'].items():
|
|
for slice_name,value in slices.items():
|
|
for seed,v in [('five_seed_mean',value['five_seed_mean']),*value['per_seed'].items()]:
|
|
lo,hi=endpoints(v['source_level']);ql,qh=endpoints(v['query_level'])
|
|
armrows.append(dict(arm=arm,slice=slice_name,seed=seed,source_equal_score=v['source_equal_point'],part_equal_score=v['part_equal_point'],source_reweight_lower95=lo,source_reweight_upper95=hi,query_lower95=ql,query_upper95=qh))
|
|
for name,rows in [('fold_summary.csv',foldrows),('part_scores.csv',partrows),('query_clipping.csv',cliprows),('control_clipping_summary.csv',controlrows),('neighbor_diagnostics.csv',neighborrows),('decision_table.csv',decisions),('seed_stability.csv',stability),('arm_summary.csv',armrows)]:csv_write(out/name,rows)
|
|
save(out/'aggregate.json',json_ready(aggregate));save(out/'shuffle_audits.json',shuffle)
|
|
verification=dict(status='PASS',folds=24,actual_logical_8NN_queries=sum(r['logical_8NN_queries'] for r in foldrows),actual_tie_radius_queries=sum(r['tie_radius_queries'] for r in foldrows),actual_tie_candidate_evaluations=total_candidates,
|
|
tree_builds=sum(r['tree_builds'] for r in foldrows),max_fold_wall_seconds=max(r['wall_seconds'] for r in foldrows),sum_fold_wall_seconds=sum(r['wall_seconds'] for r in foldrows),max_fold_RSS_MiB=max(r['peak_RSS_MiB'] for r in foldrows),max_monitor_gap_seconds=max(r['max_monitor_gap_seconds'] for r in foldrows),max_observed_processes=max(r['max_processes'] for r in foldrows),
|
|
core_source_unchanged=True,query_counts_match_all_folds=True,source_leakage=False,distinct_eight_neighbors_checked=True,all_saved_point_and_query_replica_ratios_recomputed=True,rank_map_and_weight_invariants_checked=True,fold_permutation_reuse_checked=True,
|
|
aggregation_performed_no_new_knn_queries=True,no_rank_refit=True,training=False,report_wall_seconds=time.monotonic()-start,report_peak_RSS_MiB=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024,
|
|
source_interval_label=c['source_interval_label'],source_interval_scope=c['source_interval_scope'],primary_decision=c['primary_decision'],report_code_sha256=sha(__file__))
|
|
save(out/'verification.json',verification);print(json.dumps(verification,indent=2));print(json.dumps([r for r in decisions if r['category']=='primary'],indent=2))
|
|
|
|
|
|
if __name__=='__main__':main()
|