"""Validate saved folds and aggregate fixed scores. No kNN or rank refit.""" import csv,json,resource,sys,time from pathlib import Path from types import SimpleNamespace import numpy as np from run_s1_fold import read,save,sha,unpack_tree from s1_scoring import ARMS,PERMUTATION_SEEDS,summarize_folds,digest from s1_scoring_io import json_ready def csv_write(path,rows): with Path(path).open('x') as f: w=csv.DictWriter(f,fieldnames=list(rows[0]),lineterminator='\n');w.writeheader();w.writerows(rows) def endpoints(record):return record['interval95'] if record['interval95'] is not None else [None,None] def direction(interval): ci=interval['interval95'] if ci is None:return 'NA' return 'ahead' if ci[1]<0 else ('behind' if ci[0]>0 else 'not_separated') def decision_rows(aggregate): rows=[];seedrows=[] for contrast,slices in aggregate['contrasts'].items(): for slice_name,record in slices.items(): mean=record['five_seed_mean'];kind=mean['category'];d=direction(mean['source_level']) singles=list(record['per_seed'].values());points=[x['source_equal_point'] for x in singles] lo,hi=endpoints(mean['source_level']);ql,qh=endpoints(mean['query_level']) rows.append(dict(contrast=contrast,category=kind,slice=slice_name,estimator='five_seed_mean',source_equal_difference=mean['source_equal_point'],part_equal_difference=mean['part_equal_point'], fixed_LOPO_source_reweight_lower95=lo,fixed_LOPO_source_reweight_upper95=hi,source_reweight_SE=mean['source_level']['SE'],source_valid_replicates=mean['source_level']['valid_replicates'], query_lower95=ql,query_upper95=qh,query_SE=mean['query_level']['SE'],query_valid_replicates=mean['query_level']['valid_replicates'], primary_ranking=d if kind=='primary' else 'not_a_primary_decision',secondary_direction=d if kind=='secondary' else '', seed_points_below_zero=sum(p<0 for p in points),seed_points_above_zero=sum(p>0 for p in points),seed_points_equal_zero=sum(p==0 for p in points), seed_interval_classifications_agreeing_with_mean=sum(direction(x['source_level'])==d for x in singles),seed_count=5)) for seed,x in record['per_seed'].items(): sl,sh=endpoints(x['source_level']);qlo,qhi=endpoints(x['query_level']) seedrows.append(dict(contrast=contrast,category=kind,slice=slice_name,seed=seed,source_equal_difference=x['source_equal_point'],source_lower95=sl,source_upper95=sh,query_lower95=qlo,query_upper95=qhi,use='stability_only_not_primary_decision')) return rows,seedrows def main(): start=time.monotonic();out=Path(sys.argv[1]);c=read(out/'execution_contract.json');completion=read(out/'completion.json') if completion['status']!='COMPLETE' or len(completion['records'])!=24 or any(r['status']!='PASS' for r in completion['records']):raise ValueError('Required fold missing; do not silently drop sources') for p,h in {**c['code_sha256'],**c['input_sha256']}.items(): if sha(p)!=h:raise ValueError('Frozen input/source changed: '+p) foldrows=[];partrows=[];cliprows=[];neighborrows=[];controlrows=[];parts=[];compact=[];shuffle={};total_candidates=0 for i in range(24): folder=out/'folds'/f'{i:02d}';result=read(folder/'result.json');supervisor=read(out/'folds'/f'{i:02d}.supervisor.json');plan=c['query_plan'][i] if sha(folder/'result.json')!=supervisor['result_sha256']:raise ValueError('Result receipt mismatch') for p,h in result['output_sha256'].items(): if sha(folder/p)!=h:raise ValueError('Saved array/structure changed') with np.load(folder/'arrays.npz',allow_pickle=False) as z:r=unpack_tree(read(folder/'structure.json'),z) node=r['test_nodes'];counts=node['counts'];eligible=node['eligible'];near=eligible&node['near'];boot=node['bootstrap_counts'];qcounts=counts[eligible] if digest(boot)!=r['query_bootstrap_counts_sha256']:raise ValueError('Bootstrap counts digest differs') if not np.array_equal(boot.sum(1),np.full(200,2000)):raise ValueError('Draw-slot conservation failure') if result['logical_knn_queries']!=plan['logical_8NN_queries'] or result['tie_radius_queries']!=plan['tie_radius_queries']:raise ValueError('Actual query cost differs') if len(set(map(tuple,r['physical_reference_ids'])))!=len(r['physical_reference_ids']):raise ValueError('Duplicate physical references') if node['source'] in {key[0] for key in r['physical_reference_ids']}:raise ValueError('Source leakage') for weights in r['rank_weights']: np.testing.assert_allclose(np.sum(weights),1/23,rtol=0,atol=5e-16) for m in r['rank_maps']: if not (np.all(np.diff(m['values'])>0) and np.all(np.diff(m['cumulative'])>0) and m['cumulative'][0]==0 and m['cumulative'][-1]==1):raise ValueError('Invalid rank map') for (arm,si),neighbors in r['nearest'].items(): idx=neighbors['indices'];dist=neighbors['distances'] if np.any(idx<0) or np.any(idx>=len(r['physical_reference_ids'])) or np.any(np.diff(np.sort(idx,axis=1),axis=1)==0):raise ValueError('Invalid eight distinct neighbors') np.testing.assert_array_equal(dist.mean(1),r['scores'][arm][si,eligible]) if np.any(dist<0) or np.any(dist>1) or np.any(np.diff(dist,axis=1)<0):raise ValueError('Invalid RMS distances') neighborrows.append(dict(fold=i,case=node['case'],arm=arm,seed=PERMUTATION_SEEDS[si] if arm not in ['A','B','C'] else 'all', eligible_unique_nodes=len(qcounts),eligible_draws=int(qcounts.sum()), draw_weighted_zero_distance_neighbor_fraction=float(np.dot(qcounts,neighbors['zero_distance_neighbors'])/(8*qcounts.sum())), draw_weighted_distinct_neighbor_sources=float(np.dot(qcounts,neighbors['distinct_neighbor_sources'])/qcounts.sum()), draw_weighted_largest_source_neighbor_fraction=float(np.dot(qcounts,neighbors['largest_source_neighbor_fraction'])/qcounts.sum()), tie_candidates=neighbors['tie_candidate_evaluations'])) for arm in ARMS: if arm in ['A','B','C']: for si in range(1,5): if r['scores'][arm][si].tobytes()!=r['scores'][arm][0].tobytes():raise ValueError('Cached baseline differs across seeds') for si,seed in enumerate(PERMUTATION_SEEDS): for slice_name,mask in [('whole',eligible),('near',near)]: values=r['scores'][arm][si,mask];cc=counts[mask];bb=boot[:,mask] point=float(np.dot(cc,values)/cc.sum());reps=bb@values/bb.sum(1) stored=r['estimates'][arm][si][slice_name] np.testing.assert_allclose(point,stored['point'],rtol=0,atol=1e-15) np.testing.assert_allclose(reps,stored['replicates'],rtol=0,atol=1e-15) qlo,qhi=endpoints(stored['query_level']) partrows.append(dict(fold=i,case=node['case'],source=node['source'],arm=arm,seed=seed,slice=slice_name,score=point,query_lower95=qlo,query_upper95=qhi,query_SE=stored['query_level']['SE'],valid_query_replicates=stored['query_level']['valid_replicates'],accepted_draws=stored['accepted_draws'],positive_mass_share=stored['conditional_positive_mass_share'],full_mass_share=stored['full_mass_share'],boundary_unscored_mass_share=stored['boundary_unscored_mass_share'])) for slice_name,columns in r['query_clipping'].items(): for col,v in enumerate(columns):cliprows.append(dict(fold=i,case=node['case'],slice=slice_name,column=col,**v)) for (arm,si),by_part in r['control_reference_clipping'].items(): for col in range(10): lo=[row[col]['below_fraction'] for row in by_part.values()];hi=[row[col]['above_fraction'] for row in by_part.values()] controlrows.append(dict(fold=i,arm=arm,seed=PERMUTATION_SEEDS[si],column=col,equal_part_below_fraction=float(np.mean(lo)),equal_part_above_fraction=float(np.mean(hi)),max_part_below_fraction=max(lo),max_part_above_fraction=max(hi),parts_with_any_tail=sum(a>0 or b>0 for a,b in zip(lo,hi)))) for seed,by_part in r['shuffle_audits'].items(): for case,v in by_part.items(): key=case+'|'+seed if key in shuffle and shuffle[key]!=v:raise ValueError('Part permutation/audit not reused across folds') shuffle[key]=v foldrows.append(dict(fold=i,case=node['case'],source=node['source'],status=supervisor['status'],wall_seconds=supervisor['wall_seconds'],peak_RSS_MiB=supervisor['peak_rss_bytes']/2**20,max_monitor_gap_seconds=supervisor['max_observed_poll_gap_seconds'],monitor_samples=supervisor['monitor_samples'],max_processes=supervisor['max_observed_processes'],eligible_unique_queries=int(eligible.sum()),eligible_draws=int(qcounts.sum()),eligible_near_draws=int(counts[near].sum()),training_references=len(r['physical_reference_ids']),logical_8NN_queries=result['logical_knn_queries'],tie_radius_queries=result['tie_radius_queries'],expected_logical_8NN_queries=plan['logical_8NN_queries'],tree_builds=result['tree_builds'],tie_candidate_evaluations=result['tie_candidate_evaluations'])) total_candidates+=result['tie_candidate_evaluations'];parts.append(SimpleNamespace(source=node['source']));compact.append(dict(status=r['status'],estimates=r['estimates'])) del r aggregate=summarize_folds(parts,compact) aggregate['source_interval_label']=c['source_interval_label'];aggregate['source_interval_scope']=c['source_interval_scope'];aggregate['primary_ranking_estimator']='five_seed_mean_only';aggregate['per_seed_decisions_in_raw_core_output']='diagnostic classifications only, not primary ranking';aggregate['zero_interpretation']='not separated, never equivalent' decisions,stability=decision_rows(aggregate) armrows=[] for arm,slices in aggregate['arms'].items(): for slice_name,value in slices.items(): for seed,v in [('five_seed_mean',value['five_seed_mean']),*value['per_seed'].items()]: lo,hi=endpoints(v['source_level']);ql,qh=endpoints(v['query_level']) armrows.append(dict(arm=arm,slice=slice_name,seed=seed,source_equal_score=v['source_equal_point'],part_equal_score=v['part_equal_point'],source_reweight_lower95=lo,source_reweight_upper95=hi,query_lower95=ql,query_upper95=qh)) for name,rows in [('fold_summary.csv',foldrows),('part_scores.csv',partrows),('query_clipping.csv',cliprows),('control_clipping_summary.csv',controlrows),('neighbor_diagnostics.csv',neighborrows),('decision_table.csv',decisions),('seed_stability.csv',stability),('arm_summary.csv',armrows)]:csv_write(out/name,rows) save(out/'aggregate.json',json_ready(aggregate));save(out/'shuffle_audits.json',shuffle) verification=dict(status='PASS',folds=24,actual_logical_8NN_queries=sum(r['logical_8NN_queries'] for r in foldrows),actual_tie_radius_queries=sum(r['tie_radius_queries'] for r in foldrows),actual_tie_candidate_evaluations=total_candidates, tree_builds=sum(r['tree_builds'] for r in foldrows),max_fold_wall_seconds=max(r['wall_seconds'] for r in foldrows),sum_fold_wall_seconds=sum(r['wall_seconds'] for r in foldrows),max_fold_RSS_MiB=max(r['peak_RSS_MiB'] for r in foldrows),max_monitor_gap_seconds=max(r['max_monitor_gap_seconds'] for r in foldrows),max_observed_processes=max(r['max_processes'] for r in foldrows), core_source_unchanged=True,query_counts_match_all_folds=True,source_leakage=False,distinct_eight_neighbors_checked=True,all_saved_point_and_query_replica_ratios_recomputed=True,rank_map_and_weight_invariants_checked=True,fold_permutation_reuse_checked=True, aggregation_performed_no_new_knn_queries=True,no_rank_refit=True,training=False,report_wall_seconds=time.monotonic()-start,report_peak_RSS_MiB=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024, source_interval_label=c['source_interval_label'],source_interval_scope=c['source_interval_scope'],primary_decision=c['primary_decision'],report_code_sha256=sha(__file__)) save(out/'verification.json',verification);print(json.dumps(verification,indent=2));print(json.dumps([r for r in decisions if r['category']=='primary'],indent=2)) if __name__=='__main__':main()