146 lines
9.3 KiB
Python
146 lines
9.3 KiB
Python
"""Synthetic-only acceptance of custody separation and preregistered statistics."""
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import checked,prediction_inputs,verify_prediction_index
|
|
from s2_custody_statistics import six_region,kendall_pairs,last_location,part_metrics,component_values,contrast
|
|
from run_s2_custody_score import decode_gate
|
|
from s2_models.features import BASE_COLUMNS,R2_COLUMNS,base_features
|
|
from s2_models.objectives import b0_seconds
|
|
|
|
|
|
def roster():
|
|
return [dict(case=p,component_id=g,nodes=10,included=True,exposure=dict(development_exposed=e,inventory_state='UNKNOWN'),
|
|
assignments=[dict(fold=f,role='test' if f==fold else 'learning') for f in range(5)])
|
|
for p,g,e,fold in [('a','groupA',False,0),('b','groupA',True,0),('c','groupB',False,1)]]
|
|
|
|
|
|
def rows():
|
|
return [dict(case=p,arm=arm,pool=100,seed=s,relative_l2=dict(value=value,reason=None))
|
|
for p,value in [('a',1.),('b',3.),('c',8.)] for arm in ['N','E'] for s in [0,1,2]]
|
|
|
|
|
|
class Acceptance(unittest.TestCase):
|
|
def test_six_regions_and_exact_boundaries(self):
|
|
for l,u,want in [(-.2,-.1,'harmful'),(-.1,0,'no meaningful gain'),(0,.099,'no meaningful gain'),
|
|
(0,.1,'insufficient evidence'),(.01,.099,'positive but below margin'),(.01,.1,'positive, size undetermined'),
|
|
(.1,.2,'positive, size undetermined'),(.1000000001,.2,'meaningful gain supported')]:
|
|
self.assertEqual(six_region(l,u),want)
|
|
with self.assertRaises(ValueError):six_region(.2,.1)
|
|
|
|
def test_hierarchy_is_not_part_or_fold_weighted(self):
|
|
vals,excluded,_=component_values(rows(),roster(),'N',100)
|
|
self.assertEqual(vals,{'groupA':2.,'groupB':8.});self.assertFalse(excluded)
|
|
self.assertEqual(np.mean(list(vals.values())),5.)
|
|
|
|
def test_three_seed_complete_no_two_seed_backfill(self):
|
|
rr=[x for x in rows() if not(x['case']=='a' and x['arm']=='N' and x['seed']==0)]
|
|
n,excluded,_=component_values(rr,roster(),'N',100);e,_,_=component_values(rr,roster(),'E',100)
|
|
self.assertEqual(set(n),{'groupB'});self.assertIn('groupA',excluded)
|
|
self.assertEqual(contrast(n,e)['component_ids'],['groupB'])
|
|
|
|
def test_frozen_part_exposure_keeps_mixed_component(self):
|
|
vals,_,_=component_values(rows(),roster(),'N',100,not_known_exposed=True,frozen_members={'a','c'})
|
|
self.assertEqual(vals,{'groupA':1.,'groupB':8.})
|
|
with self.assertRaises(ValueError):component_values(rows(),roster(),'N',100,not_known_exposed=True)
|
|
bad=roster();bad[0]['exposure']['development_exposed']='UNKNOWN'
|
|
with self.assertRaises(ValueError):component_values(rows(),bad,'N',100)
|
|
|
|
def test_exposure_does_not_rescue_invalid_main_component(self):
|
|
rr=[x for x in rows() if x['case']!='b']
|
|
vals,excluded,_=component_values(rr,roster(),'N',100,not_known_exposed=True,frozen_members={'a','c'})
|
|
self.assertNotIn('groupA',vals);self.assertTrue(excluded['groupA']['main_analysis_invalid'])
|
|
|
|
def test_bootstrap_matched_indices_sorted_rng_and_ratio_of_means(self):
|
|
a={'b':2.,'a':1.};b={'a':.5,'b':1.}
|
|
r=contrast(a,b);self.assertAlmostEqual(r['value'],.5);self.assertAlmostEqual(r['low'],.5);self.assertEqual(r['component_ids'],['a','b'])
|
|
b={'a':.9,'b':.5};r=contrast(a,b)
|
|
rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence(2026111408)))
|
|
ids=rng.integers(0,2,(2000,2));boot=1-np.array([.9,.5])[ids].mean(1)/np.array([1.,2.])[ids].mean(1)
|
|
np.testing.assert_array_equal([r['low'],r['high']],np.percentile(boot,[2.5,97.5],method='linear'))
|
|
|
|
def test_undefined_bootstrap_is_counted_not_replaced(self):
|
|
r=contrast({'a':0.,'b':1.},{'a':0.,'b':.5})
|
|
self.assertGreater(r['undefined_replicates'],0);self.assertIsNone(r['low'])
|
|
|
|
def test_ties_kendall_and_last_location(self):
|
|
self.assertEqual(kendall_pairs([1,1,2],[2,2,4],3)['value'],1.)
|
|
self.assertEqual(kendall_pairs([1,1,2],[4,4,2],3)['value'],-1.)
|
|
self.assertIsNone(kendall_pairs([1,1],[2,2],3)['value'])
|
|
self.assertEqual(last_location([2,2,1],[1,1,2],[20,10,30],[[0,0,0],[1,0,0],[3,0,0]]),2.)
|
|
|
|
def test_count_weighted_error_empty_slice_and_zero_norm(self):
|
|
m=part_metrics([2.,2.],[1.,2.],[3,1],[False,False],[0,1],[[0,0,0],[1,0,0]],1)
|
|
self.assertAlmostEqual(m['relative_l2']['value'],np.sqrt(3/7));self.assertIsNone(m['near_wall']['value'])
|
|
m=part_metrics([1.,1.],[0.,0.],[1,1],[True,True],[0,1],[[0,0,0],[1,0,0]],1)
|
|
self.assertIsNone(m['relative_l2']['value'])
|
|
|
|
def test_hash_change_and_no_prediction_decode_gate(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);p=root/'input';p.write_text('one');h=sha(p);p.write_text('two')
|
|
with self.assertRaises(ValueError):checked(p,h)
|
|
with patch('run_s2_custody_score.decode_custody_label',side_effect=AssertionError('label must remain sealed')) as decoder:
|
|
with self.assertRaises(FileNotFoundError):decode_gate(root,{})
|
|
decoder.assert_not_called()
|
|
|
|
def test_prediction_reader_only_uses_label_free_features(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);base=np.zeros((3,10));x=np.zeros((3,20));x[:,-1]=.4
|
|
np.save(root/'base.npy',base);np.savez(root/'r2.npz',features=x)
|
|
np.savez(root/'queries.npz',eligible_s2=[True,False,True],anchor_ids=[0,1,2],raw_node_ids=[10,11,12],
|
|
multiplicity=[2,1,3],g=np.ones((3,2)),edge_summary=np.ones((3,4)),near_wall=[True,False,False])
|
|
(root/'result.json').write_text(json.dumps(dict(base_columns=list(BASE_COLUMNS),r2=dict(columns=list(R2_COLUMNS)))))
|
|
row=dict(nodes=3,preparation=dict(preparation_root=str(root),t_set=.4,
|
|
result_sha256=sha(root/'result.json'),output_sha256={p.name:sha(p) for p in root.iterdir()}))
|
|
original=np.load;paths=[]
|
|
def audit(path,**kwargs):paths.append(Path(path).name);return original(path,**kwargs)
|
|
with patch('s2_custody_common.np.load',side_effect=audit):d=prediction_inputs(row)
|
|
self.assertEqual(set(paths),{'base.npy','r2.npz','queries.npz'});np.testing.assert_array_equal(d['raw_ids'],[10,12])
|
|
metadata=json.loads((root/'result.json').read_text());metadata['r2']['columns'][0]='wrong'
|
|
(root/'result.json').write_text(json.dumps(metadata));row['preparation']['result_sha256']=sha(root/'result.json')
|
|
with self.assertRaises(ValueError):prediction_inputs(row)
|
|
|
|
def test_named_B0_units_and_centred_coordinates_over_Rg(self):
|
|
self.assertEqual(R2_COLUMNS[0],'Vf');np.testing.assert_array_equal(b0_seconds([0,.25,1],4),[0,1,4])
|
|
x=np.array([[2.,4.,6.],[6.,10.,14.]])
|
|
base=base_features(x,[1,2],[0,1],[2,4,6],[1,2,3],2,10,1,4)
|
|
self.assertEqual(BASE_COLUMNS[:3],('x_star','y_star','z_star'))
|
|
np.testing.assert_array_equal(base[:,:3],(x-[1,2,3])/2)
|
|
self.assertEqual(last_location([1,2],[2,1],[0,1],base[:,:3]),np.linalg.norm(x[1]-x[0])/2)
|
|
|
|
def test_frozen_real_metadata_denominators_no_label_read(self):
|
|
# Only immutable JSON membership/roster metadata, never geometry or targets.
|
|
path=Path('artifacts/s2-g1-components-2026-11-15/custodian/not-known-exposed-test-subsets.json')
|
|
self.assertEqual(sha(path),'8d037786040c6fc11fa80ceb0f41dfdd58478a2576404505a64aa29a2216a22f')
|
|
members=json.loads(path.read_text());r=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text())['parts']
|
|
expected=[(104,33),(105,36),(115,39),(89,40),(104,40)]
|
|
for f in members:
|
|
candidates={x['case'] for x in r if x['assignments'][f['fold']]['role']=='test' and x['exposure']['development_exposed'] is False}
|
|
self.assertEqual(candidates,set(f['parts']))
|
|
keep=[x for x in r if x['included'] and x['case'] in set(f['parts'])]
|
|
self.assertEqual((len(keep),len({x['component_id'] for x in keep})),expected[f['fold']])
|
|
|
|
def test_prediction_guard_checks_final_recorded_peak(self):
|
|
import run_s2_custody_predictions as guard
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp)
|
|
c=dict(gpu_uuids=['synthetic'],cpu_affinities=[[0]],limits=dict(vram_bytes=15*2**30,host_bytes=16*2**30,
|
|
fold_disk_bytes=20*2**30,fold_wall_seconds=2700))
|
|
def fake_supervise(*args,**kwargs):
|
|
p=root/'folds/0';p.mkdir(parents=True)
|
|
(p/'result.json').write_text(json.dumps(dict(status='COMPLETE',peak_allocated_bytes=1,peak_reserved_bytes=16*2**30)))
|
|
return dict(status='PASS')
|
|
with patch.object(guard,'contract',return_value=c),patch.object(guard.os,'sched_setaffinity'),patch.object(guard.time,'sleep'),\
|
|
patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'tree_usage',return_value=(0,0,1)),\
|
|
patch.object(guard,'supervise',side_effect=fake_supervise),patch.object(guard,'active_command_pids',return_value=[]):
|
|
self.assertEqual(guard.guard(root,0,0),1)
|
|
r=json.loads((root/'guards/0/guard-result.json').read_text())
|
|
self.assertIn('RECORDED_VRAM_LIMIT',r['monitor']['reasons'])
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|