Files
MeltMind/tools/test_s2_custody.py
T

146 lines
9.3 KiB
Python

"""Synthetic-only acceptance of custody separation and preregistered statistics."""
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import numpy as np
from freeze_g1_components import sha
from s2_custody_common import checked,prediction_inputs,verify_prediction_index
from s2_custody_statistics import six_region,kendall_pairs,last_location,part_metrics,component_values,contrast
from run_s2_custody_score import decode_gate
from s2_models.features import BASE_COLUMNS,R2_COLUMNS,base_features
from s2_models.objectives import b0_seconds
def roster():
return [dict(case=p,component_id=g,nodes=10,included=True,exposure=dict(development_exposed=e,inventory_state='UNKNOWN'),
assignments=[dict(fold=f,role='test' if f==fold else 'learning') for f in range(5)])
for p,g,e,fold in [('a','groupA',False,0),('b','groupA',True,0),('c','groupB',False,1)]]
def rows():
return [dict(case=p,arm=arm,pool=100,seed=s,relative_l2=dict(value=value,reason=None))
for p,value in [('a',1.),('b',3.),('c',8.)] for arm in ['N','E'] for s in [0,1,2]]
class Acceptance(unittest.TestCase):
def test_six_regions_and_exact_boundaries(self):
for l,u,want in [(-.2,-.1,'harmful'),(-.1,0,'no meaningful gain'),(0,.099,'no meaningful gain'),
(0,.1,'insufficient evidence'),(.01,.099,'positive but below margin'),(.01,.1,'positive, size undetermined'),
(.1,.2,'positive, size undetermined'),(.1000000001,.2,'meaningful gain supported')]:
self.assertEqual(six_region(l,u),want)
with self.assertRaises(ValueError):six_region(.2,.1)
def test_hierarchy_is_not_part_or_fold_weighted(self):
vals,excluded,_=component_values(rows(),roster(),'N',100)
self.assertEqual(vals,{'groupA':2.,'groupB':8.});self.assertFalse(excluded)
self.assertEqual(np.mean(list(vals.values())),5.)
def test_three_seed_complete_no_two_seed_backfill(self):
rr=[x for x in rows() if not(x['case']=='a' and x['arm']=='N' and x['seed']==0)]
n,excluded,_=component_values(rr,roster(),'N',100);e,_,_=component_values(rr,roster(),'E',100)
self.assertEqual(set(n),{'groupB'});self.assertIn('groupA',excluded)
self.assertEqual(contrast(n,e)['component_ids'],['groupB'])
def test_frozen_part_exposure_keeps_mixed_component(self):
vals,_,_=component_values(rows(),roster(),'N',100,not_known_exposed=True,frozen_members={'a','c'})
self.assertEqual(vals,{'groupA':1.,'groupB':8.})
with self.assertRaises(ValueError):component_values(rows(),roster(),'N',100,not_known_exposed=True)
bad=roster();bad[0]['exposure']['development_exposed']='UNKNOWN'
with self.assertRaises(ValueError):component_values(rows(),bad,'N',100)
def test_exposure_does_not_rescue_invalid_main_component(self):
rr=[x for x in rows() if x['case']!='b']
vals,excluded,_=component_values(rr,roster(),'N',100,not_known_exposed=True,frozen_members={'a','c'})
self.assertNotIn('groupA',vals);self.assertTrue(excluded['groupA']['main_analysis_invalid'])
def test_bootstrap_matched_indices_sorted_rng_and_ratio_of_means(self):
a={'b':2.,'a':1.};b={'a':.5,'b':1.}
r=contrast(a,b);self.assertAlmostEqual(r['value'],.5);self.assertAlmostEqual(r['low'],.5);self.assertEqual(r['component_ids'],['a','b'])
b={'a':.9,'b':.5};r=contrast(a,b)
rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence(2026111408)))
ids=rng.integers(0,2,(2000,2));boot=1-np.array([.9,.5])[ids].mean(1)/np.array([1.,2.])[ids].mean(1)
np.testing.assert_array_equal([r['low'],r['high']],np.percentile(boot,[2.5,97.5],method='linear'))
def test_undefined_bootstrap_is_counted_not_replaced(self):
r=contrast({'a':0.,'b':1.},{'a':0.,'b':.5})
self.assertGreater(r['undefined_replicates'],0);self.assertIsNone(r['low'])
def test_ties_kendall_and_last_location(self):
self.assertEqual(kendall_pairs([1,1,2],[2,2,4],3)['value'],1.)
self.assertEqual(kendall_pairs([1,1,2],[4,4,2],3)['value'],-1.)
self.assertIsNone(kendall_pairs([1,1],[2,2],3)['value'])
self.assertEqual(last_location([2,2,1],[1,1,2],[20,10,30],[[0,0,0],[1,0,0],[3,0,0]]),2.)
def test_count_weighted_error_empty_slice_and_zero_norm(self):
m=part_metrics([2.,2.],[1.,2.],[3,1],[False,False],[0,1],[[0,0,0],[1,0,0]],1)
self.assertAlmostEqual(m['relative_l2']['value'],np.sqrt(3/7));self.assertIsNone(m['near_wall']['value'])
m=part_metrics([1.,1.],[0.,0.],[1,1],[True,True],[0,1],[[0,0,0],[1,0,0]],1)
self.assertIsNone(m['relative_l2']['value'])
def test_hash_change_and_no_prediction_decode_gate(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);p=root/'input';p.write_text('one');h=sha(p);p.write_text('two')
with self.assertRaises(ValueError):checked(p,h)
with patch('run_s2_custody_score.decode_custody_label',side_effect=AssertionError('label must remain sealed')) as decoder:
with self.assertRaises(FileNotFoundError):decode_gate(root,{})
decoder.assert_not_called()
def test_prediction_reader_only_uses_label_free_features(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);base=np.zeros((3,10));x=np.zeros((3,20));x[:,-1]=.4
np.save(root/'base.npy',base);np.savez(root/'r2.npz',features=x)
np.savez(root/'queries.npz',eligible_s2=[True,False,True],anchor_ids=[0,1,2],raw_node_ids=[10,11,12],
multiplicity=[2,1,3],g=np.ones((3,2)),edge_summary=np.ones((3,4)),near_wall=[True,False,False])
(root/'result.json').write_text(json.dumps(dict(base_columns=list(BASE_COLUMNS),r2=dict(columns=list(R2_COLUMNS)))))
row=dict(nodes=3,preparation=dict(preparation_root=str(root),t_set=.4,
result_sha256=sha(root/'result.json'),output_sha256={p.name:sha(p) for p in root.iterdir()}))
original=np.load;paths=[]
def audit(path,**kwargs):paths.append(Path(path).name);return original(path,**kwargs)
with patch('s2_custody_common.np.load',side_effect=audit):d=prediction_inputs(row)
self.assertEqual(set(paths),{'base.npy','r2.npz','queries.npz'});np.testing.assert_array_equal(d['raw_ids'],[10,12])
metadata=json.loads((root/'result.json').read_text());metadata['r2']['columns'][0]='wrong'
(root/'result.json').write_text(json.dumps(metadata));row['preparation']['result_sha256']=sha(root/'result.json')
with self.assertRaises(ValueError):prediction_inputs(row)
def test_named_B0_units_and_centred_coordinates_over_Rg(self):
self.assertEqual(R2_COLUMNS[0],'Vf');np.testing.assert_array_equal(b0_seconds([0,.25,1],4),[0,1,4])
x=np.array([[2.,4.,6.],[6.,10.,14.]])
base=base_features(x,[1,2],[0,1],[2,4,6],[1,2,3],2,10,1,4)
self.assertEqual(BASE_COLUMNS[:3],('x_star','y_star','z_star'))
np.testing.assert_array_equal(base[:,:3],(x-[1,2,3])/2)
self.assertEqual(last_location([1,2],[2,1],[0,1],base[:,:3]),np.linalg.norm(x[1]-x[0])/2)
def test_frozen_real_metadata_denominators_no_label_read(self):
# Only immutable JSON membership/roster metadata, never geometry or targets.
path=Path('artifacts/s2-g1-components-2026-11-15/custodian/not-known-exposed-test-subsets.json')
self.assertEqual(sha(path),'8d037786040c6fc11fa80ceb0f41dfdd58478a2576404505a64aa29a2216a22f')
members=json.loads(path.read_text());r=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text())['parts']
expected=[(104,33),(105,36),(115,39),(89,40),(104,40)]
for f in members:
candidates={x['case'] for x in r if x['assignments'][f['fold']]['role']=='test' and x['exposure']['development_exposed'] is False}
self.assertEqual(candidates,set(f['parts']))
keep=[x for x in r if x['included'] and x['case'] in set(f['parts'])]
self.assertEqual((len(keep),len({x['component_id'] for x in keep})),expected[f['fold']])
def test_prediction_guard_checks_final_recorded_peak(self):
import run_s2_custody_predictions as guard
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp)
c=dict(gpu_uuids=['synthetic'],cpu_affinities=[[0]],limits=dict(vram_bytes=15*2**30,host_bytes=16*2**30,
fold_disk_bytes=20*2**30,fold_wall_seconds=2700))
def fake_supervise(*args,**kwargs):
p=root/'folds/0';p.mkdir(parents=True)
(p/'result.json').write_text(json.dumps(dict(status='COMPLETE',peak_allocated_bytes=1,peak_reserved_bytes=16*2**30)))
return dict(status='PASS')
with patch.object(guard,'contract',return_value=c),patch.object(guard.os,'sched_setaffinity'),patch.object(guard.time,'sleep'),\
patch.object(guard,'device_usage',return_value=(0,0)),patch.object(guard,'tree_usage',return_value=(0,0,1)),\
patch.object(guard,'supervise',side_effect=fake_supervise),patch.object(guard,'active_command_pids',return_value=[]):
self.assertEqual(guard.guard(root,0,0),1)
r=json.loads((root/'guards/0/guard-result.json').read_text())
self.assertIn('RECORDED_VRAM_LIMIT',r['monitor']['reasons'])
if __name__=='__main__':unittest.main()