Files
MeltMind/tools/test_s1_scoring.py

235 lines
15 KiB
Python

"""Only generated, tiny fixtures: never opens real S1 descriptors or geometry."""
import unittest
import json
import tempfile
from pathlib import Path
from dataclasses import replace
import numpy as np
from numpy.testing import assert_allclose, assert_array_equal
import s1_scoring as s
from o2_rep import contract_direction
import s1_scoring_io as io
def part(case, values=None, counts=None, eligible=None, near=None, prefix=993, source=None):
if values is None: values=np.repeat((np.arange(1,11)/100)[:,None],10,axis=1)
values=np.asarray(values,float);n=len(values)
counts=np.ones(n,np.int64) if counts is None else np.asarray(counts,np.int64)
return s.Part(case,source or case,prefix,np.arange(n,dtype=np.int64),values,counts,
np.ones(n,bool) if eligible is None else np.asarray(eligible,bool),
np.arange(n)<3 if near is None else np.asarray(near,bool),
np.repeat(np.arange(n,dtype=np.int64),counts))
def geometric_features(points, gate, tensors, edges):
"""Independent geometric fixture: transforms points, gate and tensors together."""
n=len(points);valid=np.ones(n,bool);mu=np.zeros((n,3))
center=points.mean(0);rg=np.sqrt(np.mean(np.sum((points-center)**2,axis=1)))
g=np.trace(tensors,axis1=2,axis2=3)/np.array([1,2])
gateq=contract_direction(tensors,mu,valid,gate-points,1.,'gate')
summaries=[]
for i in range(n):
vectors=edges[i]-points[i]
q=contract_direction(np.repeat(tensors[i:i+1],len(vectors),0),np.zeros((len(vectors),3)),np.ones(len(vectors),bool),vectors,1.,'edge')['q']
summaries.append([q[:,0].mean(),q[:,0].max(),q[:,1].mean(),q[:,1].max()])
return np.column_stack([np.arange(1,n+1)/100,gateq['length']/rg,g,gateq['q'],summaries])
class ScorerTests(unittest.TestCase):
def test_weighted_midrank_hand_values(self):
def vals(x):return np.repeat(np.array(x,float)[:,None],10,1)
train=[part('p1',vals([0,1,2]),[1,2,1],[1,1,0]),
part('p2',vals([0,2]),[1,1]),part('p3',vals([1,2]),[3,1])]
weights=s.rank_weights(train)
assert_allclose(weights[0],[1/9,2/9],rtol=0,atol=1e-16)
assert_allclose(weights[1],[1/6,1/6],rtol=0,atol=1e-16)
assert_allclose(weights[2],[1/4,1/12],rtol=0,atol=1e-16)
maps,_=s.fit_maps(train)
# Masses at 0,1,2 = 5/18,17/36,1/4. Unseen 1.5 gets CDF 3/4.
assert_allclose(maps[0].transform([-1,0,1,1.5,2,3]),[0,5/36,37/72,3/4,7/8,1],rtol=0,atol=2e-16)
c=s.RankMap.fit([7,7],[.2,.8]);assert_array_equal(c.transform([6,7,8]),[0,.5,1])
audit=c.audit(np.array([6,7,8]),np.array([2,1,3]))
self.assertTrue(audit['constant']);self.assertEqual(audit['below_slots'],2);self.assertEqual(audit['above_fraction'],.5)
def test_full_hand_case_distinct_nodes_and_ratios(self):
train=[part('p1',prefix=101),part('p2',prefix=102),part('p3',prefix=103)]
test=part('test',counts=[1,2,1,1,1,1,1,1,1,1],prefix=104)
result=s.score_fold(train,test,allow_small_synthetic=True)
self.assertEqual(result['status'],'PASS')
# 3 physical nodes per descriptor. Endpoint: (3*0+3*.1+2*.2)/8=7/80.
# Interior: (3*0+5*.1)/8=1/16. Multiplicity changes ratio, not references.
expected=[7/80]+[1/16]*8+[7/80]
for arm in ['A','B','C']:
assert_allclose(result['scores'][arm],np.tile(expected,(5,1)),rtol=0,atol=2e-16)
self.assertAlmostEqual(result['estimates'][arm][0]['whole']['point'],59/880)
self.assertAlmostEqual(result['estimates'][arm][0]['near']['point'],11/160)
self.assertEqual(result['logical_knn_queries'],180)
self.assertEqual(result['tie_radius_queries'],180)
nearest=result['nearest'][('A',0)]
self.assertTrue(all(len(set(row))==8 for row in nearest['indices']))
# Physical ID tie break at endpoint, independent of training input order.
self.assertEqual(nearest['indices'][0].tolist(),[0,10,20,1,11,21,2,12])
self.assertEqual(nearest['zero_distance_neighbors'][0],3)
self.assertEqual(nearest['distinct_neighbor_sources'][0],3)
assert_allclose(s.D_p([0,0],[.3,.4]),np.sqrt(.125),rtol=0,atol=1e-16)
def test_constant_column_counterexample(self):
a=.3;keys=[('s','p',i) for i in range(8)]
one=s.knn_mean(np.zeros((8,1)),np.array([[a]]),keys)
two=s.knn_mean(np.column_stack([np.zeros(8),np.full(8,.5)]),np.array([[a,.5]]),keys)
self.assertEqual(one['scores'][0],a)
# sqrt(a*a/2) versus a/sqrt(2) can differ by one FP64 ULP.
self.assertEqual(two['scores'][0],np.sqrt(a*a/2))
assert_allclose(two['scores'][0],a/np.sqrt(2),rtol=2e-16,atol=0)
assert_array_equal(one['indices'],two['indices'])
def test_knn_against_bruteforce_with_exact_ties(self):
rng=np.random.default_rng(31)
for d in [2,6,10]:
ref=rng.integers(0,3,size=(40,d))/2;ref[10:20]=ref[:10]
q=np.concatenate([ref[:3],rng.random((7,d))]);keys=[('s','p',39-i) for i in range(40)]
got=s.knn_mean(ref,q,keys)
for i,row in enumerate(q):
dist=np.sqrt(np.sum((ref-row)**2,axis=1)/d)
indices=sorted(range(len(ref)),key=lambda k:(dist[k],keys[k]))[:8]
assert_array_equal(got['indices'][i],indices)
assert_allclose(got['scores'][i],np.mean(dist[indices]),rtol=0,atol=1e-16)
def test_na_and_identity_guards(self):
self.assertTrue(np.isnan(s.ratio([1,np.nan],[0,1],[False,False])))
self.assertEqual(s.ratio([1,np.nan],[1,0],[True,True]),1)
keys=[('s','p',i) for i in range(7)]
got=s.knn_mean(np.zeros((7,2)),np.ones((2,2)),keys)
self.assertTrue(np.isnan(got['scores']).all())
with self.assertRaises(ValueError):s.knn_mean(np.zeros((8,2)),np.ones((1,2)),[('s','p',0)]*8)
with self.assertRaises(ValueError):s.rank_weights([part('p',eligible=[False]*10)])
with self.assertRaises(ValueError):s.score_fold([part('p')],part('p'),True)
with self.assertRaises(ValueError):s.score_fold([replace(part('p'),provenance='real')],part('q'),True)
self.assertEqual(s.score_fold([part('p')],part('q'))['status'],'shortfall')
r=s.score_fold([part('p',np.ones((3,10)))],part('q'),True)
self.assertEqual(r['status'],'fewer_than_8_references')
self.assertTrue(np.isnan(r['estimates']['A'][0]['whole']['point']))
def test_coverage_and_zero_slice(self):
p=replace(part('p',counts=[2,1,1,1,1,1,1,1,1,1],eligible=[False]+[True]*9,near=[True]+[False]*9),positive_mass_fraction=.8)
got=s.slice_estimates(p,np.arange(10,dtype=float),s.query_bootstrap_counts(p,3))
self.assertEqual(got['whole']['accepted_draws'],9)
self.assertAlmostEqual(got['whole']['full_mass_share'],.8*9/11)
self.assertAlmostEqual(got['whole']['boundary_unscored_mass_share'],.2)
self.assertTrue(np.isnan(got['near']['point']))
self.assertTrue(np.isnan(got['near']['replicates']).all())
def test_source_part_aggregation_and_paired_intervals(self):
a=s.aggregate_parts([1,3,10],['a','a','b'])
self.assertEqual(a['source_equal'],6);self.assertEqual(a['part_equal'],14/3)
self.assertTrue(np.isnan(s.aggregate_parts([1,np.nan,10],['a','a','b'])['source_equal']))
ci=s.interval(2,[1,2,3]);self.assertEqual(ci['SE'],1)
assert_allclose(ci['interval95'],[1.05,2.95],rtol=0,atol=5e-16)
self.assertIsNone(s.interval(2,[np.nan,1])['interval95'])
self.assertIsNone(s.interval(np.nan,[1,2])['interval95'])
# Paired source differences [-1,-3], bootstrap chosen by hand.
indices=np.array([[0,0],[0,1],[1,0],[1,1]])
got=s.contrast_report([-1,-3],np.array([[-.5,-1,-1.5],[-2.5,-3,-3.5]]),['a','b'],indices,'primary')
self.assertEqual(got['source_equal_point'],-2)
assert_allclose(got['source_level']['interval95'],[-2.925,-1.075])
assert_allclose(got['query_level']['interval95'],[-2.475,-1.525])
self.assertEqual(got['decision'],'ranks_ahead')
for ci,expected in [([-1,0],'no_ranking_difference'),([0,1],'no_ranking_difference'),([.1,1],'ranks_behind')]:
self.assertEqual(s.ranking({'interval95':ci},'primary'),expected)
def test_plain_and_normalized_shuffle(self):
# 7 (0,0) rows: must NOT consume the stratified RNG. Other strata size 8.
gaps=np.concatenate([np.zeros((7,2)),np.tile([0,.2],(8,1)),np.tile([.3,0],(8,1)),np.column_stack([np.linspace(.1,.2,8),np.linspace(.2,.3,8)])])
n=len(gaps);v=np.linspace(.1,.4,n);norm=np.column_stack([v,v+.1,v+.2,v+.3])
x=np.zeros((n,10));x[:,2:4]=gaps;x[:,4:6]=np.arange(2*n).reshape(n,2)/100
x[:,6:10]=norm*np.repeat(gaps,2,axis=1)
for seed in s.PERMUTATION_SEEDS:
plain,strat,strata=s.permutation_maps(993,seed,gaps)
node,edge,normal,audit=s.shuffled_blocks(x,993,seed)
assert_array_equal(node[:,2:6],x[plain,2:6]);assert_array_equal(edge[:,6:10],x[plain,6:10])
assert_array_equal(normal[:7],x[:7]);assert_array_equal(strat[:7],np.arange(7))
rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence([seed,993])))
for ids in [np.arange(7,15),np.arange(15,23),np.arange(23,31)]:
assert_array_equal(strat[ids],ids[rng.permutation(8)])
self.assertEqual(audit['stratified']['tolerant_bound_violation_count'],0)
self.assertEqual(audit['stratified']['raw_cross_channel_order_violations'],0)
for ch in [0,1]:
mask=gaps[:,ch]>0
assert_allclose(normal[mask,6+2*ch:8+2*ch]/gaps[mask,ch,None],norm[strat[mask],2*ch:2*ch+2],rtol=0,atol=2e-16)
assert_array_equal(normal[~mask,6+2*ch:8+2*ch],0)
def test_seed_replay_bitwise(self):
p=part('p',counts=[1,2,1,1,1,1,1,1,1,1]);a=s.query_bootstrap_counts(p);b=s.query_bootstrap_counts(p)
self.assertEqual(a.tobytes(),b.tobytes());assert_array_equal(a.sum(1),11)
_,a=s.source_bootstrap_indices(['b','a','c']);_,b=s.source_bootstrap_indices(['c','b','a'])
self.assertEqual(a.tobytes(),b.tobytes())
train=[part('p1',prefix=101),part('p2',prefix=102),part('p3',prefix=103)];test=part('q')
a=s.score_fold(train,test,True);b=s.score_fold(train[::-1],test,True)
for arm in s.ARMS:
self.assertEqual(a['scores'][arm].tobytes(),b['scores'][arm].tobytes())
for i in range(5):
self.assertEqual(a['estimates'][arm][i]['whole']['replicates'].tobytes(),b['estimates'][arm][i]['whole']['replicates'].tobytes())
self.assertEqual(a['shuffle_audits'],b['shuffle_audits'])
def test_rigid_transform_including_gate_all_arms(self):
n=12;i=np.arange(n);points=np.column_stack([i,i%3,i%4]).astype(float);gate=np.array([-5.,2.,3.])
g=np.column_stack([.1+i/100,.2+i/200]);B=np.zeros((n,2,3,3));B[:,0,0,0]=g[:,0];B[:,1,0,0]=g[:,1];B[:,1,1,1]=g[:,1]
edges=points[:,None,:]+np.array([[1.,0.,1.],[0.,1.,2.],[1.,2.,0.]])[None,:,:]
q=np.array([[0.,-1.,0.],[1.,0.,0.],[0.,0.,1.]]);shift=np.array([10.,-4.,7.])
original=geometric_features(points,gate,B,edges)
moved=geometric_features(points@q.T+shift,gate@q.T+shift,q@B@q.T,edges@q.T+shift)
assert_allclose(original,moved,rtol=0,atol=2e-15)
train=[part('p1',original+.0001,prefix=101),part('p2',original-.0001,prefix=102),part('p3',original+.0002,prefix=103)]
a=s.score_fold(train,part('q',original),True);b=s.score_fold(train,part('q',moved),True)
for arm in s.ARMS:assert_allclose(a['scores'][arm],b['scores'][arm],rtol=0,atol=1e-15)
# Negative control: leaving gate unmoved must change geometric descriptors.
wrong=geometric_features(points@q.T+shift,gate,q@B@q.T,edges@q.T+shift)
self.assertGreater(np.max(abs(wrong[:,1]-original[:,1])),.1)
def test_shortfall_strict_endpoints(self):
for nrefs,expected in [([500,1000,1000,1000,1000],True),([499,1000,1000,1000,1000],False),
([999,1000,1000,1000,1000],True),([999,999,1000,1000,1000],False),([1000]*5,True),([None,1000],False)]:
self.assertEqual(s.shortfall_decision(nrefs)['comparable'],expected)
def test_summary_pairing_and_shared_bootstrap(self):
# Generated score estimates only; known arm differences independent of seed.
parts=[part('a'),part('b'),part('c')];results=[]
for p in parts:
estimates={}
for ai,arm in enumerate(s.ARMS):
estimates[arm]=[{slice_name:dict(point=float(ai),replicates=np.arange(200)/100+ai) for slice_name in ['whole','near']} for _ in range(5)]
results.append(dict(status='PASS',estimates=estimates))
got=s.summarize_folds(parts,results)
r=got['contrasts']['B-(A+S)']['whole']['five_seed_mean']
self.assertEqual(r['source_equal_point'],1);assert_allclose(r['query_level']['interval95'],[1,1],atol=1e-15)
assert_array_equal(r['source_level']['interval95'],[1,1]);self.assertEqual(r['decision'],'ranks_behind')
self.assertEqual(got['arms']['A']['whole']['five_seed_mean']['source_equal_point'],0)
def test_frozen_input_adapter_generated_fixture(self):
with tempfile.TemporaryDirectory(prefix='s1-synthetic-') as directory:
root=Path(directory);drawdir=root/'draw';drawdir.mkdir();o2=root/'o2';folder=o2/'panel'/'p_original';folder.mkdir(parents=True)
n=10;ids=np.arange(n);counts=np.full(n,200);inverse=np.repeat(ids,counts)
np.savez(drawdir/'draw.npz',unique_ids=ids,unique_raw_ids=ids+100,multiplicity=counts,draw_to_unique=inverse)
receipt=dict(case='p',source_key='source',numeric_prefix=993,positive_mass_fraction=.8)
(drawdir/'receipt.json').write_text(json.dumps(receipt))
d=np.arange(1,11)/100
np.savez(folder/'nodes.npz',anchor_ids=ids,raw_node_ids=ids+100,multiplicity=counts,distance=d,gate_length=np.arange(1,11),g=np.full((n,2),.1),gate_q=np.full((n,2),.05),edge_summary=np.full((n,4),.04),eligible=np.ones(n,bool),near_wall=d<=.25)
result=dict(case='p',status='COMPLETE',R_g_raw=2.,t_part=1.,n_ref=10,draw_file_sha256=io.sha(drawdir/'draw.npz'),output_sha256={'nodes.npz':io.sha(folder/'nodes.npz')})
(folder/'result.json').write_text(json.dumps(result));(o2/'panel'/'p_original.supervisor.json').write_text(json.dumps(dict(status='PASS')))
spec=dict(case='p',source_key='source',numeric_prefix=993,t_part=1.,job='p_original',anchor_file='draw/draw.npz',input_sha256={str(p.relative_to(root)):io.sha(p) for p in [drawdir/'draw.npz',drawdir/'receipt.json']})
loaded=io.load_part(spec,root,o2);self.assertEqual(loaded.provenance,'real')
assert_array_equal(loaded.features[:,0],d);assert_array_equal(loaded.features[:,1],np.arange(1,11)/2)
assert_array_equal(loaded.features[:,6:10],np.full((10,4),.04))
self.assertEqual(loaded.positive_mass_fraction,.8)
with self.assertRaises(ValueError):s.score_fold([loaded],part('q'),True)
with (folder/'nodes.npz').open('ab') as f:f.write(b'tamper')
with self.assertRaises(ValueError):io.load_part(spec,root,o2)
def test_json_na_and_tuple_diagnostics(self):
original={('A',0):dict(scores=np.array([1.,np.nan]),integer=np.int64(3),flag=np.bool_(True))}
value=io.json_ready(original);encoded=json.dumps(value,allow_nan=False)
self.assertNotIn('NaN',encoded);self.assertEqual(json.loads(encoded)[0]['value']['scores'],[1.,None])
if __name__=='__main__':unittest.main(verbosity=2)