Files
MeltMind/tools/test_s3a.py
T

177 lines
11 KiB
Python

"""Synthetic S3-A contract tests; never access actual target arrays or fit a model."""
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import numpy as np
import s3a_data as data
import s3a_statistics as stats
import run_s3a_score as scorer
from freeze_g1_components import sha
from s3a_budget import first_fit_projection, can_dispatch
from prepare_s3a import build_jobs
from run_s2_r2 import select_finished
from s2_custody_statistics import contrast, six_region, part_metrics
def roster_part(case, component, fold=0):
return dict(case=case, component_id=component, included=True, nodes=10,
exposure=dict(development_exposed=False),
assignments=[dict(fold=f, role='test' if f == fold else 'learning', learning_pools=['25','50','100']) for f in range(5)])
def score(case, arm, value, pool=100):
return dict(case=case, arm=arm, pool=pool, seed=None,
**{k: dict(value=value, reason=None) for k in ('relative_l2','near_wall','kendall_tau_b','last_location')})
class S3AChecks(unittest.TestCase):
def test_exact_feature_order_and_two_separate_arms(self):
d=dict(x=np.arange(40).reshape(2,20),edge=np.array([[41,42,43,44],[51,52,53,54]]),gaps=np.array([[61,62],[71,72]]))
e=data.features(d,'R2_E4');g=data.features(d,'R2_G2')
self.assertEqual(e.shape,(2,24));self.assertEqual(g.shape,(2,22))
np.testing.assert_array_equal(e[:,:20],d['x']);np.testing.assert_array_equal(e[:,20:],d['edge'])
np.testing.assert_array_equal(g[:,20:],d['gaps'])
self.assertEqual(data.columns('R2_E4')[-4:],['mean_q1','max_q1','mean_q2','max_q2'])
with self.assertRaises(ValueError):data.features(d,'R2')
d['edge'][0,0]=0;d['edge']=d['edge'].astype(float);d['edge'][0,0]=np.nan
with self.assertRaises(ValueError):data.features(d,'R2_E4')
def test_test_role_rejected_before_any_feature_or_cache_access(self):
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t'])
with patch.object(data,'prediction_inputs',side_effect=AssertionError('feature IO')) as io:
with self.assertRaises(PermissionError):data.read_part(dict(case='t'),roles,{},'R2_E4')
io.assert_not_called()
def test_cache_hash_ids_multiplicity_and_seconds_target(self):
with tempfile.TemporaryDirectory() as t:
p=Path(t)/'label.npz';np.savez(p,seconds=[2.,4.],raw_ids=[3,7],counts=[1,5])
entry=dict(path=str(p),sha256=sha(p))
d=dict(raw_ids=np.array([3,7]),counts=np.array([1,5]))
np.testing.assert_array_equal(data.cached_truth(entry,d),[2,4])
d['counts']=np.array([5,1])
with self.assertRaises(ValueError):data.cached_truth(entry,d)
entry['sha256']='wrong'
with self.assertRaises(ValueError):data.cached_truth(entry,d)
def test_final_loader_reads_learning_only_from_cache(self):
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t'])
rows={k:dict(case=k) for k in ['l','v']};seen=[]
def part(row,*args):
seen.append(row['case']);return dict(role='learning' if row['case']=='l' else 'validation')
c=dict(roster='r',roster_sha256='h',label_index='l',label_index_sha256='h')
with patch.object(data,'checked',side_effect=lambda p,h:p),patch.object(data,'read_json',side_effect=[{},[]]),patch.object(data,'select_cell',return_value=(rows,roles)),patch.object(data,'read_part',part):
learning,validation,_=data.load_cell(None,c,dict(arm='R2_G2',fold=0,pool=25,kind='FINAL'))
self.assertEqual(seen,['l']);self.assertEqual(validation,[])
def test_plan_is360_hpo30final_no_original_R2_refit(self):
# Real roster metadata only: no query arrays or labels are loaded.
roster=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text())
jobs,cells=build_jobs(roster)
self.assertEqual((len(jobs),len(cells)),(390,30))
self.assertEqual(sum(j['kind']=='HPO' for j in jobs),360)
self.assertTrue(all(j['kind']=='FINAL' for j in jobs[360:]))
self.assertEqual(jobs[0]['arm'],'R2_E4');self.assertEqual(jobs[12]['arm'],'R2_G2')
self.assertEqual({j['arm'] for j in jobs},set(data.ARMS))
self.assertEqual(len({(c['arm'],c['fold'],c['pool']) for c in cells}),30)
for c in cells:self.assertEqual([jobs[i]['config_index'] for i in c['hpo_indices']],list(range(12)))
def test_original_failure_and_tie_policy(self):
trials=[dict(config_index=i,status='NA',score=None) for i in range(12)]
self.assertEqual(select_finished(0,trials)['status'],'NA_ALL_HPO_FAILED')
trials[3].update(status='PASS',score=.1);trials[1].update(status='PASS',score=.1)
r=select_finished(0,trials);self.assertEqual(r['config_index'],1)
self.assertEqual(r['finished_configurations'],2)
def test_component_equal_single_fit_and_direction(self):
roster=[roster_part('a','A'),roster_part('b','A'),roster_part('c','C',1)]
rows=[score(k,arm,v) for arm,vs in [('R2',[1,3,5]),('R2_E4',[.5,1.5,2.5]),('R2_G2',[2,6,10])] for k,v in zip('abc',vs)]
tables,contrasts,_=stats.reports(rows,roster,set('abc'))
primary=next(c for c in contrasts if c['primary'])
self.assertEqual(primary['reference_mean'],3.5);self.assertEqual(primary['candidate_mean'],1.75)
self.assertEqual(primary['value'],.5);self.assertEqual(primary['region'],'meaningful gain supported')
self.assertEqual(sum(c['primary'] for c in contrasts),1)
r=next(t for t in tables if t['arm']=='R2' and t['pool']==100 and t['metric']=='relative_l2' and t['subset']=='all')
self.assertEqual(r['included_parts'],3)
def test_failure_invalidates_whole_component_not_other_arm(self):
roster=[roster_part('a','A'),roster_part('b','A'),roster_part('c','C',1)]
rows=[score(k,arm,1) for k in 'abc' for arm in ['R2','R2_G2']]+[score('a','R2_E4',.5),score('c','R2_E4',.5)]
_,contrasts,_=stats.reports(rows,roster,set('abc'))
e=next(c for c in contrasts if c['primary'])
g=next(c for c in contrasts if c['subset']=='all' and c['pool']==100 and c['candidate']=='R2_G2')
self.assertEqual(e['component_ids'],['C']);self.assertEqual(e['excluded_parts'],2)
self.assertEqual(g['components'],2)
def test_exposure_filters_parts_and_keeps_main_invalid_component_excluded(self):
roster=[roster_part('a','A'),roster_part('b','A'),roster_part('c','C',1)]
rows=[score(k,a,1) for k in 'abc' for a in ['R2','R2_E4','R2_G2']]
tables,_,_=stats.reports(rows,roster,{'a','c'})
r=next(t for t in tables if t['subset']=='not_known_exposed' and t['pool']==100 and t['arm']=='R2_E4' and t['metric']=='relative_l2')
self.assertEqual((r['components'],r['included_parts']),(2,2))
rows=[r for r in rows if not(r['case']=='b' and r['arm']=='R2_E4')]
_,contrasts,_=stats.reports(rows,roster,{'a','c'})
r=next(c for c in contrasts if c['subset']=='not_known_exposed' and c['pool']==100 and c['candidate']=='R2_E4')
self.assertEqual(r['component_ids'],['C'])
def test_undefined_bootstrap_and_six_region_boundaries_unchanged(self):
r=contrast({'a':0.,'b':1.},{'a':0.,'b':.5})
self.assertGreater(r['undefined_replicates'],0);self.assertIsNone(r['low'])
self.assertEqual(six_region(0,.1),'insufficient evidence')
self.assertEqual(six_region(.1,.2),'positive, size undetermined')
self.assertEqual(six_region(-.2,-.01),'harmful')
def test_multiplicity_once_and_negative_prediction_not_clipped(self):
r=part_metrics(np.array([-1.,2.]),np.array([1.,2.]),np.array([3,1]),np.array([True,False]),np.array([3,7]),np.zeros((2,3)),1)
self.assertAlmostEqual(r['relative_l2']['value'],np.sqrt(12/7))
self.assertEqual(r['near_wall']['value'],2.)
def test_duplicate_scores_and_seed_replication_rejected(self):
r=score('a','R2',1);roster=[roster_part('a','A')]
with self.assertRaises(ValueError):stats.reports([r,r],roster,{'a'})
r['seed']=0
with self.assertRaises(ValueError):stats.reports([r],roster,{'a'})
def test_review_gate_blocks_missing_or_wrong_contract(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'contract.json').write_text(json.dumps(dict(code_sha256={},source_sha256={})))
with self.assertRaises(FileNotFoundError):scorer.review_gate(root)
(root/'main-review.json').write_text(json.dumps(dict(approved=True,slack_ts='synthetic',git_commit='synthetic',contract_sha256='wrong')))
with self.assertRaises(PermissionError):scorer.review_gate(root)
def test_terminal_manifest_requires_all30_cells_and_model_hash(self):
cells=[dict(index=i,arm=data.ARMS[i%2],fold=i//6,pool=[25,50,100][i//2%3]) for i in range(30)]
terminals=[dict(cell=c['index'],arm=c['arm'],fold=c['fold'],pool=c['pool'],status='NA',reason='synthetic_failed') for c in cells]
self.assertEqual(scorer.expected_predictions(Path('/none'),dict(cells=cells),terminals,[]),{})
with self.assertRaises(ValueError):scorer.expected_predictions(Path('/none'),dict(cells=cells),terminals[:-1],[])
terminals[0].update(status='PASS',model_path='missing',model_sha256='wrong')
with self.assertRaises(FileNotFoundError):scorer.expected_predictions(Path('/none'),dict(cells=cells),terminals,[])
def test_budget_projection_uses_both_arms_and_stops_failed_first(self):
jobs=[dict(arm=a,cost_units=1,cost_queries=1) for a in data.ARMS for _ in range(195)]
c=dict(jobs=jobs,limits=dict(fitting_worker_cpu_seconds=64800))
first={a:dict(status='PASS',worker_cpu_seconds=100,fit_cpu_seconds=80) for a in data.ARMS}
self.assertTrue(first_fit_projection(c,first)['continue_within_cap'])
first['R2_G2']['worker_cpu_seconds']=200
self.assertFalse(first_fit_projection(c,first)['continue_within_cap'])
first['R2_E4']['status']='NA'
self.assertEqual(first_fit_projection(c,first)['reason'],'FIRST_TRIAL_FAILED_NO_RETRY')
def test_dispatch_reserves_active_and_next_caps_before_launch(self):
c=dict(limits=dict(fitting_worker_cpu_seconds=64800))
self.assertTrue(can_dispatch(c,60000,1000,[1000],1800))
self.assertFalse(can_dispatch(c,60001,1000,[2000],1800))
with self.assertRaises(ValueError):can_dispatch(c,0,float('nan'),[],1800)
def test_prediction_gate_rejects_missing_extra_before_array_IO(self):
proof=dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit='fixed',contract_sha256='hash',terminal_freeze_sha256='hash',prediction_index_sha256='hash')
frozen=dict(status='ALL_S3A_FITS_TERMINAL',contract_sha256='hash',hpo_terminal_indices=list(range(360)),batch_receipt='b',batch_receipt_sha256='hash',final_fits=[])
c=dict(roster='roster',roster_sha256='hash')
with patch.object(scorer,'sha',return_value='hash'),patch.object(scorer,'checked',side_effect=lambda p,h:p),patch.object(scorer,'read_json',side_effect=[proof,frozen,{'parts':[]},{'extra':'hash'}]),patch.object(scorer,'expected_predictions',return_value={'expected':None}):
with self.assertRaisesRegex(ValueError,'Missing or extra'):scorer.prediction_gate(Path('plan'),Path('root'),c)
if __name__=='__main__':
unittest.main(verbosity=2)