"""Synthetic S3-A contract tests; never access actual target arrays or fit a model.""" import json from pathlib import Path import tempfile import unittest from unittest.mock import patch import numpy as np import s3a_data as data import s3a_statistics as stats import run_s3a_score as scorer from freeze_g1_components import sha from s3a_budget import first_fit_projection, can_dispatch from prepare_s3a import build_jobs from run_s2_r2 import select_finished from s2_custody_statistics import contrast, six_region, part_metrics def roster_part(case, component, fold=0): return dict(case=case, component_id=component, included=True, nodes=10, exposure=dict(development_exposed=False), assignments=[dict(fold=f, role='test' if f == fold else 'learning', learning_pools=['25','50','100']) for f in range(5)]) def score(case, arm, value, pool=100): return dict(case=case, arm=arm, pool=pool, seed=None, **{k: dict(value=value, reason=None) for k in ('relative_l2','near_wall','kendall_tau_b','last_location')}) class S3AChecks(unittest.TestCase): def test_exact_feature_order_and_two_separate_arms(self): d=dict(x=np.arange(40).reshape(2,20),edge=np.array([[41,42,43,44],[51,52,53,54]]),gaps=np.array([[61,62],[71,72]])) e=data.features(d,'R2_E4');g=data.features(d,'R2_G2') self.assertEqual(e.shape,(2,24));self.assertEqual(g.shape,(2,22)) np.testing.assert_array_equal(e[:,:20],d['x']);np.testing.assert_array_equal(e[:,20:],d['edge']) np.testing.assert_array_equal(g[:,20:],d['gaps']) self.assertEqual(data.columns('R2_E4')[-4:],['mean_q1','max_q1','mean_q2','max_q2']) with self.assertRaises(ValueError):data.features(d,'R2') d['edge'][0,0]=0;d['edge']=d['edge'].astype(float);d['edge'][0,0]=np.nan with self.assertRaises(ValueError):data.features(d,'R2_E4') def test_test_role_rejected_before_any_feature_or_cache_access(self): roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t']) with patch.object(data,'prediction_inputs',side_effect=AssertionError('feature IO')) as io: with self.assertRaises(PermissionError):data.read_part(dict(case='t'),roles,{},'R2_E4') io.assert_not_called() def test_cache_hash_ids_multiplicity_and_seconds_target(self): with tempfile.TemporaryDirectory() as t: p=Path(t)/'label.npz';np.savez(p,seconds=[2.,4.],raw_ids=[3,7],counts=[1,5]) entry=dict(path=str(p),sha256=sha(p)) d=dict(raw_ids=np.array([3,7]),counts=np.array([1,5])) np.testing.assert_array_equal(data.cached_truth(entry,d),[2,4]) d['counts']=np.array([5,1]) with self.assertRaises(ValueError):data.cached_truth(entry,d) entry['sha256']='wrong' with self.assertRaises(ValueError):data.cached_truth(entry,d) def test_final_loader_reads_learning_only_from_cache(self): roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t']) rows={k:dict(case=k) for k in ['l','v']};seen=[] def part(row,*args): seen.append(row['case']);return dict(role='learning' if row['case']=='l' else 'validation') c=dict(roster='r',roster_sha256='h',label_index='l',label_index_sha256='h') with patch.object(data,'checked',side_effect=lambda p,h:p),patch.object(data,'read_json',side_effect=[{},[]]),patch.object(data,'select_cell',return_value=(rows,roles)),patch.object(data,'read_part',part): learning,validation,_=data.load_cell(None,c,dict(arm='R2_G2',fold=0,pool=25,kind='FINAL')) self.assertEqual(seen,['l']);self.assertEqual(validation,[]) def test_plan_is360_hpo30final_no_original_R2_refit(self): # Real roster metadata only: no query arrays or labels are loaded. roster=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text()) jobs,cells=build_jobs(roster) self.assertEqual((len(jobs),len(cells)),(390,30)) self.assertEqual(sum(j['kind']=='HPO' for j in jobs),360) self.assertTrue(all(j['kind']=='FINAL' for j in jobs[360:])) self.assertEqual(jobs[0]['arm'],'R2_E4');self.assertEqual(jobs[12]['arm'],'R2_G2') self.assertEqual({j['arm'] for j in jobs},set(data.ARMS)) self.assertEqual(len({(c['arm'],c['fold'],c['pool']) for c in cells}),30) for c in cells:self.assertEqual([jobs[i]['config_index'] for i in c['hpo_indices']],list(range(12))) def test_original_failure_and_tie_policy(self): trials=[dict(config_index=i,status='NA',score=None) for i in range(12)] self.assertEqual(select_finished(0,trials)['status'],'NA_ALL_HPO_FAILED') trials[3].update(status='PASS',score=.1);trials[1].update(status='PASS',score=.1) r=select_finished(0,trials);self.assertEqual(r['config_index'],1) self.assertEqual(r['finished_configurations'],2) def test_component_equal_single_fit_and_direction(self): roster=[roster_part('a','A'),roster_part('b','A'),roster_part('c','C',1)] rows=[score(k,arm,v) for arm,vs in [('R2',[1,3,5]),('R2_E4',[.5,1.5,2.5]),('R2_G2',[2,6,10])] for k,v in zip('abc',vs)] tables,contrasts,_=stats.reports(rows,roster,set('abc')) primary=next(c for c in contrasts if c['primary']) self.assertEqual(primary['reference_mean'],3.5);self.assertEqual(primary['candidate_mean'],1.75) self.assertEqual(primary['value'],.5);self.assertEqual(primary['region'],'meaningful gain supported') self.assertEqual(sum(c['primary'] for c in contrasts),1) r=next(t for t in tables if t['arm']=='R2' and t['pool']==100 and t['metric']=='relative_l2' and t['subset']=='all') self.assertEqual(r['included_parts'],3) def test_failure_invalidates_whole_component_not_other_arm(self): roster=[roster_part('a','A'),roster_part('b','A'),roster_part('c','C',1)] rows=[score(k,arm,1) for k in 'abc' for arm in ['R2','R2_G2']]+[score('a','R2_E4',.5),score('c','R2_E4',.5)] _,contrasts,_=stats.reports(rows,roster,set('abc')) e=next(c for c in contrasts if c['primary']) g=next(c for c in contrasts if c['subset']=='all' and c['pool']==100 and c['candidate']=='R2_G2') self.assertEqual(e['component_ids'],['C']);self.assertEqual(e['excluded_parts'],2) self.assertEqual(g['components'],2) def test_exposure_filters_parts_and_keeps_main_invalid_component_excluded(self): roster=[roster_part('a','A'),roster_part('b','A'),roster_part('c','C',1)] rows=[score(k,a,1) for k in 'abc' for a in ['R2','R2_E4','R2_G2']] tables,_,_=stats.reports(rows,roster,{'a','c'}) r=next(t for t in tables if t['subset']=='not_known_exposed' and t['pool']==100 and t['arm']=='R2_E4' and t['metric']=='relative_l2') self.assertEqual((r['components'],r['included_parts']),(2,2)) rows=[r for r in rows if not(r['case']=='b' and r['arm']=='R2_E4')] _,contrasts,_=stats.reports(rows,roster,{'a','c'}) r=next(c for c in contrasts if c['subset']=='not_known_exposed' and c['pool']==100 and c['candidate']=='R2_E4') self.assertEqual(r['component_ids'],['C']) def test_undefined_bootstrap_and_six_region_boundaries_unchanged(self): r=contrast({'a':0.,'b':1.},{'a':0.,'b':.5}) self.assertGreater(r['undefined_replicates'],0);self.assertIsNone(r['low']) self.assertEqual(six_region(0,.1),'insufficient evidence') self.assertEqual(six_region(.1,.2),'positive, size undetermined') self.assertEqual(six_region(-.2,-.01),'harmful') def test_multiplicity_once_and_negative_prediction_not_clipped(self): r=part_metrics(np.array([-1.,2.]),np.array([1.,2.]),np.array([3,1]),np.array([True,False]),np.array([3,7]),np.zeros((2,3)),1) self.assertAlmostEqual(r['relative_l2']['value'],np.sqrt(12/7)) self.assertEqual(r['near_wall']['value'],2.) def test_duplicate_scores_and_seed_replication_rejected(self): r=score('a','R2',1);roster=[roster_part('a','A')] with self.assertRaises(ValueError):stats.reports([r,r],roster,{'a'}) r['seed']=0 with self.assertRaises(ValueError):stats.reports([r],roster,{'a'}) def test_review_gate_blocks_missing_or_wrong_contract(self): with tempfile.TemporaryDirectory() as t: root=Path(t);(root/'contract.json').write_text(json.dumps(dict(code_sha256={},source_sha256={}))) with self.assertRaises(FileNotFoundError):scorer.review_gate(root) (root/'main-review.json').write_text(json.dumps(dict(approved=True,slack_ts='synthetic',git_commit='synthetic',contract_sha256='wrong'))) with self.assertRaises(PermissionError):scorer.review_gate(root) def test_terminal_manifest_requires_all30_cells_and_model_hash(self): cells=[dict(index=i,arm=data.ARMS[i%2],fold=i//6,pool=[25,50,100][i//2%3]) for i in range(30)] terminals=[dict(cell=c['index'],arm=c['arm'],fold=c['fold'],pool=c['pool'],status='NA',reason='synthetic_failed') for c in cells] self.assertEqual(scorer.expected_predictions(Path('/none'),dict(cells=cells),terminals,[]),{}) with self.assertRaises(ValueError):scorer.expected_predictions(Path('/none'),dict(cells=cells),terminals[:-1],[]) terminals[0].update(status='PASS',model_path='missing',model_sha256='wrong') with self.assertRaises(FileNotFoundError):scorer.expected_predictions(Path('/none'),dict(cells=cells),terminals,[]) def test_budget_projection_uses_both_arms_and_stops_failed_first(self): jobs=[dict(arm=a,cost_units=1,cost_queries=1) for a in data.ARMS for _ in range(195)] c=dict(jobs=jobs,limits=dict(fitting_worker_cpu_seconds=64800)) first={a:dict(status='PASS',worker_cpu_seconds=100,fit_cpu_seconds=80) for a in data.ARMS} self.assertTrue(first_fit_projection(c,first)['continue_within_cap']) first['R2_G2']['worker_cpu_seconds']=200 self.assertFalse(first_fit_projection(c,first)['continue_within_cap']) first['R2_E4']['status']='NA' self.assertEqual(first_fit_projection(c,first)['reason'],'FIRST_TRIAL_FAILED_NO_RETRY') def test_dispatch_reserves_active_and_next_caps_before_launch(self): c=dict(limits=dict(fitting_worker_cpu_seconds=64800)) self.assertTrue(can_dispatch(c,60000,1000,[1000],1800)) self.assertFalse(can_dispatch(c,60001,1000,[2000],1800)) with self.assertRaises(ValueError):can_dispatch(c,0,float('nan'),[],1800) def test_prediction_gate_rejects_missing_extra_before_array_IO(self): proof=dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit='fixed',contract_sha256='hash',terminal_freeze_sha256='hash',prediction_index_sha256='hash') frozen=dict(status='ALL_S3A_FITS_TERMINAL',contract_sha256='hash',hpo_terminal_indices=list(range(360)),batch_receipt='b',batch_receipt_sha256='hash',final_fits=[]) c=dict(roster='roster',roster_sha256='hash') with patch.object(scorer,'sha',return_value='hash'),patch.object(scorer,'checked',side_effect=lambda p,h:p),patch.object(scorer,'read_json',side_effect=[proof,frozen,{'parts':[]},{'extra':'hash'}]),patch.object(scorer,'expected_predictions',return_value={'expected':None}): with self.assertRaisesRegex(ValueError,'Missing or extra'):scorer.prediction_gate(Path('plan'),Path('root'),c) if __name__=='__main__': unittest.main(verbosity=2)