"""Synthetic decision directions, matching domains and custody failures.""" import unittest from pathlib import Path from unittest.mock import patch import numpy as np from phase3_followup_postfit_common import budget, checkpoint_identity, prediction_identity from phase3_followup_postfit_statistics import comparisons, decision from phase3_followup_postfit_worker import full_prediction, same_rows, score def contrast(value, high, n=204, ids=None): return dict(value=value, high=high, components=n, component_ids=ids or ['same']) def score_metrics(last, last_high, field, field_high): return dict(last_location=dict(candidate_minus_seed0=contrast(last,last_high)), relative_l2=dict(candidate_minus_seed0=contrast(field,field_high))) def seed_metrics(last, field): return dict(last_location=dict(seed1_minus_seed0=contrast(last,last)), relative_l2=dict(seed1_minus_seed0=contrast(field,field))) class PostfitContractTests(unittest.TestCase): def test_item_c_uses_hard_location_and_field_noninferiority(self): noise = seed_metrics(.004,.0002) self.assertEqual(decision('c',score_metrics(-.010,-.001,-.002,.00099),noise)['status'], 'PASSES_OLD_CORPUS_SCREEN') for m in [score_metrics(-.00999,-.001,-.002,.0001), score_metrics(-.010,0,-.002,.0001), score_metrics(-.010,-.001,-.002,.001)]: self.assertEqual(decision('c',m,noise)['status'],'DOES_NOT_PASS_FIXED_SCREEN') def test_item_b_uses_field_and_strict_seed_magnitude(self): noise = seed_metrics(.2,-.002) self.assertEqual(decision('b',score_metrics(.5,.6,-.003,-.00101),noise)['status'], 'PASSES_OLD_CORPUS_SCREEN') for m in [score_metrics(.5,.6,-.003,-.001),score_metrics(.5,.6,-.002,-.00101)]: self.assertEqual(decision('b',m,noise)['status'],'DOES_NOT_PASS_FIXED_SCREEN') def test_missing_or_different_seed_domain(self): m = score_metrics(-.02,-.01,-.003,-.002) self.assertEqual(decision('c',m,None)['status'],'WAIT_SEED_REFERENCE') seed = seed_metrics(.001,.002) seed['last_location']['seed1_minus_seed0']['component_ids'] = ['different'] with self.assertRaises(ValueError): decision('c',m,seed) def test_incomplete_components_hold(self): m=score_metrics(-.02,-.01,-.003,-.002) m['last_location']['candidate_minus_seed0']['components']=203 self.assertEqual(decision('c',m,seed_metrics(.001,.002))['status'],'INCOMPLETE_PRIMARY_PAIR_SUPPORT') def test_bootstrap_component_equal_and_signed(self): def arm(v): return {k:dict(value=v) for k in ('last_location','relative_l2','kendall_tau_b')} rows = [dict(case=f'p{i}',component_id=g,c=arm(d),seed0=arm(1.)) for i,(g,d) in enumerate([('a',0.),('a',0.),('b',1.)])] self.assertEqual(comparisons(rows,'c')['relative_l2']['candidate_minus_seed0']['value'],-.5) def test_checkpoint_seed_fold_epoch_and_contract(self): c=dict(training=dict(training_contract_sha256='fixed')) j=dict(id=1,target_steps=60000) ref=dict(seed=0,fold=1,completed_epochs=200,optimizer_steps=60000, contract_sha256='fixed',role='last_epoch_only') checkpoint_identity(ref,j,c) for field,bad in [('seed',1),('fold',0),('completed_epochs',199),('contract_sha256','foreign')]: with self.assertRaises(ValueError): checkpoint_identity(dict(ref,**{field:bad}),j,c) def test_full_prediction_identity_rejects_short_or_bad_query(self): raw=np.array([1,3,4]) ix=np.array([0,2]) seconds=np.array([0.,-1.,2.]) z=dict(raw_ids=raw,seconds=seconds,pred=seconds[ix]) e=dict(negative_full_predictions=1,negative_query_predictions=0) np.testing.assert_equal(full_prediction(z,raw,ix,e),seconds) for bad in [dict(z,pred=np.zeros(2)),dict(z,raw_ids=raw[::-1]),dict(z,seconds=seconds[:2])]: with self.assertRaises(ValueError): full_prediction(bad,raw,ix,e) def test_duplicate_or_wrong_fold_before_file_access(self): entries=[dict(fold=i%5,case=f'p{i}',component_id=f'g{i}') for i in range(552)] by={e['case']:e for e in entries} self.assertEqual(same_rows(entries,by,entries),by) with self.assertRaises(ValueError): same_rows([entries[0]]+entries[:-1],by,entries) jobs=[dict(id=i,test=[e['case'] for e in entries if e['fold']==i]) for i in range(5)] with patch('phase3_followup_postfit_common.checked',side_effect=AssertionError('early read')): with self.assertRaises(ValueError): prediction_identity(Path('/unused'),{},dict(jobs=jobs),by,[entries[0]]+entries[:-1]) def test_score_missing_review_or_predictions_stops_before_target_reader(self): root = Path('/unused') with patch('phase3_followup_postfit_worker.config',side_effect=PermissionError), \ patch('phase3_followup_postfit_worker.cached') as targets: with self.assertRaises(PermissionError): score(root) targets.assert_not_called() with patch('phase3_followup_postfit_worker.config',return_value=({'training': {}}, {}, {}, {})), \ patch('phase3_followup_postfit_worker.read',return_value={}), \ patch('phase3_followup_postfit_worker.verify',side_effect=[{},FileNotFoundError]), \ patch('phase3_followup_postfit_worker.cached') as targets: with self.assertRaises(FileNotFoundError): score(root) targets.assert_not_called() def test_budget_charges_existing_cost_once(self): c=dict(item='c',training=dict(training_gpu_seconds=105364.2), phase_reservations=dict(c=205200,b=172908),item_gpu_cap=205200) budget(c,2700,180,5) for gpu,cpu in [(2700.001,180),(2700,1080.001)]: with self.assertRaises(RuntimeError): budget(c,gpu,cpu,5) with self.assertRaises(RuntimeError): budget(dict(c,phase_reservations=dict(c=205200,b=700000)),0,180,5) if __name__=='__main__': unittest.main()