124 lines
6.1 KiB
Python
124 lines
6.1 KiB
Python
"""Synthetic decision directions, matching domains and custody failures."""
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
|
|
from phase3_followup_postfit_common import budget, checkpoint_identity, prediction_identity
|
|
from phase3_followup_postfit_statistics import comparisons, decision
|
|
from phase3_followup_postfit_worker import full_prediction, same_rows, score
|
|
|
|
|
|
def contrast(value, high, n=204, ids=None):
|
|
return dict(value=value, high=high, components=n, component_ids=ids or ['same'])
|
|
|
|
|
|
def score_metrics(last, last_high, field, field_high):
|
|
return dict(last_location=dict(candidate_minus_seed0=contrast(last,last_high)),
|
|
relative_l2=dict(candidate_minus_seed0=contrast(field,field_high)))
|
|
|
|
|
|
def seed_metrics(last, field):
|
|
return dict(last_location=dict(seed1_minus_seed0=contrast(last,last)),
|
|
relative_l2=dict(seed1_minus_seed0=contrast(field,field)))
|
|
|
|
|
|
class PostfitContractTests(unittest.TestCase):
|
|
def test_item_c_uses_hard_location_and_field_noninferiority(self):
|
|
noise = seed_metrics(.004,.0002)
|
|
self.assertEqual(decision('c',score_metrics(-.010,-.001,-.002,.00099),noise)['status'],
|
|
'PASSES_OLD_CORPUS_SCREEN')
|
|
for m in [score_metrics(-.00999,-.001,-.002,.0001),
|
|
score_metrics(-.010,0,-.002,.0001),
|
|
score_metrics(-.010,-.001,-.002,.001)]:
|
|
self.assertEqual(decision('c',m,noise)['status'],'DOES_NOT_PASS_FIXED_SCREEN')
|
|
|
|
def test_item_b_uses_field_and_strict_seed_magnitude(self):
|
|
noise = seed_metrics(.2,-.002)
|
|
self.assertEqual(decision('b',score_metrics(.5,.6,-.003,-.00101),noise)['status'],
|
|
'PASSES_OLD_CORPUS_SCREEN')
|
|
for m in [score_metrics(.5,.6,-.003,-.001),score_metrics(.5,.6,-.002,-.00101)]:
|
|
self.assertEqual(decision('b',m,noise)['status'],'DOES_NOT_PASS_FIXED_SCREEN')
|
|
|
|
def test_missing_or_different_seed_domain(self):
|
|
m = score_metrics(-.02,-.01,-.003,-.002)
|
|
self.assertEqual(decision('c',m,None)['status'],'WAIT_SEED_REFERENCE')
|
|
seed = seed_metrics(.001,.002)
|
|
seed['last_location']['seed1_minus_seed0']['component_ids'] = ['different']
|
|
with self.assertRaises(ValueError):
|
|
decision('c',m,seed)
|
|
|
|
def test_incomplete_components_hold(self):
|
|
m=score_metrics(-.02,-.01,-.003,-.002)
|
|
m['last_location']['candidate_minus_seed0']['components']=203
|
|
self.assertEqual(decision('c',m,seed_metrics(.001,.002))['status'],'INCOMPLETE_PRIMARY_PAIR_SUPPORT')
|
|
|
|
def test_bootstrap_component_equal_and_signed(self):
|
|
def arm(v):
|
|
return {k:dict(value=v) for k in ('last_location','relative_l2','kendall_tau_b')}
|
|
rows = [dict(case=f'p{i}',component_id=g,c=arm(d),seed0=arm(1.))
|
|
for i,(g,d) in enumerate([('a',0.),('a',0.),('b',1.)])]
|
|
self.assertEqual(comparisons(rows,'c')['relative_l2']['candidate_minus_seed0']['value'],-.5)
|
|
|
|
def test_checkpoint_seed_fold_epoch_and_contract(self):
|
|
c=dict(training=dict(training_contract_sha256='fixed'))
|
|
j=dict(id=1,target_steps=60000)
|
|
ref=dict(seed=0,fold=1,completed_epochs=200,optimizer_steps=60000,
|
|
contract_sha256='fixed',role='last_epoch_only')
|
|
checkpoint_identity(ref,j,c)
|
|
for field,bad in [('seed',1),('fold',0),('completed_epochs',199),('contract_sha256','foreign')]:
|
|
with self.assertRaises(ValueError):
|
|
checkpoint_identity(dict(ref,**{field:bad}),j,c)
|
|
|
|
def test_full_prediction_identity_rejects_short_or_bad_query(self):
|
|
raw=np.array([1,3,4])
|
|
ix=np.array([0,2])
|
|
seconds=np.array([0.,-1.,2.])
|
|
z=dict(raw_ids=raw,seconds=seconds,pred=seconds[ix])
|
|
e=dict(negative_full_predictions=1,negative_query_predictions=0)
|
|
np.testing.assert_equal(full_prediction(z,raw,ix,e),seconds)
|
|
for bad in [dict(z,pred=np.zeros(2)),dict(z,raw_ids=raw[::-1]),dict(z,seconds=seconds[:2])]:
|
|
with self.assertRaises(ValueError):
|
|
full_prediction(bad,raw,ix,e)
|
|
|
|
def test_duplicate_or_wrong_fold_before_file_access(self):
|
|
entries=[dict(fold=i%5,case=f'p{i}',component_id=f'g{i}') for i in range(552)]
|
|
by={e['case']:e for e in entries}
|
|
self.assertEqual(same_rows(entries,by,entries),by)
|
|
with self.assertRaises(ValueError):
|
|
same_rows([entries[0]]+entries[:-1],by,entries)
|
|
jobs=[dict(id=i,test=[e['case'] for e in entries if e['fold']==i]) for i in range(5)]
|
|
with patch('phase3_followup_postfit_common.checked',side_effect=AssertionError('early read')):
|
|
with self.assertRaises(ValueError):
|
|
prediction_identity(Path('/unused'),{},dict(jobs=jobs),by,[entries[0]]+entries[:-1])
|
|
|
|
def test_score_missing_review_or_predictions_stops_before_target_reader(self):
|
|
root = Path('/unused')
|
|
with patch('phase3_followup_postfit_worker.config',side_effect=PermissionError), \
|
|
patch('phase3_followup_postfit_worker.cached') as targets:
|
|
with self.assertRaises(PermissionError):
|
|
score(root)
|
|
targets.assert_not_called()
|
|
with patch('phase3_followup_postfit_worker.config',return_value=({'training': {}}, {}, {}, {})), \
|
|
patch('phase3_followup_postfit_worker.read',return_value={}), \
|
|
patch('phase3_followup_postfit_worker.verify',side_effect=[{},FileNotFoundError]), \
|
|
patch('phase3_followup_postfit_worker.cached') as targets:
|
|
with self.assertRaises(FileNotFoundError):
|
|
score(root)
|
|
targets.assert_not_called()
|
|
|
|
def test_budget_charges_existing_cost_once(self):
|
|
c=dict(item='c',training=dict(training_gpu_seconds=105364.2),
|
|
phase_reservations=dict(c=205200,b=172908),item_gpu_cap=205200)
|
|
budget(c,2700,180,5)
|
|
for gpu,cpu in [(2700.001,180),(2700,1080.001)]:
|
|
with self.assertRaises(RuntimeError):
|
|
budget(c,gpu,cpu,5)
|
|
with self.assertRaises(RuntimeError):
|
|
budget(dict(c,phase_reservations=dict(c=205200,b=700000)),0,180,5)
|
|
|
|
|
|
if __name__=='__main__':
|
|
unittest.main()
|