75 lines
3.8 KiB
Python
75 lines
3.8 KiB
Python
"""Synthetic checks of the four-split boundary and unchanged paired estimator."""
|
|
|
|
from pathlib import Path
|
|
from unittest import TestCase, main
|
|
from unittest.mock import patch
|
|
|
|
from track_a_postfit_rebind_common import OMITTED, SPLITS, scope, selected_training
|
|
from track_a_postfit_rebind_statistics import paired_uncertainty, summarize
|
|
from track_a_postfit_statistics import paired_uncertainty as original_uncertainty
|
|
|
|
|
|
class RebindTests(TestCase):
|
|
def setUp(self):
|
|
self.jobs = [dict(id=i, test=['a', str(i)], missing_test=[]) for i in range(5)]
|
|
self.rows = [dict(split=i, case=case, A=1., H2=2.+i)
|
|
for i in SPLITS for case in self.jobs[i]['test']]
|
|
|
|
def test_exact_split_ids_and_visit_counts(self):
|
|
original = dict(A_predictions={str(i): {str(i): {'sha256': str(i)}} for i in range(5)})
|
|
result = scope(original, {'jobs': self.jobs})
|
|
self.assertEqual([job['id'] for job in result['jobs']], list(SPLITS))
|
|
self.assertEqual(set(result['A_predictions']), {'0', '1', '2', '4'})
|
|
self.assertEqual(result['excluded_split'], OMITTED)
|
|
self.assertEqual(result['expected_pair_visits'], 8)
|
|
self.assertEqual(result['expected_unique_target_reads'], 5)
|
|
|
|
def test_four_split_summary_uses_original_part_bootstrap(self):
|
|
outcome = summarize(self.rows, selected_training({'jobs': self.jobs})['jobs'])
|
|
self.assertIs(paired_uncertainty, original_uncertainty)
|
|
self.assertEqual(outcome['descriptive_paired_uncertainty'], original_uncertainty(self.rows))
|
|
self.assertEqual(outcome['mean_of_four_split_summaries']['H2']['median'], 3.75)
|
|
self.assertEqual(outcome['excluded_split']['split'], 3)
|
|
self.assertNotIn('mean_of_five_split_summaries', outcome)
|
|
|
|
def test_incomplete_duplicate_or_extra_visits_rejected(self):
|
|
jobs = selected_training({'jobs': self.jobs})['jobs']
|
|
for rows in (self.rows[:-1], self.rows+[self.rows[0]],
|
|
self.rows+[dict(split=3, case='a', A=1., H2=4.)]):
|
|
with self.assertRaises(ValueError):
|
|
summarize(rows, jobs)
|
|
|
|
def test_mixed_source_checkpoint_identity(self):
|
|
from track_a_postfit_common import checkpoint_identity
|
|
for i in SPLITS:
|
|
training_sha = 'recovery' if i == 4 else 'original'
|
|
blob = dict(seed=0, fold=i, completed_epochs=200, optimizer_steps=62000,
|
|
contract_sha256=training_sha, role='last_epoch_only')
|
|
checkpoint_identity(blob, dict(id=i, target_steps=62000), training_sha)
|
|
with self.assertRaises(ValueError):
|
|
checkpoint_identity(blob, dict(id=i, target_steps=62000), 'other')
|
|
|
|
def test_scoring_requires_new_review_before_target_access(self):
|
|
from track_a_postfit_rebind_worker import score
|
|
with patch('track_a_postfit_rebind_worker.config', return_value=({}, {}, {})), \
|
|
patch('track_a_postfit_rebind_worker.validate_review', side_effect=PermissionError), \
|
|
patch('phase2_h2_data.cached') as targets:
|
|
with self.assertRaises(PermissionError):
|
|
score(Path('/synthetic'))
|
|
targets.assert_not_called()
|
|
|
|
def test_scoring_requires_prediction_freeze_before_target_access(self):
|
|
from track_a_postfit_rebind_worker import score
|
|
with patch('track_a_postfit_rebind_worker.config', return_value=({}, {}, {})), \
|
|
patch('track_a_postfit_rebind_worker.validate_review'), \
|
|
patch('track_a_postfit_rebind_worker.adopted'), \
|
|
patch('track_a_postfit_rebind_worker.verify', side_effect=PermissionError), \
|
|
patch('phase2_h2_data.cached') as targets:
|
|
with self.assertRaises(PermissionError):
|
|
score(Path('/synthetic'))
|
|
targets.assert_not_called()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|