104 lines
5.9 KiB
Python
104 lines
5.9 KiB
Python
"""CPU-only boundary tests for Track A paired scoring and custody."""
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
from phase2_h1_worker_v2 import sha
|
|
from track_a_postfit_common import checkpoint_identity,prediction_identity,budget
|
|
from track_a_postfit_statistics import summarize,paired_uncertainty
|
|
from phase3_item1_postfit_data import load_evaluation_part
|
|
from gre24_features_v2 import per_geometry_metrics
|
|
|
|
|
|
class PostfitTests(unittest.TestCase):
|
|
def test_fixed_last_epoch_identity(self):
|
|
j=dict(id=2,target_steps=61400);b=dict(seed=0,fold=2,completed_epochs=200,optimizer_steps=61400,contract_sha256='fixed',role='last_epoch_only')
|
|
checkpoint_identity(b,j,'fixed')
|
|
for k,v in [('seed',1),('fold',0),('completed_epochs',199),('optimizer_steps',61399),('contract_sha256','other'),('role','best_validation')]:
|
|
with self.assertRaises(ValueError):checkpoint_identity(dict(b,**{k:v}),j,'fixed')
|
|
|
|
def test_test_loader_rejects_foreign_case_before_io(self):
|
|
with patch('phase3_item1_postfit_data.np.load',side_effect=AssertionError('Array access')):
|
|
with self.assertRaises(PermissionError):load_evaluation_part(dict(folds=[dict(test=['a'])]),'b',{},'test')
|
|
|
|
def test_all_node_proxy_not_relative_l2(self):
|
|
m=per_geometry_metrics(np.array([0.,2.]),np.array([-1.,2.]))
|
|
self.assertAlmostEqual(m['rrmse_maximum'],np.sqrt(.5)/2)
|
|
self.assertNotEqual(m['rrmse_maximum'],m['rrmse_rms'])
|
|
|
|
def test_zero_truth_is_explicitly_undefined(self):
|
|
self.assertIsNone(per_geometry_metrics(np.zeros(2),np.ones(2))['rrmse_maximum'])
|
|
|
|
def fixture(self):
|
|
jobs=[dict(id=i,test=['a','b'],missing_test=['old_failure']) for i in range(5)]
|
|
rows=[dict(split=i,case=case,A=1.,H2=2. if case=='a' else 4.) for i in range(5) for case in ['a','b']]
|
|
return rows,jobs
|
|
|
|
def test_same_paired_support_five_summary_means(self):
|
|
rows,jobs=self.fixture();s=summarize(rows,jobs)
|
|
self.assertEqual(s['mean_of_five_split_summaries']['H2'],dict(min=2.,median=3.,max=4.))
|
|
self.assertFalse(s['full_requested_315_coverage'])
|
|
|
|
def test_missing_visit_not_silently_averaged(self):
|
|
rows,jobs=self.fixture()
|
|
with self.assertRaises(ValueError):summarize(rows[:-1],jobs)
|
|
|
|
def test_undefined_pairs_excluded_symmetrically(self):
|
|
rows,jobs=self.fixture();rows[0].update(A=None,H2=None)
|
|
s=summarize(rows,jobs);self.assertEqual(s['per_split'][0]['scored_pairs'],1)
|
|
self.assertEqual(s['per_split'][0]['undefined'],['a'])
|
|
|
|
def test_unique_part_bootstrap_not_visit_weighted(self):
|
|
rows=[dict(case='a',A=0.,H2=2.),dict(case='a',A=0.,H2=4.),dict(case='b',A=0.,H2=9.)]
|
|
s=paired_uncertainty(rows);self.assertEqual(s['point'],6.)
|
|
self.assertEqual(s['unique_parts'],2);self.assertEqual(s['part_visits'],{'a':2,'b':1})
|
|
self.assertEqual(s['replicates'],2000);self.assertFalse(s['gate'])
|
|
|
|
def test_bootstrap_order_invariant_and_deterministic(self):
|
|
rows,jobs=self.fixture()
|
|
self.assertEqual(paired_uncertainty(rows),paired_uncertainty(rows[::-1]))
|
|
|
|
def test_bootstrap_zero_difference_degenerate_interval(self):
|
|
s=paired_uncertainty([dict(case='a',A=.1,H2=.1),dict(case='b',A=.2,H2=.2)])
|
|
self.assertEqual(s['ci95'],[0.,0.]);self.assertEqual(s['point'],0.)
|
|
|
|
def test_undefined_candidates_and_visit_counts_retained(self):
|
|
s=paired_uncertainty([dict(case='a',A=None,H2=None),dict(case='a',A=None,H2=None),dict(case='b',A=1.,H2=2.)])
|
|
self.assertEqual(s['candidate_unique_parts'],2)
|
|
self.assertEqual(s['unique_parts'],1)
|
|
self.assertEqual(s['candidate_part_visits'],{'a':2,'b':1})
|
|
self.assertEqual(s['undefined_parts'],['a'])
|
|
|
|
def test_no_target_io_before_actual_review(self):
|
|
from track_a_postfit_worker import score
|
|
with patch('track_a_postfit_worker.config',return_value=({}, {}, {})), patch('track_a_postfit_worker.validate_review',side_effect=PermissionError('No actual PASS')), patch('phase2_h2_data.cached',side_effect=AssertionError('Target IO')) as io:
|
|
with self.assertRaises(PermissionError):score(Path('/synthetic'))
|
|
io.assert_not_called()
|
|
|
|
def test_no_target_io_before_predictions_frozen(self):
|
|
from track_a_postfit_worker import score
|
|
with patch('track_a_postfit_worker.config',return_value=({}, {}, {})), patch('track_a_postfit_worker.validate_review'), patch('track_a_postfit_worker.adopted'), patch('track_a_postfit_worker.verify',side_effect=PermissionError('Missing prediction freeze')), patch('phase2_h2_data.cached',side_effect=AssertionError('Target IO')) as io:
|
|
with self.assertRaises(PermissionError):score(Path('/synthetic'))
|
|
io.assert_not_called()
|
|
|
|
def test_postfit_reserve_and_training_charged_once(self):
|
|
budget(154800,0,180,5)
|
|
with self.assertRaises(RuntimeError):budget(154800,7201,180,0)
|
|
with self.assertRaises(RuntimeError):budget(160000,0,180,5)
|
|
with self.assertRaises(ValueError):budget(float('nan'),0,0,5)
|
|
|
|
def test_prediction_manifest_native_id_and_duplicate_rejection(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);p=root/'predictions/0/a.npz';p.parent.mkdir(parents=True);p.write_bytes(b'opaque H2')
|
|
a=root/'A.npy';a.write_bytes(b'opaque A')
|
|
tc=dict(jobs=[dict(id=0,test=['a'])],parts={'a':dict(nodes=2,output_sha256={'raw_ids.npy':'rawhash'})})
|
|
c=dict(A_predictions={'0':{'a':dict(path=str(a),sha256=sha(a),nodes=2,raw_ids_sha256='rawhash')}})
|
|
e=dict(split=0,case='a',path=str(p),sha256=sha(p),nodes=2,raw_ids_sha256='rawhash')
|
|
prediction_identity(root,c,tc,[e])
|
|
with self.assertRaises(ValueError):prediction_identity(root,c,tc,[e,e])
|
|
with self.assertRaises(ValueError):prediction_identity(root,c,tc,[dict(e,raw_ids_sha256='v5namespace')])
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|