147 lines
11 KiB
Python
147 lines
11 KiB
Python
"""CPU-only synthetic regression checks, no project data or CUDA execution."""
|
|
import copy
|
|
from pathlib import Path
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
from phase3_seed1_postfit_common import checkpoint_identity, terminal_ledger, budget, normal_identity, prediction_identity, OLD, PHASE3_PRIOR
|
|
from phase3_seed1_postfit_statistics import comparisons, summary
|
|
from phase3_seed1_postfit_worker import paired_query, predict, score
|
|
from phase3_seed1_postfit_guard import final_reasons
|
|
from phase2_h2_s1_test_data import load_test_part
|
|
from s2_custody_statistics import part_metrics
|
|
|
|
|
|
def metric(v):return {k:dict(value=v,reason=None if v is not None else 'undefined') for k in ['relative_l2','last_location','kendall_tau_b']}
|
|
|
|
|
|
class Seed1PostfitTests(unittest.TestCase):
|
|
def test_checkpoint_rejects_wrong_seed_fold_epoch_selection(self):
|
|
j=dict(id=2,target_steps=60000);b=dict(seed=1,fold=2,completed_epochs=200,optimizer_steps=60000,contract_sha256='fixed',role='last_epoch_only')
|
|
checkpoint_identity(b,j,'fixed')
|
|
for k,v in [('seed',0),('fold',3),('completed_epochs',199),('optimizer_steps',59999),('contract_sha256','other'),('role','best_validation')]:
|
|
with self.assertRaises(ValueError):checkpoint_identity(dict(b,**{k:v}),j,'fixed')
|
|
|
|
def fixture(self):
|
|
tc=dict(train_gpu_cap=160000-OLD)
|
|
d=dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING',postfit_started=False,test_targets_read=False,
|
|
entries=[dict(id=i,status='PASS',returncode=0,gpu_seconds_charged=100.) for i in range(5)],
|
|
training_gpu_seconds=500.,phase3_prior_gpu_once=PHASE3_PRIOR,cumulative_phase3_training_gpu_seconds=PHASE3_PRIOR+500.)
|
|
m=dict(models=[dict(job=i) for i in range(5)],new_fits=5,test_prediction_started=False)
|
|
return tc,d,m
|
|
|
|
def test_all_five_terminal_no_partial_adoption(self):
|
|
tc,d,m=self.fixture();self.assertEqual(terminal_ledger(tc,d,m),500.)
|
|
d['entries'].pop()
|
|
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
|
|
|
|
def test_failed_or_duplicate_terminal_rejected(self):
|
|
for key,val in [('status','STOPPED'),('returncode',1),('id',1)]:
|
|
tc,d,m=self.fixture();d['entries'][0][key]=val
|
|
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
|
|
|
|
def test_ledger_double_old_charge_rejected(self):
|
|
tc,d,m=self.fixture();d['training_gpu_seconds']+=OLD
|
|
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
|
|
tc,d,m=self.fixture();d['cumulative_phase3_training_gpu_seconds']+=OLD
|
|
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
|
|
|
|
def test_exact_cumulative_budgets_and_final_caps(self):
|
|
budget(160000-OLD,7200,2000,0)
|
|
for args in [(160000-OLD,7200.01,180,0),(160000-OLD+.01,0,180,5),(100,0,1081,5),(float('nan'),0,0,5),(100,0,0,True)]:
|
|
with self.assertRaises((ValueError,RuntimeError)):budget(*args)
|
|
|
|
def test_learning_only_normalization(self):
|
|
j=dict(learning=['a'],learning_nodes=10);s=dict(parts=['a'],nodes=10,mean=[0.]*19,scale=[1.]*19)
|
|
normal_identity(s,j)
|
|
for change in [dict(parts=['test']),dict(nodes=11),dict(scale=[0.]*19),dict(mean=[0.]*18)]:
|
|
with self.assertRaises(ValueError):normal_identity(dict(s,**change),j)
|
|
|
|
def query_fixture(self):
|
|
raw=np.array([1,3,8]);q=dict(raw_ids=np.array([1,8]),counts=np.array([1,2]),coordinates_over_Rg=np.array([[0.,0.,0.],[1.,0.,0.]]),near_wall=np.array([True,False]))
|
|
z=dict(raw_ids=raw,seconds=np.array([1.,2.,3.]),pred=np.array([1.,3.]),query_raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
|
|
return raw,q,z
|
|
|
|
def test_query_full_ids_distinct_and_multiplicity_exact(self):
|
|
raw,q,z=self.query_fixture();np.testing.assert_array_equal(paired_query(raw,q,z),[0,2])
|
|
for change in [dict(counts=np.array([1,1])),dict(pred=np.array([1.,2.])),dict(raw_ids=np.array([1,4,8])),dict(query_raw_ids=np.array([1,3]))]:
|
|
with self.assertRaises(ValueError):paired_query(raw,q,dict(z,**change))
|
|
|
|
def test_test_loader_denies_foreign_role_before_array_io(self):
|
|
with patch('phase2_h2_s1_test_data.np.load',side_effect=AssertionError('Array access')) as io:
|
|
with self.assertRaises(PermissionError):load_test_part(dict(folds=[dict(test=['a'])]),'learning',{})
|
|
io.assert_not_called()
|
|
|
|
def test_predict_needs_actual_review_before_model_or_inputs(self):
|
|
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review',side_effect=PermissionError('No review')),patch('phase3_seed1_postfit_worker.adopted') as a:
|
|
with self.assertRaises(PermissionError):predict(Path('/synthetic'),0)
|
|
a.assert_not_called()
|
|
|
|
def test_predict_needs_all_training_terminals_before_input_loader(self):
|
|
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review'),patch('phase3_seed1_postfit_worker.adopted',side_effect=ValueError('Incomplete')),patch('phase2_h2_s1_test_data.load_test_part') as io:
|
|
with self.assertRaises(ValueError):predict(Path('/synthetic'),0)
|
|
io.assert_not_called()
|
|
|
|
def test_score_no_targets_or_score_reads_before_review(self):
|
|
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review',side_effect=PermissionError('No review')),patch('phase3_seed1_postfit_worker.cached') as io,patch('phase3_seed1_postfit_worker.read') as rd:
|
|
with self.assertRaises(PermissionError):score(Path('/synthetic'))
|
|
io.assert_not_called();rd.assert_not_called()
|
|
|
|
def test_score_no_targets_before_prediction_freeze(self):
|
|
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review'),patch('phase3_seed1_postfit_worker.adopted'),patch('phase3_seed1_postfit_worker.verify',side_effect=PermissionError('No prediction freeze')),patch('phase3_seed1_postfit_worker.cached') as io,patch('phase3_seed1_postfit_worker.read') as rd:
|
|
with self.assertRaises(PermissionError):score(Path('/synthetic'))
|
|
io.assert_not_called();rd.assert_not_called()
|
|
|
|
def test_sign_and_equal_component_weight(self):
|
|
rows=[dict(case='a',component_id='x',seed0=metric(1),seed1=metric(2)),dict(case='b',component_id='x',seed0=metric(1),seed1=metric(4)),dict(case='c',component_id='y',seed0=metric(1),seed1=metric(9))]
|
|
s=comparisons(rows)['relative_l2'];self.assertEqual(s['seed1_minus_seed0']['value'],5.)
|
|
self.assertEqual(s['means'],dict(seed1=6.,seed0=1.));self.assertEqual(s['seed1_minus_seed0']['replicates'],2000)
|
|
self.assertEqual(comparisons(rows),comparisons(rows[::-1]))
|
|
|
|
def test_undefined_component_not_silently_reweighted(self):
|
|
rows=[dict(case='a',component_id='x',seed0=metric(1),seed1=metric(None)),dict(case='b',component_id='x',seed0=metric(1),seed1=metric(2)),dict(case='c',component_id='y',seed0=metric(1),seed1=metric(3))]
|
|
s=comparisons(rows)['kendall_tau_b'];self.assertEqual(s['components'],dict(seed1=1,seed0=2));self.assertEqual(s['seed1_minus_seed0']['components'],1)
|
|
self.assertEqual(s['undefined_parts']['seed1'],['a']);self.assertEqual(s['seed1_minus_seed0']['value'],2.)
|
|
|
|
def test_original_query_metric_ties_and_undefined(self):
|
|
m=part_metrics(np.zeros(3),np.zeros(3),np.ones(3),np.ones(3,bool),np.array([3,1,2]),np.array([[0.,0,0],[1.,0,0],[2.,0,0]]),6)
|
|
self.assertIsNone(m['relative_l2']['value']);self.assertIsNone(m['kendall_tau_b']['value']);self.assertEqual(m['last_location']['value'],0.)
|
|
|
|
def test_no_partial_cohort_summary(self):
|
|
with self.assertRaises(ValueError):summary([dict(case='a',component_id='x',fold=0,seed0=metric(1),seed1=metric(2))])
|
|
with self.assertRaises(ValueError):prediction_identity(Path('/synthetic'),{},dict(jobs=[]),[])
|
|
|
|
def test_manifest_requires_exact_complete_roles_and_hashes(self):
|
|
root=Path('/synthetic');rows=[];entries=[];seed0={};jobs=[dict(id=f,test=[]) for f in range(5)]
|
|
for i in range(552):
|
|
case=str(i);f=i%5;group='g'+str(i%204);jobs[f]['test'].append(case)
|
|
rows.append(dict(case=case,nodes=3,component_id=group,v5_cache=dict(output_sha256={'raw_ids.npy':'raw'})))
|
|
seed0[case]=dict(path='/old/'+case,sha256='old',query_nodes=2)
|
|
entries.append(dict(case=case,fold=f,path=str(root/'jobs'/str(f)/'prediction-records'/(case+'.npz')),sha256='new',nodes=3,query_nodes=2,component_id=group,seed0_sha256='old',raw_ids_sha256='raw',negative_full_predictions=0,negative_query_predictions=0))
|
|
c=dict(rows=rows,seed0_predictions=seed0);tc=dict(jobs=jobs)
|
|
with patch('phase3_seed1_postfit_common.checked'):
|
|
self.assertEqual(len(prediction_identity(root,c,tc,entries)),552)
|
|
for change in [dict(fold=4),dict(component_id='wrong'),dict(raw_ids_sha256='foreign'),dict(seed0_sha256='other'),dict(query_nodes=3)]:
|
|
altered=copy.deepcopy(entries);altered[0].update(change)
|
|
with self.assertRaises(ValueError):prediction_identity(root,c,tc,altered)
|
|
with self.assertRaises(ValueError):prediction_identity(root,c,tc,entries[:-1]+[entries[0]])
|
|
with patch('phase3_seed1_postfit_common.checked',side_effect=ValueError('Changed bytes')):
|
|
with self.assertRaises(ValueError):prediction_identity(root,c,tc,entries)
|
|
|
|
def test_complete_summary_has_no_performance_gate(self):
|
|
rows=[dict(case=str(i),component_id='g'+str(i%204),fold=(i%204)%5,seed0=metric(1.),seed1=metric(2.)) for i in range(552)]
|
|
r=summary(rows);self.assertIsNone(r['performance_gate']);self.assertTrue(r['descriptive_only'])
|
|
self.assertEqual(set(r['per_fold']),set(map(str,range(5))))
|
|
self.assertEqual(r['pooled']['relative_l2']['seed1_minus_seed0']['value'],1.)
|
|
|
|
def test_failed_worker_recorded_peaks_and_monitor_type(self):
|
|
class Peaks:
|
|
witnesses={'peak_allocated_bytes':1,'peak_reserved_bytes':1}
|
|
maxima={'peak_allocated_bytes':1,'peak_reserved_bytes':16*2**30}
|
|
r=dict(status='PASS',returncode=0,monitor_errors=0);result=dict(status='COMPLETE',parts=2)
|
|
self.assertIn('RECORDED_VRAM_LIMIT',final_reasons([],r,result,Peaks(),100,100,300,2))
|
|
self.assertIn('WORKER_MONITOR_ERRORS',final_reasons([],dict(r,monitor_errors=[]),result,Peaks(),100,100,300,2))
|
|
self.assertIn('INCOMPLETE_FOLD',final_reasons([],r,None,Peaks(),100,100,300,2))
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|