105 lines
9.2 KiB
Python
105 lines
9.2 KiB
Python
import unittest,tempfile,ast,copy,json
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
from phase3_item1_postfit_statistics import classify,contrasts,summarize,ARMS,METRICS
|
|
from phase3_item1_postfit_common import expected_checkpoint,require_budget,terminal_inputs,terminal_identity,MODEL_ARMS,expected_entries
|
|
from phase3_item1_postfit_worker import query_indices,score
|
|
from test_phase2_gate_outcome_postfit import SemanticReview
|
|
from phase3_item1_postfit_data import load_evaluation_part
|
|
from phase3_item1_postfit_guard import final_reasons,run_prediction
|
|
class PostfitTests(unittest.TestCase):
|
|
def wrapper_receipt(self,errors):
|
|
receipt=dict(status='PASS',cpu_seconds=.01,monitor_errors=errors,last_monitor_error=dict(operation='tree_usage',message='synthetic sampling failure') if errors else None)
|
|
original=copy.deepcopy(receipt)
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d);job=root/'jobs/0';job.mkdir(parents=True)
|
|
def synthetic_supervise(*args,**kwargs):
|
|
out=job/'prediction-records';out.mkdir()
|
|
(out/'result.json').write_text(json.dumps(dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=2048)))
|
|
return receipt
|
|
with patch('phase3_item1_postfit_guard.supervise',side_effect=synthetic_supervise) as supervisor,patch('phase3_item1_postfit_guard.device_usage',return_value=(0,0)),patch('phase3_item1_postfit_guard.tree_usage',return_value=(0,0,1)),patch('phase3_item1_postfit_guard.time.sleep'):
|
|
result=run_prediction(root,dict(gpu_uuid='synthetic-no-device',jobs=[dict(expected_parts=1)]),{},0,600,300)
|
|
supervisor.assert_called_once();self.assertEqual(receipt,original);self.assertEqual(result['worker'],original)
|
|
self.assertEqual(json.loads((job/'prediction-guard.json').read_text())['worker'],original)
|
|
return result
|
|
def test_wrapper_clean_PASS_accepted(self):
|
|
r=self.wrapper_receipt(0);self.assertEqual(r['status'],'PASS');self.assertEqual(r['reasons'],[])
|
|
def test_wrapper_PASS_with_monitor_errors_stops_retains_receipt(self):
|
|
r=self.wrapper_receipt(1);self.assertEqual(r['status'],'STOPPED');self.assertEqual(r['reasons'],['WORKER_MONITOR_ERRORS']);self.assertEqual(r['worker']['status'],'PASS')
|
|
def terminal_fixture(self):
|
|
return (dict(status='TRAIN_COMPLETE_POSTFIT_PENDING',entries=[dict(id=i,status='PASS',returncode=0,gpu_seconds_charged=100) for i in range(6)],training_gpu_seconds=600,test_targets_read=False),dict(models=[dict(job=i) for i in range(6)],test_prediction_started=False,new_fits=6,new_checkpoints=6))
|
|
def test_six_terminal_identity_and_once_ledger(self):
|
|
d,m=self.terminal_fixture();self.assertEqual(terminal_identity(d,m),600)
|
|
for change in ['partial','failed','duplicate','returncode','cost','nan','custody']:
|
|
a,b=copy.deepcopy((d,m))
|
|
if change=='partial':a['entries'].pop()
|
|
if change=='failed':a['entries'][0]['status']='FAILED'
|
|
if change=='duplicate':b['models'][0]['job']=1
|
|
if change=='returncode':a['entries'][0]['returncode']=1
|
|
if change=='cost':a['training_gpu_seconds']+=1
|
|
if change=='nan':a['entries'][0]['gpu_seconds_charged']=float('nan')
|
|
if change=='custody':a['test_targets_read']=True
|
|
with self.assertRaises(ValueError):terminal_identity(a,b)
|
|
def test_score_rejects_missing_PASS_before_targets(self):
|
|
with tempfile.TemporaryDirectory() as d,patch('phase3_item1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_item1_postfit_worker.validate_review',side_effect=PermissionError('No actual new PASS')),patch('phase2_h2_data.cached') as cached:
|
|
with self.assertRaises(PermissionError):score(Path(d))
|
|
cached.assert_not_called();self.assertFalse((Path(d)/'score').exists())
|
|
def test_score_rejects_missing_prediction_freeze_before_targets(self):
|
|
with tempfile.TemporaryDirectory() as d,patch('phase3_item1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_item1_postfit_worker.validate_review'),patch('phase3_item1_postfit_worker.adopted_inputs'),patch('phase3_item1_postfit_worker.verify',side_effect=ValueError('Prediction hash mismatch')),patch('phase2_h2_data.cached') as cached:
|
|
with self.assertRaises(ValueError):score(Path(d))
|
|
cached.assert_not_called();self.assertFalse((Path(d)/'score').exists())
|
|
def test_data_boundary(self):
|
|
self.assertEqual(classify(dict(low=.001,high=.003),dict(low=0,high=.0005)),'UNIDENTIFIED_OR_OVERFIT_SIGNS')
|
|
def test_four_classifications(self):
|
|
pos=dict(low=.002,high=.004);eq=dict(low=-.001,high=.001)
|
|
self.assertEqual(classify(pos,eq),'DATA_BENEFIT_AT_TESTED_RANGE');self.assertEqual(classify(eq,pos),'UPDATE_BUDGET_AND_SCHEDULE_BENEFIT');self.assertEqual(classify(pos,pos),'BOTH_BENEFITS');self.assertEqual(classify(eq,eq),'LOCAL_PLATEAU_ONLY')
|
|
def test_wide_not_plateau(self):self.assertEqual(classify(dict(low=-.1,high=.1),dict(low=-.1,high=.1)),'UNIDENTIFIED_OR_OVERFIT_SIGNS')
|
|
def test_incomplete_not_classified(self):self.assertEqual(classify(dict(low=.002,high=.003),dict(low=0,high=0),complete=False),'UNIDENTIFIED_OR_OVERFIT_SIGNS')
|
|
def test_actual_count_denominator(self):
|
|
a={k:.2 for k in ARMS};a['step100_2U']=.1;q=contrasts(a,{'25':34,'50':65,'100':132});self.assertAlmostEqual(q['s_2U'],.1/np.log2(132/65));self.assertNotEqual(q['s_2U'],.1)
|
|
def rows(self):
|
|
rows=[]
|
|
for i in range(5):
|
|
for arm in ARMS:
|
|
value=(i+1)*.02+(0.003 if arm.startswith('step50') else 0)
|
|
if arm in ['step50_2U','step100_2U']:value-=.002
|
|
rows.append(dict(domain='common_test',arm=arm,case=str(i),component_id=str(i),metrics={k:dict(value=value,reason=None) for k in METRICS}))
|
|
return rows
|
|
def test_joint_draw_cancellation(self):
|
|
s=summarize(self.rows(),{'25':34,'50':65,'100':132})['domains']['common_test']['relative_l2'];q=s['contrasts']['s_2U_minus_s_U'];self.assertAlmostEqual(q['low'],0);self.assertAlmostEqual(q['high'],0);self.assertTrue(s['joint_draws_all_contrasts'])
|
|
def test_component_missing_not_silently_promoted(self):
|
|
r=self.rows();r[0]['metrics']['relative_l2']=dict(value=None,reason='zero truth');s=summarize(r,{'25':34,'50':65,'100':132})['domains']['common_test'];self.assertEqual(s['relative_l2']['paired_components'],4);self.assertFalse(s['relative_l2']['complete_support']);self.assertEqual(s['field_classification'],'UNIDENTIFIED_OR_OVERFIT_SIGNS')
|
|
def test_query_order_mapping(self):
|
|
np.testing.assert_array_equal(query_indices(np.array([2,7,9]),dict(raw_ids=np.array([2,9]))),[0,2])
|
|
with self.assertRaises(ValueError):query_indices(np.array([2,7,9]),dict(raw_ids=np.array([10])))
|
|
def test_loader_denies_wrong_role_before_IO(self):
|
|
with self.assertRaises(PermissionError):load_evaluation_part(dict(folds=[dict(learning=['a'],validation=['b'],test=['c'])]),'a',{},'test')
|
|
def test_loader_no_target_branch(self):
|
|
tree=ast.parse(Path('tools/phase3_item1_postfit_data.py').read_text());literals={n.value for n in ast.walk(tree) if isinstance(n,ast.Constant) and isinstance(n.value,str)};self.assertFalse({'target','seconds','target.npy','seconds.npy'}&literals)
|
|
def test_prediction_no_label_calls(self):
|
|
t=ast.parse(Path('tools/phase3_item1_postfit_worker.py').read_text());f=next(n for n in t.body if isinstance(n,ast.FunctionDef) and n.name=='predict');calls=[n for n in ast.walk(f) if isinstance(n,ast.Call) and isinstance(n.func,ast.Name) and n.func.id=='cached'];self.assertEqual({n.args[2].value for n in calls},{'raw_ids','Vf'})
|
|
def test_checkpoint_identity(self):
|
|
j=dict(id=2,target_steps=32000,pool=50,mode='step',tag='U');m=dict(fold=0,seed=0,optimizer_steps=32000,contract_sha256='x',pool=50,mode='step',role='U');expected_checkpoint(j,m,'x')
|
|
for key,value in [('optimizer_steps',64000),('seed',1),('role','2U'),('contract_sha256','wrong')]:
|
|
with self.assertRaises(ValueError):expected_checkpoint(j,dict(m,**{key:value}),'x')
|
|
def test_S1_checkpoint_selection(self):
|
|
j=dict(id=6,target_steps=64000);m=dict(fold=0,seed=0,optimizer_steps=64000,contract_sha256='old',completed_epochs=200,role='last_epoch_only');expected_checkpoint(j,m,'old')
|
|
with self.assertRaises(ValueError):expected_checkpoint(j,dict(m,role='best_validation'),'old')
|
|
def test_training_pending_refuses_before_adoption(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
with self.assertRaisesRegex(RuntimeError,'WAIT_EXISTING'):terminal_inputs(dict(training_root=d),{})
|
|
def test_budget_fixed_remaining(self):
|
|
require_budget(112800,0,180,7)
|
|
with self.assertRaises(RuntimeError):require_budget(112800,1000,180,7)
|
|
with self.assertRaises(RuntimeError):require_budget(112800,0,600,7)
|
|
def test_budget_nonfinite(self):
|
|
with self.assertRaises(ValueError):require_budget(float('nan'),0,0,0)
|
|
def test_failure_peaks_retained(self):
|
|
class T:pass
|
|
t=T();t.witnesses={'peak_allocated_bytes':{},'peak_reserved_bytes':{}};t.maxima={'peak_allocated_bytes':1,'peak_reserved_bytes':16*2**30}
|
|
reasons=final_reasons([],dict(status='FAILED'),dict(status='FAILED',parts=1),t,20,20,300,1);self.assertIn('RECORDED_VRAM_LIMIT',reasons);self.assertIn('WORKER_NOT_PASS',reasons)
|
|
def test_every_arm_expected_domain(self):
|
|
c=dict(jobs=[dict(arm=a,pool=p) for a,p in zip(MODEL_ARMS,[25,50,50,50,100,100,100])]);tc=dict(pools={str(p):dict(learning=['l'+str(p)],validation=['v'],test=['t']) for p in [25,50,100]});e=expected_entries(c,tc);self.assertEqual(len(e),30);self.assertEqual(e[('S1_epoch100_reuse','t')]['domain'],'common_test')
|
|
if __name__=='__main__':unittest.main()
|