Files
MeltMind/tools/test_phase2_h2_s2_postfit.py
T

102 lines
8.6 KiB
Python

"""Synthetic pooled postfit boundaries; no real model prediction or target values."""
import ast,copy,inspect,json,tempfile,unittest
from pathlib import Path
from phase2_h2_s2_postfit_common import terminal_identity,terminal_inputs,checkpoint_identity,comparisons,decision,require_budget,reuse_fold0_metric
from phase2_h2_s2_postfit import predict,score,manifest_identity
from phase2_h2_s2_prediction_guard import final_reasons
from s2_main_memory_guard import PeakRecords
from test_phase2_gate_outcome_postfit import SemanticReview
def metric(v):return {k:dict(value=v) for k in ['relative_l2','last_location','kendall_tau_b']}
class TerminalAndBudget(unittest.TestCase):
def fixture(self):
tc=dict(prior_gpu_seconds=18858.21490871499,train_gpu_cap=122741.78509128501,fold0_reuse={'hash':'original'});entries=[dict(fold=f,status='PASS',returncode=0,gpu_seconds_charged=20000.) for f in range(1,5)]
done=dict(status='TRAIN_COMPLETE_POSTFIT_PENDING',postfit_started=False,terminal=entries,prior_gpu_once=tc['prior_gpu_seconds'],new_gpu_seconds_charged=80000.,cumulative_h2_gpu_seconds=tc['prior_gpu_seconds']+80000.)
ledger=dict(entries=entries,prior_gpu_once=tc['prior_gpu_seconds'],settled_gpu_seconds=80000.);models=dict(models=[dict(fold=f) for f in range(1,5)],fold0_reused_by_hash=tc['fold0_reuse'],new_fits=4,test_targets_read=False,postfit_started=False);return tc,done,ledger,models
def test_all_four_costs_and_prior_once(self):
a=self.fixture();self.assertEqual(terminal_identity(*a),98858.21490871499)
def test_partial_failed_or_wrong_role_models_rejected(self):
for kind in ['partial','failed','duplicate','predicted']:
t,d,l,m=self.fixture()
if kind=='partial':d['terminal']=d['terminal'][:3]
if kind=='failed':d['terminal'][0]['status']='FAILED'
if kind=='duplicate':m['models'][3]['fold']=1
if kind=='predicted':m['test_targets_read']=True
with self.assertRaises((ValueError,PermissionError)):terminal_identity(t,d,l,m)
def test_mismatched_or_double_counted_ledger_rejected(self):
for key in ['prior_gpu_once','new_gpu_seconds_charged','cumulative_h2_gpu_seconds']:
t,d,l,m=self.fixture();d[key]+=1
with self.assertRaises(ValueError):terminal_identity(t,d,l,m)
def test_still_training_without_model_freeze_cannot_enter(self):
with tempfile.TemporaryDirectory() as p:
with self.assertRaises(FileNotFoundError):terminal_inputs({'training_root':p},{},{})
def test_correct_new_fold_final_checkpoint_only(self):
c=dict(training_root='/train',source_sha256={'/train/contract.json':'training-fixed'});b=dict(completed_epochs=200,optimizer_steps=73200,seed=0,fold=2,role='last_epoch_only',contract_sha256='training-fixed');checkpoint_identity(b,c,2,366)
for k,v in dict(fold=0,completed_epochs=199,optimizer_steps=64000,contract_sha256='s1',role='best').items():
with self.assertRaises(ValueError):checkpoint_identity(dict(b,**{k:v}),c,2,366)
def test_budget_reserves_all_remaining_predictions_and_score(self):
require_budget(98858.,0,360,4);require_budget(98858.,300,800,1)
for args in [(142000,0,360,4),(98858,2200,500,1),(98858,0,1000,4),(98858,0,3000,0)]:
with self.assertRaises(RuntimeError):require_budget(*args)
def test_prediction_final_peaks_and_per_fold_coverage(self):
t=PeakRecords();t.witnesses={'peak_allocated_bytes':{},'peak_reserved_bytes':{}}
self.assertEqual(final_reasons([],{'status':'PASS'},dict(status='COMPLETE',parts=91),t,100,100,600,91),[])
self.assertIn('INCOMPLETE_FOLD',final_reasons([],{'status':'PASS'},dict(status='COMPLETE',parts=121),t,100,100,600,91))
t.maxima['peak_reserved_bytes']=16*2**30
self.assertIn('RECORDED_VRAM_LIMIT',final_reasons([],{'status':'FAILED'},None,t,100,100,600,91))
self.assertIn('FINAL_RESOURCE_LIMIT',final_reasons([],{'status':'PASS'},dict(status='COMPLETE',parts=91),t,601,100,600,91))
class StatisticsAndCustody(unittest.TestCase):
def test_component_equal_weights_and_b13_report_comparison(self):
rows=[dict(component_id=g,H2=metric(h),A=metric(a),B13_E1b=metric(b)) for g,h,a,b in [('x',2,1,0),('x',2,1,0),('y',1,1,0)]];d=comparisons(rows)
self.assertEqual(d['A']['relative_l2']['H2_minus_reference']['value'],.5);self.assertEqual(d['B13_E1b']['relative_l2']['H2_minus_reference']['value'],1.5)
for arm in d:
self.assertEqual(d[arm]['relative_l2']['H2_minus_reference']['replicates'],2000);self.assertEqual(d[arm]['relative_l2']['H2_minus_reference']['seed'],2026111408)
def test_original_whole_component_undefined_rule(self):
rows=[dict(component_id=g,H2=metric(h),A=metric(1),B13_E1b=metric(1)) for g,h in [('x',2),('x',None),('y',1)]];d=comparisons(rows);self.assertEqual(d['A']['relative_l2']['components'],{'H2':1,'A':2});self.assertEqual(d['A']['relative_l2']['H2_minus_reference']['components'],1)
def test_only_strict_complete_field_vs_A_confirms(self):
def result(high):return {'A':{'relative_l2':dict(components={'H2':204,'A':204},H2_minus_reference=dict(components=204,high=high))},'B13_E1b':{'relative_l2':{'arbitrarily_bad_report_only':999}}}
self.assertEqual(decision(result(-.001),552,204),'CONFIRMED')
for high in [0,.001,None,float('nan')]:self.assertEqual(decision(result(high),552,204),'NOT_CONFIRMED_FIXED_RECIPE')
self.assertEqual(decision(result(-.001),551,204),'PARK_INCOMPLETE_FIELD_SUPPORT')
r=result(-.001);r['A']['relative_l2']['components']['H2']=203;self.assertEqual(decision(r,552,204),'PARK_INCOMPLETE_FIELD_SUPPORT')
def test_fold0_metrics_are_reused_not_recomputed(self):
h=metric(.25);a=metric(.5);prior=dict(fold=0,component_id='x',new=h,A=a);entry=dict(fold=0,component_id='x');self.assertIs(reuse_fold0_metric(prior,{'A':a},entry),h)
with self.assertRaises(ValueError):reuse_fold0_metric(prior,{'A':metric(.6)},entry)
def test_fold0_prediction_forbidden_before_config_or_CUDA(self):
with self.assertRaises(PermissionError):predict(Path('/nonexistent'),0)
def test_semantic_pass_and_all_models_precede_value_reads(self):
s=inspect.getsource(score)
for token in ["c['baseline_score']","c['fold0_score']","cached(row,'seconds')"]:
self.assertLess(s.index('validate_review(root)'),s.index(token));self.assertLess(s.index('adopted_inputs('),s.index(token))
node=ast.parse(s);branch=next(n for n in ast.walk(node) if isinstance(n,ast.If) and ast.unparse(n.test)=="e['fold'] == 0")
self.assertNotIn('cached(',ast.unparse(branch.body));self.assertIn('reuse_fold0_metric',ast.unparse(branch.body));self.assertIn("cached(row, 'seconds')",ast.unparse(branch.orelse))
class ManifestReuse(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name);self.rows=[];parts={};baseline={};folds=[dict(fold=f,test=[]) for f in range(5)];self.entries=[];old=[]
for i in range(552):
case='case'+str(i);g='g'+str(i%204);f=(i%204)%5;cache={'seconds.npy':'label-digest'};folds[f]['test'].append(case);parts[case]=dict(nodes=3,clock_group=g,output_sha256=cache,t_set=.4);baseline[case]=dict(fold=f,component_id=g,sha256='baseline-'+case)
self.rows.append(dict(case=case,component_id=g,nodes=3,assignments=[dict(fold=f,role='test')],v5_cache={'output_sha256':cache},preparation={'t_set':.4}))
e=dict(case=case,component_id=g,fold=f,nodes=3,query_nodes=2,path=str(self.root/'folds'/str(f)/'predictions'/(case+'.npz')),sha256='prediction-'+case,A_source_sha256='baseline-'+case,negative_full_predictions=1,negative_query_predictions=1)
if f==0:old.append(dict(e))
self.entries.append(dict(e,origin='S1_HASH_REUSE' if f==0 else 'S2_FROZEN_CHECKPOINT'))
p=self.root/'old.json';p.write_text(json.dumps(dict(entries=old)));self.c=dict(rows=self.rows,baseline_predictions=baseline,fold0_manifest=str(p),postfit_root=str(self.root));self.pilot=dict(folds=folds,parts=parts)
def tearDown(self):self.tmp.cleanup()
def test_complete_roster_and_exact_fold0_entries(self):self.assertEqual(len(manifest_identity(self.entries,self.c,self.pilot)),552)
def test_fold0_changed_hash_rejected(self):
d=copy.deepcopy(self.entries);next(e for e in d if e['fold']==0)['sha256']='different'
with self.assertRaises(ValueError):manifest_identity(d,self.c,self.pilot)
def test_duplicate_missing_and_wrong_new_fold_path_rejected(self):
for kind in ['duplicate','missing','path']:
d=copy.deepcopy(self.entries)
if kind=='duplicate':d[-1]=d[0]
if kind=='missing':d.pop()
if kind=='path':next(e for e in d if e['fold']==1)['path']='/other/pred.npz'
with self.assertRaises(ValueError):manifest_identity(d,self.c,self.pilot)
def test_negative_counts_not_silently_changed_or_boolean(self):
for n in [-1,4,True]:
d=copy.deepcopy(self.entries);next(e for e in d if e['fold']==1)['negative_full_predictions']=n
with self.assertRaises(ValueError):manifest_identity(d,self.c,self.pilot)
if __name__=='__main__':unittest.main()