95 lines
7.2 KiB
Python
95 lines
7.2 KiB
Python
import copy,json,tempfile,unittest,subprocess
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
import phase2_home_freeze_io as fio
|
|
from phase2_h1_worker_v2 import sha
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_gate_outcome_statistics import summarize,two_metrics
|
|
from phase2_gate_outcome_postfit import predict_arrays,query_identity,remaining_cap,terminal_inputs
|
|
from s2_custody_statistics import part_metrics
|
|
|
|
def metric(v):return dict(relative_l2=dict(value=v),last_location=dict(value=v))
|
|
def row(case,component,new,old):
|
|
return dict(case=case,component_id=component,label_gate=None if new is None else {a:metric(new) for a in ['A','G']},LOG_gate=None if old is None else {a:metric(old) for a in ['A','G']})
|
|
class Statistics(unittest.TestCase):
|
|
def test_partial_component_kept_only_empty_component_drops(self):
|
|
d=summarize([row('a','x',1,1),row('b','x',None,1),row('c','y',None,1)],['x','y']);v=d['comparisons']['A']['relative_l2']
|
|
self.assertEqual(v['component_part_counts'],{'x':1});self.assertEqual(v['excluded_components'],['y']);self.assertEqual(v['paired_parts'],1);self.assertEqual(d['decision'],'PASS_OUTCOME_EQUIVALENCE')
|
|
def test_paired_parts_not_independent_arm_averages(self):
|
|
d=summarize([row('a','x',1,1),row('b','x',10,None),row('c','x',None,30)],['x']);v=d['comparisons']['A']['relative_l2'];self.assertEqual(v['label_mean'],1);self.assertEqual(v['LOG_mean'],1)
|
|
def test_equal_component_not_equal_part_weighting(self):
|
|
d=summarize([row('a','x',2,1),row('b','x',2,1),row('c','y',1,1)],['x','y']);self.assertEqual(d['comparisons']['G']['relative_l2']['label_minus_LOG']['value'],.5)
|
|
def test_both_models_both_metrics_required(self):
|
|
r=row('a','x',0,0);r['label_gate']['G']['last_location']['value']=.011;d=summarize([r],['x']);self.assertFalse(d['accepted_for_v2']);self.assertTrue(d['comparisons']['A']['last_location']['passed'])
|
|
def test_closed_band_boundaries_and_zero_support(self):
|
|
r=row('a','x',.001,0);d=summarize([r],['x']);self.assertTrue(d['accepted_for_v2']);self.assertFalse(summarize([row('a','x',None,1)],['x'])['accepted_for_v2'])
|
|
def test_duplicates_and_nonfinite_rejected(self):
|
|
r=row('a','x',0,0)
|
|
with self.assertRaises(ValueError):summarize([r,r],['x'])
|
|
with self.assertRaises(ValueError):summarize([row('a','x',float('nan'),0)],['x'])
|
|
def test_metrics_match_original_with_weighted_counts_and_raw_ties(self):
|
|
p=np.array([2.,2.,1.]);t=np.array([1.,3.,3.]);q=dict(counts=np.array([1,3,2]),raw_ids=np.array([5,2,8]),coordinates_over_Rg=np.eye(3),near_wall=np.array([True,False,True]));a=two_metrics(p,t,q);b=part_metrics(p,t,q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],1)
|
|
self.assertEqual(a['relative_l2'],b['relative_l2']);self.assertEqual(a['last_location'],b['last_location'])
|
|
def test_zero_truth_denominator_retains_undefined(self):
|
|
q=dict(counts=np.ones(2),raw_ids=np.arange(2),coordinates_over_Rg=np.zeros((2,3)));d=two_metrics(np.ones(2),np.zeros(2),q);self.assertIsNone(d['relative_l2']['value'])
|
|
class PredictAndBudget(unittest.TestCase):
|
|
def test_native_nan_query_order_and_seconds(self):
|
|
class Model:
|
|
def __init__(self,n):self.n=n
|
|
def predict(self,x):
|
|
assert x.dtype==np.float32 and x.shape[1]==self.n
|
|
if self.n==21:assert np.isnan(x[:,-2:]).all()
|
|
return x[:,0]+self.n
|
|
f=np.arange(57,dtype=float).reshape(3,19);q={'raw_ids':np.array([2,9])};out=predict_arrays(Model(19),Model(21),f,np.full((3,2),np.nan),np.array([2,5,9]),q,.4)
|
|
np.testing.assert_allclose(out['A'],np.array([19,57])*.4);np.testing.assert_allclose(out['G'],np.array([21,59])*.4)
|
|
def test_query_mismatch_fails_before_predict(self):
|
|
with self.assertRaises(ValueError):predict_arrays(None,None,np.zeros((2,19)),np.zeros((2,2)),np.array([1,2]),{'raw_ids':np.array([3])},1)
|
|
a=dict(raw_ids=np.arange(2),counts=np.ones(2),coordinates_over_Rg=np.zeros((2,3)),near_wall=np.zeros(2));b=dict(a,counts=np.array([1,2]))
|
|
with self.assertRaises(ValueError):query_identity(a,b)
|
|
def test_shared_cost_charged_once_and_both_caps_enforced(self):
|
|
self.assertEqual(remaining_cap(68000,120,600),600)
|
|
with self.assertRaises(RuntimeError):remaining_cap(71600,120,300)
|
|
with self.assertRaises(RuntimeError):remaining_cap(10000,3400,300)
|
|
def test_missing_feature_freeze_rejects_adoption(self):
|
|
with tempfile.TemporaryDirectory() as p:
|
|
with self.assertRaises(FileNotFoundError):terminal_inputs(Path(p),{'preparation_root':p},{})
|
|
class SemanticReview(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp=tempfile.TemporaryDirectory();p=Path(self.tmp.name);self.base=p/'base';self.home=p/'runs';self.store=p/'store'
|
|
for d in [self.base,self.home,self.store]:d.mkdir()
|
|
subprocess.run(['git','init','-q','-b','codex/test',str(self.store)],check=True)
|
|
self.patches=[patch.object(fio,k,v) for k,v in dict(BASE=self.base,HOME_RUNS=self.home,STORE=self.store,LOCK=p/'lock',BRANCH='codex/test').items()]
|
|
for x in self.patches:x.start()
|
|
self.root=self.home/'postfit';self.root.mkdir();self.code=self.base/'code.py';self.code.write_text('fixed code');self.comparison=self.base/'comparison.json';self.comparison.write_text('{"band":0.001}')
|
|
self.c=dict(comparison_contract=str(self.comparison),comparison_contract_sha256=sha(self.comparison),code_sha256={str(self.code):sha(self.code)})
|
|
(self.root/'contract.json').write_text(json.dumps(self.c));fio.freeze(self.root,'implementation',[self.root/'contract.json',self.code,self.comparison]);proof=fio.verify(self.root,'implementation')
|
|
self.binding=dict(reviewed_commit=proof['commit'],contract_sha256=sha(self.root/'contract.json'),implementation_proof_sha256=sha(self.root/'implementation-git-freeze.json'),comparison_contract_sha256=sha(self.comparison),code_sha256=self.c['code_sha256'])
|
|
(self.root/'expected-review-binding.json').write_text(json.dumps(self.binding));self.good=dict(verdict='PASS',**self.binding)
|
|
def tearDown(self):
|
|
for x in self.patches:x.stop()
|
|
self.tmp.cleanup()
|
|
def receipt(self,d,include=True):
|
|
(self.root/'review.json').write_text(json.dumps(d));files=[self.root/'contract.json']
|
|
if include:files.extend([self.root/'review.json',self.root/'expected-review-binding.json'])
|
|
fio.freeze(self.root,'implementation-review',files)
|
|
def test_actual_frozen_pass(self):self.receipt(self.good);self.assertEqual(validate_review(self.root)['verdict'],'PASS')
|
|
def test_fail_missing_and_unrelated_receipts(self):
|
|
for v in [dict(self.good,verdict='FAIL'),self.binding,{'unrelated':'review'}]:
|
|
self.receipt(v)
|
|
with self.assertRaises(PermissionError):validate_review(self.root)
|
|
def test_wrong_exact_identities(self):
|
|
for k in ['reviewed_commit','contract_sha256','implementation_proof_sha256','comparison_contract_sha256','code_sha256']:
|
|
v=dict(self.good);v[k]={} if k=='code_sha256' else 'wrong';self.receipt(v)
|
|
with self.assertRaises(PermissionError):validate_review(self.root)
|
|
def test_review_omitted_from_freeze(self):
|
|
self.receipt(self.good,False)
|
|
with self.assertRaises(PermissionError):validate_review(self.root)
|
|
def test_changed_source_rejected(self):
|
|
self.receipt(self.good);self.code.write_text('changed')
|
|
with self.assertRaises(ValueError):validate_review(self.root)
|
|
def test_changed_comparison_rejected(self):
|
|
self.receipt(self.good);self.comparison.write_text('changed')
|
|
with self.assertRaises(ValueError):validate_review(self.root)
|
|
if __name__=='__main__':unittest.main()
|