65 lines
3.7 KiB
Python
65 lines
3.7 KiB
Python
"""Synthetic checks for labelled post-score prediction determinism orchestration."""
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
import r4bc_determinism as d
|
|
from r4bc_common import InputViolation,save,sha,make_jobs
|
|
|
|
|
|
class DeterminismAcceptance(unittest.TestCase):
|
|
def test_exact_bytes_include_dtype_shape_and_signed_zero(self):
|
|
a=np.array([0.,1.],dtype=np.float64)
|
|
self.assertTrue(d.same_bits(a,a.copy()))
|
|
self.assertFalse(d.same_bits(a,np.array([-0.,1.],dtype=np.float64)))
|
|
self.assertFalse(d.same_bits(a,a.astype(np.float32)))
|
|
self.assertFalse(d.same_bits(a,a.reshape(1,2)))
|
|
b=a.copy();b[1]=np.nextafter(1.,2.)
|
|
self.assertFalse(d.same_bits(a,b))
|
|
|
|
def test_shared_cpu_and_wall_no_reset(self):
|
|
with patch.object(d,'read',return_value=dict(fit_cpu_seconds=97267,setup_cpu_seconds=100)):
|
|
p=dict(shared_post_start_unix=100)
|
|
self.assertEqual(d.limits(p,12000,'predict',21695)['wall_seconds'],5)
|
|
with self.assertRaises(InputViolation):d.limits(p,12000.01,'predict',200)
|
|
with self.assertRaises(InputViolation):d.limits(p,1000,'compare',21700)
|
|
|
|
def test_refuse_non100_model(self):
|
|
with patch.object(d,'pins',return_value={'models':[dict(cell=2,arm='A')]}),patch.object(d.post,'predict') as predict:
|
|
with self.assertRaises(InputViolation):d.worker('predict',1,'A')
|
|
predict.assert_not_called()
|
|
|
|
def fixture(self,root,tamper=False):
|
|
old=root/'interim';old.mkdir();models=[j for j in make_jobs() if j['kind']=='FINAL' and j['pool']==100];entries=[]
|
|
for i,m in enumerate(models):
|
|
name=f"cell-{m['cell']:02d}-{m['arm']}";out=root/'predictions'/name;out.mkdir(parents=True)
|
|
case='case'+str(m['fold']);arrays=dict(raw_ids=np.array([3,5]),counts=np.array([1,2]),seconds=np.array([0.,1.]))
|
|
prior=old/(name+'.npz');np.savez_compressed(prior,**arrays)
|
|
entries.append(dict(case=case,arm=m['arm'],array='seconds',path=str(prior),sha256=sha(prior),pool=100))
|
|
if tamper and i==0:arrays['seconds'][0]=-0.
|
|
new=out/(case+'.npz');np.savez_compressed(new,**arrays)
|
|
save(out/'index.json',{case:dict(path=str(new),sha256=sha(new),array='seconds')})
|
|
m['model_sha256']='fixed';save(out/'result.json',dict(model_sha256='fixed',index_sha256=sha(out/'index.json')))
|
|
guard=root/'prediction-guards'/name;guard.mkdir(parents=True)
|
|
save(guard/'worker.receipt.json',dict(status='PASS'))
|
|
save(old/'predictions-freeze.json',dict(entries=entries));return old,dict(models=models)
|
|
|
|
def test_all35_new_arrays_match_and_no_targets(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);old,plan=self.fixture(root)
|
|
with patch.object(d,'ROOT',root),patch.object(d.final,'INTERIM',old),patch.object(d,'pins',return_value=plan):
|
|
d.compare_arrays();r=d.read(root/'bitwise-comparison.json')
|
|
self.assertEqual(r['status'],'BITWISE_MATCH');self.assertEqual(r['models'],35);self.assertEqual(r['prediction_arrays'],35)
|
|
|
|
def test_numerically_equal_signed_zero_is_defect(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);old,plan=self.fixture(root,True)
|
|
with patch.object(d,'ROOT',root),patch.object(d.final,'INTERIM',old),patch.object(d,'pins',return_value=plan):
|
|
with self.assertRaises(InputViolation):d.compare_arrays()
|
|
r=d.read(root/'bitwise-comparison.json');self.assertEqual(r['status'],'DEFECT');self.assertEqual(len(r['defects']),1)
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|