84 lines
4.8 KiB
Python
84 lines
4.8 KiB
Python
"""Synthetic acceptance of final orchestration, without labels or predictions."""
|
|
import base64
|
|
import copy
|
|
import hashlib
|
|
from pathlib import Path
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import r4bc_final as f
|
|
from r4bc_common import make_jobs, save, InputViolation
|
|
|
|
|
|
def fixture():
|
|
jobs=make_jobs()
|
|
records=[dict(index=j['index'],global_failure=False,worker=dict(status='PASS',stop_reason=None)) for j in jobs]
|
|
records[90]['worker']=dict(status='FAILED',stop_reason='process_count_limit')
|
|
models=[dict(cell=j['cell'],arm=j['arm'],fold=j['fold'],pool=j['pool'],status='PASS',model_path=str(j['index']),model_sha256='same') for j in jobs if j['kind']=='FINAL']
|
|
done=dict(status='FIT_QUEUE_TERMINAL',global_stop=None,cut=None,accounting_complete=True,
|
|
monitor_errors=[],never_attempted=[],postfit_state='READY_FOR_GIT_CHECKPOINT_FREEZE',records=records)
|
|
return dict(jobs=jobs),done,models
|
|
|
|
|
|
class FinalAcceptance(unittest.TestCase):
|
|
def test_original_failure_and_all_terminal_gate(self):
|
|
c,d,m=fixture();f.terminal_check(c,d,m)
|
|
for field,value in [('global_stop','failure'),('accounting_complete',False),('cut','NOT_RUN_BUDGET'),('never_attempted',[1])]:
|
|
v=copy.deepcopy(d);v[field]=value
|
|
with self.assertRaises(InputViolation):f.terminal_check(c,v,m)
|
|
|
|
def test_missing_duplicate_and_changed_failure_rejected(self):
|
|
c,d,m=fixture()
|
|
for kind in ['missing','duplicate','extra_failure']:
|
|
v=copy.deepcopy(d)
|
|
if kind=='missing':v['records'].pop()
|
|
elif kind=='duplicate':v['records'][-1]=v['records'][0]
|
|
else:v['records'][91]['worker']['status']='FAILED'
|
|
with self.assertRaises(InputViolation):f.terminal_check(c,v,m)
|
|
with self.assertRaises(InputViolation):f.terminal_check(c,d,m[:-1])
|
|
|
|
def test_interim_exact_checkpoint_identity(self):
|
|
_,_,models=fixture();old=copy.deepcopy([m for m in models if m['pool']==100]);f.match_interim_models(models,old)
|
|
old[0]['model_sha256']='changed'
|
|
with self.assertRaises(InputViolation):f.match_interim_models(models,old)
|
|
|
|
def test_shared_ledger_and_score_reserve(self):
|
|
frozen=dict(shared_post_start_unix=100,fit_cpu_seconds=97000,setup_cpu_seconds=100)
|
|
self.assertEqual(f.budget(frozen,346.870387+60,1800,3600,'predict',200),3600)
|
|
with self.assertRaises(InputViolation):f.budget(frozen,4800.01,1800,3600,'predict',200)
|
|
self.assertEqual(f.budget(frozen,4800,1800,3600,'predict',200),3600)
|
|
with self.assertRaises(InputViolation):f.budget(frozen,6600.01,7200,10800,'score',200)
|
|
self.assertEqual(f.budget(frozen,600,7200,10800,'score',21690),10)
|
|
with self.assertRaises(InputViolation):f.budget(frozen,600,600,1800,'compare',21700)
|
|
|
|
def test_no_regeneration_of_interim_predictions(self):
|
|
with patch.object(f,'pins',return_value={}),patch.object(f,'read',return_value={'models':[dict(cell=2,arm='A',pool=100)]}),patch.object(f.post,'predict') as predict:
|
|
with self.assertRaises(InputViolation):f.worker('predict',2,'A')
|
|
predict.assert_not_called()
|
|
|
|
def test_no_duplicate_stage(self):
|
|
with tempfile.TemporaryDirectory(dir=f.BASE/'artifacts') as tmp:
|
|
root=Path(tmp);save(root/'predict-started.json',{'pid':1})
|
|
with patch.object(f,'ROOT',root),patch.object(f,'pins',return_value={}):
|
|
with self.assertRaises(FileExistsError):f.run('predict')
|
|
|
|
def test_git_proof_without_child_and_reject_tamper(self):
|
|
with tempfile.TemporaryDirectory(dir=f.BASE/'artifacts') as tmp:
|
|
root=Path(tmp);path=root/'contract.json';path.write_bytes(b'{"synthetic":true}\n')
|
|
rel=str(path.relative_to(f.BASE));blob=path.read_bytes();oid=f.git_oid('blob',blob);objects={}
|
|
for part in reversed(Path(rel).parts):
|
|
raw=b'100644 '+part.encode()+b'\0'+bytes.fromhex(oid)
|
|
oid=f.git_oid('tree',raw);objects[oid]=base64.b64encode(raw).decode()
|
|
raw=('tree '+oid+'\n\nsynthetic commit\n').encode();commit=f.git_oid('commit',raw);objects[commit]=base64.b64encode(raw).decode()
|
|
proof=dict(commit=commit,objects=objects,files={rel:hashlib.sha256(blob).hexdigest()});save(root/'training-git-proof.json',proof)
|
|
with patch.object(f.subprocess,'check_output',side_effect=AssertionError('No child process in guard')):
|
|
f.git_gate(root,'training-git-freeze.json',['contract.json'])
|
|
with self.assertRaises(InputViolation):f.git_gate(root,'training-git-freeze.json',['models-terminal.json'])
|
|
path.write_bytes(b'changed')
|
|
with self.assertRaises(ValueError):f.git_gate(root,'training-git-freeze.json',['contract.json'])
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|