93 lines
6.0 KiB
Python
93 lines
6.0 KiB
Python
"""Synthetic scope, common-component, ledger and cache-scoring checks."""
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import resource
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import numpy as np
|
|
from gre24_prepare_v2 import sha
|
|
from s2_recovery_supervisor import save
|
|
from run_r4a_recovery import verify_scope,CASE
|
|
import r4a_poststop_score as scorer
|
|
|
|
|
|
class Checks(unittest.TestCase):
|
|
def fixture(self):
|
|
c={'parts':[{} for _ in range(569)]};c['parts'][568]={'case':CASE}
|
|
e=dict(recovery_indices=[568],prior_charged_worker_cpu_seconds=7517.870153,new_initial_cpu_seconds=3.)
|
|
p=dict(never_attempted=[],missing_receipt_indices=[],accounting_complete=True,charged_worker_cpu_seconds=7517.870153,
|
|
records=[dict(index=568,worker=dict(returncode=-9,stop_reason=None))])
|
|
return c,e,p
|
|
|
|
def test_only_1470_once_scope(self):
|
|
c,e,p=self.fixture();verify_scope(c,e,p)
|
|
for indices in [[243],[568,568],[243,568],[]]:
|
|
bad=dict(e,recovery_indices=indices)
|
|
with self.assertRaises(ValueError):verify_scope(c,bad,p)
|
|
|
|
def test_ledger_and_never_attempted_not_reclassified(self):
|
|
c,e,p=self.fixture()
|
|
with self.assertRaises(ValueError):verify_scope(c,dict(e,prior_charged_worker_cpu_seconds=0),p)
|
|
with self.assertRaises(ValueError):verify_scope(c,e,dict(p,never_attempted=[568]))
|
|
with self.assertRaises(ValueError):verify_scope(c,e,dict(p,missing_receipt_indices=[568]))
|
|
with self.assertRaises(ValueError):verify_scope(c,dict(e,new_initial_cpu_seconds=25000),p)
|
|
|
|
def test_original_signal_evidence_not_changed(self):
|
|
c,e,p=self.fixture();before=copy.deepcopy(p);verify_scope(c,e,p);self.assertEqual(p,before)
|
|
p['records'][0]['worker']['stop_reason']='wall_limit'
|
|
with self.assertRaises(ValueError):verify_scope(c,e,p)
|
|
|
|
def test_three_arms_same_complete_components_and_new_delta_direction(self):
|
|
roster=[dict(case='a',component_id='A',nodes=4),dict(case='b',component_id='A',nodes=5),dict(case='c',component_id='C',nodes=6)]
|
|
def row(case):return dict(case=case,metrics={a:{m:{'value':v} for m in scorer.METRICS} for a,v in [('EIK',1.),('GRAPH',2.),('B0_S2',.5)]})
|
|
summary,excluded=scorer.summaries([row('a'),row('c')],roster,{'a','c'})
|
|
self.assertEqual(excluded['A']['nodes'],9)
|
|
self.assertEqual(summary['relative_l2']['component_ids'],['C'])
|
|
delta=summary['relative_l2']['contrasts'];self.assertEqual(delta['GRAPH']['value'],-1)
|
|
self.assertEqual(delta['GRAPH']['low'],-1);self.assertEqual(delta['B0_S2']['high'],.5)
|
|
self.assertNotIn('region',delta['GRAPH'])
|
|
|
|
def test_undefined_one_arm_excludes_component_for_every_arm(self):
|
|
roster=[dict(case='a',component_id='A',nodes=4)]
|
|
rows=[dict(case='a',metrics={a:{m:{'value':1.} for m in scorer.METRICS} for a in scorer.ARMS})]
|
|
rows[0]['metrics']['B0_S2']['relative_l2']['value']=None
|
|
summary,_=scorer.summaries(rows,roster,{'a'})
|
|
self.assertEqual(summary['relative_l2']['components'],0)
|
|
self.assertEqual(summary['relative_l2']['means'],{a:None for a in scorer.ARMS})
|
|
|
|
def test_frozen_cached_score_integration_and_tamper_before_labels(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root=Path(temp);(root/'contract.json').write_text('{}');save(root/'continuation-contract.json',{'code_sha256':{}})
|
|
case='00000001_synthetic';failed='00000002_synthetic'
|
|
truth=np.array([1.,2.]);raw=np.array([10,20]);counts=np.array([1,3]);labels=root/'labels.npz';np.savez(labels,seconds=truth,raw_ids=raw,counts=counts)
|
|
index=root/'labels.json';save(index,[dict(case=case,fold=0,path=str(labels),sha256=sha(labels))])
|
|
hist=root/'historical.json';save(hist,[dict(case=case,arm='B0',**{m:dict(value=.1,reason=None) for m in scorer.METRICS})])
|
|
pred=root/'pred.npz';np.savez(pred,raw_ids=raw,counts=counts,F=.8*truth,G=.6*truth)
|
|
result=root/'result.json';save(result,{'status':'PASS'})
|
|
rows=[dict(case=k,component_id=k,nodes=2,assignments=[dict(fold=0,role='test')]) for k in [case,failed]]
|
|
c=dict(parts=rows,label_index=str(index),historical_scores=str(hist))
|
|
freeze=dict(contract_sha256=sha(root/'contract.json'),continuation_contract_sha256=sha(root/'continuation-contract.json'),
|
|
canonical_failures=[dict(case=failed)],parts={case:dict(result=str(result),result_sha256=sha(result),predictions=str(pred),predictions_sha256=sha(pred))})
|
|
save(root/'pass-predictions-freeze.json',freeze)
|
|
q=dict(raw_ids=raw,counts=counts,near_wall=np.array([True,True]),coordinates_over_Rg=np.array([[0.,0,0],[1,0,0]]))
|
|
with patch.object(scorer,'contract',return_value=c),patch.object(scorer,'prediction_inputs',return_value=q):scorer.score(root)
|
|
summary=json.loads((root/'postfit/score-summary.json').read_text())
|
|
self.assertEqual(summary['pass_prediction_parts'],1);self.assertEqual(summary['new_decodes'],0)
|
|
self.assertAlmostEqual(summary['summary']['relative_l2']['contrasts']['GRAPH']['value'],-1)
|
|
pred.write_bytes(b'tampered');labels.unlink()
|
|
with patch.object(scorer,'contract',return_value=c):
|
|
with self.assertRaisesRegex(ValueError,'SHA mismatch'):scorer.score(root)
|
|
|
|
|
|
if __name__=='__main__':
|
|
out=Path(sys.argv[1]);out.mkdir(parents=True,exist_ok=False)
|
|
with (out/'tests.log').open('x') as log:
|
|
r=unittest.TextTestRunner(stream=log,verbosity=2).run(unittest.defaultTestLoader.loadTestsFromTestCase(Checks))
|
|
save(out/'receipt.json',dict(status='PASS' if r.wasSuccessful() else 'FAIL',tests=r.testsRun,failures=len(r.failures),errors=len(r.errors),
|
|
cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),scope='synthetic only; no real labels or geometry',
|
|
code_sha256={n:sha(n) for n in ['tools/r4a_poststop_score.py','tools/run_r4a_recovery.py','tools/test_r4a_recovery.py']}))
|
|
print((out/'receipt.json').read_text());raise SystemExit(0 if r.wasSuccessful() else 1)
|