Files
MeltMind/tools/test_r4a_recovery.py

93 lines
6.0 KiB
Python

"""Synthetic scope, common-component, ledger and cache-scoring checks."""
import copy
import json
from pathlib import Path
import resource
import sys
import tempfile
import unittest
from unittest.mock import patch
import numpy as np
from gre24_prepare_v2 import sha
from s2_recovery_supervisor import save
from run_r4a_recovery import verify_scope,CASE
import r4a_poststop_score as scorer
class Checks(unittest.TestCase):
def fixture(self):
c={'parts':[{} for _ in range(569)]};c['parts'][568]={'case':CASE}
e=dict(recovery_indices=[568],prior_charged_worker_cpu_seconds=7517.870153,new_initial_cpu_seconds=3.)
p=dict(never_attempted=[],missing_receipt_indices=[],accounting_complete=True,charged_worker_cpu_seconds=7517.870153,
records=[dict(index=568,worker=dict(returncode=-9,stop_reason=None))])
return c,e,p
def test_only_1470_once_scope(self):
c,e,p=self.fixture();verify_scope(c,e,p)
for indices in [[243],[568,568],[243,568],[]]:
bad=dict(e,recovery_indices=indices)
with self.assertRaises(ValueError):verify_scope(c,bad,p)
def test_ledger_and_never_attempted_not_reclassified(self):
c,e,p=self.fixture()
with self.assertRaises(ValueError):verify_scope(c,dict(e,prior_charged_worker_cpu_seconds=0),p)
with self.assertRaises(ValueError):verify_scope(c,e,dict(p,never_attempted=[568]))
with self.assertRaises(ValueError):verify_scope(c,e,dict(p,missing_receipt_indices=[568]))
with self.assertRaises(ValueError):verify_scope(c,dict(e,new_initial_cpu_seconds=25000),p)
def test_original_signal_evidence_not_changed(self):
c,e,p=self.fixture();before=copy.deepcopy(p);verify_scope(c,e,p);self.assertEqual(p,before)
p['records'][0]['worker']['stop_reason']='wall_limit'
with self.assertRaises(ValueError):verify_scope(c,e,p)
def test_three_arms_same_complete_components_and_new_delta_direction(self):
roster=[dict(case='a',component_id='A',nodes=4),dict(case='b',component_id='A',nodes=5),dict(case='c',component_id='C',nodes=6)]
def row(case):return dict(case=case,metrics={a:{m:{'value':v} for m in scorer.METRICS} for a,v in [('EIK',1.),('GRAPH',2.),('B0_S2',.5)]})
summary,excluded=scorer.summaries([row('a'),row('c')],roster,{'a','c'})
self.assertEqual(excluded['A']['nodes'],9)
self.assertEqual(summary['relative_l2']['component_ids'],['C'])
delta=summary['relative_l2']['contrasts'];self.assertEqual(delta['GRAPH']['value'],-1)
self.assertEqual(delta['GRAPH']['low'],-1);self.assertEqual(delta['B0_S2']['high'],.5)
self.assertNotIn('region',delta['GRAPH'])
def test_undefined_one_arm_excludes_component_for_every_arm(self):
roster=[dict(case='a',component_id='A',nodes=4)]
rows=[dict(case='a',metrics={a:{m:{'value':1.} for m in scorer.METRICS} for a in scorer.ARMS})]
rows[0]['metrics']['B0_S2']['relative_l2']['value']=None
summary,_=scorer.summaries(rows,roster,{'a'})
self.assertEqual(summary['relative_l2']['components'],0)
self.assertEqual(summary['relative_l2']['means'],{a:None for a in scorer.ARMS})
def test_frozen_cached_score_integration_and_tamper_before_labels(self):
with tempfile.TemporaryDirectory() as temp:
root=Path(temp);(root/'contract.json').write_text('{}');save(root/'continuation-contract.json',{'code_sha256':{}})
case='00000001_synthetic';failed='00000002_synthetic'
truth=np.array([1.,2.]);raw=np.array([10,20]);counts=np.array([1,3]);labels=root/'labels.npz';np.savez(labels,seconds=truth,raw_ids=raw,counts=counts)
index=root/'labels.json';save(index,[dict(case=case,fold=0,path=str(labels),sha256=sha(labels))])
hist=root/'historical.json';save(hist,[dict(case=case,arm='B0',**{m:dict(value=.1,reason=None) for m in scorer.METRICS})])
pred=root/'pred.npz';np.savez(pred,raw_ids=raw,counts=counts,F=.8*truth,G=.6*truth)
result=root/'result.json';save(result,{'status':'PASS'})
rows=[dict(case=k,component_id=k,nodes=2,assignments=[dict(fold=0,role='test')]) for k in [case,failed]]
c=dict(parts=rows,label_index=str(index),historical_scores=str(hist))
freeze=dict(contract_sha256=sha(root/'contract.json'),continuation_contract_sha256=sha(root/'continuation-contract.json'),
canonical_failures=[dict(case=failed)],parts={case:dict(result=str(result),result_sha256=sha(result),predictions=str(pred),predictions_sha256=sha(pred))})
save(root/'pass-predictions-freeze.json',freeze)
q=dict(raw_ids=raw,counts=counts,near_wall=np.array([True,True]),coordinates_over_Rg=np.array([[0.,0,0],[1,0,0]]))
with patch.object(scorer,'contract',return_value=c),patch.object(scorer,'prediction_inputs',return_value=q):scorer.score(root)
summary=json.loads((root/'postfit/score-summary.json').read_text())
self.assertEqual(summary['pass_prediction_parts'],1);self.assertEqual(summary['new_decodes'],0)
self.assertAlmostEqual(summary['summary']['relative_l2']['contrasts']['GRAPH']['value'],-1)
pred.write_bytes(b'tampered');labels.unlink()
with patch.object(scorer,'contract',return_value=c):
with self.assertRaisesRegex(ValueError,'SHA mismatch'):scorer.score(root)
if __name__=='__main__':
out=Path(sys.argv[1]);out.mkdir(parents=True,exist_ok=False)
with (out/'tests.log').open('x') as log:
r=unittest.TextTestRunner(stream=log,verbosity=2).run(unittest.defaultTestLoader.loadTestsFromTestCase(Checks))
save(out/'receipt.json',dict(status='PASS' if r.wasSuccessful() else 'FAIL',tests=r.testsRun,failures=len(r.failures),errors=len(r.errors),
cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),scope='synthetic only; no real labels or geometry',
code_sha256={n:sha(n) for n in ['tools/r4a_poststop_score.py','tools/run_r4a_recovery.py','tools/test_r4a_recovery.py']}))
print((out/'receipt.json').read_text());raise SystemExit(0 if r.wasSuccessful() else 1)