"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays.""" import copy import hashlib from pathlib import Path import tempfile import unittest from unittest.mock import patch from phase2_h1_worker_v2 import save, sha from phase3_item1_postfit_common import read, require_budget from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts from phase3_item1_resume import install_reuse import phase3_item1_resume_guard as guard O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18') class RecoveryTests(unittest.TestCase): def setUp(self): self.old = read(O / 'contract.json') self.done = read(O / 'complete.json') self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery', recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)), idle_wait_approval='1794981566.532619', idle_wait_seconds=20, gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36', prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'], training_gpu_once=self.done['training_gpu_seconds_once']) def test_scope_and_actual_original_stop(self): validate_scope(self.c, self.old) validate_stop(self.done) def test_no_predict0_rerun(self): self.c['resume_indices'] = list(range(7)) with self.assertRaises(PermissionError): validate_scope(self.c, self.old) def test_gpu1_only(self): for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']: self.c['gpu_uuid'] = uuid with self.assertRaises(PermissionError): validate_scope(self.c, self.old) def test_no_budget_change(self): self.c['postfit_gpu_cap'] = 7201 with self.assertRaises(ValueError): validate_scope(self.c, self.old) def test_failed_job_worker_must_be_null(self): self.done['records'][1]['guard']['worker'] = {'status':'FAILED'} with self.assertRaises(ValueError): validate_stop(self.done) def test_predict0_must_be_pass(self): self.done['records'][0]['guard']['worker']['monitor_errors'] = 1 with self.assertRaises(ValueError): validate_stop(self.done) def test_old_cost_once_not_per_job(self): a = cumulative_cost(self.c, 100., 200.) b = cumulative_cost(self.c, 300., 500.) self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200) self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300) self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300) def test_old_cost_limits_remaining_reserve(self): d = cumulative_cost(self.c,60,240) require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6) with self.assertRaises(RuntimeError): require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7) def test_original_six_receipts_currently_clean(self): self.assertEqual(len(six_receipts(self.c)['rows']),6) def test_reuse_copy_preserves_original_and_rejects_duplicate(self): with tempfile.TemporaryDirectory() as tmp: p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction') root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample') adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction') self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction']) with self.assertRaises(FileExistsError):install_reuse(root,[e]) def test_reuse_bad_hash_rejected_before_write(self): with tempfile.TemporaryDirectory() as tmp: p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir() with self.assertRaises(ValueError): install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')]) self.assertEqual(list(root.iterdir()),[]) def guarded(self, mode): timer=[0.];calls=[0] def sleep(seconds): timer[0]+=seconds def probe(uuid): calls[0]+=1 if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90 return 0,0 class Thread: def __init__(self,*a,**k):pass def start(self):pass def join(self,**k):pass def is_alive(self):return False with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True) c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)]) def supervisor(*a,**k): r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024)) return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0) with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup: result=guard.run_prediction(root,c,{},1,0,300) return result,sup.call_count,read(out/'recovery-idle-wait.json') def test_busy_then_idle_counts_wait_and_three_polls(self): r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1) self.assertEqual(r['gpu_seconds_charged'],65.) self.assertEqual(len(w['samples']),3) def test_busy_timeout_stops_without_worker(self): r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0) self.assertEqual(r['gpu_seconds_charged'],80.) self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons'])) def test_supervisor_monitor_error_still_rejected(self): r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons']) def test_wait_limit_cannot_be_relaxed(self): self.c['idle_wait_seconds']=21 with self.assertRaises(PermissionError):validate_scope(self.c,self.old) if __name__ == '__main__':unittest.main()