Files
MeltMind/tools/test_phase3_item1_resume.py
T

126 lines
6.6 KiB
Python

"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
import copy
import hashlib
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from phase2_h1_worker_v2 import save, sha
from phase3_item1_postfit_common import read, require_budget
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
from phase3_item1_resume import install_reuse
import phase3_item1_resume_guard as guard
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
class RecoveryTests(unittest.TestCase):
def setUp(self):
self.old = read(O / 'contract.json')
self.done = read(O / 'complete.json')
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
training_gpu_once=self.done['training_gpu_seconds_once'])
def test_scope_and_actual_original_stop(self):
validate_scope(self.c, self.old)
validate_stop(self.done)
def test_no_predict0_rerun(self):
self.c['resume_indices'] = list(range(7))
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_gpu1_only(self):
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
self.c['gpu_uuid'] = uuid
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_no_budget_change(self):
self.c['postfit_gpu_cap'] = 7201
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
def test_failed_job_worker_must_be_null(self):
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
with self.assertRaises(ValueError): validate_stop(self.done)
def test_predict0_must_be_pass(self):
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
with self.assertRaises(ValueError): validate_stop(self.done)
def test_old_cost_once_not_per_job(self):
a = cumulative_cost(self.c, 100., 200.)
b = cumulative_cost(self.c, 300., 500.)
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
def test_old_cost_limits_remaining_reserve(self):
d = cumulative_cost(self.c,60,240)
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
with self.assertRaises(RuntimeError):
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
def test_original_six_receipts_currently_clean(self):
self.assertEqual(len(six_receipts(self.c)['rows']),6)
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
with self.assertRaises(FileExistsError):install_reuse(root,[e])
def test_reuse_bad_hash_rejected_before_write(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
with self.assertRaises(ValueError):
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
self.assertEqual(list(root.iterdir()),[])
def guarded(self, mode):
timer=[0.];calls=[0]
def sleep(seconds): timer[0]+=seconds
def probe(uuid):
calls[0]+=1
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
return 0,0
class Thread:
def __init__(self,*a,**k):pass
def start(self):pass
def join(self,**k):pass
def is_alive(self):return False
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
def supervisor(*a,**k):
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
result=guard.run_prediction(root,c,{},1,0,300)
return result,sup.call_count,read(out/'recovery-idle-wait.json')
def test_busy_then_idle_counts_wait_and_three_polls(self):
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
self.assertEqual(r['gpu_seconds_charged'],65.)
self.assertEqual(len(w['samples']),3)
def test_busy_timeout_stops_without_worker(self):
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
self.assertEqual(r['gpu_seconds_charged'],80.)
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
def test_supervisor_monitor_error_still_rejected(self):
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
def test_wait_limit_cannot_be_relaxed(self):
self.c['idle_wait_seconds']=21
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
if __name__ == '__main__':unittest.main()