126 lines
6.6 KiB
Python
126 lines
6.6 KiB
Python
"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
|
|
import copy
|
|
import hashlib
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from phase2_h1_worker_v2 import save, sha
|
|
from phase3_item1_postfit_common import read, require_budget
|
|
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
|
|
from phase3_item1_resume import install_reuse
|
|
import phase3_item1_resume_guard as guard
|
|
|
|
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
|
|
|
|
|
|
class RecoveryTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.old = read(O / 'contract.json')
|
|
self.done = read(O / 'complete.json')
|
|
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
|
|
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
|
|
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
|
|
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
|
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
|
|
training_gpu_once=self.done['training_gpu_seconds_once'])
|
|
|
|
def test_scope_and_actual_original_stop(self):
|
|
validate_scope(self.c, self.old)
|
|
validate_stop(self.done)
|
|
|
|
def test_no_predict0_rerun(self):
|
|
self.c['resume_indices'] = list(range(7))
|
|
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
|
|
|
def test_gpu1_only(self):
|
|
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
|
|
self.c['gpu_uuid'] = uuid
|
|
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
|
|
|
def test_no_budget_change(self):
|
|
self.c['postfit_gpu_cap'] = 7201
|
|
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
|
|
|
|
def test_failed_job_worker_must_be_null(self):
|
|
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
|
|
with self.assertRaises(ValueError): validate_stop(self.done)
|
|
|
|
def test_predict0_must_be_pass(self):
|
|
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
|
|
with self.assertRaises(ValueError): validate_stop(self.done)
|
|
|
|
def test_old_cost_once_not_per_job(self):
|
|
a = cumulative_cost(self.c, 100., 200.)
|
|
b = cumulative_cost(self.c, 300., 500.)
|
|
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
|
|
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
|
|
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
|
|
|
|
def test_old_cost_limits_remaining_reserve(self):
|
|
d = cumulative_cost(self.c,60,240)
|
|
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
|
|
with self.assertRaises(RuntimeError):
|
|
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
|
|
|
|
def test_original_six_receipts_currently_clean(self):
|
|
self.assertEqual(len(six_receipts(self.c)['rows']),6)
|
|
|
|
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
|
|
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
|
|
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
|
|
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
|
|
with self.assertRaises(FileExistsError):install_reuse(root,[e])
|
|
|
|
def test_reuse_bad_hash_rejected_before_write(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
|
|
with self.assertRaises(ValueError):
|
|
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
|
|
self.assertEqual(list(root.iterdir()),[])
|
|
|
|
def guarded(self, mode):
|
|
timer=[0.];calls=[0]
|
|
def sleep(seconds): timer[0]+=seconds
|
|
def probe(uuid):
|
|
calls[0]+=1
|
|
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
|
|
return 0,0
|
|
class Thread:
|
|
def __init__(self,*a,**k):pass
|
|
def start(self):pass
|
|
def join(self,**k):pass
|
|
def is_alive(self):return False
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
|
|
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
|
|
def supervisor(*a,**k):
|
|
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
|
|
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
|
|
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
|
|
result=guard.run_prediction(root,c,{},1,0,300)
|
|
return result,sup.call_count,read(out/'recovery-idle-wait.json')
|
|
|
|
def test_busy_then_idle_counts_wait_and_three_polls(self):
|
|
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
|
|
self.assertEqual(r['gpu_seconds_charged'],65.)
|
|
self.assertEqual(len(w['samples']),3)
|
|
|
|
def test_busy_timeout_stops_without_worker(self):
|
|
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
|
|
self.assertEqual(r['gpu_seconds_charged'],80.)
|
|
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
|
|
|
|
def test_supervisor_monitor_error_still_rejected(self):
|
|
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
|
|
|
|
def test_wait_limit_cannot_be_relaxed(self):
|
|
self.c['idle_wait_seconds']=21
|
|
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
|
|
|
|
|
|
if __name__ == '__main__':unittest.main()
|