Prepare one labelled item1 postfit recovery with bounded idle wait
This commit is contained in:
1 parent
4f646232f4
commit
581208098f
24 files changed
+83789
No files matched your search
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 15,
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"cpu_seconds": 2.229610314,
|
||||
"wall_seconds": 2.240639487048611,
|
||||
"source_sha256": "1a03c6bf7b14dba448ffdff0bd9f414c15b1747ee9657b16e77a485ac95a6f37",
|
||||
"GPU_used": false,
|
||||
"real_scientific_arrays_read": false,
|
||||
"attempt01_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
|
||||
import copy
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from phase2_h1_worker_v2 import save, sha
|
||||
from phase3_item1_postfit_common import read, require_budget
|
||||
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
|
||||
from phase3_item1_resume import install_reuse
|
||||
import phase3_item1_resume_guard as guard
|
||||
|
||||
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
|
||||
|
||||
|
||||
class RecoveryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.old = read(O / 'contract.json')
|
||||
self.done = read(O / 'complete.json')
|
||||
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
|
||||
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
|
||||
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
|
||||
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
|
||||
training_gpu_once=self.done['training_gpu_seconds_once'])
|
||||
|
||||
def test_scope_and_actual_original_stop(self):
|
||||
validate_scope(self.c, self.old)
|
||||
validate_stop(self.done)
|
||||
|
||||
def test_no_predict0_rerun(self):
|
||||
self.c['resume_indices'] = list(range(7))
|
||||
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_gpu1_only(self):
|
||||
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
|
||||
self.c['gpu_uuid'] = uuid
|
||||
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_no_budget_change(self):
|
||||
self.c['postfit_gpu_cap'] = 7201
|
||||
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
|
||||
|
||||
def test_failed_job_worker_must_be_null(self):
|
||||
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
|
||||
with self.assertRaises(ValueError): validate_stop(self.done)
|
||||
|
||||
def test_predict0_must_be_pass(self):
|
||||
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
|
||||
with self.assertRaises(ValueError): validate_stop(self.done)
|
||||
|
||||
def test_old_cost_once_not_per_job(self):
|
||||
a = cumulative_cost(self.c, 100., 200.)
|
||||
b = cumulative_cost(self.c, 300., 500.)
|
||||
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
|
||||
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
|
||||
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
|
||||
|
||||
def test_old_cost_limits_remaining_reserve(self):
|
||||
d = cumulative_cost(self.c,60,180)
|
||||
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
|
||||
with self.assertRaises(RuntimeError):
|
||||
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
|
||||
|
||||
def test_original_six_receipts_currently_clean(self):
|
||||
self.assertEqual(len(six_receipts(self.c)['rows']),6)
|
||||
|
||||
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
|
||||
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
|
||||
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
|
||||
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
|
||||
with self.assertRaises(FileExistsError):install_reuse(root,[e])
|
||||
|
||||
def test_reuse_bad_hash_rejected_before_write(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
|
||||
with self.assertRaises(ValueError):
|
||||
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
|
||||
self.assertEqual(list(root.iterdir()),[])
|
||||
|
||||
def guarded(self, mode):
|
||||
timer=[0.];calls=[0]
|
||||
def sleep(seconds): timer[0]+=seconds
|
||||
def probe(uuid):
|
||||
calls[0]+=1
|
||||
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
|
||||
return 0,0
|
||||
class Thread:
|
||||
def __init__(self,*a,**k):pass
|
||||
def start(self):pass
|
||||
def join(self,**k):pass
|
||||
def is_alive(self):return False
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
|
||||
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
|
||||
def supervisor(*a,**k):
|
||||
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
|
||||
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
|
||||
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
|
||||
result=guard.run_prediction(root,c,{},1,0,300)
|
||||
return result,sup.call_count,read(out/'recovery-idle-wait.json')
|
||||
|
||||
def test_busy_then_idle_counts_wait_and_three_polls(self):
|
||||
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
|
||||
self.assertEqual(r['gpu_seconds_charged'],65.)
|
||||
self.assertEqual(len(w['samples']),3)
|
||||
|
||||
def test_busy_timeout_stops_without_worker(self):
|
||||
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
|
||||
self.assertEqual(r['gpu_seconds_charged'],80.)
|
||||
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
|
||||
|
||||
def test_supervisor_monitor_error_still_rejected(self):
|
||||
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
|
||||
|
||||
def test_wait_limit_cannot_be_relaxed(self):
|
||||
self.c['idle_wait_seconds']=21
|
||||
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
|
||||
|
||||
|
||||
if __name__ == '__main__':unittest.main()
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"status": "FAIL",
|
||||
"tests": 15,
|
||||
"failures": 1,
|
||||
"errors": 0,
|
||||
"unittest_wall_seconds": 1.478,
|
||||
"test_source_sha256": "0835350eae48f3766d8bf34caa58eefe2d0670e5ebea092a47f7df744877c4ea",
|
||||
"failure": "test_old_cost_limits_remaining_reserve expected7 remaining to exceed cap at newCPU180, but total3570.833960341 remains below3600. Incorrect synthetic assertion; no production budget change.",
|
||||
"real_scientific_execution": false
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
test_busy_then_idle_counts_wait_and_three_polls (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_busy_timeout_stops_without_worker (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_failed_job_worker_must_be_null (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_gpu1_only (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_no_budget_change (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_no_predict0_rerun (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_old_cost_limits_remaining_reserve (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_old_cost_once_not_per_job (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_original_six_receipts_currently_clean (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_predict0_must_be_pass (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_reuse_bad_hash_rejected_before_write (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_reuse_copy_preserves_original_and_rejects_duplicate (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_scope_and_actual_original_stop (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_supervisor_monitor_error_still_rejected (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
test_wait_limit_cannot_be_relaxed (test_phase3_item1_resume.RecoveryTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 15 tests in 2.240s
|
||||
|
||||
OK
|
||||
Reference in new issue
Block a user