Prepare one labelled item1 postfit recovery with bounded idle wait

This commit is contained in:
MeltAstra committed 2026-11-18 14:04:43 +08:00
1 parent 4f646232f4
commit 581208098f
24 files changed
+83789

No files matched your search

@@ -0,0 +1,12 @@
{
"status": "PASS",
"tests": 15,
"failures": 0,
"errors": 0,
"cpu_seconds": 2.229610314,
"wall_seconds": 2.240639487048611,
"source_sha256": "1a03c6bf7b14dba448ffdff0bd9f414c15b1747ee9657b16e77a485ac95a6f37",
"GPU_used": false,
"real_scientific_arrays_read": false,
"attempt01_preserved": true
}
@@ -0,0 +1,125 @@
"""Narrow recovery delta checks: synthetic files/receipts, no GPU or target arrays."""
import copy
import hashlib
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from phase2_h1_worker_v2 import save, sha
from phase3_item1_postfit_common import read, require_budget
from phase3_item1_resume_common import validate_scope, validate_stop, cumulative_cost, reuse_entries, six_receipts
from phase3_item1_resume import install_reuse
import phase3_item1_resume_guard as guard
O = Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
class RecoveryTests(unittest.TestCase):
def setUp(self):
self.old = read(O / 'contract.json')
self.done = read(O / 'complete.json')
self.c = dict(self.old, original_root=str(O), postfit_root='/tmp/synthetic-recovery',
recovery_approval='1794980643.205679', recovery_attempt=1, resume_indices=list(range(1,7)),
idle_wait_approval='1794981566.532619', idle_wait_seconds=20,
gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
prior_postfit_gpu=self.done['prediction_gpu_seconds'], prior_postfit_cpu=self.done['postfit_cpu_seconds'],
training_gpu_once=self.done['training_gpu_seconds_once'])
def test_scope_and_actual_original_stop(self):
validate_scope(self.c, self.old)
validate_stop(self.done)
def test_no_predict0_rerun(self):
self.c['resume_indices'] = list(range(7))
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_gpu1_only(self):
for uuid in ['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76']:
self.c['gpu_uuid'] = uuid
with self.assertRaises(PermissionError): validate_scope(self.c, self.old)
def test_no_budget_change(self):
self.c['postfit_gpu_cap'] = 7201
with self.assertRaises(ValueError): validate_scope(self.c, self.old)
def test_failed_job_worker_must_be_null(self):
self.done['records'][1]['guard']['worker'] = {'status':'FAILED'}
with self.assertRaises(ValueError): validate_stop(self.done)
def test_predict0_must_be_pass(self):
self.done['records'][0]['guard']['worker']['monitor_errors'] = 1
with self.assertRaises(ValueError): validate_stop(self.done)
def test_old_cost_once_not_per_job(self):
a = cumulative_cost(self.c, 100., 200.)
b = cumulative_cost(self.c, 300., 500.)
self.assertAlmostEqual(b['prediction_gpu_seconds']-a['prediction_gpu_seconds'],200)
self.assertAlmostEqual(b['postfit_cpu_seconds']-a['postfit_cpu_seconds'],300)
self.assertEqual(b['cumulative_item_gpu_seconds'],self.c['training_gpu_once']+self.c['prior_postfit_gpu']+300)
def test_old_cost_limits_remaining_reserve(self):
d = cumulative_cost(self.c,60,180)
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],6)
with self.assertRaises(RuntimeError):
require_budget(d['training_gpu_seconds_once'],d['prediction_gpu_seconds'],d['postfit_cpu_seconds'],7)
def test_original_six_receipts_currently_clean(self):
self.assertEqual(len(six_receipts(self.c)['rows']),6)
def test_reuse_copy_preserves_original_and_rejects_duplicate(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'original.npz';original.write_bytes(b'opaque synthetic prediction')
root=p/'new';root.mkdir();e=dict(path=str(original),sha256=sha(original),arm='epoch25',case='sample')
adopted=install_reuse(root,[e]);self.assertEqual(original.read_bytes(),b'opaque synthetic prediction')
self.assertEqual(sha(adopted[0]['path']),e['sha256']);self.assertTrue(adopted[0]['reused_without_prediction'])
with self.assertRaises(FileExistsError):install_reuse(root,[e])
def test_reuse_bad_hash_rejected_before_write(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp);original=p/'old.npz';original.write_bytes(b'a');root=p/'new';root.mkdir()
with self.assertRaises(ValueError):
install_reuse(root,[dict(path=str(original),sha256='0'*64,arm='epoch25',case='sample')])
self.assertEqual(list(root.iterdir()),[])
def guarded(self, mode):
timer=[0.];calls=[0]
def sleep(seconds): timer[0]+=seconds
def probe(uuid):
calls[0]+=1
if mode=='timeout' or (mode=='settles' and calls[0]<=2):return 256*2**20,90
return 0,0
class Thread:
def __init__(self,*a,**k):pass
def start(self):pass
def join(self,**k):pass
def is_alive(self):return False
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);out=root/'jobs/1';out.mkdir(parents=True)
c=dict(gpu_uuid='GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',idle_wait_seconds=20,jobs=[{},dict(expected_parts=1)])
def supervisor(*a,**k):
r=out/'prediction-records';r.mkdir();save(r/'result.json',dict(status='COMPLETE',parts=1,peak_allocated_bytes=1024,peak_reserved_bytes=1024))
return dict(status='PASS',cpu_seconds=0,monitor_errors=1 if mode=='monitor_error' else 0)
with patch.object(guard.time,'monotonic',side_effect=lambda:timer[0]),patch.object(guard.time,'sleep',side_effect=sleep),patch.object(guard,'device_usage',side_effect=probe),patch.object(guard.threading,'Thread',Thread),patch.object(guard,'supervise',side_effect=supervisor) as sup:
result=guard.run_prediction(root,c,{},1,0,300)
return result,sup.call_count,read(out/'recovery-idle-wait.json')
def test_busy_then_idle_counts_wait_and_three_polls(self):
r,n,w=self.guarded('settles');self.assertEqual(r['status'],'PASS');self.assertEqual(n,1)
self.assertEqual(r['gpu_seconds_charged'],65.)
self.assertEqual(len(w['samples']),3)
def test_busy_timeout_stops_without_worker(self):
r,n,w=self.guarded('timeout');self.assertEqual(r['status'],'STOPPED');self.assertEqual(n,0)
self.assertEqual(r['gpu_seconds_charged'],80.)
self.assertTrue(any('BOUNDED_WAIT' in x for x in r['reasons']))
def test_supervisor_monitor_error_still_rejected(self):
r,n,w=self.guarded('monitor_error');self.assertEqual(n,1);self.assertIn('WORKER_MONITOR_ERRORS',r['reasons'])
def test_wait_limit_cannot_be_relaxed(self):
self.c['idle_wait_seconds']=21
with self.assertRaises(PermissionError):validate_scope(self.c,self.old)
if __name__ == '__main__':unittest.main()
@@ -0,0 +1,10 @@
{
"status": "FAIL",
"tests": 15,
"failures": 1,
"errors": 0,
"unittest_wall_seconds": 1.478,
"test_source_sha256": "0835350eae48f3766d8bf34caa58eefe2d0670e5ebea092a47f7df744877c4ea",
"failure": "test_old_cost_limits_remaining_reserve expected7 remaining to exceed cap at newCPU180, but total3570.833960341 remains below3600. Incorrect synthetic assertion; no production budget change.",
"real_scientific_execution": false
}
@@ -0,0 +1,20 @@
test_busy_then_idle_counts_wait_and_three_polls (test_phase3_item1_resume.RecoveryTests) ... ok
test_busy_timeout_stops_without_worker (test_phase3_item1_resume.RecoveryTests) ... ok
test_failed_job_worker_must_be_null (test_phase3_item1_resume.RecoveryTests) ... ok
test_gpu1_only (test_phase3_item1_resume.RecoveryTests) ... ok
test_no_budget_change (test_phase3_item1_resume.RecoveryTests) ... ok
test_no_predict0_rerun (test_phase3_item1_resume.RecoveryTests) ... ok
test_old_cost_limits_remaining_reserve (test_phase3_item1_resume.RecoveryTests) ... ok
test_old_cost_once_not_per_job (test_phase3_item1_resume.RecoveryTests) ... ok
test_original_six_receipts_currently_clean (test_phase3_item1_resume.RecoveryTests) ... ok
test_predict0_must_be_pass (test_phase3_item1_resume.RecoveryTests) ... ok
test_reuse_bad_hash_rejected_before_write (test_phase3_item1_resume.RecoveryTests) ... ok
test_reuse_copy_preserves_original_and_rejects_duplicate (test_phase3_item1_resume.RecoveryTests) ... ok
test_scope_and_actual_original_stop (test_phase3_item1_resume.RecoveryTests) ... ok
test_supervisor_monitor_error_still_rejected (test_phase3_item1_resume.RecoveryTests) ... ok
test_wait_limit_cannot_be_relaxed (test_phase3_item1_resume.RecoveryTests) ... ok
----------------------------------------------------------------------
Ran 15 tests in 2.240s
OK