"""Synthetic-only recovery reuse, monitoring, isolation and ledger checks.""" import ast,copy,json,subprocess,sys,tempfile,unittest from pathlib import Path from unittest.mock import patch from track_a_recovery_monitor import Monitor,MonitorFailure,device_usage from track_a_recovery_common import OLD,project_total,canonical_models,accepted_guard,first_epoch_release from phase2_h1_worker_v2 import save,sha B=Path(__file__).resolve().parents[1] class RecoveryTests(unittest.TestCase): def test_two_failures_then_success_reset_and_retain_evidence(self): with tempfile.TemporaryDirectory() as t: m=Monitor(Path(t)/'errors.jsonl') def bad():raise subprocess.TimeoutExpired(['nvidia-smi'],10) self.assertEqual(m.sample('gpu',bad),(None,False));m.sample('gpu',bad) self.assertFalse(m.failed);self.assertEqual(m.sample('gpu',lambda:(12,0)),((12,0),True));self.assertEqual(m.streaks['gpu'],0) m.sample('gpu',bad);self.assertFalse(m.failed);self.assertEqual(m.errors,3) self.assertEqual(len((Path(t)/'errors.jsonl').read_text().splitlines()),3) def test_third_consecutive_failure_is_terminal(self): with tempfile.TemporaryDirectory() as t: m=Monitor(Path(t)/'errors.jsonl') for _ in range(3):m.sample('gpu',lambda:1/0) self.assertTrue(m.failed);m.sample('gpu',lambda:0);self.assertTrue(m.failed) def test_success_in_other_stream_does_not_hide_gpu_failures(self): with tempfile.TemporaryDirectory() as t: m=Monitor(Path(t)/'errors.jsonl') for _ in range(3):m.sample('gpu',lambda:1/0);m.sample('rss',lambda:100) self.assertTrue(m.failed);self.assertEqual(m.max_streaks['gpu'],3) def test_missing_is_not_zero_resource_sample(self): with tempfile.TemporaryDirectory() as t: m=Monitor(Path(t)/'errors.jsonl');value,ok=m.sample('gpu',lambda:1/0) self.assertIsNone(value);self.assertFalse(ok);self.assertFalse(m.failed) def test_final_sample_retry_bounded_at_three(self): with tempfile.TemporaryDirectory() as t: m=Monitor(Path(t)/'errors.jsonl') with self.assertRaises(MonitorFailure):m.retry('final',lambda:1/0,0) self.assertEqual(m.errors,3) def test_device_query_uses_ten_seconds_and_parses(self): with patch('track_a_recovery_monitor.subprocess.run',return_value=type('R',(),dict(stdout='4, 0\n'))()) as run: self.assertEqual(device_usage('fixed'),(4*2**20,0));self.assertEqual(run.call_args.kwargs['timeout'],10) def test_original_worker_science_ast_unchanged(self): old=ast.parse((B/'tools/track_a_h2_worker.py').read_text());new=ast.parse((B/'tools/track_a_recovery_worker.py').read_text()) for n in ast.walk(new): if isinstance(n,ast.ImportFrom) and n.module=='track_a_recovery_common':n.module='track_a_h2_common' run=next(n for n in new.body if isinstance(n,ast.FunctionDef) and n.name=='run') self.assertIsInstance(run.body[1],ast.If);run.body.pop(1) self.assertEqual(ast.dump(old,include_attributes=False),ast.dump(new,include_attributes=False)) def cfg(self):return dict(jobs=[dict(id=i,gpu_seconds=30960) for i in range(5)],gpu_cap=162000,postfit_gpu_reserve=7200) def test_old_once_and_only_two_new_jobs_in_projection(self): p=project_total(self.cfg(),{},{},{}) self.assertEqual(p['new_projected_gpu_seconds'],61920+7200) self.assertEqual(p['cumulative_projected_gpu_seconds'],OLD+61920+7200);self.assertEqual(p['status'],'PASS') with self.assertRaises(ValueError):project_total(self.cfg(),{}, {0:dict(gpu_seconds_charged=100)}, {}) def test_failed_terminal_cost_not_dropped_and_replaces_projection(self): p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=30000)},{3:dict(gpu_seconds_charged=123,status='FAILED')},{4:100}) self.assertEqual(p['new_projected_gpu_seconds'],123+30960+7200) def test_actual_charge_floor_and_nonfinite_rejection(self): p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=200)}, {},{3:500}) self.assertEqual(p['rows'][0]['projected_gpu_seconds'],500) with self.assertRaises(ValueError):project_total(self.cfg(),{}, {},{3:float('nan')}) def test_no_partial_model_adoption(self): c=dict(reused_models=[dict(job=i) for i in range(3)]) self.assertEqual(len(canonical_models(c,[dict(job=3),dict(job=4)])),5) for models in [[],[dict(job=3)],[dict(job=3),dict(job=3)]]: with self.assertRaises(ValueError):canonical_models(c,models) def test_transient_logged_monitor_errors_can_pass_but_fatal_cannot(self): w=dict(status='PASS',returncode=0,monitor_errors=2,monitor_policy=dict(failed=False,errors=2)) g=dict(status='PASS',reasons=[],monitor_errors=[{},{}],monitor_policy=dict(failed=False,errors=2),worker=w) self.assertTrue(accepted_guard(0,g));self.assertFalse(accepted_guard(1,g)) bad=copy.deepcopy(g);bad['worker']['monitor_policy']['failed']=True;self.assertFalse(accepted_guard(0,bad)) bad=copy.deepcopy(g);bad['monitor_policy']['failed']=True;self.assertFalse(accepted_guard(0,bad)) bad=copy.deepcopy(g);bad['worker']['monitor_errors']=[];self.assertFalse(accepted_guard(0,bad)) def test_actual_collect_failure_does_not_stop_other_job(self): tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='collect') with tempfile.TemporaryDirectory() as t: root=Path(t);p=root/'jobs/3';p.mkdir(parents=True) save(p/'complete.json',dict(status='STOPPED_OR_FAILED',reasons=['THREE_CONSECUTIVE_MONITOR_FAILURES'],monitor_policy=dict(failed=True))) class Proc: def poll(self):return 1 class Log: def close(self):pass terminal={};other=object();v=dict(process=Proc(),log=Log(),wait=10);active={3:v,4:other};calls=[] ns=dict(root=root,read=lambda p:json.loads(p.read_text()),accepted_guard=accepted_guard,age=lambda:20,terminal=terminal,active=active,sha=sha,ledger=lambda:calls.append('ledger')) exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_collect','exec'),ns) self.assertTrue(ns['collect'](3,v));self.assertIs(active[4],other);self.assertFalse((root/'global-stop.json').exists());self.assertEqual(terminal[3]['gpu_seconds_charged'],80) def test_own_cost_gate_stop_does_not_block_other_fold(self): with tempfile.TemporaryDirectory() as t: root=Path(t);(root/'jobs/3').mkdir(parents=True);(root/'jobs/4').mkdir();(root/'contract.json').write_text('{}') save(root/'jobs/3/stop.json',dict(reason='own')) save(root/'jobs/4/first-epoch-projection.json',dict(status='PASS',contract_sha256=sha(root/'contract.json'))) with self.assertRaises(RuntimeError):first_epoch_release(root,3) self.assertTrue(first_epoch_release(root,4)) def test_reused_fold_execution_rejected(self): with self.assertRaises(PermissionError):first_epoch_release(Path('/synthetic'),0) def test_dispatch_busy_gpu_never_launches_or_changes_other_queue(self): tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='dispatch') with tempfile.TemporaryDirectory() as t: root=Path(t);(root/'jobs/4').mkdir(parents=True) pending=[4];active={};calls=[];idle={4:Monitor(root/'jobs/4/errors.jsonl')} jobs=[dict(gpu_seconds=30960,gpu_uuid='GPU2') for _ in range(5)] ns=dict(root=root,c=dict(jobs=jobs),pending=pending,active=active,idle=idle,age=lambda:100,device_usage=lambda u:(1024**3,90),save=save,time=__import__('time'),subprocess=subprocess,sys=sys,settle_unlaunched=lambda *a:calls.append(a)) exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_dispatch','exec'),ns) with patch.object(subprocess,'Popen') as spawn:ns['dispatch']();spawn.assert_not_called() self.assertEqual(pending,[4]);self.assertFalse(calls);self.assertFalse(active) snap=json.loads((root/'jobs/4/dispatch-idle.json').read_text());self.assertTrue(snap['valid']);self.assertEqual(snap['charged_gpu_seconds'],160) def test_dispatch_three_query_failures_isolate_unlaunched_job(self): tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='dispatch') with tempfile.TemporaryDirectory() as t: root=Path(t);(root/'jobs/4').mkdir(parents=True);pending=[4];other=object();active={3:other};calls=[];idle={4:Monitor(root/'jobs/4/errors.jsonl')} def bad(u):raise subprocess.TimeoutExpired(['nvidia-smi'],10) def settle(i,why):calls.append((i,why));pending.remove(i) ns=dict(root=root,c=dict(jobs=[dict(gpu_seconds=30960,gpu_uuid='GPU2') for _ in range(5)]),pending=pending,active=active,idle=idle,age=lambda:100,device_usage=bad,save=save,time=__import__('time'),settle_unlaunched=settle) exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_dispatch','exec'),ns) ns['dispatch']();ns['dispatch']();self.assertFalse(calls);ns['dispatch']() self.assertEqual(calls,[(4,'THREE_FAILED_DISPATCH_GPU_SAMPLES')]);self.assertIs(active[3],other) def test_extreme_rejected_projection_cannot_stop_affordable_survivor(self): from types import SimpleNamespace from track_a_recovery_common import project_cell tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());node=next(n for n in ast.walk(tree) if isinstance(n,ast.For) and ast.unparse(n.iter)=='list(active)') with tempfile.TemporaryDirectory() as t: root=Path(t);save(root/'contract.json',{}) for i in [3,4]:(root/'jobs'/str(i)/'worker/history').mkdir(parents=True) save(root/'jobs/3/worker/history/001.json',dict(epoch=1,train_epoch_seconds=1000)) save(root/'jobs/4/worker/history/001.json',dict(epoch=1,train_epoch_seconds=50)) own=[];glob=[] def stop(i,why):own.append((i,why));save(root/'jobs'/str(i)/'stop.json',dict(reason=why)) ns=dict(root=root,c=self.cfg(),active={3:object(),4:object()},terminal={},estimates={},last_epoch={},read=lambda p:json.loads(p.read_text()),save=save,sha=sha,age=lambda:100,live=lambda:{3:160,4:160},project_cell=project_cell,project_total=project_total,time=SimpleNamespace(time=lambda:0),own_stop=stop,global_stop=glob.append) exec(compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),'actual_epoch_block','exec'),ns) self.assertEqual(own,[(3,'OWN_PROJECTED_CAP')]);self.assertFalse(glob) failed=json.loads((root/'jobs/3/first-epoch-projection.json').read_text());healthy=json.loads((root/'jobs/4/first-epoch-projection.json').read_text()) self.assertEqual(failed['cell']['projected_gpu_seconds'],249030);self.assertEqual(failed['status'],'STOP_OWN_COST_PROJECTION') self.assertEqual(failed['total']['cumulative_projected_gpu_seconds'],OLD+30960+30960+7200) self.assertEqual(healthy['status'],'PASS');self.assertFalse((root/'jobs/4/stop.json').exists()) self.assertEqual(ns['estimates'][3]['projected_gpu_seconds'],249030) def test_stopping_reserve_keeps_actual_and_terminal_settles_once(self): p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)}, {},{3:40000},stopping={3}) self.assertEqual(p['rows'][0]['projected_gpu_seconds'],40000) self.assertEqual(p['rows'][0]['basis'],'locally_stopping_own_cap_or_actual') p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)},{3:dict(gpu_seconds_charged=190)}, {},stopping={3}) self.assertEqual(p['rows'][0]['projected_gpu_seconds'],190);self.assertEqual(p['rows'][0]['basis'],'terminal_actual') def test_true_cumulative_overrun_still_stops_after_local_rejection(self): p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)}, {},{3:130000,4:500},stopping={3}) self.assertEqual(p['status'],'STOP_CUMULATIVE_BUDGET');self.assertGreater(p['cumulative_projected_gpu_seconds'],162000) def idle_replay(self,clock,query,cap,sleep): from types import SimpleNamespace from track_a_recovery_guard import require_prestart_window tree=ast.parse((B/'tools/track_a_recovery_guard.py').read_text());node=next(n for n in ast.walk(tree) if isinstance(n,ast.While) and ast.unparse(n.test)=='success < 3') with tempfile.TemporaryDirectory() as t: root=Path(t);monitor=Monitor(root/'errors.jsonl') ns=dict(root=root,queue=root,requested=[],monitor=monitor,success=0,samples=[],device_usage=query,job=dict(gpu_uuid='SYNTHETIC',gpu_seconds=cap),elapsed=clock,require_prestart_window=require_prestart_window,time=SimpleNamespace(time=clock,sleep=sleep),save=save) started=False try: exec(compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),'actual_idle_loop','exec'),ns) started=True except RuntimeError as exc:reason=str(exc) else:reason=None return dict(worker_would_start=started,reason=reason,policy=monitor.policy(),errors=list(monitor.records),successes=ns['success']) def test_already_expired_prestart_makes_zero_queries(self): calls=[] d=self.idle_replay(lambda:30961,lambda u:calls.append(u),30960,lambda s:None) self.assertEqual(calls,[]);self.assertFalse(d['worker_would_start']);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED') def test_alternating_two_errors_one_idle_expires_without_worker(self): clock=[60.];calls=[] def query(u): calls.append(u);clock[0]+=10 if len(calls)%3:raise TimeoutError('synthetic recoverable') return 0,0 def sleep(s):clock[0]+=s d=self.idle_replay(lambda:clock[0],query,160,sleep) self.assertFalse(d['worker_would_start']);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED') self.assertFalse(d['policy']['failed']);self.assertEqual(d['policy']['max_streaks']['gpu_usage'],2) self.assertLess(clock[0],160);self.assertLessEqual(len(calls),7);self.assertGreater(len(d['errors']),3) def test_prestart_checks_after_query_overrun(self): clock=[60.];calls=[] def query(u):calls.append(u);clock[0]=85;return 0,0 d=self.idle_replay(lambda:clock[0],query,100,lambda s:None) self.assertFalse(d['worker_would_start']);self.assertEqual(len(calls),1);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED') def test_prestart_checks_before_idle_sleep(self): clock=[60.];sleeps=[] def query(u):clock[0]=79.5;return 0,0 d=self.idle_replay(lambda:clock[0],query,100,sleeps.append) self.assertFalse(d['worker_would_start']);self.assertEqual(sleeps,[]);self.assertEqual(d['successes'],1) def test_fast_local_guard_not_blocked_on_gpu_query(self): tree=ast.parse((B/'tools/track_a_recovery_guard.py').read_text());local=next(n for n in ast.walk(tree) if isinstance(n,ast.FunctionDef) and n.name=='local_watch') self.assertNotIn('device_usage',ast.unparse(local));self.assertIn('WHOLE_TREE_RSS_LIMIT',ast.unparse(local));self.assertIn('OUTPUT_DISK_LIMIT',ast.unparse(local));self.assertIn('RECORDED_VRAM_LIMIT',(B/'tools/track_a_recovery_guard.py').read_text()) def test_supervisor_real_breach_fails_without_three_monitor_errors(self): from track_a_recovery_supervisor import supervise with tempfile.TemporaryDirectory() as t: r=supervise([sys.executable,'-c','import time; time.sleep(2)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=1,disk_bytes=1000000,max_processes=1),poll=.01) self.assertNotEqual(r['status'],'PASS');self.assertIn('rss',r['stop_reason']);self.assertFalse(r['monitor_policy']['failed']) def test_supervisor_transient_tree_failures_recover_and_log(self): import track_a_recovery_supervisor as s real=s.tree_usage;calls=[0] def sample(pid): calls[0]+=1 if calls[0]<=2:raise OSError('synthetic transient') return real(pid) with tempfile.TemporaryDirectory() as t,patch.object(s,'tree_usage',side_effect=sample): r=s.supervise([sys.executable,'-c','import time; time.sleep(.15)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=2**30,disk_bytes=1000000,max_processes=1),poll=.01) self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],2);self.assertFalse(r['monitor_policy']['failed']) def test_supervisor_three_failed_tree_samples_stop_only_child(self): import track_a_recovery_supervisor as s with tempfile.TemporaryDirectory() as t,patch.object(s,'tree_usage',side_effect=OSError('synthetic persistent')): r=s.supervise([sys.executable,'-c','import time; time.sleep(2)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=2**30,disk_bytes=1000000,max_processes=1),poll=.01) self.assertNotEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],3);self.assertEqual(r['stop_reason'],'three_consecutive_monitor_failures') if __name__=='__main__':unittest.main(verbosity=2)