Files
MeltMind/tools/test_track_a_recovery.py
T

239 lines
18 KiB
Python

"""Synthetic-only recovery reuse, monitoring, isolation and ledger checks."""
import ast,copy,json,subprocess,sys,tempfile,unittest
from pathlib import Path
from unittest.mock import patch
from track_a_recovery_monitor import Monitor,MonitorFailure,device_usage
from track_a_recovery_common import OLD,project_total,canonical_models,accepted_guard,first_epoch_release
from phase2_h1_worker_v2 import save,sha
B=Path(__file__).resolve().parents[1]
class RecoveryTests(unittest.TestCase):
def test_two_failures_then_success_reset_and_retain_evidence(self):
with tempfile.TemporaryDirectory() as t:
m=Monitor(Path(t)/'errors.jsonl')
def bad():raise subprocess.TimeoutExpired(['nvidia-smi'],10)
self.assertEqual(m.sample('gpu',bad),(None,False));m.sample('gpu',bad)
self.assertFalse(m.failed);self.assertEqual(m.sample('gpu',lambda:(12,0)),((12,0),True));self.assertEqual(m.streaks['gpu'],0)
m.sample('gpu',bad);self.assertFalse(m.failed);self.assertEqual(m.errors,3)
self.assertEqual(len((Path(t)/'errors.jsonl').read_text().splitlines()),3)
def test_third_consecutive_failure_is_terminal(self):
with tempfile.TemporaryDirectory() as t:
m=Monitor(Path(t)/'errors.jsonl')
for _ in range(3):m.sample('gpu',lambda:1/0)
self.assertTrue(m.failed);m.sample('gpu',lambda:0);self.assertTrue(m.failed)
def test_success_in_other_stream_does_not_hide_gpu_failures(self):
with tempfile.TemporaryDirectory() as t:
m=Monitor(Path(t)/'errors.jsonl')
for _ in range(3):m.sample('gpu',lambda:1/0);m.sample('rss',lambda:100)
self.assertTrue(m.failed);self.assertEqual(m.max_streaks['gpu'],3)
def test_missing_is_not_zero_resource_sample(self):
with tempfile.TemporaryDirectory() as t:
m=Monitor(Path(t)/'errors.jsonl');value,ok=m.sample('gpu',lambda:1/0)
self.assertIsNone(value);self.assertFalse(ok);self.assertFalse(m.failed)
def test_final_sample_retry_bounded_at_three(self):
with tempfile.TemporaryDirectory() as t:
m=Monitor(Path(t)/'errors.jsonl')
with self.assertRaises(MonitorFailure):m.retry('final',lambda:1/0,0)
self.assertEqual(m.errors,3)
def test_device_query_uses_ten_seconds_and_parses(self):
with patch('track_a_recovery_monitor.subprocess.run',return_value=type('R',(),dict(stdout='4, 0\n'))()) as run:
self.assertEqual(device_usage('fixed'),(4*2**20,0));self.assertEqual(run.call_args.kwargs['timeout'],10)
def test_original_worker_science_ast_unchanged(self):
old=ast.parse((B/'tools/track_a_h2_worker.py').read_text());new=ast.parse((B/'tools/track_a_recovery_worker.py').read_text())
for n in ast.walk(new):
if isinstance(n,ast.ImportFrom) and n.module=='track_a_recovery_common':n.module='track_a_h2_common'
run=next(n for n in new.body if isinstance(n,ast.FunctionDef) and n.name=='run')
self.assertIsInstance(run.body[1],ast.If);run.body.pop(1)
self.assertEqual(ast.dump(old,include_attributes=False),ast.dump(new,include_attributes=False))
def cfg(self):return dict(jobs=[dict(id=i,gpu_seconds=30960) for i in range(5)],gpu_cap=162000,postfit_gpu_reserve=7200)
def test_old_once_and_only_two_new_jobs_in_projection(self):
p=project_total(self.cfg(),{},{},{})
self.assertEqual(p['new_projected_gpu_seconds'],61920+7200)
self.assertEqual(p['cumulative_projected_gpu_seconds'],OLD+61920+7200);self.assertEqual(p['status'],'PASS')
with self.assertRaises(ValueError):project_total(self.cfg(),{}, {0:dict(gpu_seconds_charged=100)}, {})
def test_failed_terminal_cost_not_dropped_and_replaces_projection(self):
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=30000)},{3:dict(gpu_seconds_charged=123,status='FAILED')},{4:100})
self.assertEqual(p['new_projected_gpu_seconds'],123+30960+7200)
def test_actual_charge_floor_and_nonfinite_rejection(self):
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=200)}, {},{3:500})
self.assertEqual(p['rows'][0]['projected_gpu_seconds'],500)
with self.assertRaises(ValueError):project_total(self.cfg(),{}, {},{3:float('nan')})
def test_no_partial_model_adoption(self):
c=dict(reused_models=[dict(job=i) for i in range(3)])
self.assertEqual(len(canonical_models(c,[dict(job=3),dict(job=4)])),5)
for models in [[],[dict(job=3)],[dict(job=3),dict(job=3)]]:
with self.assertRaises(ValueError):canonical_models(c,models)
def test_transient_logged_monitor_errors_can_pass_but_fatal_cannot(self):
w=dict(status='PASS',returncode=0,monitor_errors=2,monitor_policy=dict(failed=False,errors=2))
g=dict(status='PASS',reasons=[],monitor_errors=[{},{}],monitor_policy=dict(failed=False,errors=2),worker=w)
self.assertTrue(accepted_guard(0,g));self.assertFalse(accepted_guard(1,g))
bad=copy.deepcopy(g);bad['worker']['monitor_policy']['failed']=True;self.assertFalse(accepted_guard(0,bad))
bad=copy.deepcopy(g);bad['monitor_policy']['failed']=True;self.assertFalse(accepted_guard(0,bad))
bad=copy.deepcopy(g);bad['worker']['monitor_errors']=[];self.assertFalse(accepted_guard(0,bad))
def test_actual_collect_failure_does_not_stop_other_job(self):
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='collect')
with tempfile.TemporaryDirectory() as t:
root=Path(t);p=root/'jobs/3';p.mkdir(parents=True)
save(p/'complete.json',dict(status='STOPPED_OR_FAILED',reasons=['THREE_CONSECUTIVE_MONITOR_FAILURES'],monitor_policy=dict(failed=True)))
class Proc:
def poll(self):return 1
class Log:
def close(self):pass
terminal={};other=object();v=dict(process=Proc(),log=Log(),wait=10);active={3:v,4:other};calls=[]
ns=dict(root=root,read=lambda p:json.loads(p.read_text()),accepted_guard=accepted_guard,age=lambda:20,terminal=terminal,active=active,sha=sha,ledger=lambda:calls.append('ledger'))
exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_collect','exec'),ns)
self.assertTrue(ns['collect'](3,v));self.assertIs(active[4],other);self.assertFalse((root/'global-stop.json').exists());self.assertEqual(terminal[3]['gpu_seconds_charged'],80)
def test_own_cost_gate_stop_does_not_block_other_fold(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'jobs/3').mkdir(parents=True);(root/'jobs/4').mkdir();(root/'contract.json').write_text('{}')
save(root/'jobs/3/stop.json',dict(reason='own'))
save(root/'jobs/4/first-epoch-projection.json',dict(status='PASS',contract_sha256=sha(root/'contract.json')))
with self.assertRaises(RuntimeError):first_epoch_release(root,3)
self.assertTrue(first_epoch_release(root,4))
def test_reused_fold_execution_rejected(self):
with self.assertRaises(PermissionError):first_epoch_release(Path('/synthetic'),0)
def test_dispatch_busy_gpu_never_launches_or_changes_other_queue(self):
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='dispatch')
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'jobs/4').mkdir(parents=True)
pending=[4];active={};calls=[];idle={4:Monitor(root/'jobs/4/errors.jsonl')}
jobs=[dict(gpu_seconds=30960,gpu_uuid='GPU2') for _ in range(5)]
ns=dict(root=root,c=dict(jobs=jobs),pending=pending,active=active,idle=idle,age=lambda:100,device_usage=lambda u:(1024**3,90),save=save,time=__import__('time'),subprocess=subprocess,sys=sys,settle_unlaunched=lambda *a:calls.append(a))
exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_dispatch','exec'),ns)
with patch.object(subprocess,'Popen') as spawn:ns['dispatch']();spawn.assert_not_called()
self.assertEqual(pending,[4]);self.assertFalse(calls);self.assertFalse(active)
snap=json.loads((root/'jobs/4/dispatch-idle.json').read_text());self.assertTrue(snap['valid']);self.assertEqual(snap['charged_gpu_seconds'],160)
def test_dispatch_three_query_failures_isolate_unlaunched_job(self):
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='dispatch')
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'jobs/4').mkdir(parents=True);pending=[4];other=object();active={3:other};calls=[];idle={4:Monitor(root/'jobs/4/errors.jsonl')}
def bad(u):raise subprocess.TimeoutExpired(['nvidia-smi'],10)
def settle(i,why):calls.append((i,why));pending.remove(i)
ns=dict(root=root,c=dict(jobs=[dict(gpu_seconds=30960,gpu_uuid='GPU2') for _ in range(5)]),pending=pending,active=active,idle=idle,age=lambda:100,device_usage=bad,save=save,time=__import__('time'),settle_unlaunched=settle)
exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_dispatch','exec'),ns)
ns['dispatch']();ns['dispatch']();self.assertFalse(calls);ns['dispatch']()
self.assertEqual(calls,[(4,'THREE_FAILED_DISPATCH_GPU_SAMPLES')]);self.assertIs(active[3],other)
def test_extreme_rejected_projection_cannot_stop_affordable_survivor(self):
from types import SimpleNamespace
from track_a_recovery_common import project_cell
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());node=next(n for n in ast.walk(tree) if isinstance(n,ast.For) and ast.unparse(n.iter)=='list(active)')
with tempfile.TemporaryDirectory() as t:
root=Path(t);save(root/'contract.json',{})
for i in [3,4]:(root/'jobs'/str(i)/'worker/history').mkdir(parents=True)
save(root/'jobs/3/worker/history/001.json',dict(epoch=1,train_epoch_seconds=1000))
save(root/'jobs/4/worker/history/001.json',dict(epoch=1,train_epoch_seconds=50))
own=[];glob=[]
def stop(i,why):own.append((i,why));save(root/'jobs'/str(i)/'stop.json',dict(reason=why))
ns=dict(root=root,c=self.cfg(),active={3:object(),4:object()},terminal={},estimates={},last_epoch={},read=lambda p:json.loads(p.read_text()),save=save,sha=sha,age=lambda:100,live=lambda:{3:160,4:160},project_cell=project_cell,project_total=project_total,time=SimpleNamespace(time=lambda:0),own_stop=stop,global_stop=glob.append)
exec(compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),'actual_epoch_block','exec'),ns)
self.assertEqual(own,[(3,'OWN_PROJECTED_CAP')]);self.assertFalse(glob)
failed=json.loads((root/'jobs/3/first-epoch-projection.json').read_text());healthy=json.loads((root/'jobs/4/first-epoch-projection.json').read_text())
self.assertEqual(failed['cell']['projected_gpu_seconds'],249030);self.assertEqual(failed['status'],'STOP_OWN_COST_PROJECTION')
self.assertEqual(failed['total']['cumulative_projected_gpu_seconds'],OLD+30960+30960+7200)
self.assertEqual(healthy['status'],'PASS');self.assertFalse((root/'jobs/4/stop.json').exists())
self.assertEqual(ns['estimates'][3]['projected_gpu_seconds'],249030)
def test_stopping_reserve_keeps_actual_and_terminal_settles_once(self):
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)}, {},{3:40000},stopping={3})
self.assertEqual(p['rows'][0]['projected_gpu_seconds'],40000)
self.assertEqual(p['rows'][0]['basis'],'locally_stopping_own_cap_or_actual')
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)},{3:dict(gpu_seconds_charged=190)}, {},stopping={3})
self.assertEqual(p['rows'][0]['projected_gpu_seconds'],190);self.assertEqual(p['rows'][0]['basis'],'terminal_actual')
def test_true_cumulative_overrun_still_stops_after_local_rejection(self):
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)}, {},{3:130000,4:500},stopping={3})
self.assertEqual(p['status'],'STOP_CUMULATIVE_BUDGET');self.assertGreater(p['cumulative_projected_gpu_seconds'],162000)
def idle_replay(self,clock,query,cap,sleep):
from types import SimpleNamespace
from track_a_recovery_guard import require_prestart_window
tree=ast.parse((B/'tools/track_a_recovery_guard.py').read_text());node=next(n for n in ast.walk(tree) if isinstance(n,ast.While) and ast.unparse(n.test)=='success < 3')
with tempfile.TemporaryDirectory() as t:
root=Path(t);monitor=Monitor(root/'errors.jsonl')
ns=dict(root=root,queue=root,requested=[],monitor=monitor,success=0,samples=[],device_usage=query,job=dict(gpu_uuid='SYNTHETIC',gpu_seconds=cap),elapsed=clock,require_prestart_window=require_prestart_window,time=SimpleNamespace(time=clock,sleep=sleep),save=save)
started=False
try:
exec(compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),'actual_idle_loop','exec'),ns)
started=True
except RuntimeError as exc:reason=str(exc)
else:reason=None
return dict(worker_would_start=started,reason=reason,policy=monitor.policy(),errors=list(monitor.records),successes=ns['success'])
def test_already_expired_prestart_makes_zero_queries(self):
calls=[]
d=self.idle_replay(lambda:30961,lambda u:calls.append(u),30960,lambda s:None)
self.assertEqual(calls,[]);self.assertFalse(d['worker_would_start']);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
def test_alternating_two_errors_one_idle_expires_without_worker(self):
clock=[60.];calls=[]
def query(u):
calls.append(u);clock[0]+=10
if len(calls)%3:raise TimeoutError('synthetic recoverable')
return 0,0
def sleep(s):clock[0]+=s
d=self.idle_replay(lambda:clock[0],query,160,sleep)
self.assertFalse(d['worker_would_start']);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
self.assertFalse(d['policy']['failed']);self.assertEqual(d['policy']['max_streaks']['gpu_usage'],2)
self.assertLess(clock[0],160);self.assertLessEqual(len(calls),7);self.assertGreater(len(d['errors']),3)
def test_prestart_checks_after_query_overrun(self):
clock=[60.];calls=[]
def query(u):calls.append(u);clock[0]=85;return 0,0
d=self.idle_replay(lambda:clock[0],query,100,lambda s:None)
self.assertFalse(d['worker_would_start']);self.assertEqual(len(calls),1);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
def test_prestart_checks_before_idle_sleep(self):
clock=[60.];sleeps=[]
def query(u):clock[0]=79.5;return 0,0
d=self.idle_replay(lambda:clock[0],query,100,sleeps.append)
self.assertFalse(d['worker_would_start']);self.assertEqual(sleeps,[]);self.assertEqual(d['successes'],1)
def test_fast_local_guard_not_blocked_on_gpu_query(self):
tree=ast.parse((B/'tools/track_a_recovery_guard.py').read_text());local=next(n for n in ast.walk(tree) if isinstance(n,ast.FunctionDef) and n.name=='local_watch')
self.assertNotIn('device_usage',ast.unparse(local));self.assertIn('WHOLE_TREE_RSS_LIMIT',ast.unparse(local));self.assertIn('OUTPUT_DISK_LIMIT',ast.unparse(local));self.assertIn('RECORDED_VRAM_LIMIT',(B/'tools/track_a_recovery_guard.py').read_text())
def test_supervisor_real_breach_fails_without_three_monitor_errors(self):
from track_a_recovery_supervisor import supervise
with tempfile.TemporaryDirectory() as t:
r=supervise([sys.executable,'-c','import time; time.sleep(2)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=1,disk_bytes=1000000,max_processes=1),poll=.01)
self.assertNotEqual(r['status'],'PASS');self.assertIn('rss',r['stop_reason']);self.assertFalse(r['monitor_policy']['failed'])
def test_supervisor_transient_tree_failures_recover_and_log(self):
import track_a_recovery_supervisor as s
real=s.tree_usage;calls=[0]
def sample(pid):
calls[0]+=1
if calls[0]<=2:raise OSError('synthetic transient')
return real(pid)
with tempfile.TemporaryDirectory() as t,patch.object(s,'tree_usage',side_effect=sample):
r=s.supervise([sys.executable,'-c','import time; time.sleep(.15)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=2**30,disk_bytes=1000000,max_processes=1),poll=.01)
self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],2);self.assertFalse(r['monitor_policy']['failed'])
def test_supervisor_three_failed_tree_samples_stop_only_child(self):
import track_a_recovery_supervisor as s
with tempfile.TemporaryDirectory() as t,patch.object(s,'tree_usage',side_effect=OSError('synthetic persistent')):
r=s.supervise([sys.executable,'-c','import time; time.sleep(2)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=2**30,disk_bytes=1000000,max_processes=1),poll=.01)
self.assertNotEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],3);self.assertEqual(r['stop_reason'],'three_consecutive_monitor_failures')
if __name__=='__main__':unittest.main(verbosity=2)