239 lines
18 KiB
Python
239 lines
18 KiB
Python
"""Synthetic-only recovery reuse, monitoring, isolation and ledger checks."""
|
|
import ast,copy,json,subprocess,sys,tempfile,unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
from track_a_recovery_monitor import Monitor,MonitorFailure,device_usage
|
|
from track_a_recovery_common import OLD,project_total,canonical_models,accepted_guard,first_epoch_release
|
|
from phase2_h1_worker_v2 import save,sha
|
|
|
|
B=Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class RecoveryTests(unittest.TestCase):
|
|
def test_two_failures_then_success_reset_and_retain_evidence(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
m=Monitor(Path(t)/'errors.jsonl')
|
|
def bad():raise subprocess.TimeoutExpired(['nvidia-smi'],10)
|
|
self.assertEqual(m.sample('gpu',bad),(None,False));m.sample('gpu',bad)
|
|
self.assertFalse(m.failed);self.assertEqual(m.sample('gpu',lambda:(12,0)),((12,0),True));self.assertEqual(m.streaks['gpu'],0)
|
|
m.sample('gpu',bad);self.assertFalse(m.failed);self.assertEqual(m.errors,3)
|
|
self.assertEqual(len((Path(t)/'errors.jsonl').read_text().splitlines()),3)
|
|
|
|
def test_third_consecutive_failure_is_terminal(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
m=Monitor(Path(t)/'errors.jsonl')
|
|
for _ in range(3):m.sample('gpu',lambda:1/0)
|
|
self.assertTrue(m.failed);m.sample('gpu',lambda:0);self.assertTrue(m.failed)
|
|
|
|
def test_success_in_other_stream_does_not_hide_gpu_failures(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
m=Monitor(Path(t)/'errors.jsonl')
|
|
for _ in range(3):m.sample('gpu',lambda:1/0);m.sample('rss',lambda:100)
|
|
self.assertTrue(m.failed);self.assertEqual(m.max_streaks['gpu'],3)
|
|
|
|
def test_missing_is_not_zero_resource_sample(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
m=Monitor(Path(t)/'errors.jsonl');value,ok=m.sample('gpu',lambda:1/0)
|
|
self.assertIsNone(value);self.assertFalse(ok);self.assertFalse(m.failed)
|
|
|
|
def test_final_sample_retry_bounded_at_three(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
m=Monitor(Path(t)/'errors.jsonl')
|
|
with self.assertRaises(MonitorFailure):m.retry('final',lambda:1/0,0)
|
|
self.assertEqual(m.errors,3)
|
|
|
|
def test_device_query_uses_ten_seconds_and_parses(self):
|
|
with patch('track_a_recovery_monitor.subprocess.run',return_value=type('R',(),dict(stdout='4, 0\n'))()) as run:
|
|
self.assertEqual(device_usage('fixed'),(4*2**20,0));self.assertEqual(run.call_args.kwargs['timeout'],10)
|
|
|
|
def test_original_worker_science_ast_unchanged(self):
|
|
old=ast.parse((B/'tools/track_a_h2_worker.py').read_text());new=ast.parse((B/'tools/track_a_recovery_worker.py').read_text())
|
|
for n in ast.walk(new):
|
|
if isinstance(n,ast.ImportFrom) and n.module=='track_a_recovery_common':n.module='track_a_h2_common'
|
|
run=next(n for n in new.body if isinstance(n,ast.FunctionDef) and n.name=='run')
|
|
self.assertIsInstance(run.body[1],ast.If);run.body.pop(1)
|
|
self.assertEqual(ast.dump(old,include_attributes=False),ast.dump(new,include_attributes=False))
|
|
|
|
def cfg(self):return dict(jobs=[dict(id=i,gpu_seconds=30960) for i in range(5)],gpu_cap=162000,postfit_gpu_reserve=7200)
|
|
|
|
def test_old_once_and_only_two_new_jobs_in_projection(self):
|
|
p=project_total(self.cfg(),{},{},{})
|
|
self.assertEqual(p['new_projected_gpu_seconds'],61920+7200)
|
|
self.assertEqual(p['cumulative_projected_gpu_seconds'],OLD+61920+7200);self.assertEqual(p['status'],'PASS')
|
|
with self.assertRaises(ValueError):project_total(self.cfg(),{}, {0:dict(gpu_seconds_charged=100)}, {})
|
|
|
|
def test_failed_terminal_cost_not_dropped_and_replaces_projection(self):
|
|
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=30000)},{3:dict(gpu_seconds_charged=123,status='FAILED')},{4:100})
|
|
self.assertEqual(p['new_projected_gpu_seconds'],123+30960+7200)
|
|
|
|
def test_actual_charge_floor_and_nonfinite_rejection(self):
|
|
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=200)}, {},{3:500})
|
|
self.assertEqual(p['rows'][0]['projected_gpu_seconds'],500)
|
|
with self.assertRaises(ValueError):project_total(self.cfg(),{}, {},{3:float('nan')})
|
|
|
|
def test_no_partial_model_adoption(self):
|
|
c=dict(reused_models=[dict(job=i) for i in range(3)])
|
|
self.assertEqual(len(canonical_models(c,[dict(job=3),dict(job=4)])),5)
|
|
for models in [[],[dict(job=3)],[dict(job=3),dict(job=3)]]:
|
|
with self.assertRaises(ValueError):canonical_models(c,models)
|
|
|
|
def test_transient_logged_monitor_errors_can_pass_but_fatal_cannot(self):
|
|
w=dict(status='PASS',returncode=0,monitor_errors=2,monitor_policy=dict(failed=False,errors=2))
|
|
g=dict(status='PASS',reasons=[],monitor_errors=[{},{}],monitor_policy=dict(failed=False,errors=2),worker=w)
|
|
self.assertTrue(accepted_guard(0,g));self.assertFalse(accepted_guard(1,g))
|
|
bad=copy.deepcopy(g);bad['worker']['monitor_policy']['failed']=True;self.assertFalse(accepted_guard(0,bad))
|
|
bad=copy.deepcopy(g);bad['monitor_policy']['failed']=True;self.assertFalse(accepted_guard(0,bad))
|
|
bad=copy.deepcopy(g);bad['worker']['monitor_errors']=[];self.assertFalse(accepted_guard(0,bad))
|
|
|
|
def test_actual_collect_failure_does_not_stop_other_job(self):
|
|
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='collect')
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);p=root/'jobs/3';p.mkdir(parents=True)
|
|
save(p/'complete.json',dict(status='STOPPED_OR_FAILED',reasons=['THREE_CONSECUTIVE_MONITOR_FAILURES'],monitor_policy=dict(failed=True)))
|
|
class Proc:
|
|
def poll(self):return 1
|
|
class Log:
|
|
def close(self):pass
|
|
terminal={};other=object();v=dict(process=Proc(),log=Log(),wait=10);active={3:v,4:other};calls=[]
|
|
ns=dict(root=root,read=lambda p:json.loads(p.read_text()),accepted_guard=accepted_guard,age=lambda:20,terminal=terminal,active=active,sha=sha,ledger=lambda:calls.append('ledger'))
|
|
exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_collect','exec'),ns)
|
|
self.assertTrue(ns['collect'](3,v));self.assertIs(active[4],other);self.assertFalse((root/'global-stop.json').exists());self.assertEqual(terminal[3]['gpu_seconds_charged'],80)
|
|
|
|
def test_own_cost_gate_stop_does_not_block_other_fold(self):
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);(root/'jobs/3').mkdir(parents=True);(root/'jobs/4').mkdir();(root/'contract.json').write_text('{}')
|
|
save(root/'jobs/3/stop.json',dict(reason='own'))
|
|
save(root/'jobs/4/first-epoch-projection.json',dict(status='PASS',contract_sha256=sha(root/'contract.json')))
|
|
with self.assertRaises(RuntimeError):first_epoch_release(root,3)
|
|
self.assertTrue(first_epoch_release(root,4))
|
|
|
|
def test_reused_fold_execution_rejected(self):
|
|
with self.assertRaises(PermissionError):first_epoch_release(Path('/synthetic'),0)
|
|
|
|
def test_dispatch_busy_gpu_never_launches_or_changes_other_queue(self):
|
|
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='dispatch')
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);(root/'jobs/4').mkdir(parents=True)
|
|
pending=[4];active={};calls=[];idle={4:Monitor(root/'jobs/4/errors.jsonl')}
|
|
jobs=[dict(gpu_seconds=30960,gpu_uuid='GPU2') for _ in range(5)]
|
|
ns=dict(root=root,c=dict(jobs=jobs),pending=pending,active=active,idle=idle,age=lambda:100,device_usage=lambda u:(1024**3,90),save=save,time=__import__('time'),subprocess=subprocess,sys=sys,settle_unlaunched=lambda *a:calls.append(a))
|
|
exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_dispatch','exec'),ns)
|
|
with patch.object(subprocess,'Popen') as spawn:ns['dispatch']();spawn.assert_not_called()
|
|
self.assertEqual(pending,[4]);self.assertFalse(calls);self.assertFalse(active)
|
|
snap=json.loads((root/'jobs/4/dispatch-idle.json').read_text());self.assertTrue(snap['valid']);self.assertEqual(snap['charged_gpu_seconds'],160)
|
|
|
|
def test_dispatch_three_query_failures_isolate_unlaunched_job(self):
|
|
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());run=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run');fn=next(n for n in run.body if isinstance(n,ast.FunctionDef) and n.name=='dispatch')
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);(root/'jobs/4').mkdir(parents=True);pending=[4];other=object();active={3:other};calls=[];idle={4:Monitor(root/'jobs/4/errors.jsonl')}
|
|
def bad(u):raise subprocess.TimeoutExpired(['nvidia-smi'],10)
|
|
def settle(i,why):calls.append((i,why));pending.remove(i)
|
|
ns=dict(root=root,c=dict(jobs=[dict(gpu_seconds=30960,gpu_uuid='GPU2') for _ in range(5)]),pending=pending,active=active,idle=idle,age=lambda:100,device_usage=bad,save=save,time=__import__('time'),settle_unlaunched=settle)
|
|
exec(compile(ast.Module(body=[fn],type_ignores=[]),'actual_dispatch','exec'),ns)
|
|
ns['dispatch']();ns['dispatch']();self.assertFalse(calls);ns['dispatch']()
|
|
self.assertEqual(calls,[(4,'THREE_FAILED_DISPATCH_GPU_SAMPLES')]);self.assertIs(active[3],other)
|
|
|
|
def test_extreme_rejected_projection_cannot_stop_affordable_survivor(self):
|
|
from types import SimpleNamespace
|
|
from track_a_recovery_common import project_cell
|
|
tree=ast.parse((B/'tools/track_a_recovery_queue.py').read_text());node=next(n for n in ast.walk(tree) if isinstance(n,ast.For) and ast.unparse(n.iter)=='list(active)')
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);save(root/'contract.json',{})
|
|
for i in [3,4]:(root/'jobs'/str(i)/'worker/history').mkdir(parents=True)
|
|
save(root/'jobs/3/worker/history/001.json',dict(epoch=1,train_epoch_seconds=1000))
|
|
save(root/'jobs/4/worker/history/001.json',dict(epoch=1,train_epoch_seconds=50))
|
|
own=[];glob=[]
|
|
def stop(i,why):own.append((i,why));save(root/'jobs'/str(i)/'stop.json',dict(reason=why))
|
|
ns=dict(root=root,c=self.cfg(),active={3:object(),4:object()},terminal={},estimates={},last_epoch={},read=lambda p:json.loads(p.read_text()),save=save,sha=sha,age=lambda:100,live=lambda:{3:160,4:160},project_cell=project_cell,project_total=project_total,time=SimpleNamespace(time=lambda:0),own_stop=stop,global_stop=glob.append)
|
|
exec(compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),'actual_epoch_block','exec'),ns)
|
|
self.assertEqual(own,[(3,'OWN_PROJECTED_CAP')]);self.assertFalse(glob)
|
|
failed=json.loads((root/'jobs/3/first-epoch-projection.json').read_text());healthy=json.loads((root/'jobs/4/first-epoch-projection.json').read_text())
|
|
self.assertEqual(failed['cell']['projected_gpu_seconds'],249030);self.assertEqual(failed['status'],'STOP_OWN_COST_PROJECTION')
|
|
self.assertEqual(failed['total']['cumulative_projected_gpu_seconds'],OLD+30960+30960+7200)
|
|
self.assertEqual(healthy['status'],'PASS');self.assertFalse((root/'jobs/4/stop.json').exists())
|
|
self.assertEqual(ns['estimates'][3]['projected_gpu_seconds'],249030)
|
|
|
|
def test_stopping_reserve_keeps_actual_and_terminal_settles_once(self):
|
|
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)}, {},{3:40000},stopping={3})
|
|
self.assertEqual(p['rows'][0]['projected_gpu_seconds'],40000)
|
|
self.assertEqual(p['rows'][0]['basis'],'locally_stopping_own_cap_or_actual')
|
|
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)},{3:dict(gpu_seconds_charged=190)}, {},stopping={3})
|
|
self.assertEqual(p['rows'][0]['projected_gpu_seconds'],190);self.assertEqual(p['rows'][0]['basis'],'terminal_actual')
|
|
|
|
def test_true_cumulative_overrun_still_stops_after_local_rejection(self):
|
|
p=project_total(self.cfg(),{3:dict(projected_gpu_seconds=249030)}, {},{3:130000,4:500},stopping={3})
|
|
self.assertEqual(p['status'],'STOP_CUMULATIVE_BUDGET');self.assertGreater(p['cumulative_projected_gpu_seconds'],162000)
|
|
|
|
def idle_replay(self,clock,query,cap,sleep):
|
|
from types import SimpleNamespace
|
|
from track_a_recovery_guard import require_prestart_window
|
|
tree=ast.parse((B/'tools/track_a_recovery_guard.py').read_text());node=next(n for n in ast.walk(tree) if isinstance(n,ast.While) and ast.unparse(n.test)=='success < 3')
|
|
with tempfile.TemporaryDirectory() as t:
|
|
root=Path(t);monitor=Monitor(root/'errors.jsonl')
|
|
ns=dict(root=root,queue=root,requested=[],monitor=monitor,success=0,samples=[],device_usage=query,job=dict(gpu_uuid='SYNTHETIC',gpu_seconds=cap),elapsed=clock,require_prestart_window=require_prestart_window,time=SimpleNamespace(time=clock,sleep=sleep),save=save)
|
|
started=False
|
|
try:
|
|
exec(compile(ast.fix_missing_locations(ast.Module(body=[node],type_ignores=[])),'actual_idle_loop','exec'),ns)
|
|
started=True
|
|
except RuntimeError as exc:reason=str(exc)
|
|
else:reason=None
|
|
return dict(worker_would_start=started,reason=reason,policy=monitor.policy(),errors=list(monitor.records),successes=ns['success'])
|
|
|
|
def test_already_expired_prestart_makes_zero_queries(self):
|
|
calls=[]
|
|
d=self.idle_replay(lambda:30961,lambda u:calls.append(u),30960,lambda s:None)
|
|
self.assertEqual(calls,[]);self.assertFalse(d['worker_would_start']);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
|
|
|
|
def test_alternating_two_errors_one_idle_expires_without_worker(self):
|
|
clock=[60.];calls=[]
|
|
def query(u):
|
|
calls.append(u);clock[0]+=10
|
|
if len(calls)%3:raise TimeoutError('synthetic recoverable')
|
|
return 0,0
|
|
def sleep(s):clock[0]+=s
|
|
d=self.idle_replay(lambda:clock[0],query,160,sleep)
|
|
self.assertFalse(d['worker_would_start']);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
|
|
self.assertFalse(d['policy']['failed']);self.assertEqual(d['policy']['max_streaks']['gpu_usage'],2)
|
|
self.assertLess(clock[0],160);self.assertLessEqual(len(calls),7);self.assertGreater(len(d['errors']),3)
|
|
|
|
def test_prestart_checks_after_query_overrun(self):
|
|
clock=[60.];calls=[]
|
|
def query(u):calls.append(u);clock[0]=85;return 0,0
|
|
d=self.idle_replay(lambda:clock[0],query,100,lambda s:None)
|
|
self.assertFalse(d['worker_would_start']);self.assertEqual(len(calls),1);self.assertEqual(d['reason'],'PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
|
|
|
|
def test_prestart_checks_before_idle_sleep(self):
|
|
clock=[60.];sleeps=[]
|
|
def query(u):clock[0]=79.5;return 0,0
|
|
d=self.idle_replay(lambda:clock[0],query,100,sleeps.append)
|
|
self.assertFalse(d['worker_would_start']);self.assertEqual(sleeps,[]);self.assertEqual(d['successes'],1)
|
|
|
|
def test_fast_local_guard_not_blocked_on_gpu_query(self):
|
|
tree=ast.parse((B/'tools/track_a_recovery_guard.py').read_text());local=next(n for n in ast.walk(tree) if isinstance(n,ast.FunctionDef) and n.name=='local_watch')
|
|
self.assertNotIn('device_usage',ast.unparse(local));self.assertIn('WHOLE_TREE_RSS_LIMIT',ast.unparse(local));self.assertIn('OUTPUT_DISK_LIMIT',ast.unparse(local));self.assertIn('RECORDED_VRAM_LIMIT',(B/'tools/track_a_recovery_guard.py').read_text())
|
|
|
|
def test_supervisor_real_breach_fails_without_three_monitor_errors(self):
|
|
from track_a_recovery_supervisor import supervise
|
|
with tempfile.TemporaryDirectory() as t:
|
|
r=supervise([sys.executable,'-c','import time; time.sleep(2)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=1,disk_bytes=1000000,max_processes=1),poll=.01)
|
|
self.assertNotEqual(r['status'],'PASS');self.assertIn('rss',r['stop_reason']);self.assertFalse(r['monitor_policy']['failed'])
|
|
|
|
def test_supervisor_transient_tree_failures_recover_and_log(self):
|
|
import track_a_recovery_supervisor as s
|
|
real=s.tree_usage;calls=[0]
|
|
def sample(pid):
|
|
calls[0]+=1
|
|
if calls[0]<=2:raise OSError('synthetic transient')
|
|
return real(pid)
|
|
with tempfile.TemporaryDirectory() as t,patch.object(s,'tree_usage',side_effect=sample):
|
|
r=s.supervise([sys.executable,'-c','import time; time.sleep(.15)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=2**30,disk_bytes=1000000,max_processes=1),poll=.01)
|
|
self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],2);self.assertFalse(r['monitor_policy']['failed'])
|
|
|
|
def test_supervisor_three_failed_tree_samples_stop_only_child(self):
|
|
import track_a_recovery_supervisor as s
|
|
with tempfile.TemporaryDirectory() as t,patch.object(s,'tree_usage',side_effect=OSError('synthetic persistent')):
|
|
r=s.supervise([sys.executable,'-c','import time; time.sleep(2)'],Path(t),'synthetic',dict(total_cpu_seconds=10,job_cpu_seconds=10,job_wall_seconds=5,tree_rss_bytes=2**30,disk_bytes=1000000,max_processes=1),poll=.01)
|
|
self.assertNotEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],3);self.assertEqual(r['stop_reason'],'three_consecutive_monitor_failures')
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|