Files
MeltMind/tools/test_s2_recovery_infrastructure.py

88 lines
5.4 KiB
Python

import json
from pathlib import Path
import sys
import tempfile
import unittest
from unittest.mock import patch
from s2_recovery_monitoring import disk_usage,budget_reason
import s2_e1_supervisor as old
import s2_recovery_supervisor as fixed
import run_s2_recovery_preparation as coordinator
class InfrastructureTests(unittest.TestCase):
def test_old_tmp_replace_race_reproduced_new_scan_survives(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);pending=root/'progress.json.tmp';final=root/'progress.json';pending.write_text('new');final.write_text('old')
stat=Path.stat;calls=0
def racing_stat(path,*a,**kw):
nonlocal calls
if path==pending:
calls+=1
if calls==2:pending.replace(final)
return stat(path,*a,**kw)
with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',racing_stat):
with self.assertRaises(FileNotFoundError):old.disk_usage(root)
pending.write_text('replacement')
def disappearing_stat(path,*a,**kw):
if path==pending:pending.replace(final)
return stat(path,*a,**kw)
with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',disappearing_stat):
measured=disk_usage(root)
self.assertEqual(measured,max(final.stat().st_size,final.stat().st_blocks*512))
def test_live_guard_monitor_exception_does_not_kill_child(self):
with tempfile.TemporaryDirectory() as tmp:
real=fixed.disk_usage;calls=0
def once(root):
nonlocal calls
calls+=1
if calls==2:raise OSError('synthetic live monitoring failure')
return real(root)
limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=2,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1)
with patch.object(fixed,'disk_usage',once):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(.12)'],tmp,'worker',limits,poll=.01)
self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],1);self.assertEqual(r['returncode'],0)
def test_wall_cap_still_stops_with_monitor_faults(self):
with tempfile.TemporaryDirectory() as tmp:
real=fixed.disk_usage;calls=0
def intermittent(root):
nonlocal calls
calls+=1
if calls in [2,3]:raise OSError('synthetic sample failure')
return real(root)
limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=.08,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1)
with patch.object(fixed,'disk_usage',intermittent):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(2)'],tmp,'worker',limits,poll=.01)
self.assertEqual(r['stop_reason'],'wall_limit');self.assertLess(r['returncode'],0);self.assertGreater(r['monitor_errors'],0)
def test_cumulative_cpu_counts_incident_once_and_real_caps_stop(self):
self.assertIsNone(budget_reason(23.78094*3600,3600,100,100))
self.assertEqual(budget_reason(23.78094*3600,36.21906*3600,1,0),'GLOBAL_CPU_LIMIT')
self.assertEqual(budget_reason(0,0,0,100*2**30),'GLOBAL_DISK_LIMIT')
def test_coordinator_repolls_fault_with_active_workers_without_killing_them(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);spawned=[];terminations=[];fault=False
c=dict(cases=[dict(case=str(i)) for i in range(134)],original_indices=list(range(134)),
projected_cumulative_worker_cpu_hours=1,projected_cumulative_gib=1,prior_worker_cpu_seconds=0.,
limits=dict(worker_cpu_hours=60,disk_bytes=100*2**30),budget_roots=[str(root)])
(root/'contract.json').write_text(json.dumps(c))
class FakeProcess:
def __init__(self,index):self.index=index;self.pid=1000000+index;self.polls=0
def poll(self):
self.polls+=1
if self.polls==1:return None
path=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json'
path.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=0.)))
return 0
def spawn(command,**kwargs):
p=FakeProcess(int(command[-1]));spawned.append(p);return p
def disk(_):
nonlocal fault
if spawned and not fault:
fault=True;raise OSError('synthetic coordinator accounting failure')
return 100
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(.01,100,1)),patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda active:terminations.append(list(active))),patch('builtins.print'):
coordinator.run(root)
r=json.loads((root/'batch-complete.json').read_text());self.assertEqual(r['status'],'ATTEMPTS_COMPLETE')
self.assertEqual(len(r['jobs']),134);self.assertEqual(r['monitor_errors'],1);self.assertTrue(all(not ids for ids in terminations))
if __name__=='__main__':unittest.main(verbosity=2)