import json from pathlib import Path import sys import tempfile import unittest from unittest.mock import patch from s2_recovery_monitoring import disk_usage,budget_reason import s2_e1_supervisor as old import s2_recovery_supervisor as fixed import run_s2_recovery_preparation as coordinator class InfrastructureTests(unittest.TestCase): def test_old_tmp_replace_race_reproduced_new_scan_survives(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);pending=root/'progress.json.tmp';final=root/'progress.json';pending.write_text('new');final.write_text('old') stat=Path.stat;calls=0 def racing_stat(path,*a,**kw): nonlocal calls if path==pending: calls+=1 if calls==2:pending.replace(final) return stat(path,*a,**kw) with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',racing_stat): with self.assertRaises(FileNotFoundError):old.disk_usage(root) pending.write_text('replacement') def disappearing_stat(path,*a,**kw): if path==pending:pending.replace(final) return stat(path,*a,**kw) with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',disappearing_stat): measured=disk_usage(root) self.assertEqual(measured,max(final.stat().st_size,final.stat().st_blocks*512)) def test_live_guard_monitor_exception_does_not_kill_child(self): with tempfile.TemporaryDirectory() as tmp: real=fixed.disk_usage;calls=0 def once(root): nonlocal calls calls+=1 if calls==2:raise OSError('synthetic live monitoring failure') return real(root) limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=2,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1) with patch.object(fixed,'disk_usage',once):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(.12)'],tmp,'worker',limits,poll=.01) self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],1);self.assertEqual(r['returncode'],0) def test_wall_cap_still_stops_with_monitor_faults(self): with tempfile.TemporaryDirectory() as tmp: real=fixed.disk_usage;calls=0 def intermittent(root): nonlocal calls calls+=1 if calls in [2,3]:raise OSError('synthetic sample failure') return real(root) limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=.08,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1) with patch.object(fixed,'disk_usage',intermittent):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(2)'],tmp,'worker',limits,poll=.01) self.assertEqual(r['stop_reason'],'wall_limit');self.assertLess(r['returncode'],0);self.assertGreater(r['monitor_errors'],0) def test_cumulative_cpu_counts_incident_once_and_real_caps_stop(self): self.assertIsNone(budget_reason(23.78094*3600,3600,100,100)) self.assertEqual(budget_reason(23.78094*3600,36.21906*3600,1,0),'GLOBAL_CPU_LIMIT') self.assertEqual(budget_reason(0,0,0,100*2**30),'GLOBAL_DISK_LIMIT') def test_coordinator_repolls_fault_with_active_workers_without_killing_them(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);spawned=[];terminations=[];fault=False c=dict(cases=[dict(case=str(i)) for i in range(134)],original_indices=list(range(134)), projected_cumulative_worker_cpu_hours=1,projected_cumulative_gib=1,prior_worker_cpu_seconds=0., limits=dict(worker_cpu_hours=60,disk_bytes=100*2**30),budget_roots=[str(root)]) (root/'contract.json').write_text(json.dumps(c)) class FakeProcess: def __init__(self,index):self.index=index;self.pid=1000000+index;self.polls=0 def poll(self): self.polls+=1 if self.polls==1:return None path=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json' path.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=0.))) return 0 def spawn(command,**kwargs): p=FakeProcess(int(command[-1]));spawned.append(p);return p def disk(_): nonlocal fault if spawned and not fault: fault=True;raise OSError('synthetic coordinator accounting failure') return 100 with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(.01,100,1)),patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda active:terminations.append(list(active))),patch('builtins.print'): coordinator.run(root) r=json.loads((root/'batch-complete.json').read_text());self.assertEqual(r['status'],'ATTEMPTS_COMPLETE') self.assertEqual(len(r['jobs']),134);self.assertEqual(r['monitor_errors'],1);self.assertTrue(all(not ids for ids in terminations)) if __name__=='__main__':unittest.main(verbosity=2)