88 lines
5.4 KiB
Python
88 lines
5.4 KiB
Python
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from s2_recovery_monitoring import disk_usage,budget_reason
|
|
import s2_e1_supervisor as old
|
|
import s2_recovery_supervisor as fixed
|
|
import run_s2_recovery_preparation as coordinator
|
|
|
|
|
|
class InfrastructureTests(unittest.TestCase):
|
|
def test_old_tmp_replace_race_reproduced_new_scan_survives(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);pending=root/'progress.json.tmp';final=root/'progress.json';pending.write_text('new');final.write_text('old')
|
|
stat=Path.stat;calls=0
|
|
def racing_stat(path,*a,**kw):
|
|
nonlocal calls
|
|
if path==pending:
|
|
calls+=1
|
|
if calls==2:pending.replace(final)
|
|
return stat(path,*a,**kw)
|
|
with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',racing_stat):
|
|
with self.assertRaises(FileNotFoundError):old.disk_usage(root)
|
|
pending.write_text('replacement')
|
|
def disappearing_stat(path,*a,**kw):
|
|
if path==pending:pending.replace(final)
|
|
return stat(path,*a,**kw)
|
|
with patch.object(Path,'rglob',return_value=iter([pending,final])),patch.object(Path,'stat',disappearing_stat):
|
|
measured=disk_usage(root)
|
|
self.assertEqual(measured,max(final.stat().st_size,final.stat().st_blocks*512))
|
|
def test_live_guard_monitor_exception_does_not_kill_child(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
real=fixed.disk_usage;calls=0
|
|
def once(root):
|
|
nonlocal calls
|
|
calls+=1
|
|
if calls==2:raise OSError('synthetic live monitoring failure')
|
|
return real(root)
|
|
limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=2,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1)
|
|
with patch.object(fixed,'disk_usage',once):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(.12)'],tmp,'worker',limits,poll=.01)
|
|
self.assertEqual(r['status'],'PASS');self.assertEqual(r['monitor_errors'],1);self.assertEqual(r['returncode'],0)
|
|
def test_wall_cap_still_stops_with_monitor_faults(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
real=fixed.disk_usage;calls=0
|
|
def intermittent(root):
|
|
nonlocal calls
|
|
calls+=1
|
|
if calls in [2,3]:raise OSError('synthetic sample failure')
|
|
return real(root)
|
|
limits=dict(total_cpu_seconds=3,job_cpu_seconds=2,job_wall_seconds=.08,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1)
|
|
with patch.object(fixed,'disk_usage',intermittent):r=fixed.supervise([sys.executable,'-c','import time; time.sleep(2)'],tmp,'worker',limits,poll=.01)
|
|
self.assertEqual(r['stop_reason'],'wall_limit');self.assertLess(r['returncode'],0);self.assertGreater(r['monitor_errors'],0)
|
|
def test_cumulative_cpu_counts_incident_once_and_real_caps_stop(self):
|
|
self.assertIsNone(budget_reason(23.78094*3600,3600,100,100))
|
|
self.assertEqual(budget_reason(23.78094*3600,36.21906*3600,1,0),'GLOBAL_CPU_LIMIT')
|
|
self.assertEqual(budget_reason(0,0,0,100*2**30),'GLOBAL_DISK_LIMIT')
|
|
def test_coordinator_repolls_fault_with_active_workers_without_killing_them(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);spawned=[];terminations=[];fault=False
|
|
c=dict(cases=[dict(case=str(i)) for i in range(134)],original_indices=list(range(134)),
|
|
projected_cumulative_worker_cpu_hours=1,projected_cumulative_gib=1,prior_worker_cpu_seconds=0.,
|
|
limits=dict(worker_cpu_hours=60,disk_bytes=100*2**30),budget_roots=[str(root)])
|
|
(root/'contract.json').write_text(json.dumps(c))
|
|
class FakeProcess:
|
|
def __init__(self,index):self.index=index;self.pid=1000000+index;self.polls=0
|
|
def poll(self):
|
|
self.polls+=1
|
|
if self.polls==1:return None
|
|
path=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json'
|
|
path.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=0.)))
|
|
return 0
|
|
def spawn(command,**kwargs):
|
|
p=FakeProcess(int(command[-1]));spawned.append(p);return p
|
|
def disk(_):
|
|
nonlocal fault
|
|
if spawned and not fault:
|
|
fault=True;raise OSError('synthetic coordinator accounting failure')
|
|
return 100
|
|
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(.01,100,1)),patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda active:terminations.append(list(active))),patch('builtins.print'):
|
|
coordinator.run(root)
|
|
r=json.loads((root/'batch-complete.json').read_text());self.assertEqual(r['status'],'ATTEMPTS_COMPLETE')
|
|
self.assertEqual(len(r['jobs']),134);self.assertEqual(r['monitor_errors'],1);self.assertTrue(all(not ids for ids in terminations))
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|