77 lines
4.6 KiB
Python
77 lines
4.6 KiB
Python
"""Approved exit-before-receipt repair and seven-only dispatch checks."""
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import psutil
|
|
from s3b_v04_stage_guard_v2 import identity_decision,stage_charge
|
|
from run_s3b_v04_continue import run
|
|
from s2_recovery_supervisor import save
|
|
from s3b_runtime import read
|
|
|
|
|
|
class RecoveryTests(unittest.TestCase):
|
|
def test_identity_with_completed_result_before_wait4_receipt(self):
|
|
observation=dict(pid=123,ppid=7,status='running',cmdline=[])
|
|
self.assertEqual(identity_decision(observation,7,['worker'],False,True,False),'EXITED_AWAITING_RECEIPT')
|
|
self.assertEqual(identity_decision(observation,7,['worker'],True,False,False),'EXITED_AWAITING_RECEIPT')
|
|
self.assertEqual(identity_decision(observation,7,['worker'],False,False,True),'EXITED_AWAITING_RECEIPT')
|
|
self.assertEqual(identity_decision(observation,7,['worker'],False,False,False),'UNEXPLAINED_IDENTITY_CHANGE')
|
|
|
|
def test_actual_worker_exit_before_parent_reap(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
result=Path(temp)/'result.json'
|
|
proc=subprocess.Popen([sys.executable,'-c','from pathlib import Path; import sys; Path(sys.argv[1]).write_text("{}")',str(result)])
|
|
try:
|
|
worker=psutil.Process(proc.pid);until=time.monotonic()+5
|
|
while worker.status()!=psutil.STATUS_ZOMBIE and time.monotonic()<until:time.sleep(.01)
|
|
observation=dict(pid=proc.pid,ppid=worker.ppid(),status=worker.status(),cmdline=[])
|
|
self.assertTrue(result.exists());self.assertFalse((Path(temp)/'worker.receipt.json').exists())
|
|
self.assertEqual(identity_decision(observation,os.getpid(),['worker'],False,True,False),'EXITED_AWAITING_RECEIPT')
|
|
finally:proc.wait(timeout=5)
|
|
|
|
def test_live_expected_worker_and_known_gone(self):
|
|
o=dict(ppid=7,status='running',cmdline=['python','worker','root','3','splu'])
|
|
self.assertEqual(identity_decision(o,7,['worker','root','3','splu'],False,False,False),'ACTIVE')
|
|
self.assertEqual(identity_decision(dict(o,status='gone',cmdline=[]),7,['worker'],False,False,False),'EXITED_AWAITING_RECEIPT')
|
|
|
|
def test_wait4_replaces_live_cpu_once(self):
|
|
self.assertAlmostEqual(stage_charge({'part-02':565.748434},{'part-02':565.54},3.893839),569.642273,places=9)
|
|
|
|
def fixture(self,prior_cpu=2.):
|
|
calls=[]
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root=Path(temp);save(root/'contract.json',{});save(root/'code-freeze.json',dict(git_commit='synthetic'))
|
|
prior=[dict(method='splu',index=i,case=str(i),worker=dict(cpu_seconds=prior_cpu),classification=dict(category='PASS')) for i in range(3)]
|
|
c=dict(coordinator_cpus=[0],worker_cpu=0,old_worker_CPU_s=3239.962177,new_synthetic_worker_CPU_s=3.,
|
|
candidate_order=['splu','gmres_ilu'],reused_receipts=prior,ten_parts=[dict(case=str(i)) for i in range(10)],
|
|
limits=dict(part_CPU_s=1800,part_wall_s=1800,part_RSS_bytes=4*2**30,ten_disk_bytes=5*2**30,
|
|
total_worker_CPU_s=72000,postfit_reserve_CPU_s=7200))
|
|
def fake(command,dest,name,limits,**kw):
|
|
calls.append((command[-1],int(command[-2])));(dest/'output').mkdir()
|
|
save(dest/'output/result.json',dict(status='COMPLETE',pressure_calls=10,early_resolves=9))
|
|
return dict(status='PASS',monitor_errors=0,stop_reason=None,cpu_seconds=2.)
|
|
with patch('run_s3b_v04_continue.contract',return_value=c),patch('run_s3b_v04_continue.supervise',side_effect=fake),patch('run_s3b_v04_continue.os.sched_setaffinity'),patch('run_s3b_v04_continue.signal.signal'):
|
|
code=run(root)
|
|
return code,calls,read(root/'complete.json')
|
|
|
|
def test_only_seven_unattempted_are_dispatched(self):
|
|
code,calls,r=self.fixture()
|
|
self.assertEqual(code,0);self.assertEqual(calls,[('splu',i) for i in range(3,10)])
|
|
self.assertEqual(r['new_attempts'],7);self.assertEqual(r['new_worker_CPU_s'],23.)
|
|
self.assertEqual(r['candidate_CPU_s']['splu'],23.)
|
|
self.assertEqual(r['candidates'][0]['attempts'][-1]['dead_end_share'],.9)
|
|
|
|
def test_existing_spend_counts_against_stage_full_job_reservation(self):
|
|
code,calls,r=self.fixture(prior_cpu=3001.)
|
|
self.assertEqual(code,1);self.assertEqual(calls,[])
|
|
self.assertEqual(r['stop_reason'],'CANDIDATE_STAGE_FULL_JOB_RESERVATION')
|
|
self.assertIsNone(r['selected_solver'])
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|