Bind GINO cleanup to unreaped worker group and replay actual signals
This commit is contained in:
1 parent
8af022e434
commit
ee8e01937e
9 files changed
+1185
-21
No files matched your search
@@ -15,6 +15,9 @@ import track_b_gino_formal_worker as worker
|
||||
from track_b_gino_shapenet_entry import read, save, sha
|
||||
|
||||
|
||||
REPLAYS = {}
|
||||
|
||||
|
||||
class GuardReplay(unittest.TestCase):
|
||||
def replay(self, ordering):
|
||||
uuid = 'GPU-fixture'
|
||||
@@ -32,16 +35,28 @@ class GuardReplay(unittest.TestCase):
|
||||
"root=Path(sys.argv[1])\n"
|
||||
"while not (root/'exit').exists(): time.sleep(.01)\n"
|
||||
"(root/'worker-records/result.json').write_text("
|
||||
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder]
|
||||
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n"
|
||||
"if sys.argv[2] == 'failed-exit': raise SystemExit(7)\n", folder, ordering]
|
||||
observed = []
|
||||
stopped = []
|
||||
real_stop = gpu_guard.stop_group
|
||||
syscalls = []
|
||||
real_kill = os.killpg
|
||||
real_wait = os.wait4
|
||||
real_tree = gpu_guard.process_tree
|
||||
ticket = dict(ordinal='', dispatch_before_unix=time.time() + 15)
|
||||
|
||||
def stop_owned(pid):
|
||||
stopped.append(pid)
|
||||
real_stop(pid)
|
||||
def record_kill(pid, signal):
|
||||
own = read(root / 'actual-worker-spawn.json')['process']
|
||||
is_reaped = any(e == dict(event='wait4_reaped', pid=pid) for e in syscalls)
|
||||
syscalls.append(dict(event='killpg', pid=pid, signal=int(signal), leader_reaped=is_reaped))
|
||||
if is_reaped or pid != own['pid']:
|
||||
raise AssertionError('Invalid group signal intercepted before delivery')
|
||||
return real_kill(pid, signal)
|
||||
|
||||
def record_wait(pid, options):
|
||||
receipt = real_wait(pid, options)
|
||||
if receipt[0]:
|
||||
syscalls.append(dict(event='wait4_reaped', pid=receipt[0]))
|
||||
return receipt
|
||||
|
||||
def exit_and_wait():
|
||||
(root / 'exit').touch()
|
||||
@@ -74,13 +89,14 @@ class GuardReplay(unittest.TestCase):
|
||||
occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')])
|
||||
if ordering == 'crossed-after-spawn':
|
||||
ticket['dispatch_before_unix'] = time.time() - 1
|
||||
if ordering in ('gpu-first', 'crossed-after-spawn'):
|
||||
if ordering in ('gpu-first', 'crossed-after-spawn', 'failed-exit'):
|
||||
exit_and_wait()
|
||||
return sample
|
||||
|
||||
with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot),
|
||||
patch.object(gpu_guard, 'process_tree', side_effect=tree),
|
||||
patch.object(gpu_guard, 'stop_group', side_effect=stop_owned),
|
||||
patch.object(os, 'killpg', side_effect=record_kill),
|
||||
patch.object(os, 'wait4', side_effect=record_wait),
|
||||
patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}),
|
||||
patch.object(gpu_guard, 'dispatch_environment',
|
||||
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))):
|
||||
@@ -88,10 +104,13 @@ class GuardReplay(unittest.TestCase):
|
||||
ticket['guard_started_monotonic'] = began
|
||||
outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15,
|
||||
ticket=ticket, validate_result=lambda: sha(result))
|
||||
REPLAYS[ordering] = dict(status=outcome['status'], trigger=outcome['trigger'],
|
||||
syscalls=syscalls, lifecycle=outcome['lifecycle'])
|
||||
if ordering == 'expired-before-spawn':
|
||||
self.assertEqual(outcome['status'], 'STOPPED')
|
||||
self.assertEqual(outcome['trigger'], 'CARD_RESERVATION_EXPIRED_BEFORE_SPAWN')
|
||||
self.assertFalse((root / 'actual-worker-spawn.json').exists())
|
||||
self.assertEqual(syscalls, [])
|
||||
return observed
|
||||
spawn = read(root / 'actual-worker-spawn.json')['process']
|
||||
if ordering == 'foreign':
|
||||
@@ -102,11 +121,16 @@ class GuardReplay(unittest.TestCase):
|
||||
self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid))
|
||||
self.assertEqual(row['occupants'][0]['pid'], os.getpid())
|
||||
self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python')
|
||||
self.assertEqual(stopped, [spawn['pid']])
|
||||
self.assertEqual(syscalls, [
|
||||
dict(event='killpg', pid=spawn['pid'], signal=9, leader_reaped=False),
|
||||
dict(event='wait4_reaped', pid=spawn['pid'])])
|
||||
elif ordering == 'failed-exit':
|
||||
self.assertEqual((outcome['status'], outcome['worker']['returncode']), ('STOPPED', 7))
|
||||
self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])])
|
||||
else:
|
||||
self.assertEqual(outcome['status'], 'COMPLETE', outcome)
|
||||
self.assertFalse((root / 'foreign-occupant.json').exists())
|
||||
self.assertEqual(stopped, [])
|
||||
self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])])
|
||||
self.assertEqual(outcome['result_sha256'], sha(result))
|
||||
return observed
|
||||
|
||||
@@ -127,6 +151,23 @@ class GuardReplay(unittest.TestCase):
|
||||
def test_running_guard_continues_after_reservation_end(self):
|
||||
self.replay('crossed-after-spawn')
|
||||
|
||||
def test_failed_worker_exit_has_no_stale_group_signal(self):
|
||||
self.replay('failed-exit')
|
||||
|
||||
def test_simulated_reused_pgid_cannot_be_signalled(self):
|
||||
group = gpu_guard.OwnedGroup()
|
||||
group.pid = 12
|
||||
group.identity = dict(pid=12, created=1.)
|
||||
group.reaped = True
|
||||
reused = dict(pid=12, created=2.)
|
||||
with (patch.object(gpu_guard, 'current_identity', return_value=reused),
|
||||
patch.object(os, 'waitid') as observe,
|
||||
patch.object(os, 'killpg') as intercept):
|
||||
group.stop(12)
|
||||
observe.assert_not_called()
|
||||
intercept.assert_not_called()
|
||||
REPLAYS['pgid-reuse'] = dict(reaped=group.reaped, simulated_new_identity=reused, signals=0)
|
||||
|
||||
def test_reused_worker_pid_is_foreign(self):
|
||||
with TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
|
||||
Reference in new issue
Block a user