Bind GINO cleanup to unreaped worker group and replay actual signals

This commit is contained in:
NewMeltAssist committed 2026-11-19 16:18:05 +08:00
1 parent 8af022e434
commit ee8e01937e
9 files changed
+1185 -21

No files matched your search

+51 -10
View File
@@ -15,6 +15,9 @@ import track_b_gino_formal_worker as worker
from track_b_gino_shapenet_entry import read, save, sha
REPLAYS = {}
class GuardReplay(unittest.TestCase):
def replay(self, ordering):
uuid = 'GPU-fixture'
@@ -32,16 +35,28 @@ class GuardReplay(unittest.TestCase):
"root=Path(sys.argv[1])\n"
"while not (root/'exit').exists(): time.sleep(.01)\n"
"(root/'worker-records/result.json').write_text("
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder]
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n"
"if sys.argv[2] == 'failed-exit': raise SystemExit(7)\n", folder, ordering]
observed = []
stopped = []
real_stop = gpu_guard.stop_group
syscalls = []
real_kill = os.killpg
real_wait = os.wait4
real_tree = gpu_guard.process_tree
ticket = dict(ordinal='', dispatch_before_unix=time.time() + 15)
def stop_owned(pid):
stopped.append(pid)
real_stop(pid)
def record_kill(pid, signal):
own = read(root / 'actual-worker-spawn.json')['process']
is_reaped = any(e == dict(event='wait4_reaped', pid=pid) for e in syscalls)
syscalls.append(dict(event='killpg', pid=pid, signal=int(signal), leader_reaped=is_reaped))
if is_reaped or pid != own['pid']:
raise AssertionError('Invalid group signal intercepted before delivery')
return real_kill(pid, signal)
def record_wait(pid, options):
receipt = real_wait(pid, options)
if receipt[0]:
syscalls.append(dict(event='wait4_reaped', pid=receipt[0]))
return receipt
def exit_and_wait():
(root / 'exit').touch()
@@ -74,13 +89,14 @@ class GuardReplay(unittest.TestCase):
occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')])
if ordering == 'crossed-after-spawn':
ticket['dispatch_before_unix'] = time.time() - 1
if ordering in ('gpu-first', 'crossed-after-spawn'):
if ordering in ('gpu-first', 'crossed-after-spawn', 'failed-exit'):
exit_and_wait()
return sample
with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot),
patch.object(gpu_guard, 'process_tree', side_effect=tree),
patch.object(gpu_guard, 'stop_group', side_effect=stop_owned),
patch.object(os, 'killpg', side_effect=record_kill),
patch.object(os, 'wait4', side_effect=record_wait),
patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}),
patch.object(gpu_guard, 'dispatch_environment',
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))):
@@ -88,10 +104,13 @@ class GuardReplay(unittest.TestCase):
ticket['guard_started_monotonic'] = began
outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15,
ticket=ticket, validate_result=lambda: sha(result))
REPLAYS[ordering] = dict(status=outcome['status'], trigger=outcome['trigger'],
syscalls=syscalls, lifecycle=outcome['lifecycle'])
if ordering == 'expired-before-spawn':
self.assertEqual(outcome['status'], 'STOPPED')
self.assertEqual(outcome['trigger'], 'CARD_RESERVATION_EXPIRED_BEFORE_SPAWN')
self.assertFalse((root / 'actual-worker-spawn.json').exists())
self.assertEqual(syscalls, [])
return observed
spawn = read(root / 'actual-worker-spawn.json')['process']
if ordering == 'foreign':
@@ -102,11 +121,16 @@ class GuardReplay(unittest.TestCase):
self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid))
self.assertEqual(row['occupants'][0]['pid'], os.getpid())
self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python')
self.assertEqual(stopped, [spawn['pid']])
self.assertEqual(syscalls, [
dict(event='killpg', pid=spawn['pid'], signal=9, leader_reaped=False),
dict(event='wait4_reaped', pid=spawn['pid'])])
elif ordering == 'failed-exit':
self.assertEqual((outcome['status'], outcome['worker']['returncode']), ('STOPPED', 7))
self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])])
else:
self.assertEqual(outcome['status'], 'COMPLETE', outcome)
self.assertFalse((root / 'foreign-occupant.json').exists())
self.assertEqual(stopped, [])
self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])])
self.assertEqual(outcome['result_sha256'], sha(result))
return observed
@@ -127,6 +151,23 @@ class GuardReplay(unittest.TestCase):
def test_running_guard_continues_after_reservation_end(self):
self.replay('crossed-after-spawn')
def test_failed_worker_exit_has_no_stale_group_signal(self):
self.replay('failed-exit')
def test_simulated_reused_pgid_cannot_be_signalled(self):
group = gpu_guard.OwnedGroup()
group.pid = 12
group.identity = dict(pid=12, created=1.)
group.reaped = True
reused = dict(pid=12, created=2.)
with (patch.object(gpu_guard, 'current_identity', return_value=reused),
patch.object(os, 'waitid') as observe,
patch.object(os, 'killpg') as intercept):
group.stop(12)
observe.assert_not_called()
intercept.assert_not_called()
REPLAYS['pgid-reuse'] = dict(reaped=group.reaped, simulated_new_identity=reused, signals=0)
def test_reused_worker_pid_is_foreign(self):
with TemporaryDirectory() as folder:
root = Path(folder)