222 lines
11 KiB
Python
222 lines
11 KiB
Python
"""CPU replay of the production GINO guard and formal custody gates."""
|
|
|
|
from pathlib import Path
|
|
import os
|
|
import sys
|
|
from tempfile import TemporaryDirectory
|
|
import time
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import track_b_gino_formal as formal
|
|
import track_b_gino_formal_guard as gpu_guard
|
|
import track_b_gino_formal_worker as worker
|
|
from track_b_gino_shapenet_entry import read, save, sha
|
|
|
|
|
|
class GuardReplay(unittest.TestCase):
|
|
def replay(self, ordering):
|
|
uuid = 'GPU-fixture'
|
|
limit = dict(disk_bytes=2**20, idle_wait_seconds=10, cleanup_seconds=2, cpu_seconds=10,
|
|
rss_bytes=2**30, max_processes=1, device_vram_bytes=2**30,
|
|
recorded_vram_bytes=2**30)
|
|
with TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
result = root / 'worker-records/result.json'
|
|
result.parent.mkdir()
|
|
# This subprocess has no GPU code. It waits until the replay lets
|
|
# it finish or until the production guard stops its process group.
|
|
command = [sys.executable, '-c',
|
|
"from pathlib import Path\nimport time,sys\n"
|
|
"root=Path(sys.argv[1])\n"
|
|
"while not (root/'exit').exists(): time.sleep(.01)\n"
|
|
"(root/'worker-records/result.json').write_text("
|
|
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder]
|
|
observed = []
|
|
stopped = []
|
|
real_stop = gpu_guard.stop_group
|
|
real_tree = gpu_guard.process_tree
|
|
ticket = dict(ordinal='', dispatch_before_unix=time.time() + 15)
|
|
|
|
def stop_owned(pid):
|
|
stopped.append(pid)
|
|
real_stop(pid)
|
|
|
|
def exit_and_wait():
|
|
(root / 'exit').touch()
|
|
pid = read(root / 'actual-worker-spawn.json')['process']['pid']
|
|
by = time.monotonic() + 4
|
|
while gpu_guard.current_identity(pid) is not None and time.monotonic() < by:
|
|
time.sleep(.01)
|
|
if gpu_guard.current_identity(pid) is not None:
|
|
raise RuntimeError('fixture worker did not exit and get reaped')
|
|
observed.append('worker-reaped')
|
|
|
|
def tree(identity):
|
|
if (root / 'actual-worker-spawn.json').exists() and (
|
|
ordering == 'tree-first' and 'worker-reaped' not in observed):
|
|
exit_and_wait()
|
|
observed.append('tree')
|
|
return real_tree(identity)
|
|
|
|
def snapshot(card):
|
|
if card != uuid:
|
|
raise AssertionError(card)
|
|
spawn = root / 'actual-worker-spawn.json'
|
|
if not spawn.exists():
|
|
if ordering == 'expired-before-spawn':
|
|
ticket['dispatch_before_unix'] = time.time() - 1
|
|
return dict(idle=True, occupants=[], unix=10., memory_MiB=0)
|
|
observed.append('gpu-sample')
|
|
pid = os.getpid() if ordering == 'foreign' else read(spawn)['process']['pid']
|
|
sample = dict(idle=False, unix=42.125, memory_MiB=4,
|
|
occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')])
|
|
if ordering == 'crossed-after-spawn':
|
|
ticket['dispatch_before_unix'] = time.time() - 1
|
|
if ordering in ('gpu-first', 'crossed-after-spawn'):
|
|
exit_and_wait()
|
|
return sample
|
|
|
|
with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot),
|
|
patch.object(gpu_guard, 'process_tree', side_effect=tree),
|
|
patch.object(gpu_guard, 'stop_group', side_effect=stop_owned),
|
|
patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}),
|
|
patch.object(gpu_guard, 'dispatch_environment',
|
|
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))):
|
|
began = time.monotonic()
|
|
ticket['guard_started_monotonic'] = began
|
|
outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15,
|
|
ticket=ticket, validate_result=lambda: sha(result))
|
|
if ordering == 'expired-before-spawn':
|
|
self.assertEqual(outcome['status'], 'STOPPED')
|
|
self.assertEqual(outcome['trigger'], 'CARD_RESERVATION_EXPIRED_BEFORE_SPAWN')
|
|
self.assertFalse((root / 'actual-worker-spawn.json').exists())
|
|
return observed
|
|
spawn = read(root / 'actual-worker-spawn.json')['process']
|
|
if ordering == 'foreign':
|
|
self.assertEqual(outcome['status'], 'STOPPED')
|
|
self.assertEqual(outcome['trigger'], 'FOREIGN_GPU_OCCUPANT')
|
|
self.assertIsNone(outcome['result_sha256'])
|
|
row = read(root / 'foreign-occupant.json')
|
|
self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid))
|
|
self.assertEqual(row['occupants'][0]['pid'], os.getpid())
|
|
self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python')
|
|
self.assertEqual(stopped, [spawn['pid']])
|
|
else:
|
|
self.assertEqual(outcome['status'], 'COMPLETE', outcome)
|
|
self.assertFalse((root / 'foreign-occupant.json').exists())
|
|
self.assertEqual(stopped, [])
|
|
self.assertEqual(outcome['result_sha256'], sha(result))
|
|
return observed
|
|
|
|
def test_gpu_list_before_worker_exit(self):
|
|
order = self.replay('gpu-first')
|
|
self.assertLess(order.index('gpu-sample'), order.index('worker-reaped'))
|
|
|
|
def test_worker_already_exited_before_gpu_list(self):
|
|
order = self.replay('tree-first')
|
|
self.assertLess(order.index('worker-reaped'), order.index('gpu-sample'))
|
|
|
|
def test_real_foreign_identity_has_exact_stop_receipt(self):
|
|
self.replay('foreign')
|
|
|
|
def test_reservation_expires_after_idle_but_before_spawn(self):
|
|
self.replay('expired-before-spawn')
|
|
|
|
def test_running_guard_continues_after_reservation_end(self):
|
|
self.replay('crossed-after-spawn')
|
|
|
|
def test_reused_worker_pid_is_foreign(self):
|
|
with TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
original = dict(pid=12, created=1.)
|
|
current = dict(pid=12, created=2.)
|
|
sample = dict(unix=42., occupants=[dict(pid=12, process_name='new-process')], memory_MiB=4)
|
|
monitor = gpu_guard.OccupancyMonitor(root, 'fixture', original, 2**30)
|
|
with (patch.object(gpu_guard, 'snapshot', return_value=sample),
|
|
patch.object(gpu_guard, 'current_identity', return_value=current)):
|
|
with self.assertRaisesRegex(RuntimeError, 'FOREIGN_GPU_OCCUPANT'):
|
|
monitor.check()
|
|
row = read(root / 'foreign-occupant.json')
|
|
self.assertEqual(row['occupants'][0]['observed_process_identity'], current)
|
|
|
|
|
|
class FormalBinding(unittest.TestCase):
|
|
def test_inclusive_gate_and_all_three_seeds_required(self):
|
|
for endpoint in (6.62, 7.62):
|
|
self.assertEqual(formal.pool([endpoint] * 3, [6.62, 7.62]), (endpoint, True))
|
|
self.assertEqual(formal.pool([7.12, 7.12, None], [6.62, 7.62]), (None, None))
|
|
self.assertFalse(formal.pool([7.63] * 3, [6.62, 7.62])[1])
|
|
|
|
def test_original_split_score_keeps_order_and_rejects_nan(self):
|
|
ids = list(range(111))
|
|
complete = [dict(case=i, relative_l2=.0712) for i in ids]
|
|
self.assertAlmostEqual(formal.score_mean(complete, ids), 7.12)
|
|
with self.assertRaises(ValueError):
|
|
formal.score_mean(complete[::-1], ids)
|
|
complete[0]['relative_l2'] = float('nan')
|
|
with self.assertRaises(ValueError):
|
|
formal.score_mean(complete, ids)
|
|
|
|
def test_scoring_needs_all_checkpoints_before_access(self):
|
|
with TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
(root / 'checkpoints-frozen.json').write_text('{}')
|
|
with patch.object(formal, 'all_checkpoints', side_effect=PermissionError('missing fit')):
|
|
with self.assertRaisesRegex(PermissionError, 'missing fit'):
|
|
worker.score({}, 0, None, root, None)
|
|
self.assertFalse((root / 'heldout-target-access.jsonl').exists())
|
|
|
|
def test_score_opens_one_111_target_pass_and_rejects_a_second(self):
|
|
import torch
|
|
|
|
with TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
path = root / 'checkpoint.pt'
|
|
torch.save(dict(state={}, pressure_mean=torch.tensor([2.]), pressure_std=torch.tensor([3.])), path)
|
|
roster = {'0': dict(checkpoint=dict(path=str(path), sha256=sha(path)))}
|
|
save(root / 'checkpoints-frozen.json', roster)
|
|
ids = list(range(111))
|
|
reads = []
|
|
|
|
def load(ids_argument, role, log):
|
|
self.assertEqual((ids_argument, role), (ids, 'heldout'))
|
|
for case in ids_argument:
|
|
reads.append(case)
|
|
log.write(f'{case}\n')
|
|
return object()
|
|
|
|
class Normalizer:
|
|
pass
|
|
|
|
data = SimpleNamespace(test_ids=ids, normalizer_class=Normalizer, data=load)
|
|
torch_cpu_fixture = SimpleNamespace(load=torch.load, isfinite=torch.isfinite,
|
|
cuda=SimpleNamespace(synchronize=lambda: None))
|
|
model = SimpleNamespace(to=lambda _: model, load_state_dict=lambda _state, **kw: None)
|
|
rows = [dict(case=i, relative_l2=.0712) for i in ids]
|
|
with (patch.object(formal, 'all_checkpoints', return_value=roster),
|
|
patch.object(worker, 'paper_model', return_value=model),
|
|
patch.object(worker, 'case_scores', return_value=rows)):
|
|
result = worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture)
|
|
with self.assertRaises(FileExistsError):
|
|
worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture)
|
|
self.assertEqual(reads, ids)
|
|
self.assertEqual(result['target_reads'], 111)
|
|
self.assertAlmostEqual(result['mean_percent'], 7.12)
|
|
|
|
def test_card_reservation_expiry_and_total_budget(self):
|
|
binding = formal.expected_contract()['formal']
|
|
budgets = binding['budget_per_seed_stage_gpu_seconds']
|
|
self.assertEqual(sum(sum(b.values()) for b in budgets.values()), 40000)
|
|
self.assertEqual(binding['device_pci_ordinals'], {'0': 2, '1': 3, '2': 0})
|
|
end = formal.RESERVATION_END.timestamp()
|
|
formal.check_reservation(end - 1)
|
|
for when in (end, formal.RESERVATION_START.timestamp() - 1):
|
|
with self.assertRaises(PermissionError):
|
|
formal.check_reservation(when)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|