Files
MeltMind/tests/test_track_b_gino_formal.py
T

205 lines
10 KiB
Python

"""CPU replay of the production GINO guard and formal custody gates."""
from pathlib import Path
import os
import sys
from tempfile import TemporaryDirectory
import time
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import track_b_gino_formal as formal
import track_b_gino_formal_guard as gpu_guard
import track_b_gino_formal_worker as worker
from track_b_gino_shapenet_entry import read, save, sha
class GuardReplay(unittest.TestCase):
def replay(self, ordering):
uuid = 'GPU-fixture'
limit = dict(disk_bytes=2**20, idle_wait_seconds=10, cleanup_seconds=2, cpu_seconds=10,
rss_bytes=2**30, max_processes=1, device_vram_bytes=2**30,
recorded_vram_bytes=2**30)
with TemporaryDirectory() as folder:
root = Path(folder)
result = root / 'worker-records/result.json'
result.parent.mkdir()
# This subprocess has no GPU code. It waits until the replay lets
# it finish or until the production guard stops its process group.
command = [sys.executable, '-c',
"from pathlib import Path\nimport time,sys\n"
"root=Path(sys.argv[1])\n"
"while not (root/'exit').exists(): time.sleep(.01)\n"
"(root/'worker-records/result.json').write_text("
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder]
observed = []
stopped = []
real_stop = gpu_guard.stop_group
real_tree = gpu_guard.process_tree
def stop_owned(pid):
stopped.append(pid)
real_stop(pid)
def exit_and_wait():
(root / 'exit').touch()
pid = read(root / 'actual-worker-spawn.json')['process']['pid']
by = time.monotonic() + 4
while gpu_guard.current_identity(pid) is not None and time.monotonic() < by:
time.sleep(.01)
if gpu_guard.current_identity(pid) is not None:
raise RuntimeError('fixture worker did not exit and get reaped')
observed.append('worker-reaped')
def tree(identity):
if (root / 'actual-worker-spawn.json').exists() and (
ordering == 'tree-first' and 'worker-reaped' not in observed):
exit_and_wait()
observed.append('tree')
return real_tree(identity)
def snapshot(card):
if card != uuid:
raise AssertionError(card)
spawn = root / 'actual-worker-spawn.json'
if not spawn.exists():
return dict(idle=True, occupants=[], unix=10., memory_MiB=0)
observed.append('gpu-sample')
pid = os.getpid() if ordering == 'foreign' else read(spawn)['process']['pid']
sample = dict(idle=False, unix=42.125, memory_MiB=4,
occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')])
if ordering == 'gpu-first':
exit_and_wait()
return sample
with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot),
patch.object(gpu_guard, 'process_tree', side_effect=tree),
patch.object(gpu_guard, 'stop_group', side_effect=stop_owned),
patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}),
patch.object(gpu_guard, 'dispatch_environment',
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))):
began = time.monotonic()
outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15,
ticket=dict(ordinal='', guard_started_monotonic=began), validate_result=lambda: sha(result))
spawn = read(root / 'actual-worker-spawn.json')['process']
if ordering == 'foreign':
self.assertEqual(outcome['status'], 'STOPPED')
self.assertEqual(outcome['trigger'], 'FOREIGN_GPU_OCCUPANT')
self.assertIsNone(outcome['result_sha256'])
row = read(root / 'foreign-occupant.json')
self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid))
self.assertEqual(row['occupants'][0]['pid'], os.getpid())
self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python')
self.assertEqual(stopped, [spawn['pid']])
else:
self.assertEqual(outcome['status'], 'COMPLETE', outcome)
self.assertFalse((root / 'foreign-occupant.json').exists())
self.assertEqual(stopped, [])
self.assertEqual(outcome['result_sha256'], sha(result))
return observed
def test_gpu_list_before_worker_exit(self):
order = self.replay('gpu-first')
self.assertLess(order.index('gpu-sample'), order.index('worker-reaped'))
def test_worker_already_exited_before_gpu_list(self):
order = self.replay('tree-first')
self.assertLess(order.index('worker-reaped'), order.index('gpu-sample'))
def test_real_foreign_identity_has_exact_stop_receipt(self):
self.replay('foreign')
def test_reused_worker_pid_is_foreign(self):
with TemporaryDirectory() as folder:
root = Path(folder)
original = dict(pid=12, created=1.)
current = dict(pid=12, created=2.)
sample = dict(unix=42., occupants=[dict(pid=12, process_name='new-process')], memory_MiB=4)
monitor = gpu_guard.OccupancyMonitor(root, 'fixture', original, 2**30)
with (patch.object(gpu_guard, 'snapshot', return_value=sample),
patch.object(gpu_guard, 'current_identity', return_value=current)):
with self.assertRaisesRegex(RuntimeError, 'FOREIGN_GPU_OCCUPANT'):
monitor.check()
row = read(root / 'foreign-occupant.json')
self.assertEqual(row['occupants'][0]['observed_process_identity'], current)
class FormalBinding(unittest.TestCase):
def test_inclusive_gate_and_all_three_seeds_required(self):
for endpoint in (6.62, 7.62):
self.assertEqual(formal.pool([endpoint] * 3, [6.62, 7.62]), (endpoint, True))
self.assertEqual(formal.pool([7.12, 7.12, None], [6.62, 7.62]), (None, None))
self.assertFalse(formal.pool([7.63] * 3, [6.62, 7.62])[1])
def test_original_split_score_keeps_order_and_rejects_nan(self):
ids = list(range(111))
complete = [dict(case=i, relative_l2=.0712) for i in ids]
self.assertAlmostEqual(formal.score_mean(complete, ids), 7.12)
with self.assertRaises(ValueError):
formal.score_mean(complete[::-1], ids)
complete[0]['relative_l2'] = float('nan')
with self.assertRaises(ValueError):
formal.score_mean(complete, ids)
def test_scoring_needs_all_checkpoints_before_access(self):
with TemporaryDirectory() as folder:
root = Path(folder)
(root / 'checkpoints-frozen.json').write_text('{}')
with patch.object(formal, 'all_checkpoints', side_effect=PermissionError('missing fit')):
with self.assertRaisesRegex(PermissionError, 'missing fit'):
worker.score({}, 0, None, root, None)
self.assertFalse((root / 'heldout-target-access.jsonl').exists())
def test_score_opens_one_111_target_pass_and_rejects_a_second(self):
import torch
with TemporaryDirectory() as folder:
root = Path(folder)
path = root / 'checkpoint.pt'
torch.save(dict(state={}, pressure_mean=torch.tensor([2.]), pressure_std=torch.tensor([3.])), path)
roster = {'0': dict(checkpoint=dict(path=str(path), sha256=sha(path)))}
save(root / 'checkpoints-frozen.json', roster)
ids = list(range(111))
reads = []
def load(ids_argument, role, log):
self.assertEqual((ids_argument, role), (ids, 'heldout'))
for case in ids_argument:
reads.append(case)
log.write(f'{case}\n')
return object()
class Normalizer:
pass
data = SimpleNamespace(test_ids=ids, normalizer_class=Normalizer, data=load)
torch_cpu_fixture = SimpleNamespace(load=torch.load, isfinite=torch.isfinite,
cuda=SimpleNamespace(synchronize=lambda: None))
model = SimpleNamespace(to=lambda _: model, load_state_dict=lambda _state, **kw: None)
rows = [dict(case=i, relative_l2=.0712) for i in ids]
with (patch.object(formal, 'all_checkpoints', return_value=roster),
patch.object(worker, 'paper_model', return_value=model),
patch.object(worker, 'case_scores', return_value=rows)):
result = worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture)
with self.assertRaises(FileExistsError):
worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture)
self.assertEqual(reads, ids)
self.assertEqual(result['target_reads'], 111)
self.assertAlmostEqual(result['mean_percent'], 7.12)
def test_card_reservation_expiry_and_total_budget(self):
binding = formal.expected_contract()['formal']
budgets = binding['budget_per_seed_stage_gpu_seconds']
self.assertEqual(sum(sum(b.values()) for b in budgets.values()), 40000)
self.assertEqual(binding['device_pci_ordinals'], {'0': 2, '1': 3, '2': 0})
latest = formal.RESERVATION_END.timestamp() - 300
formal.check_reservation(latest - 1, 300)
for when in (latest, formal.RESERVATION_END.timestamp(), formal.RESERVATION_START.timestamp() - 1):
with self.assertRaises(PermissionError):
formal.check_reservation(when, 300)
if __name__ == '__main__':
unittest.main()