Files
MeltMind/tests/test_track_b_h2_pressure_combined.py
T

215 lines
10 KiB
Python

"""CPU-only checks of the combined production entry, timing and custody bindings."""
from pathlib import Path
from tempfile import TemporaryDirectory
from types import SimpleNamespace
import time
import unittest
from unittest.mock import patch
import numpy as np
import torch
from s2_gpu_idle_gate import wait_idle
import track_b_gino_formal_guard as real_guard
import track_b_h2_pressure_entry as entry
from track_b_h2_pressure_prepare import surface
import track_b_h2_pressure_worker as worker
from track_b_gino_shapenet_entry import read, save, sha
EVIDENCE = {}
class CombinedGuard(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.binding = entry.contract()
cls.contract_bytes = (entry.ROOT / 'contract.json').read_bytes()
def replay(self, mode):
with TemporaryDirectory() as folder:
root = Path(folder)
(root / 'contract.json').write_bytes(self.contract_bytes)
fixture = root / 'track_b_h2_pressure_worker.py'
fixture.write_text(
'import json, pathlib, sys\n'
'seed, phase = sys.argv[1:]\n'
'root = pathlib.Path(__file__).parent / f"seed-{seed}" / phase\n'
'records = root / "worker-records"\n'
'records.mkdir()\n'
f'if {mode == "over"}:\n'
' (root / "first-epoch-projection.json").write_text('
'json.dumps({"within_cap":False}))\n'
' raise SystemExit(2)\n'
'path = records / "result.json"\n'
'temp = path.with_suffix(".tmp")\n'
'temp.write_text(json.dumps(dict(peak_allocated_bytes=1, peak_reserved_bytes=2)))\n'
'temp.rename(path)\n')
admission = []
clock = [0.]
selected = entry.DEVICES['0']
def probe(uuid):
self.assertEqual(uuid, selected)
admission.append(uuid)
return dict(uuid=uuid, idle=True, unix=10., occupants=[], memory_MiB=0)
def sample(uuid):
return dict(probe(uuid), idle=mode != 'busy')
def cpu_wait(uuid, limit, record, *, sample):
return wait_idle(uuid, limit, record, sample=sample, clock=lambda: clock[0],
sleep=lambda seconds: clock.__setitem__(0, clock[0] + seconds))
builds = []
author = dict(eligible_ordinals=['0'], dispatch_before_unix=time.time()+60)
def capture_result(c, seed, phase):
builds.append((seed, phase))
return sha(entry.run_root(seed, phase) / 'worker-records/result.json')
def fake_roster(_):
self.assertEqual(builds, [(s, 'fit') for s in entry.SEEDS])
return {str(seed): {'synthetic': True} for seed in entry.SEEDS}
with (patch.object(entry, 'ROOT', root), patch.object(entry, 'SOURCES', root),
patch.object(entry, 'authorize', return_value=author),
patch.object(entry, 'snapshot', side_effect=probe),
patch.object(entry, 'validate_result', side_effect=capture_result),
patch.object(entry, 'all_checkpoints', side_effect=fake_roster),
patch.object(real_guard, 'snapshot', side_effect=sample),
patch.object(real_guard, 'wait_idle', side_effect=cpu_wait),
patch.object(real_guard, 'dispatch_environment',
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES='0'), []))):
code = entry.run(self.binding)
with self.assertRaises(FileExistsError):
entry.run(self.binding)
stages = sorted(root.glob('seed-*/*/complete.json'))
terminals = [read(p) for p in stages]
starts = len(list(root.glob('seed-*/*/actual-worker-spawn.json')))
e = read(root / 'complete.json')
if mode == 'ok':
self.assertEqual((code, e['status'], starts), (0, 'COMPLETE', 6), e)
self.assertEqual(builds, [(s, phase) for phase in ('fit', 'score') for s in entry.SEEDS])
self.assertTrue(all(x['lifecycle'][0]['event'] == 'leader_reaped' for x in terminals))
else:
expected = 'STOP_PROJECTED_CAP' if mode == 'over' else 'Seed 0 fit HOLD: IDLE_DEADLINE'
self.assertEqual((code, e['trigger'], starts), (1, expected, int(mode == 'over')))
self.assertEqual(len(stages), 1)
self.assertFalse((root / 'checkpoints-frozen.json').exists())
EVIDENCE[mode] = dict(combined=e['status'], trigger=e['trigger'], actual_cpu_children=starts,
phase_order=builds, stage_terminals=[x['status'] for x in terminals])
def test_actual_guard_and_combined_order(self):
self.replay('ok')
def test_busy_after_initial_admission_starts_no_worker(self):
self.replay('busy')
def test_actual_sequential_guard_stops_after_cost_failure(self):
self.replay('over')
def test_missing_independent_review_refuses_before_device_probe(self):
with patch.object(entry, 'snapshot') as probe:
with self.assertRaises(FileNotFoundError):
entry.run(self.binding)
probe.assert_not_called()
class PressureWork(unittest.TestCase):
def test_first_epoch_at_two_sides_of_cap(self):
row = dict(stage='EPOCH_COMPLETE', seed=0, epoch=1, examples=500, seconds=90.)
ticket = dict(guard_started_monotonic=100.)
low = entry.first_epoch_projection(ticket, row, 220.)
self.assertTrue(low['within_cap'])
self.assertEqual(low['booked_seconds_so_far'], 120)
self.assertEqual(low['remaining_train_epochs'], 299)
row['seconds'] = 115.
high = entry.first_epoch_projection(ticket, row, 245.)
self.assertFalse(high['within_cap'])
with self.assertRaises(ValueError):
entry.first_epoch_projection(ticket, dict(row, examples=499), 245.)
def test_production_fit_callback_raises_on_first_full_epoch(self):
def fixture_train(c, data, seed, epochs, path):
save(path, dict(stage='EPOCH_COMPLETE', seed=seed, epoch=1, examples=500, seconds=120.))
raise AssertionError('Second epoch should never be entered')
with TemporaryDirectory() as folder, patch.object(worker, 'train_seed', fixture_train):
root = Path(folder)
data = SimpleNamespace(train_ids=[1], data=lambda *_: [1])
with self.assertRaisesRegex(RuntimeError, 'STOP_PROJECTED_CAP'):
worker.fit({'protocol': {'epochs_per_seed': 100}}, 0, data, root, root,
{'guard_started_monotonic': time.monotonic() - 140})
self.assertFalse(read(root/'first-epoch-projection.json')['within_cap'])
self.assertFalse((root/'epoch-100.pt').exists())
def test_h2_model_forward_and_backward_on_original_runtime(self):
torch.set_num_threads(1)
x = np.array([[0., 0., 0.], [1., 0., 0.], [0., 1., 0.], [0., 0., 1.]])
triangles = np.array([[0,1,2], [0,2,3], [1,2,3], [0,1,3]])
graph = surface(x, triangles, [-1]*3, [2]*3)
model = worker.PressureModel().train()
item = dict(graph=[a[None] for a in graph], pressure=torch.tensor([[1., 2., 3., 4.]]))
loss = model.loss_dict(item, torch.nn.functional.mse_loss)['loss']
self.assertTrue(bool(torch.isfinite(loss)))
loss.backward()
self.assertTrue(bool(torch.any(model.head[-1].weight.grad != 0)))
def test_real_score_wrap_requires_custody_and_reads_only_once(self):
with TemporaryDirectory() as folder:
root = Path(folder)
path = root/'epoch-100.pt'
torch.save(dict(state={}, pressure_mean=torch.tensor([2.]), pressure_std=torch.tensor([3.])), path)
roster = {'0': dict(checkpoint=dict(path=str(path), sha256=sha(path)))}
save(root/'checkpoints-frozen.json', roster)
c = dict(protocol={'pressure_epsilon': 1e-6})
seen = []
def load(ids, role, f):
self.assertEqual(role, 'heldout')
for case in ids:
seen.append(case)
f.write(str(case) + '\n')
return object()
ids = list(range(111))
data = SimpleNamespace(test_ids=ids, normalizer_class=type('Normalizer', (), {}), data=load)
model = SimpleNamespace(load_state_dict=lambda *_a, **_kw: None, to=lambda _: model)
scored = [dict(case=case, relative_l2=.07) for case in ids]
with patch.object(entry, 'all_checkpoints', side_effect=PermissionError('unfrozen')):
with self.assertRaises(PermissionError):
worker.score(c, 0, data, root)
self.assertEqual(seen, [])
with (patch.object(entry, 'all_checkpoints', return_value=roster),
patch.object(worker, 'paper_model', return_value=model),
patch.object(worker.gino, 'case_scores', return_value=scored),
patch.object(torch.cuda, 'synchronize')):
result = worker.score(c, 0, data, root)
with self.assertRaises(FileExistsError):
worker.score(c, 0, data, root)
self.assertEqual(seen, ids)
self.assertEqual(result['target_reads'], 111)
def test_paired_components_bootstrap_and_gino_gate(self):
ids = list(range(111))
c = dict(heldout_cases=ids, heldout_components=[ids[:2]]+[[i] for i in ids[2:]])
def arms(error):
return [[dict(case=i, relative_l2=error) for i in ids] for _ in entry.SEEDS]
stats = entry.paired_bootstrap(c, arms(.06), arms(.07))
self.assertAlmostEqual(stats['paired_mean_difference_pp'], -1.)
self.assertEqual(stats['decision'], 'H2_P_IMPROVEMENT_PASS')
self.assertEqual(entry.paired_bootstrap(c, arms(.07), arms(.07))['decision'],
'H2_P_IMPROVEMENT_NOT_ESTABLISHED')
self.assertEqual(entry.paired_bootstrap(c, arms(.06), arms(.08))['decision'],
'GINO_REPRODUCTION_GATE_NOT_MET')
with self.assertRaises(ValueError):
entry.paired_bootstrap(c, arms(.06)[:2], arms(.07))
if __name__ == '__main__':
unittest.main()