215 lines
10 KiB
Python
215 lines
10 KiB
Python
"""CPU-only checks of the combined production entry, timing and custody bindings."""
|
|
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from types import SimpleNamespace
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from s2_gpu_idle_gate import wait_idle
|
|
import track_b_gino_formal_guard as real_guard
|
|
import track_b_h2_pressure_entry as entry
|
|
from track_b_h2_pressure_prepare import surface
|
|
import track_b_h2_pressure_worker as worker
|
|
from track_b_gino_shapenet_entry import read, save, sha
|
|
|
|
|
|
EVIDENCE = {}
|
|
|
|
|
|
class CombinedGuard(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.binding = entry.contract()
|
|
cls.contract_bytes = (entry.ROOT / 'contract.json').read_bytes()
|
|
|
|
def replay(self, mode):
|
|
with TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
(root / 'contract.json').write_bytes(self.contract_bytes)
|
|
fixture = root / 'track_b_h2_pressure_worker.py'
|
|
fixture.write_text(
|
|
'import json, pathlib, sys\n'
|
|
'seed, phase = sys.argv[1:]\n'
|
|
'root = pathlib.Path(__file__).parent / f"seed-{seed}" / phase\n'
|
|
'records = root / "worker-records"\n'
|
|
'records.mkdir()\n'
|
|
f'if {mode == "over"}:\n'
|
|
' (root / "first-epoch-projection.json").write_text('
|
|
'json.dumps({"within_cap":False}))\n'
|
|
' raise SystemExit(2)\n'
|
|
'path = records / "result.json"\n'
|
|
'temp = path.with_suffix(".tmp")\n'
|
|
'temp.write_text(json.dumps(dict(peak_allocated_bytes=1, peak_reserved_bytes=2)))\n'
|
|
'temp.rename(path)\n')
|
|
admission = []
|
|
clock = [0.]
|
|
selected = entry.DEVICES['0']
|
|
|
|
def probe(uuid):
|
|
self.assertEqual(uuid, selected)
|
|
admission.append(uuid)
|
|
return dict(uuid=uuid, idle=True, unix=10., occupants=[], memory_MiB=0)
|
|
|
|
def sample(uuid):
|
|
return dict(probe(uuid), idle=mode != 'busy')
|
|
|
|
def cpu_wait(uuid, limit, record, *, sample):
|
|
return wait_idle(uuid, limit, record, sample=sample, clock=lambda: clock[0],
|
|
sleep=lambda seconds: clock.__setitem__(0, clock[0] + seconds))
|
|
|
|
builds = []
|
|
author = dict(eligible_ordinals=['0'], dispatch_before_unix=time.time()+60)
|
|
|
|
def capture_result(c, seed, phase):
|
|
builds.append((seed, phase))
|
|
return sha(entry.run_root(seed, phase) / 'worker-records/result.json')
|
|
|
|
def fake_roster(_):
|
|
self.assertEqual(builds, [(s, 'fit') for s in entry.SEEDS])
|
|
return {str(seed): {'synthetic': True} for seed in entry.SEEDS}
|
|
|
|
with (patch.object(entry, 'ROOT', root), patch.object(entry, 'SOURCES', root),
|
|
patch.object(entry, 'authorize', return_value=author),
|
|
patch.object(entry, 'snapshot', side_effect=probe),
|
|
patch.object(entry, 'validate_result', side_effect=capture_result),
|
|
patch.object(entry, 'all_checkpoints', side_effect=fake_roster),
|
|
patch.object(real_guard, 'snapshot', side_effect=sample),
|
|
patch.object(real_guard, 'wait_idle', side_effect=cpu_wait),
|
|
patch.object(real_guard, 'dispatch_environment',
|
|
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES='0'), []))):
|
|
code = entry.run(self.binding)
|
|
with self.assertRaises(FileExistsError):
|
|
entry.run(self.binding)
|
|
stages = sorted(root.glob('seed-*/*/complete.json'))
|
|
terminals = [read(p) for p in stages]
|
|
starts = len(list(root.glob('seed-*/*/actual-worker-spawn.json')))
|
|
e = read(root / 'complete.json')
|
|
if mode == 'ok':
|
|
self.assertEqual((code, e['status'], starts), (0, 'COMPLETE', 6), e)
|
|
self.assertEqual(builds, [(s, phase) for phase in ('fit', 'score') for s in entry.SEEDS])
|
|
self.assertTrue(all(x['lifecycle'][0]['event'] == 'leader_reaped' for x in terminals))
|
|
else:
|
|
expected = 'STOP_PROJECTED_CAP' if mode == 'over' else 'Seed 0 fit HOLD: IDLE_DEADLINE'
|
|
self.assertEqual((code, e['trigger'], starts), (1, expected, int(mode == 'over')))
|
|
self.assertEqual(len(stages), 1)
|
|
self.assertFalse((root / 'checkpoints-frozen.json').exists())
|
|
EVIDENCE[mode] = dict(combined=e['status'], trigger=e['trigger'], actual_cpu_children=starts,
|
|
phase_order=builds, stage_terminals=[x['status'] for x in terminals])
|
|
|
|
def test_actual_guard_and_combined_order(self):
|
|
self.replay('ok')
|
|
|
|
def test_busy_after_initial_admission_starts_no_worker(self):
|
|
self.replay('busy')
|
|
|
|
def test_actual_sequential_guard_stops_after_cost_failure(self):
|
|
self.replay('over')
|
|
|
|
def test_missing_independent_review_refuses_before_device_probe(self):
|
|
with patch.object(entry, 'snapshot') as probe:
|
|
with self.assertRaises(FileNotFoundError):
|
|
entry.run(self.binding)
|
|
probe.assert_not_called()
|
|
|
|
|
|
class PressureWork(unittest.TestCase):
|
|
def test_first_epoch_at_two_sides_of_cap(self):
|
|
row = dict(stage='EPOCH_COMPLETE', seed=0, epoch=1, examples=500, seconds=90.)
|
|
ticket = dict(guard_started_monotonic=100.)
|
|
low = entry.first_epoch_projection(ticket, row, 220.)
|
|
self.assertTrue(low['within_cap'])
|
|
self.assertEqual(low['booked_seconds_so_far'], 120)
|
|
self.assertEqual(low['remaining_train_epochs'], 299)
|
|
row['seconds'] = 115.
|
|
high = entry.first_epoch_projection(ticket, row, 245.)
|
|
self.assertFalse(high['within_cap'])
|
|
with self.assertRaises(ValueError):
|
|
entry.first_epoch_projection(ticket, dict(row, examples=499), 245.)
|
|
|
|
def test_production_fit_callback_raises_on_first_full_epoch(self):
|
|
def fixture_train(c, data, seed, epochs, path):
|
|
save(path, dict(stage='EPOCH_COMPLETE', seed=seed, epoch=1, examples=500, seconds=120.))
|
|
raise AssertionError('Second epoch should never be entered')
|
|
|
|
with TemporaryDirectory() as folder, patch.object(worker, 'train_seed', fixture_train):
|
|
root = Path(folder)
|
|
data = SimpleNamespace(train_ids=[1], data=lambda *_: [1])
|
|
with self.assertRaisesRegex(RuntimeError, 'STOP_PROJECTED_CAP'):
|
|
worker.fit({'protocol': {'epochs_per_seed': 100}}, 0, data, root, root,
|
|
{'guard_started_monotonic': time.monotonic() - 140})
|
|
self.assertFalse(read(root/'first-epoch-projection.json')['within_cap'])
|
|
self.assertFalse((root/'epoch-100.pt').exists())
|
|
|
|
def test_h2_model_forward_and_backward_on_original_runtime(self):
|
|
torch.set_num_threads(1)
|
|
x = np.array([[0., 0., 0.], [1., 0., 0.], [0., 1., 0.], [0., 0., 1.]])
|
|
triangles = np.array([[0,1,2], [0,2,3], [1,2,3], [0,1,3]])
|
|
graph = surface(x, triangles, [-1]*3, [2]*3)
|
|
model = worker.PressureModel().train()
|
|
item = dict(graph=[a[None] for a in graph], pressure=torch.tensor([[1., 2., 3., 4.]]))
|
|
loss = model.loss_dict(item, torch.nn.functional.mse_loss)['loss']
|
|
self.assertTrue(bool(torch.isfinite(loss)))
|
|
loss.backward()
|
|
self.assertTrue(bool(torch.any(model.head[-1].weight.grad != 0)))
|
|
|
|
def test_real_score_wrap_requires_custody_and_reads_only_once(self):
|
|
with TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
path = root/'epoch-100.pt'
|
|
torch.save(dict(state={}, pressure_mean=torch.tensor([2.]), pressure_std=torch.tensor([3.])), path)
|
|
roster = {'0': dict(checkpoint=dict(path=str(path), sha256=sha(path)))}
|
|
save(root/'checkpoints-frozen.json', roster)
|
|
c = dict(protocol={'pressure_epsilon': 1e-6})
|
|
seen = []
|
|
|
|
def load(ids, role, f):
|
|
self.assertEqual(role, 'heldout')
|
|
for case in ids:
|
|
seen.append(case)
|
|
f.write(str(case) + '\n')
|
|
return object()
|
|
|
|
ids = list(range(111))
|
|
data = SimpleNamespace(test_ids=ids, normalizer_class=type('Normalizer', (), {}), data=load)
|
|
model = SimpleNamespace(load_state_dict=lambda *_a, **_kw: None, to=lambda _: model)
|
|
scored = [dict(case=case, relative_l2=.07) for case in ids]
|
|
with patch.object(entry, 'all_checkpoints', side_effect=PermissionError('unfrozen')):
|
|
with self.assertRaises(PermissionError):
|
|
worker.score(c, 0, data, root)
|
|
self.assertEqual(seen, [])
|
|
with (patch.object(entry, 'all_checkpoints', return_value=roster),
|
|
patch.object(worker, 'paper_model', return_value=model),
|
|
patch.object(worker.gino, 'case_scores', return_value=scored),
|
|
patch.object(torch.cuda, 'synchronize')):
|
|
result = worker.score(c, 0, data, root)
|
|
with self.assertRaises(FileExistsError):
|
|
worker.score(c, 0, data, root)
|
|
self.assertEqual(seen, ids)
|
|
self.assertEqual(result['target_reads'], 111)
|
|
|
|
def test_paired_components_bootstrap_and_gino_gate(self):
|
|
ids = list(range(111))
|
|
c = dict(heldout_cases=ids, heldout_components=[ids[:2]]+[[i] for i in ids[2:]])
|
|
|
|
def arms(error):
|
|
return [[dict(case=i, relative_l2=error) for i in ids] for _ in entry.SEEDS]
|
|
|
|
stats = entry.paired_bootstrap(c, arms(.06), arms(.07))
|
|
self.assertAlmostEqual(stats['paired_mean_difference_pp'], -1.)
|
|
self.assertEqual(stats['decision'], 'H2_P_IMPROVEMENT_PASS')
|
|
self.assertEqual(entry.paired_bootstrap(c, arms(.07), arms(.07))['decision'],
|
|
'H2_P_IMPROVEMENT_NOT_ESTABLISHED')
|
|
self.assertEqual(entry.paired_bootstrap(c, arms(.06), arms(.08))['decision'],
|
|
'GINO_REPRODUCTION_GATE_NOT_MET')
|
|
with self.assertRaises(ValueError):
|
|
entry.paired_bootstrap(c, arms(.06)[:2], arms(.07))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|