201 lines
9.4 KiB
Python
201 lines
9.4 KiB
Python
"""CPU synthetic regression checks. No real feature/target arrays, CUDA or fits."""
|
|
import copy
|
|
from contextlib import ExitStack
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from phase2_h1_worker_v2 import save
|
|
from phase2_h2_data import load_part
|
|
from track_a_h2_common import (dispatchable, execution_command, first_epoch_release,
|
|
project_cell, project_total, read, validate_roles)
|
|
import track_a_h2_guard as guard
|
|
|
|
ROOT = Path('/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18')
|
|
|
|
|
|
class EntryTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.c = read(ROOT / 'contract.json')
|
|
cls.entry = read(cls.c['reuse_entry'])
|
|
|
|
def test_01_exact_original_roles(self):
|
|
validate_roles(self.c, self.entry)
|
|
self.assertEqual([len(j['learning']) for j in self.c['jobs']], [311, 304, 307, 306, 310])
|
|
|
|
def test_02_test_to_train_rejected(self):
|
|
c = copy.deepcopy(self.c)
|
|
c['jobs'][0]['learning'][0] = c['jobs'][0]['test'][0]
|
|
with self.assertRaises(ValueError):
|
|
validate_roles(c, self.entry)
|
|
|
|
def test_03_extra_validation_rejected(self):
|
|
c = copy.deepcopy(self.c)
|
|
c['jobs'][0]['validation'] = [c['jobs'][0]['learning'][0]]
|
|
with self.assertRaises(ValueError):
|
|
validate_roles(c, self.entry)
|
|
|
|
def test_04_test_loader_rejects_before_io(self):
|
|
with patch('phase2_h2_data.np.load', side_effect=AssertionError('Unexpected array read')):
|
|
with self.assertRaises(PermissionError):
|
|
load_part(dict(folds=[dict(learning=['a'], validation=[])]), 'b', {}, 'test', Path('/tmp'))
|
|
|
|
def test_05_clock_cannot_be_changed(self):
|
|
c = copy.deepcopy(self.c)
|
|
c['jobs'][0]['c'] += .01
|
|
with self.assertRaises(ValueError):
|
|
validate_roles(c, self.entry)
|
|
|
|
def test_06_two_slots_do_not_require_five_first_epochs(self):
|
|
jobs = self.c['jobs']
|
|
self.assertEqual([j['id'] for j in dispatchable(jobs, [])], [0, 1])
|
|
self.assertEqual([j['id'] for j in dispatchable(jobs[2:], [jobs[1]])], [2])
|
|
|
|
def test_07_item1_gpu_protected(self):
|
|
j = copy.deepcopy(self.c['jobs'][0])
|
|
j['gpu_uuid'] = 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c'
|
|
with self.assertRaises(ValueError):
|
|
dispatchable([j], [])
|
|
|
|
def test_08_existing_item1_is_not_duplicate_track_a(self):
|
|
self.assertFalse(execution_command(['python', 'tools/phase3_data_sufficiency_v2_worker.py']))
|
|
self.assertTrue(execution_command(['python', 'tools/track_a_h2_queue.py']))
|
|
|
|
def test_09_pending_caps_reserved(self):
|
|
p = project_total(self.c, {}, {})
|
|
self.assertEqual(p['projected_gpu_seconds'], 162000)
|
|
self.assertEqual(p['status'], 'PASS')
|
|
|
|
def test_10_terminal_replaces_estimate_once(self):
|
|
p = project_total(self.c, {0: dict(projected_gpu_seconds=30000)},
|
|
{0: dict(gpu_seconds_charged=10000)})
|
|
self.assertEqual(p['projected_gpu_seconds'], 162000 - 30960 + 10000)
|
|
|
|
def test_11_active_waits_cannot_disappear(self):
|
|
p = project_total(self.c, {0: dict(projected_gpu_seconds=100)}, {}, {0: 31000})
|
|
self.assertEqual(p['status'], 'STOP_BUDGET_PROJECTION')
|
|
|
|
def test_12_slow_first_epoch_stops(self):
|
|
j = self.c['jobs'][0]
|
|
self.assertFalse(project_cell(j, dict(epoch=1, max_epoch_seconds=200, charged=300))['within_cap'])
|
|
self.assertTrue(project_cell(j, dict(epoch=1, max_epoch_seconds=50, charged=100))['within_cap'])
|
|
|
|
def test_13_invalid_projection_rejected(self):
|
|
for d in [float('nan'), -1, 0, float('inf')]:
|
|
with self.assertRaises(ValueError):
|
|
project_cell(self.c['jobs'][0], dict(epoch=1, max_epoch_seconds=d, charged=100))
|
|
|
|
def test_14_first_epoch_global_stop(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
self.assertFalse(first_epoch_release(root, 0))
|
|
save(root / 'global-stop.json', dict(reason='synthetic'))
|
|
with self.assertRaises(RuntimeError):
|
|
first_epoch_release(root, 0)
|
|
|
|
def guard_case(self, *, monitor_errors=None, peak=1024, worker_status='PASS'):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / 'jobs/0').mkdir(parents=True)
|
|
c = copy.deepcopy(self.c)
|
|
# No GPU query/process is used; preserve real guard decision paths.
|
|
save(root / 'contract.json', c)
|
|
def fake_supervise(*args, **kwargs):
|
|
w = root / 'jobs/0/worker'
|
|
w.mkdir()
|
|
save(w / 'result.json', dict(status='COMPLETE', job=0,
|
|
optimizer_steps=c['jobs'][0]['target_steps'], completed_epochs=200,
|
|
peak_allocated_bytes=1024, peak_reserved_bytes=1024))
|
|
if peak > 1024:
|
|
save(w / 'failure.json', dict(peak_reserved_bytes=peak))
|
|
return dict(status=worker_status, cpu_seconds=0,
|
|
monitor_errors=monitor_errors or [])
|
|
with patch.object(guard, 'config', return_value=(c, c)), \
|
|
patch.object(guard, 'device_usage', return_value=(0, 0)), \
|
|
patch.object(guard, 'supervise', side_effect=fake_supervise), \
|
|
patch.object(guard.os, 'sched_setaffinity'), patch.object(guard.time, 'sleep'):
|
|
rc = guard.run(root, 0)
|
|
return rc, read(root / 'jobs/0/complete.json')
|
|
|
|
def test_15_guard_success_path(self):
|
|
rc, r = self.guard_case()
|
|
self.assertEqual(rc, 0)
|
|
self.assertEqual(r['status'], 'PASS')
|
|
|
|
def test_16_worker_monitor_error_overrides_pass(self):
|
|
rc, r = self.guard_case(monitor_errors=[dict(reason='synthetic monitor error')])
|
|
self.assertEqual(rc, 1)
|
|
self.assertIn('WORKER_MONITOR_ERRORS', r['reasons'])
|
|
|
|
def test_17_failure_peak_remains_enforced(self):
|
|
rc, r = self.guard_case(peak=16 * 2**30, worker_status='FAILED')
|
|
self.assertEqual(rc, 1)
|
|
self.assertIn('RECORDED_VRAM_LIMIT', r['reasons'])
|
|
|
|
def test_18_foreign_ledger_job_rejected(self):
|
|
with self.assertRaises(ValueError):
|
|
project_total(self.c, {5: dict(projected_gpu_seconds=1)}, {})
|
|
|
|
def test_19_geometry_cannot_mix_versions(self):
|
|
c = copy.deepcopy(self.c)
|
|
case = next(iter(c['geometry']))
|
|
c['geometry'][case]['protocol'] = 'grouped'
|
|
with self.assertRaises(ValueError):
|
|
validate_roles(c, self.entry)
|
|
|
|
def test_20_worker_all_epochs_no_validation_or_test(self):
|
|
import track_a_h2_worker as worker
|
|
import os
|
|
class Model:
|
|
def cuda(self): return self
|
|
def parameters(self): return []
|
|
def state_dict(self): return {}
|
|
class Optimizer:
|
|
param_groups = [{'lr': worker.lr_epoch(1)}]
|
|
def state_dict(self): return {}
|
|
job = dict(id=0, learning=['a', 'b'], validation=[], test=['sealed'], c=1.,
|
|
learning_nodes=5, clock_sha256='synthetic', worker_cpus=[0], gpu_uuid='synthetic')
|
|
c = dict(jobs=[job])
|
|
accesses = []
|
|
def load(view, case, stats, role, out):
|
|
accesses.append((case, role))
|
|
if case not in ['a', 'b'] or role != 'learning':
|
|
raise AssertionError('Test/validation accessed')
|
|
return dict(nodes=2 if case == 'a' else 3)
|
|
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
|
|
root = Path(tmp)
|
|
(root / 'jobs/0').mkdir(parents=True)
|
|
save(root / 'contract.json', c)
|
|
stack.enter_context(patch.dict(os.environ, CUDA_VISIBLE_DEVICES='synthetic',
|
|
PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
|
|
replacements = dict(config=lambda _: (c, c), standardization=lambda view, out: {},
|
|
H2Geo=Model, optimizer=lambda _: Optimizer(), seed=lambda: None,
|
|
peaks=lambda: dict(peak_allocated_bytes=0, peak_reserved_bytes=0),
|
|
load_part=load, to_device=lambda data, device: data,
|
|
train_step=lambda *a: dict(loss=0., gradient_norm_before_clip=0.),
|
|
first_epoch_release=lambda *a: True)
|
|
for name, value in replacements.items():
|
|
stack.enter_context(patch.object(worker, name, value))
|
|
for name in ['set_device', 'init', 'reset_peak_memory_stats']:
|
|
stack.enter_context(patch.object(worker.torch.cuda, name))
|
|
stack.enter_context(patch.object(worker.torch.cuda.memory, '_snapshot', return_value={}))
|
|
stack.enter_context(patch.object(worker.os, 'sched_setaffinity'))
|
|
stack.enter_context(patch.object(worker.torch, 'set_num_threads'))
|
|
stack.enter_context(patch.object(worker.torch, 'set_num_interop_threads'))
|
|
stack.enter_context(patch.object(worker.torch, 'save', side_effect=lambda v, p: p.write_bytes(b'synthetic')))
|
|
worker.run(root, 0)
|
|
result = read(root / 'jobs/0/worker/result.json')
|
|
self.assertEqual(result['completed_epochs'], 200)
|
|
self.assertEqual(result['optimizer_steps'], 400)
|
|
self.assertEqual(accesses.count(('a', 'learning')), 200)
|
|
self.assertEqual(accesses.count(('b', 'learning')), 200)
|
|
self.assertFalse(result['test_targets_read'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|