"""CPU synthetic regression checks. No real feature/target arrays, CUDA or fits.""" import copy from contextlib import ExitStack import json from pathlib import Path import tempfile import time import unittest from unittest.mock import patch from phase2_h1_worker_v2 import save from phase2_h2_data import load_part from track_a_h2_common import (dispatchable, execution_command, first_epoch_release, project_cell, project_total, read, validate_roles) import track_a_h2_guard as guard ROOT = Path('/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18') class EntryTests(unittest.TestCase): @classmethod def setUpClass(cls): cls.c = read(ROOT / 'contract.json') cls.entry = read(cls.c['reuse_entry']) def test_01_exact_original_roles(self): validate_roles(self.c, self.entry) self.assertEqual([len(j['learning']) for j in self.c['jobs']], [311, 304, 307, 306, 310]) def test_02_test_to_train_rejected(self): c = copy.deepcopy(self.c) c['jobs'][0]['learning'][0] = c['jobs'][0]['test'][0] with self.assertRaises(ValueError): validate_roles(c, self.entry) def test_03_extra_validation_rejected(self): c = copy.deepcopy(self.c) c['jobs'][0]['validation'] = [c['jobs'][0]['learning'][0]] with self.assertRaises(ValueError): validate_roles(c, self.entry) def test_04_test_loader_rejects_before_io(self): with patch('phase2_h2_data.np.load', side_effect=AssertionError('Unexpected array read')): with self.assertRaises(PermissionError): load_part(dict(folds=[dict(learning=['a'], validation=[])]), 'b', {}, 'test', Path('/tmp')) def test_05_clock_cannot_be_changed(self): c = copy.deepcopy(self.c) c['jobs'][0]['c'] += .01 with self.assertRaises(ValueError): validate_roles(c, self.entry) def test_06_two_slots_do_not_require_five_first_epochs(self): jobs = self.c['jobs'] self.assertEqual([j['id'] for j in dispatchable(jobs, [])], [0, 1]) self.assertEqual([j['id'] for j in dispatchable(jobs[2:], [jobs[1]])], [2]) def test_07_item1_gpu_protected(self): j = copy.deepcopy(self.c['jobs'][0]) j['gpu_uuid'] = 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c' with self.assertRaises(ValueError): dispatchable([j], []) def test_08_existing_item1_is_not_duplicate_track_a(self): self.assertFalse(execution_command(['python', 'tools/phase3_data_sufficiency_v2_worker.py'])) self.assertTrue(execution_command(['python', 'tools/track_a_h2_queue.py'])) def test_09_pending_caps_reserved(self): p = project_total(self.c, {}, {}) self.assertEqual(p['projected_gpu_seconds'], 162000) self.assertEqual(p['status'], 'PASS') def test_10_terminal_replaces_estimate_once(self): p = project_total(self.c, {0: dict(projected_gpu_seconds=30000)}, {0: dict(gpu_seconds_charged=10000)}) self.assertEqual(p['projected_gpu_seconds'], 162000 - 30960 + 10000) def test_11_active_waits_cannot_disappear(self): p = project_total(self.c, {0: dict(projected_gpu_seconds=100)}, {}, {0: 31000}) self.assertEqual(p['status'], 'STOP_BUDGET_PROJECTION') def test_12_slow_first_epoch_stops(self): j = self.c['jobs'][0] self.assertFalse(project_cell(j, dict(epoch=1, max_epoch_seconds=200, charged=300))['within_cap']) self.assertTrue(project_cell(j, dict(epoch=1, max_epoch_seconds=50, charged=100))['within_cap']) def test_13_invalid_projection_rejected(self): for d in [float('nan'), -1, 0, float('inf')]: with self.assertRaises(ValueError): project_cell(self.c['jobs'][0], dict(epoch=1, max_epoch_seconds=d, charged=100)) def test_14_first_epoch_global_stop(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) self.assertFalse(first_epoch_release(root, 0)) save(root / 'global-stop.json', dict(reason='synthetic')) with self.assertRaises(RuntimeError): first_epoch_release(root, 0) def guard_case(self, *, monitor_errors=None, peak=1024, worker_status='PASS'): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / 'jobs/0').mkdir(parents=True) c = copy.deepcopy(self.c) # No GPU query/process is used; preserve real guard decision paths. save(root / 'contract.json', c) def fake_supervise(*args, **kwargs): w = root / 'jobs/0/worker' w.mkdir() save(w / 'result.json', dict(status='COMPLETE', job=0, optimizer_steps=c['jobs'][0]['target_steps'], completed_epochs=200, peak_allocated_bytes=1024, peak_reserved_bytes=1024)) if peak > 1024: save(w / 'failure.json', dict(peak_reserved_bytes=peak)) return dict(status=worker_status, cpu_seconds=0, monitor_errors=monitor_errors or []) with patch.object(guard, 'config', return_value=(c, c)), \ patch.object(guard, 'device_usage', return_value=(0, 0)), \ patch.object(guard, 'supervise', side_effect=fake_supervise), \ patch.object(guard.os, 'sched_setaffinity'), patch.object(guard.time, 'sleep'): rc = guard.run(root, 0) return rc, read(root / 'jobs/0/complete.json') def test_15_guard_success_path(self): rc, r = self.guard_case() self.assertEqual(rc, 0) self.assertEqual(r['status'], 'PASS') def test_16_worker_monitor_error_overrides_pass(self): rc, r = self.guard_case(monitor_errors=[dict(reason='synthetic monitor error')]) self.assertEqual(rc, 1) self.assertIn('WORKER_MONITOR_ERRORS', r['reasons']) def test_17_failure_peak_remains_enforced(self): rc, r = self.guard_case(peak=16 * 2**30, worker_status='FAILED') self.assertEqual(rc, 1) self.assertIn('RECORDED_VRAM_LIMIT', r['reasons']) def test_18_foreign_ledger_job_rejected(self): with self.assertRaises(ValueError): project_total(self.c, {5: dict(projected_gpu_seconds=1)}, {}) def test_19_geometry_cannot_mix_versions(self): c = copy.deepcopy(self.c) case = next(iter(c['geometry'])) c['geometry'][case]['protocol'] = 'grouped' with self.assertRaises(ValueError): validate_roles(c, self.entry) def test_20_worker_all_epochs_no_validation_or_test(self): import track_a_h2_worker as worker import os class Model: def cuda(self): return self def parameters(self): return [] def state_dict(self): return {} class Optimizer: param_groups = [{'lr': worker.lr_epoch(1)}] def state_dict(self): return {} job = dict(id=0, learning=['a', 'b'], validation=[], test=['sealed'], c=1., learning_nodes=5, clock_sha256='synthetic', worker_cpus=[0], gpu_uuid='synthetic') c = dict(jobs=[job]) accesses = [] def load(view, case, stats, role, out): accesses.append((case, role)) if case not in ['a', 'b'] or role != 'learning': raise AssertionError('Test/validation accessed') return dict(nodes=2 if case == 'a' else 3) with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack: root = Path(tmp) (root / 'jobs/0').mkdir(parents=True) save(root / 'contract.json', c) stack.enter_context(patch.dict(os.environ, CUDA_VISIBLE_DEVICES='synthetic', PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True')) replacements = dict(config=lambda _: (c, c), standardization=lambda view, out: {}, H2Geo=Model, optimizer=lambda _: Optimizer(), seed=lambda: None, peaks=lambda: dict(peak_allocated_bytes=0, peak_reserved_bytes=0), load_part=load, to_device=lambda data, device: data, train_step=lambda *a: dict(loss=0., gradient_norm_before_clip=0.), first_epoch_release=lambda *a: True) for name, value in replacements.items(): stack.enter_context(patch.object(worker, name, value)) for name in ['set_device', 'init', 'reset_peak_memory_stats']: stack.enter_context(patch.object(worker.torch.cuda, name)) stack.enter_context(patch.object(worker.torch.cuda.memory, '_snapshot', return_value={})) stack.enter_context(patch.object(worker.os, 'sched_setaffinity')) stack.enter_context(patch.object(worker.torch, 'set_num_threads')) stack.enter_context(patch.object(worker.torch, 'set_num_interop_threads')) stack.enter_context(patch.object(worker.torch, 'save', side_effect=lambda v, p: p.write_bytes(b'synthetic'))) worker.run(root, 0) result = read(root / 'jobs/0/worker/result.json') self.assertEqual(result['completed_epochs'], 200) self.assertEqual(result['optimizer_steps'], 400) self.assertEqual(accesses.count(('a', 'learning')), 200) self.assertEqual(accesses.count(('b', 'learning')), 200) self.assertFalse(result['test_targets_read']) if __name__ == '__main__': unittest.main()