"""Exact training/prediction barriers for the five fixed v4 random splits.""" import math from pathlib import Path from phase2_h1_worker_v2 import checked, sha from phase2_home_freeze_io import verify from track_a_h2_common import read, config as training_config def config(root): c = read(root / 'contract.json') verify(root, 'implementation') if (c['approval'] != '1794975970.156219' or c['budget_approval'] != '1794982020.974879' or c['estimator_approval'] != '1794982202.809969' or c['gpu_cap'] != 162000 or c['postfit_gpu_cap'] != 7200 or c['postfit_cpu_cap'] != 3600 or c['gpu_uuid'] != 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76' or c['idle_wait_seconds'] != 20 or not c['no_retry']): raise PermissionError('Frozen Track A postfit scope') for p, h in {**c['source_sha256'], **c['code_sha256']}.items(): checked(p, h) if c['bootstrap'] != dict(replicates=2000, seed=2026111408, confidence=.95, unit='unique_part_mean_of_split_visit_differences', interval='percentile_linear'): raise ValueError('Frozen descriptive bootstrap') tc, _ = training_config(Path(c['training_root'])) entry = read(checked(tc['reuse_entry'], c['comparison_contract_sha256'])) if len(c['jobs']) != 5 or c['jobs'] != [dict(id=j['id'], expected_parts=len(j['test'])) for j in tc['jobs']]: raise ValueError('Five original test lists') return c, tc, entry def terminal(root, c, tc): import psutil train = Path(c['training_root']) verify(train, 'terminal') verify(train, 'models') if (train / 'global-stop.json').exists(): raise ValueError('Training global stop') done = read(train / 'complete.json') models = read(train / 'models-terminal.json')['models'] if (done['status'] != 'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING' or len(done['entries']) != 5 or {e['id'] for e in done['entries']} != set(range(5)) or len(models) != 5 or {m['job'] for m in models} != set(range(5)) or done['test_targets_read']): raise ValueError('Five complete training endpoints required') if any(e['status'] != 'PASS' or e['returncode'] != 0 for e in done['entries']): raise ValueError('Training failure remains failure; no missing-run averaging') prior = done['training_gpu_seconds'] if (not math.isfinite(prior) or prior < 0 or prior > 154800 or abs(prior - sum(e['gpu_seconds_charged'] for e in done['entries'])) > 1e-8): raise ValueError('Training ledger identity') for p in psutil.process_iter(['cmdline']): if any(Path(a).name in {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'} for a in p.info['cmdline'] or []): raise RuntimeError('Original training processes still present') result = [] for j in tc['jobs']: i = j['id'];w = train / 'jobs' / str(i) / 'worker' e = next(e for e in done['entries'] if e['id'] == i) g = read(checked(w.parent / 'complete.json', e['guard_complete_sha256'])) receipt = read(w.parent / 'guards/train.receipt.json') if (g['status'] != 'PASS' or g['monitor_errors'] or g['worker'] != receipt or type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0 or receipt['status'] != 'PASS' or receipt['returncode'] != 0): raise ValueError('Training monitoring errors') r = read(checked(w / 'result.json', g['worker_result_sha256'])) m = next(m for m in models if m['job'] == i) if (r['status'] != 'COMPLETE' or r['job'] != i or r['fold'] != i or r['completed_epochs'] != 200 or r['optimizer_steps'] != j['target_steps'] or r['test_targets_read'] or r['test_prediction_started'] or m['path'] != r['checkpoint'] or m['sha256'] != r['checkpoint_sha256'] or m['standardization'] != str(w / 'standardization.json') or m['standardization_sha256'] != r['standardization_sha256']): raise ValueError('Final model identity') checked(m['path'], m['sha256']) stats = read(checked(m['standardization'], m['standardization_sha256'])) if stats['parts'] != j['learning'] or stats['nodes'] != j['learning_nodes']: raise ValueError('Learning-only normalization identity') result.append(m) return dict(models=result, training_gpu_seconds_once=prior, training_contract_sha256=sha(train / 'contract.json'), training_terminal_sha256=sha(train / 'complete.json')) def adopted(root, tc): verify(root, 'models') d = read(root / 'models-adopted.json') if {m['job'] for m in d['models']} != set(range(5)) or len(d['models']) != 5: raise ValueError('Five distinct adopted models') for m in d['models']: checked(m['path'], m['sha256']) s = read(checked(m['standardization'], m['standardization_sha256'])) if s['parts'] != tc['jobs'][m['job']]['learning']: raise ValueError('Adopted learning normalization') return d def checkpoint_identity(blob, j, training_sha): if (blob.get('seed') != 0 or blob.get('fold') != j['id'] or blob.get('completed_epochs') != 200 or blob.get('optimizer_steps') != j['target_steps'] or blob.get('contract_sha256') != training_sha or blob.get('role') != 'last_epoch_only'): raise ValueError('Original last-epoch checkpoint identity') def prediction_identity(root, c, tc, entries): expected = {(j['id'], case) for j in tc['jobs'] for case in j['test']} keys = [(e['split'], e['case']) for e in entries] if len(keys) != len(set(keys)) or set(keys) != expected: raise ValueError('All original valid test visits required before scoring') for e in entries: p = tc['parts'][e['case']] expected_path = root / 'predictions' / str(e['split']) / (e['case'] + '.npz') if (e['path'] != str(expected_path) or e['nodes'] != p['nodes'] or e['raw_ids_sha256'] != p['output_sha256']['raw_ids.npy']): raise ValueError('Prediction path/native node identity') checked(e['path'], e['sha256']) a = c['A_predictions'][str(e['split'])][e['case']] if a['nodes'] != e['nodes'] or a['raw_ids_sha256'] != e['raw_ids_sha256']: raise ValueError('Paired original A domain identity') checked(a['path'], a['sha256']) return expected def budget(prior, gpu, cpu, remaining): if any(not math.isfinite(x) or x < 0 for x in [prior,gpu,cpu]) or remaining not in range(6): raise ValueError('Invalid ledger') if gpu + 900*remaining > 7200 or prior + gpu + 900*remaining > 162000: raise RuntimeError('Track A total GPU reserve exhausted') if cpu + 300*remaining + 900 + 120 > 3600: raise RuntimeError('Track A postfit CPU reserve exhausted')