"""Frozen candidate, roster, resource and custody checks for phase 3 postfit.""" import math from pathlib import Path import numpy as np import psutil from phase2_gate_outcome_review import validate_review from phase2_h1_worker_v2 import checked, sha from phase2_h2_s2_postfit_common import roster from phase2_home_freeze_io import verify from phase3_item_b_shared_common import ROOT as B_TRAIN, config as b_config from phase3_item_c_common import ROOT as C_TRAIN, config as c_config, read from phase3_seed1_postfit_common import normal_identity RUNS = C_TRAIN.parent BASELINE = RUNS / 'phase2-h2-s2-postfit-2026-11-17' SEED = RUNS / 'phase3-seed1-postfit-2026-11-18' GPU = 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36' RULING = '1795055299' TRAINING = { 'c': (C_TRAIN, c_config, '2ddecef3fac432ccc0267c468d5df4d3203e53ef', 'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING'), 'b': (B_TRAIN, b_config, '961ce62360003e7621f4fad6ae58648ef96d82af', 'TRAIN_COMPLETE_POSTFIT_PENDING'), } POSTFIT_GPU = 7200 POSTFIT_CPU = 3600 FOLD_GPU = 900 FOLD_CPU = 300 SCORE_CPU = 900 SETUP_CPU = 180 CLEANUP_CPU = 120 def root_for(item): return RUNS / f'phase3-item-{item}-postfit-2026-11-19' def frozen_file(proof, path, digest): path = Path(path) if proof['files'].get(str(path.resolve())) != digest: raise ValueError(f'File omitted from the required freeze: {path}') return checked(path, digest) def training_terminal(item, tc): train, _, commit, expected_status = TRAINING[item] if verify(train, 'implementation')['commit'] != commit: raise ValueError('Candidate training source changed') if (train / 'global-stop.json').exists(): raise PermissionError('Original training stopped') for proc in psutil.process_iter(['cmdline']): args = proc.info['cmdline'] or [] if str(train) in args and any(Path(a).name.startswith(f'phase3_item_{item}') for a in args): raise RuntimeError('Candidate training process still alive') proof = verify(train, 'training-terminal') complete = train / 'complete.json' done = read(frozen_file(proof, complete, sha(complete))) manifest = train / 'models.json' models = read(frozen_file(proof, manifest, sha(manifest))) tail_path = train / 'post-terminal-cost.json' tail = read(tail_path) entries = {e['id']: e for e in done['entries']} by_fold = {m['fold']: m for m in models} if (done['status'] != expected_status or done['failure'] is not None or done['never_attempted'] or any(done[field] is not False for field in ('prediction_started', 'scoring_started', 'test_targets_read')) or len(done['entries']) != 5 or len(models) != 5 or set(entries) != set(range(5)) or set(by_fold) != set(range(5))): raise ValueError('Exactly five clean original terminal models required') costs = [entries[i]['gpu_seconds_charged'] for i in range(5)] expected = tc['preparation_charge_gpu_seconds'] + sum(costs) final = tail['cumulative_gpu_seconds'] if (not all(math.isfinite(x) and x >= 0 for x in costs + [final]) or not math.isclose(expected, done['cumulative_gpu_seconds'], abs_tol=1e-8, rel_tol=0) or not expected - 1e-8 <= final <= tc['gpu_cap'] - POSTFIT_GPU or done['postfit_gpu_reserve'] != POSTFIT_GPU or tail['postfit_gpu_reserve'] != POSTFIT_GPU or tail['training_complete_sha256'] != sha(complete) or tail['within_cap'] is not True or tail['freeze_tail_charged_once'] is not True or tail['postfit_started'] is not False): raise ValueError('Invalid training or freeze-tail accounting') bound_models = [] for j in tc['jobs']: i = j['id'] jr = train / 'jobs' / str(i) e = entries[i] guard = read(frozen_file(proof, jr / 'complete.json', e['guard_sha256'])) receipt = guard['worker'] if (e['status'] != 'PASS' or e['returncode'] != 0 or guard['status'] != 'PASS' or guard['reasons'] or guard['monitor_errors'] != [] or receipt['status'] != 'PASS' or receipt['returncode'] != 0 or type(receipt['monitor_errors']) is not int or receipt['monitor_errors'] != 0 or receipt != read(jr / 'guards/train.receipt.json')): raise ValueError(f'Fold {i} guard or worker failure') result = read(frozen_file(proof, jr / 'worker/result.json', guard['worker_result_sha256'])) weights = jr / 'worker/last-epoch200.pt' std = jr / 'worker/standardization.json' if (result != by_fold[i] or result['status'] != 'COMPLETE' or result['fold'] != i or result['completed_epochs'] != 200 or result['optimizer_steps'] != j['target_steps'] or result['learning_parts'] != len(j['learning']) or result['learning_nodes_per_epoch'] != j['learning_nodes'] or any(result[field] is not False for field in ('test_prediction_started', 'test_targets_read', 'validation_used_for_selection')) or result['checkpoint'] != str(weights)): raise ValueError(f'Fold {i} final checkpoint identity') frozen_file(proof, weights, result['checkpoint_sha256']) stats = read(frozen_file(proof, std, result['standardization_sha256'])) normal_identity(stats, j) if stats['labels_read'] is not False: raise ValueError('Learning normalization may not use labels') if item == 'b': context = stats['context'] expected_context = [dict(case=case, nodes=tc['context'][case]['nodes'], valid_nodes=tc['context'][case]['nodes'] if tc['context'][case]['status'] == 'GEOMETRY_COMPLETE' else 0) for case in j['learning']] if (context['parts'] != expected_context or context['labels_read'] is not False or context['valid_nodes'] != sum(r['valid_nodes'] for r in expected_context) or len(context['mean']) != 2 or len(context['scale']) != 2 or not np.isfinite(context['mean'] + context['scale']).all() or min(context['scale']) <= 0): raise ValueError('Original learning-only context normalization') bound_models.append(dict(fold=i, path=str(weights), sha256=result['checkpoint_sha256'], standardization=str(std), standardization_sha256=result['standardization_sha256'])) return dict(models=bound_models, training_gpu_seconds=final, training_terminal_commit=proof['commit'], training_complete_sha256=sha(complete), training_cost_sha256=sha(tail_path), training_contract_sha256=sha(train / 'contract.json')) def config(root, *, review=True): c = read(root / 'contract.json') item = c['item'] train, load_training, _, _ = TRAINING[item] if (root.resolve() != root_for(item) or c['ruling'] != RULING or c['gpu_uuid'] != GPU or c['no_retry'] is not True or c['new_fits'] != 0 or c['new_decodes'] != 0 or c['worker_cpus'] != [2, 3] or c['coordinator_cpus'] != [16, 17]): raise PermissionError('Frozen candidate postfit scope') verify(root, 'implementation') if review: validate_review(root) for path, digest in {**c['source_sha256'], **c['code_sha256']}.items(): checked(path, digest) tc, pilot = load_training(train) if c['item_gpu_cap'] != tc['gpu_cap'] or c['postfit_gpu_cap'] != POSTFIT_GPU or c['postfit_cpu_cap'] != POSTFIT_CPU: raise PermissionError('Original item budget required') if c['jobs'] != [dict(id=j['id'], expected_parts=len(j['test'])) for j in tc['jobs']]: raise ValueError('Same five fixed test folds required') by = roster(read(BASELINE / 'contract.json'), pilot) if set(c['seed0_predictions']) != set(by): raise ValueError('Same 552 baseline predictions required') if item == 'b': statuses = [row['status'] for row in tc['context'].values()] if (set(tc['context']) != set(by) or len(statuses) != 552 or statuses.count('GEOMETRY_COMPLETE') != 510 or statuses.count('SCIENTIFIC_MISSING') != 42): raise ValueError('Original context coverage and missingness required') return c, tc, pilot, by def budget(c, gpu, cpu, remaining): if not all(math.isfinite(x) and x >= 0 for x in (gpu, cpu)) or type(remaining) is not int or remaining not in range(6): raise ValueError('Invalid postfit accounting') reserved = gpu + FOLD_GPU * remaining train = c['training']['training_gpu_seconds'] other = sum(v for item, v in c['phase_reservations'].items() if item != c['item']) if reserved > POSTFIT_GPU or train + reserved > c['item_gpu_cap'] or other + train + reserved > 720000: raise RuntimeError('Original item/phase 3 GPU envelope exhausted') if cpu + FOLD_CPU * remaining + SCORE_CPU + CLEANUP_CPU > POSTFIT_CPU: raise RuntimeError('Original postfit CPU envelope exhausted') def checkpoint_identity(blob, job, c): expected = dict(seed=0, fold=job['id'], completed_epochs=200, optimizer_steps=job['target_steps'], contract_sha256=c['training']['training_contract_sha256'], role='last_epoch_only') if any(blob.get(k) != v for k, v in expected.items()): raise ValueError('Candidate last-epoch checkpoint identity') def prediction_identity(root, c, tc, by, entries): expected = {(j['id'], case) for j in tc['jobs'] for case in j['test']} keys = {(e['fold'], e['case']) for e in entries} if len(entries) != 552 or len(keys) != 552 or keys != expected: raise ValueError('Full same-domain roster requires 552 distinct predictions') for e in entries: case = e['case'] base = c['seed0_predictions'][case] row = by[case] expected_path = root / 'jobs' / str(e['fold']) / 'prediction-records' / f'{case}.npz' if (e['component_id'] != row['component_id'] or e['path'] != str(expected_path) or e['nodes'] != row['nodes'] or e['query_nodes'] != base['query_nodes'] or e['seed0_sha256'] != base['sha256'] or e['raw_ids_sha256'] != row['v5_cache']['output_sha256']['raw_ids.npy']): raise ValueError('Prediction pairing or original raw ID mismatch') for key, count in [('negative_full_predictions', e['nodes']), ('negative_query_predictions', e['query_nodes'])]: if type(e[key]) is not int or not 0 <= e[key] <= count: raise ValueError('Invalid negative count') checked(expected_path, e['sha256']) checked(base['path'], base['sha256'])