Files
MeltMind/tools/phase3_followup_postfit_common.py
T

193 lines
10 KiB
Python

"""Frozen candidate, roster, resource and custody checks for phase 3 postfit."""
import math
from pathlib import Path
import numpy as np
import psutil
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, sha
from phase2_h2_s2_postfit_common import roster
from phase2_home_freeze_io import verify
from phase3_item_b_shared_common import ROOT as B_TRAIN, config as b_config
from phase3_item_c_common import ROOT as C_TRAIN, config as c_config, read
from phase3_seed1_postfit_common import normal_identity
RUNS = C_TRAIN.parent
BASELINE = RUNS / 'phase2-h2-s2-postfit-2026-11-17'
SEED = RUNS / 'phase3-seed1-postfit-2026-11-18'
GPU = 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36'
RULING = '1795055299'
TRAINING = {
'c': (C_TRAIN, c_config, '2ddecef3fac432ccc0267c468d5df4d3203e53ef',
'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING'),
'b': (B_TRAIN, b_config, '961ce62360003e7621f4fad6ae58648ef96d82af', 'TRAIN_COMPLETE_POSTFIT_PENDING'),
}
POSTFIT_GPU = 7200
POSTFIT_CPU = 3600
FOLD_GPU = 900
FOLD_CPU = 300
SCORE_CPU = 900
SETUP_CPU = 180
CLEANUP_CPU = 120
def root_for(item):
return RUNS / f'phase3-item-{item}-postfit-2026-11-19'
def frozen_file(proof, path, digest):
path = Path(path)
if proof['files'].get(str(path.resolve())) != digest:
raise ValueError(f'File omitted from the required freeze: {path}')
return checked(path, digest)
def training_terminal(item, tc):
train, _, commit, expected_status = TRAINING[item]
if verify(train, 'implementation')['commit'] != commit:
raise ValueError('Candidate training source changed')
if (train / 'global-stop.json').exists():
raise PermissionError('Original training stopped')
for proc in psutil.process_iter(['cmdline']):
args = proc.info['cmdline'] or []
if str(train) in args and any(Path(a).name.startswith(f'phase3_item_{item}') for a in args):
raise RuntimeError('Candidate training process still alive')
proof = verify(train, 'training-terminal')
complete = train / 'complete.json'
done = read(frozen_file(proof, complete, sha(complete)))
manifest = train / 'models.json'
models = read(frozen_file(proof, manifest, sha(manifest)))
tail_path = train / 'post-terminal-cost.json'
tail = read(tail_path)
entries = {e['id']: e for e in done['entries']}
by_fold = {m['fold']: m for m in models}
if (done['status'] != expected_status or done['failure'] is not None or done['never_attempted']
or any(done[field] is not False for field in ('prediction_started', 'scoring_started', 'test_targets_read'))
or len(done['entries']) != 5 or len(models) != 5
or set(entries) != set(range(5)) or set(by_fold) != set(range(5))):
raise ValueError('Exactly five clean original terminal models required')
costs = [entries[i]['gpu_seconds_charged'] for i in range(5)]
expected = tc['preparation_charge_gpu_seconds'] + sum(costs)
final = tail['cumulative_gpu_seconds']
if (not all(math.isfinite(x) and x >= 0 for x in costs + [final])
or not math.isclose(expected, done['cumulative_gpu_seconds'], abs_tol=1e-8, rel_tol=0)
or not expected - 1e-8 <= final <= tc['gpu_cap'] - POSTFIT_GPU
or done['postfit_gpu_reserve'] != POSTFIT_GPU or tail['postfit_gpu_reserve'] != POSTFIT_GPU
or tail['training_complete_sha256'] != sha(complete) or tail['within_cap'] is not True
or tail['freeze_tail_charged_once'] is not True or tail['postfit_started'] is not False):
raise ValueError('Invalid training or freeze-tail accounting')
bound_models = []
for j in tc['jobs']:
i = j['id']
jr = train / 'jobs' / str(i)
e = entries[i]
guard = read(frozen_file(proof, jr / 'complete.json', e['guard_sha256']))
receipt = guard['worker']
if (e['status'] != 'PASS' or e['returncode'] != 0 or guard['status'] != 'PASS'
or guard['reasons'] or guard['monitor_errors'] != [] or receipt['status'] != 'PASS'
or receipt['returncode'] != 0 or type(receipt['monitor_errors']) is not int
or receipt['monitor_errors'] != 0 or receipt != read(jr / 'guards/train.receipt.json')):
raise ValueError(f'Fold {i} guard or worker failure')
result = read(frozen_file(proof, jr / 'worker/result.json', guard['worker_result_sha256']))
weights = jr / 'worker/last-epoch200.pt'
std = jr / 'worker/standardization.json'
if (result != by_fold[i] or result['status'] != 'COMPLETE' or result['fold'] != i
or result['completed_epochs'] != 200 or result['optimizer_steps'] != j['target_steps']
or result['learning_parts'] != len(j['learning'])
or result['learning_nodes_per_epoch'] != j['learning_nodes']
or any(result[field] is not False for field in
('test_prediction_started', 'test_targets_read', 'validation_used_for_selection'))
or result['checkpoint'] != str(weights)):
raise ValueError(f'Fold {i} final checkpoint identity')
frozen_file(proof, weights, result['checkpoint_sha256'])
stats = read(frozen_file(proof, std, result['standardization_sha256']))
normal_identity(stats, j)
if stats['labels_read'] is not False:
raise ValueError('Learning normalization may not use labels')
if item == 'b':
context = stats['context']
expected_context = [dict(case=case, nodes=tc['context'][case]['nodes'],
valid_nodes=tc['context'][case]['nodes'] if tc['context'][case]['status'] == 'GEOMETRY_COMPLETE' else 0)
for case in j['learning']]
if (context['parts'] != expected_context or context['labels_read'] is not False
or context['valid_nodes'] != sum(r['valid_nodes'] for r in expected_context)
or len(context['mean']) != 2 or len(context['scale']) != 2
or not np.isfinite(context['mean'] + context['scale']).all() or min(context['scale']) <= 0):
raise ValueError('Original learning-only context normalization')
bound_models.append(dict(fold=i, path=str(weights), sha256=result['checkpoint_sha256'],
standardization=str(std), standardization_sha256=result['standardization_sha256']))
return dict(models=bound_models, training_gpu_seconds=final, training_terminal_commit=proof['commit'],
training_complete_sha256=sha(complete), training_cost_sha256=sha(tail_path),
training_contract_sha256=sha(train / 'contract.json'))
def config(root, *, review=True):
c = read(root / 'contract.json')
item = c['item']
train, load_training, _, _ = TRAINING[item]
if (root.resolve() != root_for(item) or c['ruling'] != RULING or c['gpu_uuid'] != GPU
or c['no_retry'] is not True or c['new_fits'] != 0 or c['new_decodes'] != 0
or c['worker_cpus'] != [2, 3] or c['coordinator_cpus'] != [16, 17]):
raise PermissionError('Frozen candidate postfit scope')
verify(root, 'implementation')
if review:
validate_review(root)
for path, digest in {**c['source_sha256'], **c['code_sha256']}.items():
checked(path, digest)
tc, pilot = load_training(train)
if c['item_gpu_cap'] != tc['gpu_cap'] or c['postfit_gpu_cap'] != POSTFIT_GPU or c['postfit_cpu_cap'] != POSTFIT_CPU:
raise PermissionError('Original item budget required')
if c['jobs'] != [dict(id=j['id'], expected_parts=len(j['test'])) for j in tc['jobs']]:
raise ValueError('Same five fixed test folds required')
by = roster(read(BASELINE / 'contract.json'), pilot)
if set(c['seed0_predictions']) != set(by):
raise ValueError('Same 552 baseline predictions required')
if item == 'b':
statuses = [row['status'] for row in tc['context'].values()]
if (set(tc['context']) != set(by) or len(statuses) != 552
or statuses.count('GEOMETRY_COMPLETE') != 510 or statuses.count('SCIENTIFIC_MISSING') != 42):
raise ValueError('Original context coverage and missingness required')
return c, tc, pilot, by
def budget(c, gpu, cpu, remaining):
if not all(math.isfinite(x) and x >= 0 for x in (gpu, cpu)) or type(remaining) is not int or remaining not in range(6):
raise ValueError('Invalid postfit accounting')
reserved = gpu + FOLD_GPU * remaining
train = c['training']['training_gpu_seconds']
other = sum(v for item, v in c['phase_reservations'].items() if item != c['item'])
if reserved > POSTFIT_GPU or train + reserved > c['item_gpu_cap'] or other + train + reserved > 720000:
raise RuntimeError('Original item/phase 3 GPU envelope exhausted')
if cpu + FOLD_CPU * remaining + SCORE_CPU + CLEANUP_CPU > POSTFIT_CPU:
raise RuntimeError('Original postfit CPU envelope exhausted')
def checkpoint_identity(blob, job, c):
expected = dict(seed=0, fold=job['id'], completed_epochs=200, optimizer_steps=job['target_steps'],
contract_sha256=c['training']['training_contract_sha256'], role='last_epoch_only')
if any(blob.get(k) != v for k, v in expected.items()):
raise ValueError('Candidate last-epoch checkpoint identity')
def prediction_identity(root, c, tc, by, entries):
expected = {(j['id'], case) for j in tc['jobs'] for case in j['test']}
keys = {(e['fold'], e['case']) for e in entries}
if len(entries) != 552 or len(keys) != 552 or keys != expected:
raise ValueError('Full same-domain roster requires 552 distinct predictions')
for e in entries:
case = e['case']
base = c['seed0_predictions'][case]
row = by[case]
expected_path = root / 'jobs' / str(e['fold']) / 'prediction-records' / f'{case}.npz'
if (e['component_id'] != row['component_id'] or e['path'] != str(expected_path)
or e['nodes'] != row['nodes'] or e['query_nodes'] != base['query_nodes']
or e['seed0_sha256'] != base['sha256']
or e['raw_ids_sha256'] != row['v5_cache']['output_sha256']['raw_ids.npy']):
raise ValueError('Prediction pairing or original raw ID mismatch')
for key, count in [('negative_full_predictions', e['nodes']), ('negative_query_predictions', e['query_nodes'])]:
if type(e[key]) is not int or not 0 <= e[key] <= count:
raise ValueError('Invalid negative count')
checked(expected_path, e['sha256'])
checked(base['path'], base['sha256'])