193 lines
10 KiB
Python
193 lines
10 KiB
Python
"""Frozen candidate, roster, resource and custody checks for phase 3 postfit."""
|
|
import math
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import psutil
|
|
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h1_worker_v2 import checked, sha
|
|
from phase2_h2_s2_postfit_common import roster
|
|
from phase2_home_freeze_io import verify
|
|
from phase3_item_b_shared_common import ROOT as B_TRAIN, config as b_config
|
|
from phase3_item_c_common import ROOT as C_TRAIN, config as c_config, read
|
|
from phase3_seed1_postfit_common import normal_identity
|
|
|
|
RUNS = C_TRAIN.parent
|
|
BASELINE = RUNS / 'phase2-h2-s2-postfit-2026-11-17'
|
|
SEED = RUNS / 'phase3-seed1-postfit-2026-11-18'
|
|
GPU = 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36'
|
|
RULING = '1795055299'
|
|
TRAINING = {
|
|
'c': (C_TRAIN, c_config, '2ddecef3fac432ccc0267c468d5df4d3203e53ef',
|
|
'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING'),
|
|
'b': (B_TRAIN, b_config, '961ce62360003e7621f4fad6ae58648ef96d82af', 'TRAIN_COMPLETE_POSTFIT_PENDING'),
|
|
}
|
|
POSTFIT_GPU = 7200
|
|
POSTFIT_CPU = 3600
|
|
FOLD_GPU = 900
|
|
FOLD_CPU = 300
|
|
SCORE_CPU = 900
|
|
SETUP_CPU = 180
|
|
CLEANUP_CPU = 120
|
|
|
|
|
|
def root_for(item):
|
|
return RUNS / f'phase3-item-{item}-postfit-2026-11-19'
|
|
|
|
|
|
def frozen_file(proof, path, digest):
|
|
path = Path(path)
|
|
if proof['files'].get(str(path.resolve())) != digest:
|
|
raise ValueError(f'File omitted from the required freeze: {path}')
|
|
return checked(path, digest)
|
|
|
|
|
|
def training_terminal(item, tc):
|
|
train, _, commit, expected_status = TRAINING[item]
|
|
if verify(train, 'implementation')['commit'] != commit:
|
|
raise ValueError('Candidate training source changed')
|
|
if (train / 'global-stop.json').exists():
|
|
raise PermissionError('Original training stopped')
|
|
for proc in psutil.process_iter(['cmdline']):
|
|
args = proc.info['cmdline'] or []
|
|
if str(train) in args and any(Path(a).name.startswith(f'phase3_item_{item}') for a in args):
|
|
raise RuntimeError('Candidate training process still alive')
|
|
proof = verify(train, 'training-terminal')
|
|
complete = train / 'complete.json'
|
|
done = read(frozen_file(proof, complete, sha(complete)))
|
|
manifest = train / 'models.json'
|
|
models = read(frozen_file(proof, manifest, sha(manifest)))
|
|
tail_path = train / 'post-terminal-cost.json'
|
|
tail = read(tail_path)
|
|
entries = {e['id']: e for e in done['entries']}
|
|
by_fold = {m['fold']: m for m in models}
|
|
if (done['status'] != expected_status or done['failure'] is not None or done['never_attempted']
|
|
or any(done[field] is not False for field in ('prediction_started', 'scoring_started', 'test_targets_read'))
|
|
or len(done['entries']) != 5 or len(models) != 5
|
|
or set(entries) != set(range(5)) or set(by_fold) != set(range(5))):
|
|
raise ValueError('Exactly five clean original terminal models required')
|
|
costs = [entries[i]['gpu_seconds_charged'] for i in range(5)]
|
|
expected = tc['preparation_charge_gpu_seconds'] + sum(costs)
|
|
final = tail['cumulative_gpu_seconds']
|
|
if (not all(math.isfinite(x) and x >= 0 for x in costs + [final])
|
|
or not math.isclose(expected, done['cumulative_gpu_seconds'], abs_tol=1e-8, rel_tol=0)
|
|
or not expected - 1e-8 <= final <= tc['gpu_cap'] - POSTFIT_GPU
|
|
or done['postfit_gpu_reserve'] != POSTFIT_GPU or tail['postfit_gpu_reserve'] != POSTFIT_GPU
|
|
or tail['training_complete_sha256'] != sha(complete) or tail['within_cap'] is not True
|
|
or tail['freeze_tail_charged_once'] is not True or tail['postfit_started'] is not False):
|
|
raise ValueError('Invalid training or freeze-tail accounting')
|
|
bound_models = []
|
|
for j in tc['jobs']:
|
|
i = j['id']
|
|
jr = train / 'jobs' / str(i)
|
|
e = entries[i]
|
|
guard = read(frozen_file(proof, jr / 'complete.json', e['guard_sha256']))
|
|
receipt = guard['worker']
|
|
if (e['status'] != 'PASS' or e['returncode'] != 0 or guard['status'] != 'PASS'
|
|
or guard['reasons'] or guard['monitor_errors'] != [] or receipt['status'] != 'PASS'
|
|
or receipt['returncode'] != 0 or type(receipt['monitor_errors']) is not int
|
|
or receipt['monitor_errors'] != 0 or receipt != read(jr / 'guards/train.receipt.json')):
|
|
raise ValueError(f'Fold {i} guard or worker failure')
|
|
result = read(frozen_file(proof, jr / 'worker/result.json', guard['worker_result_sha256']))
|
|
weights = jr / 'worker/last-epoch200.pt'
|
|
std = jr / 'worker/standardization.json'
|
|
if (result != by_fold[i] or result['status'] != 'COMPLETE' or result['fold'] != i
|
|
or result['completed_epochs'] != 200 or result['optimizer_steps'] != j['target_steps']
|
|
or result['learning_parts'] != len(j['learning'])
|
|
or result['learning_nodes_per_epoch'] != j['learning_nodes']
|
|
or any(result[field] is not False for field in
|
|
('test_prediction_started', 'test_targets_read', 'validation_used_for_selection'))
|
|
or result['checkpoint'] != str(weights)):
|
|
raise ValueError(f'Fold {i} final checkpoint identity')
|
|
frozen_file(proof, weights, result['checkpoint_sha256'])
|
|
stats = read(frozen_file(proof, std, result['standardization_sha256']))
|
|
normal_identity(stats, j)
|
|
if stats['labels_read'] is not False:
|
|
raise ValueError('Learning normalization may not use labels')
|
|
if item == 'b':
|
|
context = stats['context']
|
|
expected_context = [dict(case=case, nodes=tc['context'][case]['nodes'],
|
|
valid_nodes=tc['context'][case]['nodes'] if tc['context'][case]['status'] == 'GEOMETRY_COMPLETE' else 0)
|
|
for case in j['learning']]
|
|
if (context['parts'] != expected_context or context['labels_read'] is not False
|
|
or context['valid_nodes'] != sum(r['valid_nodes'] for r in expected_context)
|
|
or len(context['mean']) != 2 or len(context['scale']) != 2
|
|
or not np.isfinite(context['mean'] + context['scale']).all() or min(context['scale']) <= 0):
|
|
raise ValueError('Original learning-only context normalization')
|
|
bound_models.append(dict(fold=i, path=str(weights), sha256=result['checkpoint_sha256'],
|
|
standardization=str(std), standardization_sha256=result['standardization_sha256']))
|
|
return dict(models=bound_models, training_gpu_seconds=final, training_terminal_commit=proof['commit'],
|
|
training_complete_sha256=sha(complete), training_cost_sha256=sha(tail_path),
|
|
training_contract_sha256=sha(train / 'contract.json'))
|
|
|
|
|
|
def config(root, *, review=True):
|
|
c = read(root / 'contract.json')
|
|
item = c['item']
|
|
train, load_training, _, _ = TRAINING[item]
|
|
if (root.resolve() != root_for(item) or c['ruling'] != RULING or c['gpu_uuid'] != GPU
|
|
or c['no_retry'] is not True or c['new_fits'] != 0 or c['new_decodes'] != 0
|
|
or c['worker_cpus'] != [2, 3] or c['coordinator_cpus'] != [16, 17]):
|
|
raise PermissionError('Frozen candidate postfit scope')
|
|
verify(root, 'implementation')
|
|
if review:
|
|
validate_review(root)
|
|
for path, digest in {**c['source_sha256'], **c['code_sha256']}.items():
|
|
checked(path, digest)
|
|
tc, pilot = load_training(train)
|
|
if c['item_gpu_cap'] != tc['gpu_cap'] or c['postfit_gpu_cap'] != POSTFIT_GPU or c['postfit_cpu_cap'] != POSTFIT_CPU:
|
|
raise PermissionError('Original item budget required')
|
|
if c['jobs'] != [dict(id=j['id'], expected_parts=len(j['test'])) for j in tc['jobs']]:
|
|
raise ValueError('Same five fixed test folds required')
|
|
by = roster(read(BASELINE / 'contract.json'), pilot)
|
|
if set(c['seed0_predictions']) != set(by):
|
|
raise ValueError('Same 552 baseline predictions required')
|
|
if item == 'b':
|
|
statuses = [row['status'] for row in tc['context'].values()]
|
|
if (set(tc['context']) != set(by) or len(statuses) != 552
|
|
or statuses.count('GEOMETRY_COMPLETE') != 510 or statuses.count('SCIENTIFIC_MISSING') != 42):
|
|
raise ValueError('Original context coverage and missingness required')
|
|
return c, tc, pilot, by
|
|
|
|
|
|
def budget(c, gpu, cpu, remaining):
|
|
if not all(math.isfinite(x) and x >= 0 for x in (gpu, cpu)) or type(remaining) is not int or remaining not in range(6):
|
|
raise ValueError('Invalid postfit accounting')
|
|
reserved = gpu + FOLD_GPU * remaining
|
|
train = c['training']['training_gpu_seconds']
|
|
other = sum(v for item, v in c['phase_reservations'].items() if item != c['item'])
|
|
if reserved > POSTFIT_GPU or train + reserved > c['item_gpu_cap'] or other + train + reserved > 720000:
|
|
raise RuntimeError('Original item/phase 3 GPU envelope exhausted')
|
|
if cpu + FOLD_CPU * remaining + SCORE_CPU + CLEANUP_CPU > POSTFIT_CPU:
|
|
raise RuntimeError('Original postfit CPU envelope exhausted')
|
|
|
|
|
|
def checkpoint_identity(blob, job, c):
|
|
expected = dict(seed=0, fold=job['id'], completed_epochs=200, optimizer_steps=job['target_steps'],
|
|
contract_sha256=c['training']['training_contract_sha256'], role='last_epoch_only')
|
|
if any(blob.get(k) != v for k, v in expected.items()):
|
|
raise ValueError('Candidate last-epoch checkpoint identity')
|
|
|
|
|
|
def prediction_identity(root, c, tc, by, entries):
|
|
expected = {(j['id'], case) for j in tc['jobs'] for case in j['test']}
|
|
keys = {(e['fold'], e['case']) for e in entries}
|
|
if len(entries) != 552 or len(keys) != 552 or keys != expected:
|
|
raise ValueError('Full same-domain roster requires 552 distinct predictions')
|
|
for e in entries:
|
|
case = e['case']
|
|
base = c['seed0_predictions'][case]
|
|
row = by[case]
|
|
expected_path = root / 'jobs' / str(e['fold']) / 'prediction-records' / f'{case}.npz'
|
|
if (e['component_id'] != row['component_id'] or e['path'] != str(expected_path)
|
|
or e['nodes'] != row['nodes'] or e['query_nodes'] != base['query_nodes']
|
|
or e['seed0_sha256'] != base['sha256']
|
|
or e['raw_ids_sha256'] != row['v5_cache']['output_sha256']['raw_ids.npy']):
|
|
raise ValueError('Prediction pairing or original raw ID mismatch')
|
|
for key, count in [('negative_full_predictions', e['nodes']), ('negative_query_predictions', e['query_nodes'])]:
|
|
if type(e[key]) is not int or not 0 <= e[key] <= count:
|
|
raise ValueError('Invalid negative count')
|
|
checked(expected_path, e['sha256'])
|
|
checked(base['path'], base['sha256'])
|