Files
MeltMind/tools/track_a_postfit_common.py
T

122 lines
6.7 KiB
Python

"""Exact training/prediction barriers for the five fixed v4 random splits."""
import math
from pathlib import Path
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
from track_a_h2_common import read, config as training_config
def config(root):
c = read(root / 'contract.json')
verify(root, 'implementation')
if (c['approval'] != '1794975970.156219' or c['budget_approval'] != '1794982020.974879' or c['estimator_approval'] != '1794982202.809969' or c['gpu_cap'] != 162000 or c['postfit_gpu_cap'] != 7200
or c['postfit_cpu_cap'] != 3600 or c['gpu_uuid'] != 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76'
or c['idle_wait_seconds'] != 20 or not c['no_retry']):
raise PermissionError('Frozen Track A postfit scope')
for p, h in {**c['source_sha256'], **c['code_sha256']}.items():
checked(p, h)
if c['bootstrap'] != dict(replicates=2000, seed=2026111408, confidence=.95, unit='unique_part_mean_of_split_visit_differences', interval='percentile_linear'):
raise ValueError('Frozen descriptive bootstrap')
tc, _ = training_config(Path(c['training_root']))
entry = read(checked(tc['reuse_entry'], c['comparison_contract_sha256']))
if len(c['jobs']) != 5 or c['jobs'] != [dict(id=j['id'], expected_parts=len(j['test'])) for j in tc['jobs']]:
raise ValueError('Five original test lists')
return c, tc, entry
def terminal(root, c, tc):
import psutil
train = Path(c['training_root'])
verify(train, 'terminal')
verify(train, 'models')
if (train / 'global-stop.json').exists():
raise ValueError('Training global stop')
done = read(train / 'complete.json')
models = read(train / 'models-terminal.json')['models']
if (done['status'] != 'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING' or len(done['entries']) != 5
or {e['id'] for e in done['entries']} != set(range(5)) or len(models) != 5
or {m['job'] for m in models} != set(range(5)) or done['test_targets_read']):
raise ValueError('Five complete training endpoints required')
if any(e['status'] != 'PASS' or e['returncode'] != 0 for e in done['entries']):
raise ValueError('Training failure remains failure; no missing-run averaging')
prior = done['training_gpu_seconds']
if (not math.isfinite(prior) or prior < 0 or prior > 154800
or abs(prior - sum(e['gpu_seconds_charged'] for e in done['entries'])) > 1e-8):
raise ValueError('Training ledger identity')
for p in psutil.process_iter(['cmdline']):
if any(Path(a).name in {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'} for a in p.info['cmdline'] or []):
raise RuntimeError('Original training processes still present')
result = []
for j in tc['jobs']:
i = j['id'];w = train / 'jobs' / str(i) / 'worker'
e = next(e for e in done['entries'] if e['id'] == i)
g = read(checked(w.parent / 'complete.json', e['guard_complete_sha256']))
receipt = read(w.parent / 'guards/train.receipt.json')
if (g['status'] != 'PASS' or g['monitor_errors'] or g['worker'] != receipt
or type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0
or receipt['status'] != 'PASS' or receipt['returncode'] != 0):
raise ValueError('Training monitoring errors')
r = read(checked(w / 'result.json', g['worker_result_sha256']))
m = next(m for m in models if m['job'] == i)
if (r['status'] != 'COMPLETE' or r['job'] != i or r['fold'] != i or r['completed_epochs'] != 200
or r['optimizer_steps'] != j['target_steps'] or r['test_targets_read'] or r['test_prediction_started']
or m['path'] != r['checkpoint'] or m['sha256'] != r['checkpoint_sha256']
or m['standardization'] != str(w / 'standardization.json')
or m['standardization_sha256'] != r['standardization_sha256']):
raise ValueError('Final model identity')
checked(m['path'], m['sha256'])
stats = read(checked(m['standardization'], m['standardization_sha256']))
if stats['parts'] != j['learning'] or stats['nodes'] != j['learning_nodes']:
raise ValueError('Learning-only normalization identity')
result.append(m)
return dict(models=result, training_gpu_seconds_once=prior, training_contract_sha256=sha(train / 'contract.json'),
training_terminal_sha256=sha(train / 'complete.json'))
def adopted(root, tc):
verify(root, 'models')
d = read(root / 'models-adopted.json')
if {m['job'] for m in d['models']} != set(range(5)) or len(d['models']) != 5:
raise ValueError('Five distinct adopted models')
for m in d['models']:
checked(m['path'], m['sha256'])
s = read(checked(m['standardization'], m['standardization_sha256']))
if s['parts'] != tc['jobs'][m['job']]['learning']:
raise ValueError('Adopted learning normalization')
return d
def checkpoint_identity(blob, j, training_sha):
if (blob.get('seed') != 0 or blob.get('fold') != j['id'] or blob.get('completed_epochs') != 200
or blob.get('optimizer_steps') != j['target_steps'] or blob.get('contract_sha256') != training_sha
or blob.get('role') != 'last_epoch_only'):
raise ValueError('Original last-epoch checkpoint identity')
def prediction_identity(root, c, tc, entries):
expected = {(j['id'], case) for j in tc['jobs'] for case in j['test']}
keys = [(e['split'], e['case']) for e in entries]
if len(keys) != len(set(keys)) or set(keys) != expected:
raise ValueError('All original valid test visits required before scoring')
for e in entries:
p = tc['parts'][e['case']]
expected_path = root / 'predictions' / str(e['split']) / (e['case'] + '.npz')
if (e['path'] != str(expected_path) or e['nodes'] != p['nodes']
or e['raw_ids_sha256'] != p['output_sha256']['raw_ids.npy']):
raise ValueError('Prediction path/native node identity')
checked(e['path'], e['sha256'])
a = c['A_predictions'][str(e['split'])][e['case']]
if a['nodes'] != e['nodes'] or a['raw_ids_sha256'] != e['raw_ids_sha256']:
raise ValueError('Paired original A domain identity')
checked(a['path'], a['sha256'])
return expected
def budget(prior, gpu, cpu, remaining):
if any(not math.isfinite(x) or x < 0 for x in [prior,gpu,cpu]) or remaining not in range(6):
raise ValueError('Invalid ledger')
if gpu + 900*remaining > 7200 or prior + gpu + 900*remaining > 162000:
raise RuntimeError('Track A total GPU reserve exhausted')
if cpu + 300*remaining + 900 + 120 > 3600:
raise RuntimeError('Track A postfit CPU reserve exhausted')