122 lines
6.7 KiB
Python
122 lines
6.7 KiB
Python
"""Exact training/prediction barriers for the five fixed v4 random splits."""
|
|
import math
|
|
from pathlib import Path
|
|
from phase2_h1_worker_v2 import checked, sha
|
|
from phase2_home_freeze_io import verify
|
|
from track_a_h2_common import read, config as training_config
|
|
|
|
|
|
def config(root):
|
|
c = read(root / 'contract.json')
|
|
verify(root, 'implementation')
|
|
if (c['approval'] != '1794975970.156219' or c['budget_approval'] != '1794982020.974879' or c['estimator_approval'] != '1794982202.809969' or c['gpu_cap'] != 162000 or c['postfit_gpu_cap'] != 7200
|
|
or c['postfit_cpu_cap'] != 3600 or c['gpu_uuid'] != 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76'
|
|
or c['idle_wait_seconds'] != 20 or not c['no_retry']):
|
|
raise PermissionError('Frozen Track A postfit scope')
|
|
for p, h in {**c['source_sha256'], **c['code_sha256']}.items():
|
|
checked(p, h)
|
|
if c['bootstrap'] != dict(replicates=2000, seed=2026111408, confidence=.95, unit='unique_part_mean_of_split_visit_differences', interval='percentile_linear'):
|
|
raise ValueError('Frozen descriptive bootstrap')
|
|
tc, _ = training_config(Path(c['training_root']))
|
|
entry = read(checked(tc['reuse_entry'], c['comparison_contract_sha256']))
|
|
if len(c['jobs']) != 5 or c['jobs'] != [dict(id=j['id'], expected_parts=len(j['test'])) for j in tc['jobs']]:
|
|
raise ValueError('Five original test lists')
|
|
return c, tc, entry
|
|
|
|
|
|
def terminal(root, c, tc):
|
|
import psutil
|
|
train = Path(c['training_root'])
|
|
verify(train, 'terminal')
|
|
verify(train, 'models')
|
|
if (train / 'global-stop.json').exists():
|
|
raise ValueError('Training global stop')
|
|
done = read(train / 'complete.json')
|
|
models = read(train / 'models-terminal.json')['models']
|
|
if (done['status'] != 'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING' or len(done['entries']) != 5
|
|
or {e['id'] for e in done['entries']} != set(range(5)) or len(models) != 5
|
|
or {m['job'] for m in models} != set(range(5)) or done['test_targets_read']):
|
|
raise ValueError('Five complete training endpoints required')
|
|
if any(e['status'] != 'PASS' or e['returncode'] != 0 for e in done['entries']):
|
|
raise ValueError('Training failure remains failure; no missing-run averaging')
|
|
prior = done['training_gpu_seconds']
|
|
if (not math.isfinite(prior) or prior < 0 or prior > 154800
|
|
or abs(prior - sum(e['gpu_seconds_charged'] for e in done['entries'])) > 1e-8):
|
|
raise ValueError('Training ledger identity')
|
|
for p in psutil.process_iter(['cmdline']):
|
|
if any(Path(a).name in {'track_a_h2_queue.py','track_a_h2_guard.py','track_a_h2_worker.py'} for a in p.info['cmdline'] or []):
|
|
raise RuntimeError('Original training processes still present')
|
|
result = []
|
|
for j in tc['jobs']:
|
|
i = j['id'];w = train / 'jobs' / str(i) / 'worker'
|
|
e = next(e for e in done['entries'] if e['id'] == i)
|
|
g = read(checked(w.parent / 'complete.json', e['guard_complete_sha256']))
|
|
receipt = read(w.parent / 'guards/train.receipt.json')
|
|
if (g['status'] != 'PASS' or g['monitor_errors'] or g['worker'] != receipt
|
|
or type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0
|
|
or receipt['status'] != 'PASS' or receipt['returncode'] != 0):
|
|
raise ValueError('Training monitoring errors')
|
|
r = read(checked(w / 'result.json', g['worker_result_sha256']))
|
|
m = next(m for m in models if m['job'] == i)
|
|
if (r['status'] != 'COMPLETE' or r['job'] != i or r['fold'] != i or r['completed_epochs'] != 200
|
|
or r['optimizer_steps'] != j['target_steps'] or r['test_targets_read'] or r['test_prediction_started']
|
|
or m['path'] != r['checkpoint'] or m['sha256'] != r['checkpoint_sha256']
|
|
or m['standardization'] != str(w / 'standardization.json')
|
|
or m['standardization_sha256'] != r['standardization_sha256']):
|
|
raise ValueError('Final model identity')
|
|
checked(m['path'], m['sha256'])
|
|
stats = read(checked(m['standardization'], m['standardization_sha256']))
|
|
if stats['parts'] != j['learning'] or stats['nodes'] != j['learning_nodes']:
|
|
raise ValueError('Learning-only normalization identity')
|
|
result.append(m)
|
|
return dict(models=result, training_gpu_seconds_once=prior, training_contract_sha256=sha(train / 'contract.json'),
|
|
training_terminal_sha256=sha(train / 'complete.json'))
|
|
|
|
|
|
def adopted(root, tc):
|
|
verify(root, 'models')
|
|
d = read(root / 'models-adopted.json')
|
|
if {m['job'] for m in d['models']} != set(range(5)) or len(d['models']) != 5:
|
|
raise ValueError('Five distinct adopted models')
|
|
for m in d['models']:
|
|
checked(m['path'], m['sha256'])
|
|
s = read(checked(m['standardization'], m['standardization_sha256']))
|
|
if s['parts'] != tc['jobs'][m['job']]['learning']:
|
|
raise ValueError('Adopted learning normalization')
|
|
return d
|
|
|
|
|
|
def checkpoint_identity(blob, j, training_sha):
|
|
if (blob.get('seed') != 0 or blob.get('fold') != j['id'] or blob.get('completed_epochs') != 200
|
|
or blob.get('optimizer_steps') != j['target_steps'] or blob.get('contract_sha256') != training_sha
|
|
or blob.get('role') != 'last_epoch_only'):
|
|
raise ValueError('Original last-epoch checkpoint identity')
|
|
|
|
|
|
def prediction_identity(root, c, tc, entries):
|
|
expected = {(j['id'], case) for j in tc['jobs'] for case in j['test']}
|
|
keys = [(e['split'], e['case']) for e in entries]
|
|
if len(keys) != len(set(keys)) or set(keys) != expected:
|
|
raise ValueError('All original valid test visits required before scoring')
|
|
for e in entries:
|
|
p = tc['parts'][e['case']]
|
|
expected_path = root / 'predictions' / str(e['split']) / (e['case'] + '.npz')
|
|
if (e['path'] != str(expected_path) or e['nodes'] != p['nodes']
|
|
or e['raw_ids_sha256'] != p['output_sha256']['raw_ids.npy']):
|
|
raise ValueError('Prediction path/native node identity')
|
|
checked(e['path'], e['sha256'])
|
|
a = c['A_predictions'][str(e['split'])][e['case']]
|
|
if a['nodes'] != e['nodes'] or a['raw_ids_sha256'] != e['raw_ids_sha256']:
|
|
raise ValueError('Paired original A domain identity')
|
|
checked(a['path'], a['sha256'])
|
|
return expected
|
|
|
|
|
|
def budget(prior, gpu, cpu, remaining):
|
|
if any(not math.isfinite(x) or x < 0 for x in [prior,gpu,cpu]) or remaining not in range(6):
|
|
raise ValueError('Invalid ledger')
|
|
if gpu + 900*remaining > 7200 or prior + gpu + 900*remaining > 162000:
|
|
raise RuntimeError('Track A total GPU reserve exhausted')
|
|
if cpu + 300*remaining + 900 + 120 > 3600:
|
|
raise RuntimeError('Track A postfit CPU reserve exhausted')
|