150 lines
9.2 KiB
Python
150 lines
9.2 KiB
Python
"""Seed1 custody barriers; same S2 test roles and no performance decision gate."""
|
|
import math
|
|
from pathlib import Path
|
|
import numpy as np
|
|
from phase2_h1_worker_v2 import checked, sha
|
|
from phase2_home_freeze_io import verify
|
|
from phase3_seed1_v2_common import config as training_config, execution_command
|
|
from track_a_h2_common import read
|
|
from phase2_h2_s2_postfit_common import roster
|
|
|
|
OLD = 8482.313407897018
|
|
PHASE3_PRIOR = 80903.62940728385
|
|
GPU = 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c'
|
|
TRAINING = '8603801c4644068db5102227878e1af28c3b5f5e'
|
|
BOOTSTRAP = dict(replicates=2000, seed=2026111408, confidence=.95,
|
|
unit='component_equal_mean_of_part_differences', interval='percentile_linear')
|
|
|
|
|
|
def config(root):
|
|
c = read(root / 'contract.json'); verify(root, 'implementation')
|
|
if (c['approval'] != '1794992262.078969' or c['gpu_cap'] != 167200
|
|
or c['postfit_gpu_cap'] != 7200 or c['postfit_cpu_cap'] != 3600
|
|
or c['phase3_gpu_cap'] != 612000 or c['old_seed1_gpu_once'] != OLD
|
|
or c['phase3_prior_gpu_once'] != PHASE3_PRIOR or c['gpu_uuid'] != GPU
|
|
or c['idle_wait_seconds'] != 20 or not c['no_retry'] or c['performance_gate'] is not None
|
|
or c['new_fits'] != 0 or c['new_decodes'] != 0 or c['bootstrap'] != BOOTSTRAP):
|
|
raise PermissionError('Fixed seed1 postfit scope')
|
|
for p, h in {**c['source_sha256'], **c['code_sha256']}.items(): checked(p, h)
|
|
train = Path(c['training_root'])
|
|
if verify(train, 'implementation')['commit'] != TRAINING: raise ValueError('Training implementation differs')
|
|
tc, pilot = training_config(train)
|
|
if sha(train / 'contract.json') != c['training_contract_sha256']: raise ValueError('Training contract differs')
|
|
if c['jobs'] != [dict(id=j['id'], expected_parts=len(j['test'])) for j in tc['jobs']]:
|
|
raise ValueError('All original five test lists')
|
|
for name in ['predictions', 'terminal']: verify(Path(c['seed0_root']), name)
|
|
by = roster(c, pilot)
|
|
seed0 = read(c['seed0_manifest'])['entries']
|
|
if len(seed0) != 552 or len({e['case'] for e in seed0}) != 552 or c['seed0_predictions'] != {e['case']:e for e in seed0}:
|
|
raise ValueError('Original seed0 manifest identity')
|
|
for case, e in c['seed0_predictions'].items():
|
|
row = by[case]
|
|
if (e['fold'] != next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
|
or e['component_id'] != row['component_id'] or e['nodes'] != row['nodes']):
|
|
raise ValueError('Seed0 same test roles')
|
|
return c, tc, pilot
|
|
|
|
|
|
def terminal_ledger(tc, done, manifest):
|
|
entries = done.get('entries', []); models = manifest.get('models', [])
|
|
if (done.get('status') != 'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING'
|
|
or done.get('postfit_started') is not False or done.get('test_targets_read') is not False
|
|
or len(entries) != 5 or {e['id'] for e in entries} != set(range(5))
|
|
or any(e['status'] != 'PASS' or e['returncode'] != 0 for e in entries)
|
|
or len(models) != 5 or {m['job'] for m in models} != set(range(5))
|
|
or manifest.get('new_fits') != 5 or manifest.get('test_prediction_started') is not False):
|
|
raise ValueError('All five original terminal checkpoints required')
|
|
costs = [e['gpu_seconds_charged'] for e in entries]
|
|
value = done['training_gpu_seconds']
|
|
if (any(not math.isfinite(x) or x < 0 or x > 40000 for x in costs)
|
|
or not math.isfinite(value) or abs(value-sum(costs)) > 1e-8
|
|
or value > tc['train_gpu_cap'] or done['phase3_prior_gpu_once'] != PHASE3_PRIOR
|
|
or abs(done['cumulative_phase3_training_gpu_seconds']-PHASE3_PRIOR-value)>1e-8):
|
|
raise ValueError('Training ledger must include old costs exactly once')
|
|
budget(value, 0, 180, 5)
|
|
return value
|
|
|
|
|
|
def normal_identity(stats, job):
|
|
if (stats['parts'] != job['learning'] or stats['nodes'] != job['learning_nodes']
|
|
or len(stats['mean']) != 19 or len(stats['scale']) != 19
|
|
or not np.isfinite(stats['mean']).all() or not np.isfinite(stats['scale']).all()
|
|
or min(stats['scale']) <= 0): raise ValueError('Learning-only 19-column normalization')
|
|
|
|
|
|
def terminal(root, c, tc):
|
|
import psutil
|
|
train = Path(c['training_root'])
|
|
tp = verify(train, 'terminal'); mp = verify(train, 'models')
|
|
if (train/'global-stop.json').exists(): raise PermissionError('Training global stop')
|
|
done = read(train/'complete.json'); manifest = read(train/'models-terminal.json')
|
|
value = terminal_ledger(tc, done, manifest)
|
|
for p in psutil.process_iter(['cmdline']):
|
|
if execution_command(p.info['cmdline'] or []): raise RuntimeError('Seed1 training process still alive')
|
|
for j in tc['jobs']:
|
|
i=j['id']; w=train/'jobs'/str(i)/'worker'; e=next(e for e in done['entries'] if e['id']==i)
|
|
assignment=read(checked(w.parent/'assignment.json',e['assignment_sha256']))
|
|
if assignment['id']!=i or assignment['contract_sha256']!=c['training_contract_sha256'] or assignment['gpu_uuid']!=e['gpu_uuid']:
|
|
raise ValueError('Original assignment identity')
|
|
g=read(checked(w.parent/'complete.json',e['guard_complete_sha256']))
|
|
receipt=read(w.parent/'guards/train.receipt.json')
|
|
if (g['status']!='PASS' or g['reasons'] or g['monitor_errors'] or g['worker']!=receipt
|
|
or type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors']!=0
|
|
or receipt['status']!='PASS' or receipt['returncode']!=0): raise ValueError('Training monitoring failure')
|
|
r=read(checked(w/'result.json',g['worker_result_sha256'])); m=next(m for m in manifest['models'] if m['job']==i)
|
|
if (r['status']!='COMPLETE' or r['fold']!=i or r['completed_epochs']!=200
|
|
or r['optimizer_steps']!=j['target_steps'] or r['learning_parts']!=len(j['learning'])
|
|
or r['learning_nodes_per_epoch']!=j['learning_nodes'] or r['test_targets_read'] is not False
|
|
or r['test_prediction_started'] is not False or r['validation_used_for_selection'] is not False
|
|
or m['path']!=str(w/'last-epoch200.pt') or m['path']!=r['checkpoint']
|
|
or m['sha256']!=r['checkpoint_sha256'] or m['standardization']!=str(w/'standardization.json')
|
|
or m['standardization_sha256']!=r['standardization_sha256']): raise ValueError('Final checkpoint/roles differ')
|
|
for p,h in [(m['path'],m['sha256']),(m['standardization'],m['standardization_sha256'])]:
|
|
checked(p,h)
|
|
if mp['files'].get(str(Path(p).resolve()))!=h: raise ValueError('Model omitted from freeze')
|
|
normal_identity(read(m['standardization']),j)
|
|
return dict(models=manifest['models'],new_training_gpu_seconds_once=value,
|
|
old_seed1_gpu_seconds_once=OLD,cumulative_seed1_training_gpu_seconds=OLD+value,
|
|
cumulative_phase3_training_gpu_seconds=PHASE3_PRIOR+value,
|
|
training_contract_sha256=c['training_contract_sha256'],training_terminal_sha256=sha(train/'complete.json'),
|
|
training_terminal_commit=tp['commit'],training_models_commit=mp['commit'])
|
|
|
|
|
|
def adopted(root, c, tc):
|
|
verify(root,'models'); d=read(root/'models-adopted.json')
|
|
if d!=terminal(root,c,tc): raise ValueError('Adopted model/terminal identity differs')
|
|
return d
|
|
|
|
|
|
def checkpoint_identity(blob, j, training_sha):
|
|
if (blob.get('seed')!=1 or blob.get('fold')!=j['id'] or blob.get('completed_epochs')!=200
|
|
or blob.get('optimizer_steps')!=j['target_steps'] or blob.get('contract_sha256')!=training_sha
|
|
or blob.get('role')!='last_epoch_only'): raise ValueError('Seed1 final checkpoint identity')
|
|
|
|
|
|
def prediction_identity(root, c, tc, entries):
|
|
expected={(j['id'],case) for j in tc['jobs'] for case in j['test']}
|
|
keys=[(e['fold'],e['case']) for e in entries]
|
|
if len(keys)!=552 or len(set(keys))!=552 or set(keys)!=expected: raise ValueError('All552 distinct test predictions required')
|
|
rows={r['case']:r for r in c['rows']}
|
|
for e in entries:
|
|
row=rows[e['case']]; old=c['seed0_predictions'][e['case']]
|
|
if (e['path']!=str(root/'jobs'/str(e['fold'])/'prediction-records'/(e['case']+'.npz'))
|
|
or e['nodes']!=row['nodes'] or e['query_nodes']!=old['query_nodes']
|
|
or e['component_id']!=row['component_id'] or e['seed0_sha256']!=old['sha256']
|
|
or e['raw_ids_sha256']!=row['v5_cache']['output_sha256']['raw_ids.npy']):
|
|
raise ValueError('Prediction path/query/pair identity')
|
|
for k,limit in [('negative_full_predictions',e['nodes']),('negative_query_predictions',e['query_nodes'])]:
|
|
if type(e[k]) is not int or not 0<=e[k]<=limit: raise ValueError('Invalid negative counts')
|
|
checked(e['path'],e['sha256']); checked(old['path'],old['sha256'])
|
|
return rows
|
|
|
|
|
|
def budget(new_train, gpu, cpu, remaining):
|
|
if any(not math.isfinite(x) or x<0 for x in [new_train,gpu,cpu]) or type(remaining) is not int or remaining not in range(6):
|
|
raise ValueError('Invalid ledger')
|
|
reserve=gpu+900*remaining
|
|
if new_train>160000-OLD or reserve>7200 or OLD+new_train+reserve>167200 or PHASE3_PRIOR+new_train+reserve>612000:
|
|
raise RuntimeError('Seed1/phase3 cumulative GPU reserve exhausted')
|
|
if cpu+300*remaining+900+120>3600: raise RuntimeError('Postfit CPU reserve exhausted')
|