146 lines
8.0 KiB
Python
146 lines
8.0 KiB
Python
"""Identity and custody for the four completed splits of the Track A rebind."""
|
|
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from phase2_h1_worker_v2 import checked, sha
|
|
from phase2_home_freeze_io import verify
|
|
from track_a_h2_common import read
|
|
from track_a_postfit_common import config as original_postfit_config
|
|
from track_a_recovery_common import ORIGINAL, ROOT as RECOVERY, accepted_guard, execution_command, reuse_identity
|
|
|
|
|
|
RULING = '1795022093.308479'
|
|
ROOT = ORIGINAL.with_name('track-a-postfit-four-split-2026-11-19')
|
|
ORIGINAL_POSTFIT = ORIGINAL.with_name('track-a-postfit-2026-11-18')
|
|
SPLITS = (0, 1, 2, 4)
|
|
OMITTED = dict(split=3, status='NOT_TRAINED_TO_COMPLETION_BUDGET_STOP',
|
|
reason='OWN_PROJECTED_CAP', recovery_completed_epochs=1)
|
|
ENTRYPOINTS = {
|
|
'track_a_postfit.py', 'track_a_postfit_worker.py',
|
|
'track_a_postfit_rebind.py', 'track_a_postfit_rebind_worker.py',
|
|
}
|
|
|
|
|
|
def selected_training(training):
|
|
return dict(training, jobs=[training['jobs'][split] for split in SPLITS])
|
|
|
|
|
|
def scope(original, training):
|
|
jobs = selected_training(training)['jobs']
|
|
if tuple(job['id'] for job in jobs) != SPLITS:
|
|
raise ValueError('Original split identities changed')
|
|
return dict(
|
|
jobs=[dict(id=j['id'], expected_parts=len(j['test'])) for j in jobs],
|
|
A_predictions={str(split): original['A_predictions'][str(split)] for split in SPLITS},
|
|
expected_pair_visits=sum(len(j['test']) for j in jobs),
|
|
expected_unique_target_reads=len({part for j in jobs for part in j['test']}),
|
|
excluded_split=OMITTED,
|
|
)
|
|
|
|
|
|
def config(root):
|
|
if root.resolve() != ROOT:
|
|
raise PermissionError('Only the four-split rebind root is approved')
|
|
contract = read(root / 'contract.json')
|
|
verify(root, 'implementation')
|
|
original, training, entry = original_postfit_config(ORIGINAL_POSTFIT)
|
|
expected = scope(original, training)
|
|
for key in ('approval', 'budget_approval', 'estimator_approval', 'training_root',
|
|
'training_contract_sha256', 'comparison_contract', 'comparison_contract_sha256',
|
|
'gpu_uuid', 'worker_cpus', 'coordinator_cpus', 'gpu_cap', 'postfit_gpu_cap',
|
|
'postfit_cpu_cap', 'idle_wait_seconds', 'no_retry', 'bootstrap',
|
|
'all_predictions_frozen_before_targets', 'A_refit', 'H2_refit', 'new_decodes',
|
|
'metric', 'original_failed'):
|
|
expected[key] = original[key]
|
|
if (contract['rebind_ruling'] != RULING or contract['version'] != 'track-a-postfit-four-split'
|
|
or not contract['no_auto_launch'] or any(contract[k] != v for k, v in expected.items())
|
|
or (contract['expected_pair_visits'], contract['expected_unique_target_reads']) != (1228, 571)):
|
|
raise PermissionError('Frozen four-of-five postfit scope')
|
|
for p, h in {**contract['source_sha256'], **contract['code_sha256']}.items():
|
|
checked(p, h)
|
|
return contract, training, entry
|
|
|
|
|
|
def terminal(training):
|
|
import psutil
|
|
|
|
for process in psutil.process_iter(['pid', 'cmdline']):
|
|
args = process.info['cmdline'] or []
|
|
if execution_command(args) or (process.pid != os.getpid()
|
|
and any(Path(a).name in ENTRYPOINTS for a in args)):
|
|
raise RuntimeError('Track A training or another postfit process still present')
|
|
verify(RECOVERY, 'terminal')
|
|
model_proof = verify(RECOVERY, 'incomplete-models')
|
|
recovered = read(RECOVERY / 'complete.json')
|
|
incomplete = read(RECOVERY / 'available-models-incomplete.json')
|
|
recovery = read(RECOVERY / 'contract.json')
|
|
originals = reuse_identity(recovery)
|
|
if (recovered['status'] != 'RECOVERY_INCOMPLETE_NO_FURTHER_RECOVERY'
|
|
or recovered['available_folds'] != list(SPLITS) or not recovered['no_further_recovery']
|
|
or recovered['postfit_started'] or recovered['test_targets_read']
|
|
or incomplete['all_five_complete'] or incomplete['postfit_started']
|
|
or (RECOVERY / 'global-stop.json').exists() or (ORIGINAL_POSTFIT / 'started.json').exists()):
|
|
raise ValueError('Track A terminal scope differs')
|
|
entries = {e['id']: e for e in recovered['entries']}
|
|
if len(recovered['entries']) != 2 or set(entries) != {3, 4}:
|
|
raise ValueError('Exact two recovery receipts required')
|
|
fold3, fold4 = (RECOVERY / 'jobs' / str(i) for i in (3, 4))
|
|
stopped = read(checked(fold3 / 'complete.json', entries[3]['guard_complete_sha256']))
|
|
stop = read(fold3 / 'stop.json')
|
|
projection = read(fold3 / 'first-epoch-projection.json')
|
|
if (entries[3]['status'] != 'STOPPED_OR_FAILED' or 'OWN_FOLD_STOP' not in stopped['reasons']
|
|
or stop['id'] != 3 or stop['reason'] != OMITTED['reason']
|
|
or projection['cell']['id'] != 3 or projection['cell']['within_cap']
|
|
or projection['cell']['projected_gpu_seconds'] <= recovery['jobs'][3]['gpu_seconds']
|
|
or (fold3 / 'worker' / 'result.json').exists()):
|
|
raise ValueError('Split 3 budget stop identity')
|
|
g = read(checked(fold4 / 'complete.json', entries[4]['guard_complete_sha256']))
|
|
if (entries[4]['status'] != 'PASS' or not accepted_guard(entries[4]['returncode'], g)
|
|
or g['worker'] != read(fold4 / 'guards' / 'train.receipt.json')
|
|
or g['worker']['monitor_errors'] != 0 or g['monitor_errors']):
|
|
raise ValueError('Split 4 clean recovery PASS required')
|
|
result = read(checked(fold4 / 'worker' / 'result.json', g['worker_result_sha256']))
|
|
job = training['jobs'][4]
|
|
if (result['status'] != 'COMPLETE' or result['fold'] != 4 or result['job'] != 4
|
|
or result['completed_epochs'] != 200 or result['optimizer_steps'] != job['target_steps']
|
|
or result['test_targets_read'] or result['test_prediction_started']
|
|
or result['validation_used_for_selection']):
|
|
raise ValueError('Split 4 last epoch identity')
|
|
model = dict(job=4, path=result['checkpoint'], sha256=result['checkpoint_sha256'],
|
|
standardization=str(fold4 / 'worker' / 'standardization.json'),
|
|
standardization_sha256=result['standardization_sha256'],
|
|
origin='RECOVERY_FRESH_SEED0', training_contract_sha256=sha(RECOVERY / 'contract.json'),
|
|
guard_complete_sha256=entries[4]['guard_complete_sha256'])
|
|
models = originals + [model]
|
|
if models != incomplete['models'] or tuple(m['job'] for m in models) != SPLITS:
|
|
raise ValueError('Mixed-source manifest differs from recovered models')
|
|
for m in models:
|
|
for p, h in ((m['path'], m['sha256']), (m['standardization'], m['standardization_sha256'])):
|
|
checked(p, h)
|
|
if model_proof['files'].get(str(Path(p).resolve())) != h:
|
|
raise ValueError('Model or normalization is missing from the recovery freeze')
|
|
stats = read(m['standardization'])
|
|
j = training['jobs'][m['job']]
|
|
if stats['parts'] != j['learning'] or stats['nodes'] != j['learning_nodes']:
|
|
raise ValueError('Learning-only normalization identity')
|
|
old = read(ORIGINAL / 'complete.json')['training_gpu_seconds']
|
|
new = sum(e['gpu_seconds_charged'] for e in entries.values())
|
|
prior = recovered['cumulative_training_gpu_seconds']
|
|
if (old != recovered['old_gpu_seconds_once'] or new != recovered['new_training_gpu_seconds']
|
|
or not math.isclose(prior, old + new, rel_tol=0, abs_tol=1e-8) or not 0 <= prior <= 154800):
|
|
raise ValueError('Old and recovery training charges must be included exactly once')
|
|
return dict(models=models, excluded_split=OMITTED, training_gpu_seconds_once=prior,
|
|
original_gpu_seconds_once=old, recovery_gpu_seconds_once=new,
|
|
original_terminal_sha256=sha(ORIGINAL / 'complete.json'),
|
|
recovery_terminal_sha256=sha(RECOVERY / 'complete.json'))
|
|
|
|
|
|
def adopted(root, contract):
|
|
verify(root, 'models')
|
|
receipt = read(root / 'models-adopted.json')
|
|
if receipt != contract['training_evidence']:
|
|
raise ValueError('Adopted evidence differs from the frozen binding')
|
|
return receipt
|