160 lines
9.7 KiB
Python
160 lines
9.7 KiB
Python
"""Freeze b's pre-launch budget/queue delta using metadata from the live c run."""
|
|
|
|
import ast
|
|
import fcntl
|
|
import math
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h1_worker_v2 import checked, save, sha
|
|
from phase2_home_freeze_io import freeze, verify
|
|
from phase3_item_b_common import ROOT as ORIGINAL
|
|
from phase3_item_b_shared_budget import MARGIN, RULING, USER_CAP, full_fold_bound
|
|
from phase3_item_b_shared_common import ROOT, config, project, read
|
|
from track_a_h2_prepare import local_code_closure
|
|
|
|
BASE = Path(__file__).resolve().parents[1]
|
|
RECORDS = BASE / 'artifacts/phase3-item-b-shared-budget-2026-11-19'
|
|
C_ROOT = ORIGINAL.with_name('phase3-item-c-2026-11-18')
|
|
PARKED = ORIGINAL.with_name('phase3-item-1b-2026-11-18')
|
|
|
|
|
|
def measured_bounds(old_jobs):
|
|
c = read(C_ROOT / 'contract.json')
|
|
if sha(C_ROOT / 'contract.json') != read(C_ROOT / 'started.json')['contract_sha256']:
|
|
raise ValueError('c training identity changed')
|
|
for p, h in c['code_sha256'].items():
|
|
checked(p, h)
|
|
jobs = {i: C_ROOT / 'jobs' / str(i) for i in range(5)}
|
|
assignments = {i: read(job / 'assignment.json') for i, job in jobs.items()}
|
|
workers = {i: read(job / 'worker/worker-identity.json') for i, job in jobs.items()}
|
|
samples = {i: read(job / 'gpu-sample.json') for i, job in jobs.items()}
|
|
rows = []
|
|
for i, old in enumerate(old_jobs):
|
|
assignment = assignments[i]
|
|
if assignment['id'] != i or assignment['contract_sha256'] != sha(C_ROOT / 'contract.json'):
|
|
raise ValueError('c fold assignment identity')
|
|
sample = samples[i]
|
|
seen = {r['pid'] for r in sample['rows']}
|
|
worker = workers[i]
|
|
if worker['pid'] not in seen or worker['pid'] not in sample['own_pids']:
|
|
raise ValueError('c worker missing from GPU sample')
|
|
peers = [j for j in jobs if j != i and assignments[j]['gpu_uuid'] == assignment['gpu_uuid']
|
|
and workers[j]['pid'] in seen]
|
|
if not peers:
|
|
raise ValueError('A live same-card c peer must corroborate the overlap')
|
|
overlap_start = max(worker['created'], min(workers[j]['created'] for j in peers))
|
|
candidates = []
|
|
for p in sorted((jobs[i] / 'worker/history').glob('*.json')):
|
|
mtime = p.stat().st_mtime
|
|
if mtime > sample['unix']:
|
|
continue
|
|
row = read(p)
|
|
seconds = row['train_epoch_seconds']
|
|
if (row['fold'] != i or row['epoch_nodes'] != old['learning_nodes']
|
|
or row['epoch_parts'] != len(old['learning']) or not math.isfinite(seconds) or seconds <= 0):
|
|
raise ValueError('c and b fold workload or epoch identity differs')
|
|
if mtime - seconds > overlap_start + 2:
|
|
candidates.append(dict(epoch=row['epoch'], epoch_seconds=seconds, file=str(p),
|
|
sha256=sha(p), mtime=mtime))
|
|
if not candidates:
|
|
raise ValueError('No c epoch lies fully inside the same-card window')
|
|
maximum = max(candidates, key=lambda t: t['epoch_seconds'])
|
|
rows.append(dict(fold=i, gpu_uuid=assignment['gpu_uuid'], worker=worker,
|
|
shared_peers={str(j): workers[j] for j in peers}, gpu_sample=sample,
|
|
overlap_start=overlap_start, candidates=candidates, worst_epoch=maximum,
|
|
max_train_epoch_seconds=maximum['epoch_seconds'],
|
|
validation_seconds_bound=old['validation_seconds_bound']))
|
|
return rows
|
|
|
|
|
|
def main():
|
|
start = time.process_time()
|
|
if ROOT.exists() or (ORIGINAL / 'started.json').exists() or (ORIGINAL / 'launch-attempt.json').exists():
|
|
raise FileExistsError('Only a never-launched b may be rebound')
|
|
validate_review(ORIGINAL)
|
|
old = read(ORIGINAL / 'contract.json')
|
|
stop = read(PARKED / 'complete.json')
|
|
verify(PARKED, 'training-stop')
|
|
tail = read(PARKED / 'post-terminal-cost.json')
|
|
if (stop['status'] != 'STOPPED_INCOMPLETE' or stop['prediction_started'] or stop['scoring_started']
|
|
or any(e['status'] == 'PASS' for e in stop['entries'])
|
|
or tail['training_complete_sha256'] != sha(PARKED / 'complete.json')
|
|
or tail['postfit_started'] or not tail['freeze_tail_charged_once']):
|
|
raise ValueError('1b terminal identity changed')
|
|
rows = measured_bounds(old['jobs'])
|
|
jobs = []
|
|
for job, row in zip(old['jobs'], rows, strict=True):
|
|
epoch = row['max_train_epoch_seconds']
|
|
jobs.append(dict(job, shared_epoch_seconds_bound=epoch,
|
|
full_training_gpu_seconds_bound=full_fold_bound(epoch, job['validation_seconds_bound'])))
|
|
bounds = [j['full_training_gpu_seconds_bound'] for j in jobs]
|
|
wait_reserve = max(bounds)
|
|
cap = math.ceil(old['preparation_charge_gpu_seconds'] + old['postfit_gpu_reserve'] + sum(bounds) + wait_reserve)
|
|
reserves = dict(old['phase_reservations'], item1b=tail['cumulative_gpu_seconds'], b=cap)
|
|
if cap > USER_CAP or sum(reserves.values()) > old['phase3_gpu_cap']:
|
|
raise ValueError('Shared-mode cap does not fit the approved GPU limits')
|
|
original_worker = BASE / 'tools/phase3_item_b_worker.py'
|
|
new_worker = BASE / 'tools/phase3_item_b_shared_worker.py'
|
|
normalized = new_worker.read_text().replace('from phase3_item_b_shared_common import',
|
|
'from phase3_item_b_common import')
|
|
if ast.dump(ast.parse(normalized)) != ast.dump(ast.parse(original_worker.read_text())):
|
|
raise ValueError('b worker differs beyond the run identity import')
|
|
files = [Path(__file__), *sorted((BASE / 'tools').glob('phase3_item_b_shared_*.py')),
|
|
BASE / 'tools/test_phase3_item_b_shared_budget.py']
|
|
codes = {**old['code_sha256'], **local_code_closure(files)}
|
|
proof_sources = [ORIGINAL / x for x in ('contract.json', 'expected-review-binding.json',
|
|
'implementation-git-freeze.json', 'review.json')]
|
|
proof_sources += [PARKED / x for x in ('complete.json', 'post-terminal-cost.json', 'training-stop-git-freeze.json')]
|
|
proof_sources += [C_ROOT / 'contract.json', C_ROOT / 'started.json']
|
|
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
evidence = RECORDS / 'shared-cost-evidence.json'
|
|
if evidence.exists():
|
|
raise FileExistsError('A shared calibration receipt already exists')
|
|
save(evidence, dict(ruling=RULING, c_root=str(C_ROOT), c_contract_sha256=sha(C_ROOT / 'contract.json'),
|
|
folds=rows, margin=MARGIN, full_fold_training_bounds=bounds,
|
|
aggregate_wait_reserve_gpu_seconds=wait_reserve,
|
|
wait_reserve_basis='one longest full-fold bound across all waiting charges',
|
|
gpu_cap=cap, user_cap=USER_CAP, phase_reservations=reserves,
|
|
parked_1b_actual_including_tail=tail['cumulative_gpu_seconds'],
|
|
phase3_unreserved_gpu_seconds=old['phase3_gpu_cap']-sum(reserves.values()),
|
|
validation_bound_basis='original b diagnostic bounds; no shared diagnostic measurement',
|
|
worker_same_except_rebind_import=True))
|
|
sources = dict(old['source_sha256'])
|
|
sources.update({str(p): sha(p) for p in proof_sources + [evidence]})
|
|
contract = dict(old, version='phase3-item-b-shared-budget', shared_budget_ruling=RULING,
|
|
shared_budget_evidence=str(evidence), gpu_cap=cap, phase_reservations=reserves,
|
|
waiting_charge='Pre-dispatch waiting charged once to total; excluded from per-fold training',
|
|
projection='Measured c shared-mode per-fold times +25%; actual b maxima may raise projections',
|
|
jobs=jobs, code_sha256=codes, source_sha256=sources)
|
|
ROOT.mkdir()
|
|
save(ROOT / 'contract.json', contract)
|
|
initial = project(contract, dict.fromkeys(range(5), 60.), dict.fromkeys(range(5), 0.), {}, {})
|
|
if initial['status'] != 'PASS':
|
|
raise ValueError('Initial shared-mode projection cannot fit')
|
|
save(ROOT / 'preparation-audit.json', dict(status='WAIT_INDEPENDENT_DELTA_REVIEW',
|
|
base_implementation=verify(ORIGINAL, 'implementation')['commit'],
|
|
gpu_cap=cap, phase_reservations=reserves, initial_projection=initial,
|
|
code_count=len(codes), source_count=len(sources), metadata_cpu_seconds=time.process_time()-start,
|
|
comparison_unchanged=True, scientific_worker_unchanged=True,
|
|
started=False, test_targets_read=False, review_adopted=False))
|
|
freeze(ROOT, 'implementation', [ROOT / 'contract.json', ROOT / 'preparation-audit.json',
|
|
*(Path(p) for p in codes), evidence, RECORDS / 'acceptance.log',
|
|
BASE / 'docs/phase3-item-b-shared-budget-2026-11-19.md'])
|
|
frozen = verify(ROOT, 'implementation')
|
|
config(ROOT)
|
|
save(ROOT / 'expected-review-binding.json', dict(
|
|
reviewed_commit=frozen['commit'], contract_sha256=sha(ROOT / 'contract.json'),
|
|
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
|
|
comparison_contract_sha256=contract['comparison_contract_sha256'], code_sha256=codes))
|
|
print(dict(implementation=frozen['commit'], contract=sha(ROOT / 'contract.json'),
|
|
binding_sha256=sha(ROOT / 'expected-review-binding.json'), gpu_cap=cap,
|
|
initial_projected_gpu_seconds=initial['projected_gpu_seconds'],
|
|
phase3_reserved_gpu_seconds=sum(reserves.values()), code_count=len(codes)))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|