Files
MeltMind/tools/prepare_phase3_item_b_shared.py
T

160 lines
9.7 KiB
Python

"""Freeze b's pre-launch budget/queue delta using metadata from the live c run."""
import ast
import fcntl
import math
import time
from pathlib import Path
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze, verify
from phase3_item_b_common import ROOT as ORIGINAL
from phase3_item_b_shared_budget import MARGIN, RULING, USER_CAP, full_fold_bound
from phase3_item_b_shared_common import ROOT, config, project, read
from track_a_h2_prepare import local_code_closure
BASE = Path(__file__).resolve().parents[1]
RECORDS = BASE / 'artifacts/phase3-item-b-shared-budget-2026-11-19'
C_ROOT = ORIGINAL.with_name('phase3-item-c-2026-11-18')
PARKED = ORIGINAL.with_name('phase3-item-1b-2026-11-18')
def measured_bounds(old_jobs):
c = read(C_ROOT / 'contract.json')
if sha(C_ROOT / 'contract.json') != read(C_ROOT / 'started.json')['contract_sha256']:
raise ValueError('c training identity changed')
for p, h in c['code_sha256'].items():
checked(p, h)
jobs = {i: C_ROOT / 'jobs' / str(i) for i in range(5)}
assignments = {i: read(job / 'assignment.json') for i, job in jobs.items()}
workers = {i: read(job / 'worker/worker-identity.json') for i, job in jobs.items()}
samples = {i: read(job / 'gpu-sample.json') for i, job in jobs.items()}
rows = []
for i, old in enumerate(old_jobs):
assignment = assignments[i]
if assignment['id'] != i or assignment['contract_sha256'] != sha(C_ROOT / 'contract.json'):
raise ValueError('c fold assignment identity')
sample = samples[i]
seen = {r['pid'] for r in sample['rows']}
worker = workers[i]
if worker['pid'] not in seen or worker['pid'] not in sample['own_pids']:
raise ValueError('c worker missing from GPU sample')
peers = [j for j in jobs if j != i and assignments[j]['gpu_uuid'] == assignment['gpu_uuid']
and workers[j]['pid'] in seen]
if not peers:
raise ValueError('A live same-card c peer must corroborate the overlap')
overlap_start = max(worker['created'], min(workers[j]['created'] for j in peers))
candidates = []
for p in sorted((jobs[i] / 'worker/history').glob('*.json')):
mtime = p.stat().st_mtime
if mtime > sample['unix']:
continue
row = read(p)
seconds = row['train_epoch_seconds']
if (row['fold'] != i or row['epoch_nodes'] != old['learning_nodes']
or row['epoch_parts'] != len(old['learning']) or not math.isfinite(seconds) or seconds <= 0):
raise ValueError('c and b fold workload or epoch identity differs')
if mtime - seconds > overlap_start + 2:
candidates.append(dict(epoch=row['epoch'], epoch_seconds=seconds, file=str(p),
sha256=sha(p), mtime=mtime))
if not candidates:
raise ValueError('No c epoch lies fully inside the same-card window')
maximum = max(candidates, key=lambda t: t['epoch_seconds'])
rows.append(dict(fold=i, gpu_uuid=assignment['gpu_uuid'], worker=worker,
shared_peers={str(j): workers[j] for j in peers}, gpu_sample=sample,
overlap_start=overlap_start, candidates=candidates, worst_epoch=maximum,
max_train_epoch_seconds=maximum['epoch_seconds'],
validation_seconds_bound=old['validation_seconds_bound']))
return rows
def main():
start = time.process_time()
if ROOT.exists() or (ORIGINAL / 'started.json').exists() or (ORIGINAL / 'launch-attempt.json').exists():
raise FileExistsError('Only a never-launched b may be rebound')
validate_review(ORIGINAL)
old = read(ORIGINAL / 'contract.json')
stop = read(PARKED / 'complete.json')
verify(PARKED, 'training-stop')
tail = read(PARKED / 'post-terminal-cost.json')
if (stop['status'] != 'STOPPED_INCOMPLETE' or stop['prediction_started'] or stop['scoring_started']
or any(e['status'] == 'PASS' for e in stop['entries'])
or tail['training_complete_sha256'] != sha(PARKED / 'complete.json')
or tail['postfit_started'] or not tail['freeze_tail_charged_once']):
raise ValueError('1b terminal identity changed')
rows = measured_bounds(old['jobs'])
jobs = []
for job, row in zip(old['jobs'], rows, strict=True):
epoch = row['max_train_epoch_seconds']
jobs.append(dict(job, shared_epoch_seconds_bound=epoch,
full_training_gpu_seconds_bound=full_fold_bound(epoch, job['validation_seconds_bound'])))
bounds = [j['full_training_gpu_seconds_bound'] for j in jobs]
wait_reserve = max(bounds)
cap = math.ceil(old['preparation_charge_gpu_seconds'] + old['postfit_gpu_reserve'] + sum(bounds) + wait_reserve)
reserves = dict(old['phase_reservations'], item1b=tail['cumulative_gpu_seconds'], b=cap)
if cap > USER_CAP or sum(reserves.values()) > old['phase3_gpu_cap']:
raise ValueError('Shared-mode cap does not fit the approved GPU limits')
original_worker = BASE / 'tools/phase3_item_b_worker.py'
new_worker = BASE / 'tools/phase3_item_b_shared_worker.py'
normalized = new_worker.read_text().replace('from phase3_item_b_shared_common import',
'from phase3_item_b_common import')
if ast.dump(ast.parse(normalized)) != ast.dump(ast.parse(original_worker.read_text())):
raise ValueError('b worker differs beyond the run identity import')
files = [Path(__file__), *sorted((BASE / 'tools').glob('phase3_item_b_shared_*.py')),
BASE / 'tools/test_phase3_item_b_shared_budget.py']
codes = {**old['code_sha256'], **local_code_closure(files)}
proof_sources = [ORIGINAL / x for x in ('contract.json', 'expected-review-binding.json',
'implementation-git-freeze.json', 'review.json')]
proof_sources += [PARKED / x for x in ('complete.json', 'post-terminal-cost.json', 'training-stop-git-freeze.json')]
proof_sources += [C_ROOT / 'contract.json', C_ROOT / 'started.json']
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
evidence = RECORDS / 'shared-cost-evidence.json'
if evidence.exists():
raise FileExistsError('A shared calibration receipt already exists')
save(evidence, dict(ruling=RULING, c_root=str(C_ROOT), c_contract_sha256=sha(C_ROOT / 'contract.json'),
folds=rows, margin=MARGIN, full_fold_training_bounds=bounds,
aggregate_wait_reserve_gpu_seconds=wait_reserve,
wait_reserve_basis='one longest full-fold bound across all waiting charges',
gpu_cap=cap, user_cap=USER_CAP, phase_reservations=reserves,
parked_1b_actual_including_tail=tail['cumulative_gpu_seconds'],
phase3_unreserved_gpu_seconds=old['phase3_gpu_cap']-sum(reserves.values()),
validation_bound_basis='original b diagnostic bounds; no shared diagnostic measurement',
worker_same_except_rebind_import=True))
sources = dict(old['source_sha256'])
sources.update({str(p): sha(p) for p in proof_sources + [evidence]})
contract = dict(old, version='phase3-item-b-shared-budget', shared_budget_ruling=RULING,
shared_budget_evidence=str(evidence), gpu_cap=cap, phase_reservations=reserves,
waiting_charge='Pre-dispatch waiting charged once to total; excluded from per-fold training',
projection='Measured c shared-mode per-fold times +25%; actual b maxima may raise projections',
jobs=jobs, code_sha256=codes, source_sha256=sources)
ROOT.mkdir()
save(ROOT / 'contract.json', contract)
initial = project(contract, dict.fromkeys(range(5), 60.), dict.fromkeys(range(5), 0.), {}, {})
if initial['status'] != 'PASS':
raise ValueError('Initial shared-mode projection cannot fit')
save(ROOT / 'preparation-audit.json', dict(status='WAIT_INDEPENDENT_DELTA_REVIEW',
base_implementation=verify(ORIGINAL, 'implementation')['commit'],
gpu_cap=cap, phase_reservations=reserves, initial_projection=initial,
code_count=len(codes), source_count=len(sources), metadata_cpu_seconds=time.process_time()-start,
comparison_unchanged=True, scientific_worker_unchanged=True,
started=False, test_targets_read=False, review_adopted=False))
freeze(ROOT, 'implementation', [ROOT / 'contract.json', ROOT / 'preparation-audit.json',
*(Path(p) for p in codes), evidence, RECORDS / 'acceptance.log',
BASE / 'docs/phase3-item-b-shared-budget-2026-11-19.md'])
frozen = verify(ROOT, 'implementation')
config(ROOT)
save(ROOT / 'expected-review-binding.json', dict(
reviewed_commit=frozen['commit'], contract_sha256=sha(ROOT / 'contract.json'),
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
comparison_contract_sha256=contract['comparison_contract_sha256'], code_sha256=codes))
print(dict(implementation=frozen['commit'], contract=sha(ROOT / 'contract.json'),
binding_sha256=sha(ROOT / 'expected-review-binding.json'), gpu_cap=cap,
initial_projected_gpu_seconds=initial['projected_gpu_seconds'],
phase3_reserved_gpu_seconds=sum(reserves.values()), code_count=len(codes)))
if __name__ == '__main__':
main()