Files
MeltMind/tools/track_a_h2_common.py
T

145 lines
6.3 KiB
Python

"""Track A random-split identity and cost gates; no array or target reads."""
import json
import math
from pathlib import Path
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
GPU_SECONDS = 162000
PROTECTED_GPUS = {
'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
}
def read(path):
return json.loads(Path(path).read_text())
def validate_roles(c, entry):
if len(c['parts']) != 615 or len(c['jobs']) != 5:
raise ValueError('Five original splits and 615 v4 parts required')
if c['original_failed'] != entry['original_failed']:
raise ValueError('Original failures changed')
for case, original in entry['H2_inputs'].items():
f = original['features_and_targets']
expected = dict(case=case, output=f['output'], nodes=f['nodes'],
t_set=f['t_scale_seconds'], output_sha256=f['output_sha256'])
if c['parts'].get(case) != expected or c['geometry'].get(case) != original['geometry']:
raise ValueError('Frozen native v4 input identity differs')
for j, original in zip(c['jobs'], entry['jobs']):
if j['id'] != original['split'] or j['split_seed'] != original['split_seed']:
raise ValueError('Original split identity changed')
for role in ('learning', 'test'):
if j[role] != original[role] or len(set(j[role])) != len(j[role]):
raise ValueError('Original membership/order changed: ' + role)
if j['validation'] or set(j['learning']) & set(j['test']):
raise ValueError('No new holdout or test leakage')
clock = read(checked(j['clock_path'], j['clock_sha256']))
if set(clock['parts']) != set(j['learning']) or len(clock['parts']) != len(j['learning']):
raise ValueError('Clock learning membership differs')
if clock['groups'] != ['part:' + x for x in clock['parts']]:
raise ValueError('Original v4 equal-part clock groups differ')
if j['c'] != clock['c'] or not math.isfinite(j['c']) or j['c'] <= 0:
raise ValueError('Clock value differs')
if j['learning_nodes'] != sum(c['parts'][x]['nodes'] for x in j['learning']):
raise ValueError('Learning node count differs')
if j['gpu_uuid'] in PROTECTED_GPUS or j['target_steps'] != 200 * len(j['learning']):
raise ValueError('Protected GPU or epoch count')
def config(root):
root = Path(root)
c = read(root / 'contract.json')
verify(root, 'implementation')
if (c['approval'] != '1794973932.596749' or c['gpu_cap'] != GPU_SECONDS
or c['new_fits'] != 5 or c['seed'] != 0 or c['epochs'] != 200
or not c['no_retry'] or c['train_gpu_cap'] + c['postfit_gpu_reserve'] != GPU_SECONDS):
raise PermissionError('Frozen Track A scope differs')
if c['protocol_ruling'] != '1794975093.432709':
raise PermissionError('Actual protocol ruling differs')
for p, h in {**c['source_sha256'], **c['code_sha256']}.items():
checked(p, h)
if sum(j['gpu_seconds'] for j in c['jobs']) != c['train_gpu_cap']:
raise ValueError('Five job budgets must partition training cap')
entry = read(c['reuse_entry'])
validate_roles(c, entry)
import numpy as np
import torch
if np.__version__ != c['versions']['numpy'] or torch.__version__ != c['versions']['torch']:
raise ValueError('Original library versions differ')
return c, c
def view_for(c, job):
# The shared loader rejects every case outside this split's learning role.
return dict(c, folds=[job])
def project_cell(job, observation):
epoch = observation['epoch']
duration = observation['max_epoch_seconds']
charged = observation['charged']
if (not isinstance(epoch, int) or not 1 <= epoch <= 200
or not math.isfinite(duration) or duration <= 0
or not math.isfinite(charged) or charged < 0):
raise ValueError('Invalid full-epoch cost evidence')
projected = charged + 1.25 * (200 - epoch) * duration + 120
return dict(id=job['id'], projected_gpu_seconds=projected,
within_cap=projected <= job['gpu_seconds'])
def project_total(c, estimates, terminal, active_charges=None):
"""Pending jobs retain full reserved caps; terminal costs replace estimates."""
active_charges = active_charges or {}
valid = {j['id'] for j in c['jobs']}
if (set(estimates) | set(terminal) | set(active_charges)) - valid:
raise ValueError('Unknown job in ledger')
total = c['postfit_gpu_reserve']
rows = []
for j in c['jobs']:
i = j['id']
value = (terminal[i]['gpu_seconds_charged'] if i in terminal else
max(active_charges.get(i, 0), estimates.get(i, {}).get('projected_gpu_seconds', j['gpu_seconds'])))
if not math.isfinite(value) or value < 0:
raise ValueError('Invalid ledger value')
total += value
rows.append(dict(id=i, projected_gpu_seconds=value, within_cap=value <= j['gpu_seconds']))
return dict(status='PASS' if total <= c['gpu_cap'] and all(r['within_cap'] for r in rows)
else 'STOP_BUDGET_PROJECTION', rows=rows, projected_gpu_seconds=total,
postfit_reserve=c['postfit_gpu_reserve'])
def first_epoch_release(root, jobid):
if (root / 'global-stop.json').exists():
raise RuntimeError('Track A global stop')
p = root / 'jobs' / str(jobid) / 'first-epoch-projection.json'
if not p.exists():
return False
r = read(p)
if r['status'] != 'PASS' or r['contract_sha256'] != sha(root / 'contract.json'):
raise RuntimeError('First-epoch cost gate rejected')
return True
def dispatchable(pending, active):
occupied = {j['gpu_uuid'] for j in active}
if len(occupied) != len(active) or len(active) > 2:
raise ValueError('Active GPU collision')
selected = []
for j in pending:
if len(active) + len(selected) == 2:
break
if j['gpu_uuid'] in PROTECTED_GPUS:
raise ValueError('Item1 GPU reservation')
if j['gpu_uuid'] not in occupied:
selected.append(j)
occupied.add(j['gpu_uuid'])
return selected
def execution_command(args):
return any(Path(a).name in {'track_a_h2_queue.py', 'track_a_h2_guard.py',
'track_a_h2_worker.py'} for a in args)