145 lines
6.3 KiB
Python
145 lines
6.3 KiB
Python
"""Track A random-split identity and cost gates; no array or target reads."""
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
|
|
from phase2_h1_worker_v2 import checked, sha
|
|
from phase2_home_freeze_io import verify
|
|
|
|
GPU_SECONDS = 162000
|
|
PROTECTED_GPUS = {
|
|
'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
|
|
'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
|
}
|
|
|
|
|
|
def read(path):
|
|
return json.loads(Path(path).read_text())
|
|
|
|
|
|
def validate_roles(c, entry):
|
|
if len(c['parts']) != 615 or len(c['jobs']) != 5:
|
|
raise ValueError('Five original splits and 615 v4 parts required')
|
|
if c['original_failed'] != entry['original_failed']:
|
|
raise ValueError('Original failures changed')
|
|
for case, original in entry['H2_inputs'].items():
|
|
f = original['features_and_targets']
|
|
expected = dict(case=case, output=f['output'], nodes=f['nodes'],
|
|
t_set=f['t_scale_seconds'], output_sha256=f['output_sha256'])
|
|
if c['parts'].get(case) != expected or c['geometry'].get(case) != original['geometry']:
|
|
raise ValueError('Frozen native v4 input identity differs')
|
|
for j, original in zip(c['jobs'], entry['jobs']):
|
|
if j['id'] != original['split'] or j['split_seed'] != original['split_seed']:
|
|
raise ValueError('Original split identity changed')
|
|
for role in ('learning', 'test'):
|
|
if j[role] != original[role] or len(set(j[role])) != len(j[role]):
|
|
raise ValueError('Original membership/order changed: ' + role)
|
|
if j['validation'] or set(j['learning']) & set(j['test']):
|
|
raise ValueError('No new holdout or test leakage')
|
|
clock = read(checked(j['clock_path'], j['clock_sha256']))
|
|
if set(clock['parts']) != set(j['learning']) or len(clock['parts']) != len(j['learning']):
|
|
raise ValueError('Clock learning membership differs')
|
|
if clock['groups'] != ['part:' + x for x in clock['parts']]:
|
|
raise ValueError('Original v4 equal-part clock groups differ')
|
|
if j['c'] != clock['c'] or not math.isfinite(j['c']) or j['c'] <= 0:
|
|
raise ValueError('Clock value differs')
|
|
if j['learning_nodes'] != sum(c['parts'][x]['nodes'] for x in j['learning']):
|
|
raise ValueError('Learning node count differs')
|
|
if j['gpu_uuid'] in PROTECTED_GPUS or j['target_steps'] != 200 * len(j['learning']):
|
|
raise ValueError('Protected GPU or epoch count')
|
|
|
|
|
|
def config(root):
|
|
root = Path(root)
|
|
c = read(root / 'contract.json')
|
|
verify(root, 'implementation')
|
|
if (c['approval'] != '1794973932.596749' or c['gpu_cap'] != GPU_SECONDS
|
|
or c['new_fits'] != 5 or c['seed'] != 0 or c['epochs'] != 200
|
|
or not c['no_retry'] or c['train_gpu_cap'] + c['postfit_gpu_reserve'] != GPU_SECONDS):
|
|
raise PermissionError('Frozen Track A scope differs')
|
|
if c['protocol_ruling'] != '1794975093.432709':
|
|
raise PermissionError('Actual protocol ruling differs')
|
|
for p, h in {**c['source_sha256'], **c['code_sha256']}.items():
|
|
checked(p, h)
|
|
if sum(j['gpu_seconds'] for j in c['jobs']) != c['train_gpu_cap']:
|
|
raise ValueError('Five job budgets must partition training cap')
|
|
entry = read(c['reuse_entry'])
|
|
validate_roles(c, entry)
|
|
import numpy as np
|
|
import torch
|
|
if np.__version__ != c['versions']['numpy'] or torch.__version__ != c['versions']['torch']:
|
|
raise ValueError('Original library versions differ')
|
|
return c, c
|
|
|
|
|
|
def view_for(c, job):
|
|
# The shared loader rejects every case outside this split's learning role.
|
|
return dict(c, folds=[job])
|
|
|
|
|
|
def project_cell(job, observation):
|
|
epoch = observation['epoch']
|
|
duration = observation['max_epoch_seconds']
|
|
charged = observation['charged']
|
|
if (not isinstance(epoch, int) or not 1 <= epoch <= 200
|
|
or not math.isfinite(duration) or duration <= 0
|
|
or not math.isfinite(charged) or charged < 0):
|
|
raise ValueError('Invalid full-epoch cost evidence')
|
|
projected = charged + 1.25 * (200 - epoch) * duration + 120
|
|
return dict(id=job['id'], projected_gpu_seconds=projected,
|
|
within_cap=projected <= job['gpu_seconds'])
|
|
|
|
|
|
def project_total(c, estimates, terminal, active_charges=None):
|
|
"""Pending jobs retain full reserved caps; terminal costs replace estimates."""
|
|
active_charges = active_charges or {}
|
|
valid = {j['id'] for j in c['jobs']}
|
|
if (set(estimates) | set(terminal) | set(active_charges)) - valid:
|
|
raise ValueError('Unknown job in ledger')
|
|
total = c['postfit_gpu_reserve']
|
|
rows = []
|
|
for j in c['jobs']:
|
|
i = j['id']
|
|
value = (terminal[i]['gpu_seconds_charged'] if i in terminal else
|
|
max(active_charges.get(i, 0), estimates.get(i, {}).get('projected_gpu_seconds', j['gpu_seconds'])))
|
|
if not math.isfinite(value) or value < 0:
|
|
raise ValueError('Invalid ledger value')
|
|
total += value
|
|
rows.append(dict(id=i, projected_gpu_seconds=value, within_cap=value <= j['gpu_seconds']))
|
|
return dict(status='PASS' if total <= c['gpu_cap'] and all(r['within_cap'] for r in rows)
|
|
else 'STOP_BUDGET_PROJECTION', rows=rows, projected_gpu_seconds=total,
|
|
postfit_reserve=c['postfit_gpu_reserve'])
|
|
|
|
|
|
def first_epoch_release(root, jobid):
|
|
if (root / 'global-stop.json').exists():
|
|
raise RuntimeError('Track A global stop')
|
|
p = root / 'jobs' / str(jobid) / 'first-epoch-projection.json'
|
|
if not p.exists():
|
|
return False
|
|
r = read(p)
|
|
if r['status'] != 'PASS' or r['contract_sha256'] != sha(root / 'contract.json'):
|
|
raise RuntimeError('First-epoch cost gate rejected')
|
|
return True
|
|
|
|
|
|
def dispatchable(pending, active):
|
|
occupied = {j['gpu_uuid'] for j in active}
|
|
if len(occupied) != len(active) or len(active) > 2:
|
|
raise ValueError('Active GPU collision')
|
|
selected = []
|
|
for j in pending:
|
|
if len(active) + len(selected) == 2:
|
|
break
|
|
if j['gpu_uuid'] in PROTECTED_GPUS:
|
|
raise ValueError('Item1 GPU reservation')
|
|
if j['gpu_uuid'] not in occupied:
|
|
selected.append(j)
|
|
occupied.add(j['gpu_uuid'])
|
|
return selected
|
|
|
|
|
|
def execution_command(args):
|
|
return any(Path(a).name in {'track_a_h2_queue.py', 'track_a_h2_guard.py',
|
|
'track_a_h2_worker.py'} for a in args)
|