Files
MeltMind/tools/r4bc_common.py

232 lines
9.4 KiB
Python

"""R4bc fixed protocol and checked inputs; no archive reader or fitting here."""
import hashlib
import json
import math
from collections import Counter
from pathlib import Path
import numpy as np
from s2_models.features import R2_COLUMNS, histgbrt_grid
from s2_recovery_supervisor import save
ARMS = ('H0p', 'H1', 'H2', 'H3', 'H4', 'A', 'Q')
HIST = ARMS[:5]
POOLS = (100, 50, 25)
class InputViolation(ValueError):
"""Identity/role/hash failures are global stops, never a failed HPO choice."""
def read(path):
return json.loads(Path(path).read_text())
def sha(path):
h = hashlib.sha256()
with Path(path).open('rb') as f:
for block in iter(lambda: f.read(4 * 1024 * 1024), b''):
h.update(block)
return h.hexdigest()
def checked(path, digest):
if sha(path) != digest:
raise InputViolation('Frozen hash mismatch: ' + str(path))
return Path(path)
def exact(a, b, name):
if not np.array_equal(a, b):
raise InputViolation('Identity mismatch: ' + name)
def fingerprint(a):
a = np.ascontiguousarray(a)
return dict(shape=list(a.shape), dtype=a.dtype.str,
sha256=hashlib.sha256(a.tobytes()).hexdigest())
def contract(root, *, inputs=True):
c = read(Path(root) / 'contract.json')
for path, digest in c['code_sha256'].items():
checked(path, digest)
if inputs:
for path, digest in c['source_sha256'].items():
checked(path, digest)
return c
def require_role(row, fold, role, pool):
a = next(x for x in row['assignments'] if x['fold'] == fold)
if a['role'] != role or (role == 'learning' and str(pool) not in a['learning_pools']):
raise InputViolation('Role denied: ' + row['case'] + '/' + role)
def role_rows(c, fold, pool, role):
result = []
for row in c['parts']:
a = next(x for x in row['assignments'] if x['fold'] == fold)
if a['role'] == role and (role != 'learning' or str(pool) in a['learning_pools']):
result.append(row)
return sorted(result, key=lambda r: r['case'])
def make_jobs():
jobs = []
for pool in POOLS:
for fold in range(5):
cell = fold * 3 + (25, 50, 100).index(pool)
for arm in ARMS:
for config in (range(12) if arm in HIST else []):
jobs.append(dict(index=len(jobs), cell=cell, fold=fold, pool=pool,
arm=arm, kind='HPO', config_index=config))
jobs.append(dict(index=len(jobs), cell=cell, fold=fold, pool=pool,
arm=arm, kind='FINAL', config_index=None))
assert len(jobs) == 1005
return jobs
def spec(arm):
if arm not in ARMS:
raise ValueError(arm)
return dict(columns=20 if arm == 'H0p' else 19,
geometry='historical' if arm in ('H0p', 'H1') else 'tet',
support='all' if arm in ('H4', 'A') else 'query',
weights='hierarchy' if arm in ('H0p', 'H1', 'H2', 'Q') else 'unit',
dtype='float32' if arm in ('A', 'Q') else 'float64',
learner='XGB' if arm in ('A', 'Q') else 'Hist')
def hierarchy_weights(counts, components, *, xgb=False):
if len(counts) != len(components) or not counts:
raise InputViolation('Weight parts mismatch')
groups = Counter(components)
ws = []
for cc, group in zip(counts, components):
cc = np.asarray(cc, dtype=np.float64)
if not len(cc) or not np.isfinite(cc).all() or np.any(cc <= 0):
raise InputViolation('Invalid query counts')
ws.append(cc / cc.sum() / groups[group] / len(groups))
w = np.concatenate(ws)
if not np.isclose(w.sum(), 1., rtol=0, atol=1e-12):
raise InputViolation('Hierarchy sum differs from one')
return w * len(w) if xgb else w
def selection(trials):
if len(trials) != 12 or sorted(t['config_index'] for t in trials) != list(range(12)):
raise InputViolation('Selection requires all 12 terminal HPO attempts')
good = [t for t in trials if t['status'] == 'PASS' and t.get('score') is not None
and math.isfinite(t['score'])]
best = min(good, key=lambda t: (t['score'], t['config_index'])) if good else None
return dict(status='SELECTED' if best else 'NA_ALL_HPO_FAILED',
config_index=best['config_index'] if best else None, trials=trials,
finished_configurations=len(good), failed_configurations=12-len(good))
def can_reserve(spent, active, cap, per_job=1800.):
return math.isfinite(spent) and spent >= 0 and spent + (active + 1) * per_job <= cap
def selection_path(root, job):
return Path(root) / 'selections' / f"cell-{job['cell']:02d}-{job['arm']}.json"
def job_dir(root, index):
return Path(root) / 'jobs' / f'job-{index:04d}'
def relative_l2(pred, truth, counts):
denominator = float(np.sum(counts * truth * truth))
if denominator <= 0:
return None
return float(np.sqrt(np.sum(counts * (pred - truth)**2) / denominator))
def query_inputs(row):
prep = row['preparation']; source = Path(prep['preparation_root'])
with np.load(checked(source/'queries.npz', prep['output_sha256']['queries.npz']), allow_pickle=False) as z:
keep = z['eligible_s2']
out = dict(ids=z['anchor_ids'][keep].copy(), raw_ids=z['raw_node_ids'][keep].copy(),
counts=z['multiplicity'][keep].copy(), near_wall=z['near_wall'][keep].copy())
if not len(out['ids']) or np.any(np.diff(out['raw_ids']) <= 0) or np.any(out['counts'] <= 0):
raise InputViolation('Invalid distinct query IDs/counts')
return out
def tet_indices(row, query):
v = row['v5_cache']; path = Path(v['output'])/'raw_ids.npy'
raw = np.load(checked(path, v['output_sha256']['raw_ids.npy']), allow_pickle=False)
if len(raw) != row['nodes'] or np.any(np.diff(raw) <= 0):
raise InputViolation('Tet node identity')
ids = np.searchsorted(raw, query['raw_ids'])
if np.any(ids >= len(raw)):
raise InputViolation('Query outside tet domain')
exact(raw[ids], query['raw_ids'], 'tet queries')
return ids
def features(row, arm, *, full=False):
q = query_inputs(row); cfg = spec(arm); v = row['v5_cache']; prep = row['preparation']
if cfg['geometry'] == 'historical':
with np.load(checked(Path(prep['preparation_root'])/'r2.npz', prep['output_sha256']['r2.npz']), allow_pickle=False) as z:
data = z['features']
if data.shape != (row['nodes'], 20):
raise InputViolation('Historical 20-column shape')
x = data[q['ids']].copy()
exact(x[:, -1], np.full(len(x), prep['t_set']), 'historical t_set')
if arm == 'H1':
x = x[:, :-1]
else:
path = checked(Path(v['output'])/'features.npy', v['output_sha256']['features.npy'])
data = np.load(path, mmap_mode='r', allow_pickle=False)
if data.shape != (row['nodes'], 19):
raise InputViolation('Tet 19-column shape')
x = data if full else data[tet_indices(row, q)]
x = np.asarray(x, dtype=cfg['dtype'])
if not np.isfinite(x).all():
raise InputViolation('Nonfinite features')
return x, q
def targets(row, job, role, *, full, access):
if role not in ('learning', 'validation') or (role == 'validation' and job['kind'] != 'HPO'):
raise InputViolation('Target role forbidden before prediction freeze')
require_role(row, job['fold'], role, job['pool'])
q = query_inputs(row); v = row['v5_cache']; label = row['label']
# Record intended accesses before any target values are loaded.
access.append(dict(case=row['case'], fold=job['fold'], role=role, full=full,
cached_label_path=label['path'], cached_label_sha256=label['sha256'],
v5_target_sha256=v['output_sha256']['target.npy']))
with np.load(checked(label['path'], label['sha256']), allow_pickle=False) as z:
exact(z['raw_ids'], q['raw_ids'], 'label query IDs')
exact(z['counts'], q['counts'], 'label counts')
truth = z['seconds'].copy()
path = checked(Path(v['output'])/'target.npy', v['output_sha256']['target.npy'])
yfull = np.load(path, mmap_mode='r', allow_pickle=False)
iq = tet_indices(row, q)
scale = float(row['preparation']['t_set'])
if scale <= 0 or scale != float(v['t_scale_seconds']):
raise InputViolation('t_set mismatch')
exact(yfull[iq], truth / scale, 'S2 and v5 normalized targets')
y = np.array(yfull if full else truth/scale, dtype=spec(job['arm'])['dtype'])
if not np.isfinite(y).all() or np.any(y < 0):
raise InputViolation('Invalid cached target')
return y, truth
def paired_difference(before, after, *, seed=2026111408, replicates=2000):
ids = sorted(set(before) & set(after))
if not ids:
return dict(value=None, low=None, high=None, components=0, undefined_replicates=replicates)
d = np.array([before[g] - after[g] for g in ids], dtype=np.float64)
if not np.isfinite(d).all():
raise InputViolation('Undefined component passed to bootstrap')
rng = np.random.default_rng(seed)
values = np.array([d[rng.integers(0, len(d), len(d))].mean() for _ in range(replicates)])
low, high = np.percentile(values, [2.5, 97.5], method='linear')
return dict(value=float(d.mean()), low=float(low), high=float(high), components=len(ids),
component_ids=ids, replicates=replicates, seed=seed, undefined_replicates=0,
formula='mean(before)-mean(after)', interval='component-paired percentile95; descriptive')