"""R4bc fixed protocol and checked inputs; no archive reader or fitting here.""" import hashlib import json import math from collections import Counter from pathlib import Path import numpy as np from s2_models.features import R2_COLUMNS, histgbrt_grid from s2_recovery_supervisor import save ARMS = ('H0p', 'H1', 'H2', 'H3', 'H4', 'A', 'Q') HIST = ARMS[:5] POOLS = (100, 50, 25) class InputViolation(ValueError): """Identity/role/hash failures are global stops, never a failed HPO choice.""" def read(path): return json.loads(Path(path).read_text()) def sha(path): h = hashlib.sha256() with Path(path).open('rb') as f: for block in iter(lambda: f.read(4 * 1024 * 1024), b''): h.update(block) return h.hexdigest() def checked(path, digest): if sha(path) != digest: raise InputViolation('Frozen hash mismatch: ' + str(path)) return Path(path) def exact(a, b, name): if not np.array_equal(a, b): raise InputViolation('Identity mismatch: ' + name) def fingerprint(a): a = np.ascontiguousarray(a) return dict(shape=list(a.shape), dtype=a.dtype.str, sha256=hashlib.sha256(a.tobytes()).hexdigest()) def contract(root, *, inputs=True): c = read(Path(root) / 'contract.json') for path, digest in c['code_sha256'].items(): checked(path, digest) if inputs: for path, digest in c['source_sha256'].items(): checked(path, digest) return c def require_role(row, fold, role, pool): a = next(x for x in row['assignments'] if x['fold'] == fold) if a['role'] != role or (role == 'learning' and str(pool) not in a['learning_pools']): raise InputViolation('Role denied: ' + row['case'] + '/' + role) def role_rows(c, fold, pool, role): result = [] for row in c['parts']: a = next(x for x in row['assignments'] if x['fold'] == fold) if a['role'] == role and (role != 'learning' or str(pool) in a['learning_pools']): result.append(row) return sorted(result, key=lambda r: r['case']) def make_jobs(): jobs = [] for pool in POOLS: for fold in range(5): cell = fold * 3 + (25, 50, 100).index(pool) for arm in ARMS: for config in (range(12) if arm in HIST else []): jobs.append(dict(index=len(jobs), cell=cell, fold=fold, pool=pool, arm=arm, kind='HPO', config_index=config)) jobs.append(dict(index=len(jobs), cell=cell, fold=fold, pool=pool, arm=arm, kind='FINAL', config_index=None)) assert len(jobs) == 1005 return jobs def spec(arm): if arm not in ARMS: raise ValueError(arm) return dict(columns=20 if arm == 'H0p' else 19, geometry='historical' if arm in ('H0p', 'H1') else 'tet', support='all' if arm in ('H4', 'A') else 'query', weights='hierarchy' if arm in ('H0p', 'H1', 'H2', 'Q') else 'unit', dtype='float32' if arm in ('A', 'Q') else 'float64', learner='XGB' if arm in ('A', 'Q') else 'Hist') def hierarchy_weights(counts, components, *, xgb=False): if len(counts) != len(components) or not counts: raise InputViolation('Weight parts mismatch') groups = Counter(components) ws = [] for cc, group in zip(counts, components): cc = np.asarray(cc, dtype=np.float64) if not len(cc) or not np.isfinite(cc).all() or np.any(cc <= 0): raise InputViolation('Invalid query counts') ws.append(cc / cc.sum() / groups[group] / len(groups)) w = np.concatenate(ws) if not np.isclose(w.sum(), 1., rtol=0, atol=1e-12): raise InputViolation('Hierarchy sum differs from one') return w * len(w) if xgb else w def selection(trials): if len(trials) != 12 or sorted(t['config_index'] for t in trials) != list(range(12)): raise InputViolation('Selection requires all 12 terminal HPO attempts') good = [t for t in trials if t['status'] == 'PASS' and t.get('score') is not None and math.isfinite(t['score'])] best = min(good, key=lambda t: (t['score'], t['config_index'])) if good else None return dict(status='SELECTED' if best else 'NA_ALL_HPO_FAILED', config_index=best['config_index'] if best else None, trials=trials, finished_configurations=len(good), failed_configurations=12-len(good)) def can_reserve(spent, active, cap, per_job=1800.): return math.isfinite(spent) and spent >= 0 and spent + (active + 1) * per_job <= cap def selection_path(root, job): return Path(root) / 'selections' / f"cell-{job['cell']:02d}-{job['arm']}.json" def job_dir(root, index): return Path(root) / 'jobs' / f'job-{index:04d}' def relative_l2(pred, truth, counts): denominator = float(np.sum(counts * truth * truth)) if denominator <= 0: return None return float(np.sqrt(np.sum(counts * (pred - truth)**2) / denominator)) def query_inputs(row): prep = row['preparation']; source = Path(prep['preparation_root']) with np.load(checked(source/'queries.npz', prep['output_sha256']['queries.npz']), allow_pickle=False) as z: keep = z['eligible_s2'] out = dict(ids=z['anchor_ids'][keep].copy(), raw_ids=z['raw_node_ids'][keep].copy(), counts=z['multiplicity'][keep].copy(), near_wall=z['near_wall'][keep].copy()) if not len(out['ids']) or np.any(np.diff(out['raw_ids']) <= 0) or np.any(out['counts'] <= 0): raise InputViolation('Invalid distinct query IDs/counts') return out def tet_indices(row, query): v = row['v5_cache']; path = Path(v['output'])/'raw_ids.npy' raw = np.load(checked(path, v['output_sha256']['raw_ids.npy']), allow_pickle=False) if len(raw) != row['nodes'] or np.any(np.diff(raw) <= 0): raise InputViolation('Tet node identity') ids = np.searchsorted(raw, query['raw_ids']) if np.any(ids >= len(raw)): raise InputViolation('Query outside tet domain') exact(raw[ids], query['raw_ids'], 'tet queries') return ids def features(row, arm, *, full=False): q = query_inputs(row); cfg = spec(arm); v = row['v5_cache']; prep = row['preparation'] if cfg['geometry'] == 'historical': with np.load(checked(Path(prep['preparation_root'])/'r2.npz', prep['output_sha256']['r2.npz']), allow_pickle=False) as z: data = z['features'] if data.shape != (row['nodes'], 20): raise InputViolation('Historical 20-column shape') x = data[q['ids']].copy() exact(x[:, -1], np.full(len(x), prep['t_set']), 'historical t_set') if arm == 'H1': x = x[:, :-1] else: path = checked(Path(v['output'])/'features.npy', v['output_sha256']['features.npy']) data = np.load(path, mmap_mode='r', allow_pickle=False) if data.shape != (row['nodes'], 19): raise InputViolation('Tet 19-column shape') x = data if full else data[tet_indices(row, q)] x = np.asarray(x, dtype=cfg['dtype']) if not np.isfinite(x).all(): raise InputViolation('Nonfinite features') return x, q def targets(row, job, role, *, full, access): if role not in ('learning', 'validation') or (role == 'validation' and job['kind'] != 'HPO'): raise InputViolation('Target role forbidden before prediction freeze') require_role(row, job['fold'], role, job['pool']) q = query_inputs(row); v = row['v5_cache']; label = row['label'] # Record intended accesses before any target values are loaded. access.append(dict(case=row['case'], fold=job['fold'], role=role, full=full, cached_label_path=label['path'], cached_label_sha256=label['sha256'], v5_target_sha256=v['output_sha256']['target.npy'])) with np.load(checked(label['path'], label['sha256']), allow_pickle=False) as z: exact(z['raw_ids'], q['raw_ids'], 'label query IDs') exact(z['counts'], q['counts'], 'label counts') truth = z['seconds'].copy() path = checked(Path(v['output'])/'target.npy', v['output_sha256']['target.npy']) yfull = np.load(path, mmap_mode='r', allow_pickle=False) iq = tet_indices(row, q) scale = float(row['preparation']['t_set']) if scale <= 0 or scale != float(v['t_scale_seconds']): raise InputViolation('t_set mismatch') exact(yfull[iq], truth / scale, 'S2 and v5 normalized targets') y = np.array(yfull if full else truth/scale, dtype=spec(job['arm'])['dtype']) if not np.isfinite(y).all() or np.any(y < 0): raise InputViolation('Invalid cached target') return y, truth def paired_difference(before, after, *, seed=2026111408, replicates=2000): ids = sorted(set(before) & set(after)) if not ids: return dict(value=None, low=None, high=None, components=0, undefined_replicates=replicates) d = np.array([before[g] - after[g] for g in ids], dtype=np.float64) if not np.isfinite(d).all(): raise InputViolation('Undefined component passed to bootstrap') rng = np.random.default_rng(seed) values = np.array([d[rng.integers(0, len(d), len(d))].mean() for _ in range(replicates)]) low, high = np.percentile(values, [2.5, 97.5], method='linear') return dict(value=float(d.mean()), low=float(low), high=float(high), components=len(ids), component_ids=ids, replicates=replicates, seed=seed, undefined_replicates=0, formula='mean(before)-mean(after)', interval='component-paired percentile95; descriptive')