"""S3-A fixed feature extensions and cache-only targets; no archive decoder.""" import numpy as np from s2_custody_common import checked, read_json, prediction_inputs from s2_main_data import select_cell from s2_pilot_data import permitted_role from s2_models.features import R2_COLUMNS from s2_r2_data import learning_arrays, validation_score ARMS = ('R2_E4', 'R2_G2') EXTRA_COLUMNS = {'R2_E4': ('mean_q1', 'max_q1', 'mean_q2', 'max_q2'), 'R2_G2': ('g12', 'g23')} def columns(arm): if arm not in ARMS: raise ValueError('Only the two new arms may be fitted; never refit R2') return list(R2_COLUMNS + EXTRA_COLUMNS[arm]) def features(d, arm): expected = columns(arm) x = np.asarray(d['x']) extra = np.asarray(d['edge' if arm == 'R2_E4' else 'gaps']) if x.ndim != 2 or x.shape[1] != 20 or extra.shape != (len(x), len(expected)-20): raise ValueError('Frozen feature order/shape mismatch') if not len(x) or not np.isfinite(x).all() or not np.isfinite(extra).all(): raise ValueError('Nonfinite or empty common query features') return np.column_stack((x, extra)) def cached_truth(entry, d): with np.load(checked(entry['path'], entry['sha256']), allow_pickle=False) as z: if not np.array_equal(z['raw_ids'], d['raw_ids']) or not np.array_equal(z['counts'], d['counts']): raise ValueError('Custody cache/query IDs or multiplicity mismatch') truth = z['seconds'].copy() if truth.shape != np.asarray(d['raw_ids']).shape or not np.isfinite(truth).all() or np.any(truth < 0): raise ValueError('Invalid cached target') return truth def read_part(row, roles, label_index, arm): role = permitted_role(roles, row['case']) # reject current test before any IO columns(arm) d = prediction_inputs(row) entry = label_index[row['case']] truth = cached_truth(entry, d) return dict(case=row['case'], component_id=row['component_id'], role=role, x=features(d, arm), y=truth/d['t_set'], seconds=truth, counts=d['counts'], t_set=d['t_set'], query_ids=d['ids'], label_sha256=entry['sha256'], label_mode='S2_custody_cache_role_checked_no_decode') def load_cell(root, c, job): columns(job['arm']) rows, roles = select_cell(read_json(checked(c['roster'], c['roster_sha256'])), job['fold'], job['pool']) entries = read_json(checked(c['label_index'], c['label_index_sha256'])) labels = {r['case']: r for r in entries} if len(labels) != len(entries): raise ValueError('Duplicate cached label case') cases = roles['learning_parts'] + (roles['validation_parts'] if job['kind'] == 'HPO' else []) if job['kind'] not in ('HPO', 'FINAL'): raise ValueError('Unknown job kind') parts = [read_part(rows[k], roles, labels, job['arm']) for k in sorted(cases)] return [p for p in parts if p['role'] == 'learning'], [p for p in parts if p['role'] == 'validation'], roles