65 lines
2.9 KiB
Python
65 lines
2.9 KiB
Python
"""S3-A fixed feature extensions and cache-only targets; no archive decoder."""
|
|
import numpy as np
|
|
from s2_custody_common import checked, read_json, prediction_inputs
|
|
from s2_main_data import select_cell
|
|
from s2_pilot_data import permitted_role
|
|
from s2_models.features import R2_COLUMNS
|
|
from s2_r2_data import learning_arrays, validation_score
|
|
|
|
ARMS = ('R2_E4', 'R2_G2')
|
|
EXTRA_COLUMNS = {'R2_E4': ('mean_q1', 'max_q1', 'mean_q2', 'max_q2'),
|
|
'R2_G2': ('g12', 'g23')}
|
|
|
|
|
|
def columns(arm):
|
|
if arm not in ARMS:
|
|
raise ValueError('Only the two new arms may be fitted; never refit R2')
|
|
return list(R2_COLUMNS + EXTRA_COLUMNS[arm])
|
|
|
|
|
|
def features(d, arm):
|
|
expected = columns(arm)
|
|
x = np.asarray(d['x'])
|
|
extra = np.asarray(d['edge' if arm == 'R2_E4' else 'gaps'])
|
|
if x.ndim != 2 or x.shape[1] != 20 or extra.shape != (len(x), len(expected)-20):
|
|
raise ValueError('Frozen feature order/shape mismatch')
|
|
if not len(x) or not np.isfinite(x).all() or not np.isfinite(extra).all():
|
|
raise ValueError('Nonfinite or empty common query features')
|
|
return np.column_stack((x, extra))
|
|
|
|
|
|
def cached_truth(entry, d):
|
|
with np.load(checked(entry['path'], entry['sha256']), allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'], d['raw_ids']) or not np.array_equal(z['counts'], d['counts']):
|
|
raise ValueError('Custody cache/query IDs or multiplicity mismatch')
|
|
truth = z['seconds'].copy()
|
|
if truth.shape != np.asarray(d['raw_ids']).shape or not np.isfinite(truth).all() or np.any(truth < 0):
|
|
raise ValueError('Invalid cached target')
|
|
return truth
|
|
|
|
|
|
def read_part(row, roles, label_index, arm):
|
|
role = permitted_role(roles, row['case']) # reject current test before any IO
|
|
columns(arm)
|
|
d = prediction_inputs(row)
|
|
entry = label_index[row['case']]
|
|
truth = cached_truth(entry, d)
|
|
return dict(case=row['case'], component_id=row['component_id'], role=role,
|
|
x=features(d, arm), y=truth/d['t_set'], seconds=truth, counts=d['counts'],
|
|
t_set=d['t_set'], query_ids=d['ids'], label_sha256=entry['sha256'],
|
|
label_mode='S2_custody_cache_role_checked_no_decode')
|
|
|
|
|
|
def load_cell(root, c, job):
|
|
columns(job['arm'])
|
|
rows, roles = select_cell(read_json(checked(c['roster'], c['roster_sha256'])), job['fold'], job['pool'])
|
|
entries = read_json(checked(c['label_index'], c['label_index_sha256']))
|
|
labels = {r['case']: r for r in entries}
|
|
if len(labels) != len(entries):
|
|
raise ValueError('Duplicate cached label case')
|
|
cases = roles['learning_parts'] + (roles['validation_parts'] if job['kind'] == 'HPO' else [])
|
|
if job['kind'] not in ('HPO', 'FINAL'):
|
|
raise ValueError('Unknown job kind')
|
|
parts = [read_part(rows[k], roles, labels, job['arm']) for k in sorted(cases)]
|
|
return [p for p in parts if p['role'] == 'learning'], [p for p in parts if p['role'] == 'validation'], roles
|