Files
MeltMind/tools/s3a_data.py
T

65 lines
2.9 KiB
Python

"""S3-A fixed feature extensions and cache-only targets; no archive decoder."""
import numpy as np
from s2_custody_common import checked, read_json, prediction_inputs
from s2_main_data import select_cell
from s2_pilot_data import permitted_role
from s2_models.features import R2_COLUMNS
from s2_r2_data import learning_arrays, validation_score
ARMS = ('R2_E4', 'R2_G2')
EXTRA_COLUMNS = {'R2_E4': ('mean_q1', 'max_q1', 'mean_q2', 'max_q2'),
'R2_G2': ('g12', 'g23')}
def columns(arm):
if arm not in ARMS:
raise ValueError('Only the two new arms may be fitted; never refit R2')
return list(R2_COLUMNS + EXTRA_COLUMNS[arm])
def features(d, arm):
expected = columns(arm)
x = np.asarray(d['x'])
extra = np.asarray(d['edge' if arm == 'R2_E4' else 'gaps'])
if x.ndim != 2 or x.shape[1] != 20 or extra.shape != (len(x), len(expected)-20):
raise ValueError('Frozen feature order/shape mismatch')
if not len(x) or not np.isfinite(x).all() or not np.isfinite(extra).all():
raise ValueError('Nonfinite or empty common query features')
return np.column_stack((x, extra))
def cached_truth(entry, d):
with np.load(checked(entry['path'], entry['sha256']), allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'], d['raw_ids']) or not np.array_equal(z['counts'], d['counts']):
raise ValueError('Custody cache/query IDs or multiplicity mismatch')
truth = z['seconds'].copy()
if truth.shape != np.asarray(d['raw_ids']).shape or not np.isfinite(truth).all() or np.any(truth < 0):
raise ValueError('Invalid cached target')
return truth
def read_part(row, roles, label_index, arm):
role = permitted_role(roles, row['case']) # reject current test before any IO
columns(arm)
d = prediction_inputs(row)
entry = label_index[row['case']]
truth = cached_truth(entry, d)
return dict(case=row['case'], component_id=row['component_id'], role=role,
x=features(d, arm), y=truth/d['t_set'], seconds=truth, counts=d['counts'],
t_set=d['t_set'], query_ids=d['ids'], label_sha256=entry['sha256'],
label_mode='S2_custody_cache_role_checked_no_decode')
def load_cell(root, c, job):
columns(job['arm'])
rows, roles = select_cell(read_json(checked(c['roster'], c['roster_sha256'])), job['fold'], job['pool'])
entries = read_json(checked(c['label_index'], c['label_index_sha256']))
labels = {r['case']: r for r in entries}
if len(labels) != len(entries):
raise ValueError('Duplicate cached label case')
cases = roles['learning_parts'] + (roles['validation_parts'] if job['kind'] == 'HPO' else [])
if job['kind'] not in ('HPO', 'FINAL'):
raise ValueError('Unknown job kind')
parts = [read_part(rows[k], roles, labels, job['arm']) for k in sorted(cases)]
return [p for p in parts if p['role'] == 'learning'], [p for p in parts if p['role'] == 'validation'], roles