Files
MeltMind/tools/s2_r2_data.py

66 lines
3.4 KiB
Python

"""R2 uses frozen query rows, direct targets and the same component/part weights."""
from collections import Counter
import json
from pathlib import Path
import numpy as np
from freeze_g1_components import sha
from s2_main_data import select_cell,ensure_targets
from s2_pilot_data import permitted_role
from s2_models.features import R2_COLUMNS
from s2_models.objectives import relative_l2
def read_part(root,c,row,roles,old_labels):
role=permitted_role(roles,row['case'])
prep=row['preparation'];source=Path(prep['preparation_root'])
label,label_sha,mode=ensure_targets(root,c,row,roles,old_labels)
for name in ['r2.npz','queries.npz']:assert sha(source/name)==prep['output_sha256'][name]
with np.load(source/'queries.npz',allow_pickle=False) as q:
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];counts=q['multiplicity'][mask]
with np.load(source/'r2.npz',allow_pickle=False) as z:
x=z['features'];assert x.shape==(row['nodes'],len(R2_COLUMNS));x=x[ids].copy()
assert sha(label)==label_sha
with np.load(label,allow_pickle=False) as z:
assert np.array_equal(z['anchor_ids'],ids) and np.array_equal(z['multiplicity'],counts)
y=z['target_t_star'].copy();seconds=z['seconds'].copy()
assert len(y)>0 and np.isfinite(x).all() and np.isfinite(y).all() and (y>=0).all()
assert np.all(x[:,-1]==prep['t_set'])
assert np.array_equal(y,seconds/prep['t_set'])
return dict(case=row['case'],component_id=row['component_id'],role=role,x=x,y=y,seconds=seconds,
counts=counts,t_set=prep['t_set'],query_ids=ids,label_mode=mode,label_sha256=label_sha)
def learning_arrays(parts):
assert parts and all(p['role']=='learning' for p in parts)
groups=Counter(p['component_id'] for p in parts);ng=len(groups)
x=[];y=[];weights=[]
for p in parts:
counts=np.asarray(p['counts'],dtype=np.float64)
assert len(counts)==len(p['y'])==len(p['x']) and np.all(counts>0)
x.append(p['x']);y.append(p['y'])
weights.append(counts/counts.sum()/groups[p['component_id']]/ng)
w=np.concatenate(weights);assert np.isclose(w.sum(),1,rtol=0,atol=1e-12)
return np.concatenate(x),np.concatenate(y),w
def validation_score(model,parts):
assert parts and all(p['role']=='validation' for p in parts)
groups={};errors={}
for p in parts:
predicted=model.predict(p['x'])
value=relative_l2(predicted*p['t_set'],p['seconds'],p['counts'],np.ones(len(p['counts']),bool))['value']
if value is None or not np.isfinite(value):raise ValueError('Invalid validation relative L2')
errors[p['case']]=value;groups.setdefault(p['component_id'],[]).append(value)
return float(np.mean([np.mean(v) for v in groups.values()])),errors
def load_cell(root,c,job):
assert sha(c['roster'])==c['roster_sha256']
roster=json.loads(Path(c['roster']).read_text());rows,roles=select_cell(roster,job['fold'],job['pool'])
assert sha(c['old_labels_manifest'])==c['old_labels_manifest_sha256']
old={p['case']:p for p in json.loads(Path(c['old_labels_manifest']).read_text())['ready_parts']}
# Final refit reads learning only; validation was already used for HPO selection.
cases=roles['learning_parts']+(roles['validation_parts'] if job['kind']=='HPO' else [])
parts=[read_part(root,c,rows[k],roles,old) for k in sorted(cases)]
return [p for p in parts if p['role']=='learning'],[p for p in parts if p['role']=='validation'],roles