"""R2 uses frozen query rows, direct targets and the same component/part weights.""" from collections import Counter import json from pathlib import Path import numpy as np from freeze_g1_components import sha from s2_main_data import select_cell,ensure_targets from s2_pilot_data import permitted_role from s2_models.features import R2_COLUMNS from s2_models.objectives import relative_l2 def read_part(root,c,row,roles,old_labels): role=permitted_role(roles,row['case']) prep=row['preparation'];source=Path(prep['preparation_root']) label,label_sha,mode=ensure_targets(root,c,row,roles,old_labels) for name in ['r2.npz','queries.npz']:assert sha(source/name)==prep['output_sha256'][name] with np.load(source/'queries.npz',allow_pickle=False) as q: mask=q['eligible_s2'];ids=q['anchor_ids'][mask];counts=q['multiplicity'][mask] with np.load(source/'r2.npz',allow_pickle=False) as z: x=z['features'];assert x.shape==(row['nodes'],len(R2_COLUMNS));x=x[ids].copy() assert sha(label)==label_sha with np.load(label,allow_pickle=False) as z: assert np.array_equal(z['anchor_ids'],ids) and np.array_equal(z['multiplicity'],counts) y=z['target_t_star'].copy();seconds=z['seconds'].copy() assert len(y)>0 and np.isfinite(x).all() and np.isfinite(y).all() and (y>=0).all() assert np.all(x[:,-1]==prep['t_set']) assert np.array_equal(y,seconds/prep['t_set']) return dict(case=row['case'],component_id=row['component_id'],role=role,x=x,y=y,seconds=seconds, counts=counts,t_set=prep['t_set'],query_ids=ids,label_mode=mode,label_sha256=label_sha) def learning_arrays(parts): assert parts and all(p['role']=='learning' for p in parts) groups=Counter(p['component_id'] for p in parts);ng=len(groups) x=[];y=[];weights=[] for p in parts: counts=np.asarray(p['counts'],dtype=np.float64) assert len(counts)==len(p['y'])==len(p['x']) and np.all(counts>0) x.append(p['x']);y.append(p['y']) weights.append(counts/counts.sum()/groups[p['component_id']]/ng) w=np.concatenate(weights);assert np.isclose(w.sum(),1,rtol=0,atol=1e-12) return np.concatenate(x),np.concatenate(y),w def validation_score(model,parts): assert parts and all(p['role']=='validation' for p in parts) groups={};errors={} for p in parts: predicted=model.predict(p['x']) value=relative_l2(predicted*p['t_set'],p['seconds'],p['counts'],np.ones(len(p['counts']),bool))['value'] if value is None or not np.isfinite(value):raise ValueError('Invalid validation relative L2') errors[p['case']]=value;groups.setdefault(p['component_id'],[]).append(value) return float(np.mean([np.mean(v) for v in groups.values()])),errors def load_cell(root,c,job): assert sha(c['roster'])==c['roster_sha256'] roster=json.loads(Path(c['roster']).read_text());rows,roles=select_cell(roster,job['fold'],job['pool']) assert sha(c['old_labels_manifest'])==c['old_labels_manifest_sha256'] old={p['case']:p for p in json.loads(Path(c['old_labels_manifest']).read_text())['ready_parts']} # Final refit reads learning only; validation was already used for HPO selection. cases=roles['learning_parts']+(roles['validation_parts'] if job['kind']=='HPO' else []) parts=[read_part(root,c,rows[k],roles,old) for k in sorted(cases)] return [p for p in parts if p['role']=='learning'],[p for p in parts if p['role']=='validation'],roles