Files
MeltMind/tools/phase3_item_b_data.py
T

72 lines
3.4 KiB
Python

"""Reuse frozen B13 readout; learning-only normalization, joint missingness, no partition."""
import json,time
from pathlib import Path
import numpy as np
import torch
from phase2_h1_worker_v2 import checked,save
from phase2_h2_data import standardization as original_standardization,load_part as original_load
def valid_rows(extra,status,nodes):
x=np.asarray(extra)
if x.shape!=(nodes,2) or np.isinf(x).any():
raise ValueError('Context shape/infinity input inconsistency')
finite=np.isfinite(x)
if np.any(finite[:,0]!=finite[:,1]):
raise ValueError('Unexpected single-column missingness')
if status=='GEOMETRY_COMPLETE' and finite.all():return np.ones(nodes,dtype=bool)
if status=='SCIENTIFIC_MISSING' and np.isnan(x).all():return np.zeros(nodes,dtype=bool)
raise ValueError('Frozen whole-part context status inconsistent with array')
def read_context(old,c,case):
p=c['context'][case];r=old['parts'][case]
if p['nodes']!=r['nodes'] or p['raw_ids_sha256']!=r['output_sha256']['raw_ids.npy'] or p['features_sha256']!=r['output_sha256']['features.npy']:
raise ValueError('Context original node/feature identity')
result=json.loads(Path(checked(p['result'],p['result_sha256'])).read_text())
if result['case']!=case or result['status']!=p['status'] or result['extra_sha256']!=p['extra_sha256']:
raise ValueError('Context result identity')
x=np.load(checked(p['extra'],p['extra_sha256']),allow_pickle=False)
return np.asarray(x,np.float64),valid_rows(x,p['status'],p['nodes'])
def moments(chunks):
n=0;mean=np.zeros(2,np.float64);m2=np.zeros(2,np.float64)
for x,valid in chunks:
values=x[valid];k=len(values)
if not k:continue
avg=values.mean(0);delta=avg-mean
m2+=np.square(values-avg).sum(0)+delta*delta*n*k/(n+k)
mean+=delta*k/(n+k);n+=k
if n==0:raise ValueError('No valid learning nodes; no invented normalization')
std=np.sqrt(m2/n);scale=np.where(std==0,1.,std)
if not np.isfinite(mean).all() or not np.isfinite(scale).all():raise ValueError('Context normalization overflow')
return dict(mean=mean.tolist(),scale=scale.tolist(),population_std=std.tolist(),valid_nodes=n)
def encode(extra,valid,stats):
out=np.zeros((len(extra),3),np.float32)
out[valid,:2]=((extra[valid]-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
out[valid,2]=1
if not np.isfinite(out).all():raise ValueError('Context standardized overflow')
return out
def standardization(old,c,root):
start=time.process_time();stats=original_standardization(old,root);seen=[]
def chunks():
for case in old['folds'][0]['learning']:
x,valid=read_context(old,c,case);seen.append(dict(case=case,nodes=len(x),valid_nodes=int(valid.sum())))
yield x,valid
context=moments(chunks());context.update(parts=seen,weighting='unit supervised node as original H2',labels_read=False,cpu_seconds=time.process_time()-start)
stats['context']=context;save(root/'standardization.json',stats);return stats
def load_part(old,c,case,stats,role,root):
# The original role guard runs before any extra context read.
data=original_load(old,case,stats,role,root)
extra,valid=read_context(old,c,case)
data['x']=torch.cat([data['x'],torch.from_numpy(encode(extra,valid,stats['context']))],dim=1)
data['context_valid_nodes']=int(valid.sum())
return data