Files
MeltMind/tools/s2_models/normalization.py

92 lines
4.3 KiB
Python

"""Frozen per-fold/pool base-input statistics. Caller supplies the learning manifest.
No I/O or dataset selection occurs here. Statistics use all learning tet nodes
equally, population std (ddof=0), accumulated by part without concatenating the
whole pool. This is input preprocessing, not fitting model parameters.
"""
from dataclasses import dataclass
import hashlib
import json
import numpy as np
from .features import BASE_COLUMNS, finite_array
@dataclass(frozen=True)
class BaseStandardizer:
fold: str
pool: str
learning_part_ids: tuple
node_count: int
mean: tuple
std: tuple
def __post_init__(self):
if not isinstance(self.fold, str) or not self.fold or not isinstance(self.pool, str) or not self.pool:
raise ValueError("fold and pool identifiers required")
if not isinstance(self.learning_part_ids, tuple) or not self.learning_part_ids or any(not isinstance(p, str) or not p for p in self.learning_part_ids) or len(set(self.learning_part_ids)) != len(self.learning_part_ids):
raise ValueError("unique immutable learning part IDs required")
if not isinstance(self.node_count, int) or self.node_count <= 0:
raise ValueError("positive learning node count required")
if not isinstance(self.mean, tuple) or not isinstance(self.std, tuple):
raise ValueError("statistics must be immutable tuples")
finite_array(self.mean, (len(BASE_COLUMNS),), "learning mean")
std = finite_array(self.std, (len(BASE_COLUMNS),), "learning std")
if np.any(std < 0):
raise ValueError("negative std")
@property
def scale(self):
# Constant learning columns become zero. Unseen values are not imputed.
return tuple(s if s > 0 else 1. for s in self.std)
def as_dict(self):
return dict(fold=self.fold, pool=self.pool, learning_part_ids=list(self.learning_part_ids),
node_count=self.node_count, mean=list(self.mean), std=list(self.std),
columns=list(BASE_COLUMNS), weighting="equal_learning_nodes", ddof=0,
constant_column_scale=1.)
@property
def fingerprint(self):
return hashlib.sha256(json.dumps(self.as_dict(), sort_keys=True, allow_nan=False).encode()).hexdigest()
def transform(self, base, *, fold, pool):
if str(fold) != self.fold or str(pool) != self.pool:
raise ValueError("standardizer belongs to another fold/pool")
a = finite_array(base, name="base input")
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
raise ValueError("nonempty ten-column base input required")
return finite_array((a-np.asarray(self.mean))/np.asarray(self.scale), name="standardized base")
def fit_base_standardizer(part_base, *, learning_part_ids, fold, pool):
"""Fit only an exact, explicitly enumerated learning pool; reject extra IDs.
part_base maps part ID to its full-node base_features output. Validation/test
arrays are passed only to the frozen transform, never to this function.
"""
ids = tuple(learning_part_ids)
if not ids or any(not isinstance(p, str) or not p for p in ids) or len(ids) != len(set(ids)):
raise ValueError("nonempty unique learning part IDs required")
if set(part_base) != set(ids):
raise ValueError("base arrays must match learning pool exactly; no held-out or missing parts")
count, mean, m2 = 0, np.zeros(len(BASE_COLUMNS)), np.zeros(len(BASE_COLUMNS))
for part in sorted(ids):
a = finite_array(part_base[part], name=f"learning base {part}")
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
raise ValueError("nonempty ten-column learning base required")
# Shift before summation: an exactly constant column must have exactly
# its original mean and zero variance, not roundoff-sized pseudo-variance.
local_mean = a[0] + (a-a[0]).mean(axis=0)
local_m2 = ((a-local_mean)**2).sum(axis=0)
delta = local_mean - mean
n = len(a)
total = count + n
mean += delta * (n/total)
m2 += local_m2 + delta*delta*(count*n/total)
count = total
std = np.sqrt(m2/count)
return BaseStandardizer(str(fold), str(pool), tuple(sorted(ids)), count,
tuple(float(v) for v in mean), tuple(float(v) for v in std))