92 lines
4.3 KiB
Python
92 lines
4.3 KiB
Python
"""Frozen per-fold/pool base-input statistics. Caller supplies the learning manifest.
|
|
|
|
No I/O or dataset selection occurs here. Statistics use all learning tet nodes
|
|
equally, population std (ddof=0), accumulated by part without concatenating the
|
|
whole pool. This is input preprocessing, not fitting model parameters.
|
|
"""
|
|
from dataclasses import dataclass
|
|
import hashlib
|
|
import json
|
|
|
|
import numpy as np
|
|
|
|
from .features import BASE_COLUMNS, finite_array
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BaseStandardizer:
|
|
fold: str
|
|
pool: str
|
|
learning_part_ids: tuple
|
|
node_count: int
|
|
mean: tuple
|
|
std: tuple
|
|
|
|
def __post_init__(self):
|
|
if not isinstance(self.fold, str) or not self.fold or not isinstance(self.pool, str) or not self.pool:
|
|
raise ValueError("fold and pool identifiers required")
|
|
if not isinstance(self.learning_part_ids, tuple) or not self.learning_part_ids or any(not isinstance(p, str) or not p for p in self.learning_part_ids) or len(set(self.learning_part_ids)) != len(self.learning_part_ids):
|
|
raise ValueError("unique immutable learning part IDs required")
|
|
if not isinstance(self.node_count, int) or self.node_count <= 0:
|
|
raise ValueError("positive learning node count required")
|
|
if not isinstance(self.mean, tuple) or not isinstance(self.std, tuple):
|
|
raise ValueError("statistics must be immutable tuples")
|
|
finite_array(self.mean, (len(BASE_COLUMNS),), "learning mean")
|
|
std = finite_array(self.std, (len(BASE_COLUMNS),), "learning std")
|
|
if np.any(std < 0):
|
|
raise ValueError("negative std")
|
|
|
|
@property
|
|
def scale(self):
|
|
# Constant learning columns become zero. Unseen values are not imputed.
|
|
return tuple(s if s > 0 else 1. for s in self.std)
|
|
|
|
def as_dict(self):
|
|
return dict(fold=self.fold, pool=self.pool, learning_part_ids=list(self.learning_part_ids),
|
|
node_count=self.node_count, mean=list(self.mean), std=list(self.std),
|
|
columns=list(BASE_COLUMNS), weighting="equal_learning_nodes", ddof=0,
|
|
constant_column_scale=1.)
|
|
|
|
@property
|
|
def fingerprint(self):
|
|
return hashlib.sha256(json.dumps(self.as_dict(), sort_keys=True, allow_nan=False).encode()).hexdigest()
|
|
|
|
def transform(self, base, *, fold, pool):
|
|
if str(fold) != self.fold or str(pool) != self.pool:
|
|
raise ValueError("standardizer belongs to another fold/pool")
|
|
a = finite_array(base, name="base input")
|
|
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
|
|
raise ValueError("nonempty ten-column base input required")
|
|
return finite_array((a-np.asarray(self.mean))/np.asarray(self.scale), name="standardized base")
|
|
|
|
|
|
def fit_base_standardizer(part_base, *, learning_part_ids, fold, pool):
|
|
"""Fit only an exact, explicitly enumerated learning pool; reject extra IDs.
|
|
|
|
part_base maps part ID to its full-node base_features output. Validation/test
|
|
arrays are passed only to the frozen transform, never to this function.
|
|
"""
|
|
ids = tuple(learning_part_ids)
|
|
if not ids or any(not isinstance(p, str) or not p for p in ids) or len(ids) != len(set(ids)):
|
|
raise ValueError("nonempty unique learning part IDs required")
|
|
if set(part_base) != set(ids):
|
|
raise ValueError("base arrays must match learning pool exactly; no held-out or missing parts")
|
|
count, mean, m2 = 0, np.zeros(len(BASE_COLUMNS)), np.zeros(len(BASE_COLUMNS))
|
|
for part in sorted(ids):
|
|
a = finite_array(part_base[part], name=f"learning base {part}")
|
|
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
|
|
raise ValueError("nonempty ten-column learning base required")
|
|
# Shift before summation: an exactly constant column must have exactly
|
|
# its original mean and zero variance, not roundoff-sized pseudo-variance.
|
|
local_mean = a[0] + (a-a[0]).mean(axis=0)
|
|
local_m2 = ((a-local_mean)**2).sum(axis=0)
|
|
delta = local_mean - mean
|
|
n = len(a)
|
|
total = count + n
|
|
mean += delta * (n/total)
|
|
m2 += local_m2 + delta*delta*(count*n/total)
|
|
count = total
|
|
std = np.sqrt(m2/count)
|
|
return BaseStandardizer(str(fold), str(pool), tuple(sorted(ids)), count,
|
|
tuple(float(v) for v in mean), tuple(float(v) for v in std))
|