Normalize neural base inputs with frozen learning-pool statistics
This commit is contained in:
1 parent
bb09e8e164
commit
f3532962b0
10 files changed
+326
-15
No files matched your search
@@ -145,10 +145,11 @@ def r2_features(points, tets, wall_distance, gate, t_set):
|
||||
|
||||
|
||||
BASE_COLUMNS = ("x_star", "y_star", "z_star", "d_over_t", "gate_distance_over_Rg",
|
||||
"geodesic_over_Rg", "Rg", "volume", "t_part", "t_set")
|
||||
"geodesic_over_Rg", "log_Rg", "log_volume", "log_t_part", "log_t_set")
|
||||
|
||||
|
||||
def base_features(points, wall_distance, geodesic_distance, gate, center, rg, volume, t_part, t_set):
|
||||
"""Ten columns before learning-pool standardisation; scales use natural log."""
|
||||
x = finite_array(points, name="points")
|
||||
if x.ndim != 2 or x.shape[1] != 3 or not len(x):
|
||||
raise ValueError("invalid node coordinates")
|
||||
@@ -159,7 +160,7 @@ def base_features(points, wall_distance, geodesic_distance, gate, center, rg, vo
|
||||
if np.any(scales <= 0) or np.any(dw < 0) or np.any(dg < 0):
|
||||
raise ValueError("nonpositive scale or negative distance")
|
||||
return np.column_stack([(x-center)/rg, dw/t_part, np.linalg.norm(x-gate, axis=1)/rg,
|
||||
dg/rg, np.tile(scales, (len(x), 1))])
|
||||
dg/rg, np.tile(np.log(scales), (len(x), 1))])
|
||||
|
||||
|
||||
def histgbrt_grid():
|
||||
|
||||
@@ -7,6 +7,7 @@ from torch import nn
|
||||
|
||||
from .blocks import MLP, Transolver_block
|
||||
from .features import BASE_COLUMNS
|
||||
from .normalization import BaseStandardizer
|
||||
|
||||
|
||||
TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
@@ -20,10 +21,15 @@ def check_tensor(tensor, shape, name):
|
||||
|
||||
|
||||
class WholePartEncoder(nn.Module):
|
||||
def __init__(self, width):
|
||||
def __init__(self, width, standardizer):
|
||||
super().__init__()
|
||||
if width not in (128, 256):
|
||||
raise ValueError("S2 permits widths 128 or 256 only")
|
||||
if not isinstance(standardizer, BaseStandardizer):
|
||||
raise ValueError("frozen learning-pool standardizer required")
|
||||
self.standardizer = standardizer
|
||||
self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
|
||||
self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
|
||||
self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
dropout=0., mlp_ratio=1, slice_num=32,
|
||||
@@ -46,6 +52,10 @@ class WholePartEncoder(nn.Module):
|
||||
if base.ndim != 2 or base.shape[0] == 0:
|
||||
raise ValueError("one whole part with all tet nodes required")
|
||||
check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
# Keep normalisation in FP32 even under future mixed-precision training.
|
||||
base = (base.float()-self.base_mean.float()) / self.base_scale.float()
|
||||
if not torch.isfinite(base).all():
|
||||
raise ValueError("nonfinite standardized base input")
|
||||
fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
for block in self.blocks:
|
||||
fx = block(fx)
|
||||
@@ -55,12 +65,12 @@ class WholePartEncoder(nn.Module):
|
||||
|
||||
|
||||
class QueryModel(nn.Module):
|
||||
def __init__(self, arm, width=128):
|
||||
def __init__(self, arm, width=128, *, standardizer):
|
||||
super().__init__()
|
||||
if arm not in ("N", "E"):
|
||||
raise ValueError("arm must be N or E")
|
||||
self.arm, self.width = arm, width
|
||||
self.encoder = WholePartEncoder(width)
|
||||
self.encoder = WholePartEncoder(width, standardizer)
|
||||
query_dim = 2 if arm == "N" else 6
|
||||
self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Frozen per-fold/pool base-input statistics. Caller supplies the learning manifest.
|
||||
|
||||
No I/O or dataset selection occurs here. Statistics use all learning tet nodes
|
||||
equally, population std (ddof=0), accumulated by part without concatenating the
|
||||
whole pool. This is input preprocessing, not fitting model parameters.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .features import BASE_COLUMNS, finite_array
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BaseStandardizer:
|
||||
fold: str
|
||||
pool: str
|
||||
learning_part_ids: tuple
|
||||
node_count: int
|
||||
mean: tuple
|
||||
std: tuple
|
||||
|
||||
def __post_init__(self):
|
||||
if not isinstance(self.fold, str) or not self.fold or not isinstance(self.pool, str) or not self.pool:
|
||||
raise ValueError("fold and pool identifiers required")
|
||||
if not isinstance(self.learning_part_ids, tuple) or not self.learning_part_ids or any(not isinstance(p, str) or not p for p in self.learning_part_ids) or len(set(self.learning_part_ids)) != len(self.learning_part_ids):
|
||||
raise ValueError("unique immutable learning part IDs required")
|
||||
if not isinstance(self.node_count, int) or self.node_count <= 0:
|
||||
raise ValueError("positive learning node count required")
|
||||
if not isinstance(self.mean, tuple) or not isinstance(self.std, tuple):
|
||||
raise ValueError("statistics must be immutable tuples")
|
||||
finite_array(self.mean, (len(BASE_COLUMNS),), "learning mean")
|
||||
std = finite_array(self.std, (len(BASE_COLUMNS),), "learning std")
|
||||
if np.any(std < 0):
|
||||
raise ValueError("negative std")
|
||||
|
||||
@property
|
||||
def scale(self):
|
||||
# Constant learning columns become zero. Unseen values are not imputed.
|
||||
return tuple(s if s > 0 else 1. for s in self.std)
|
||||
|
||||
def as_dict(self):
|
||||
return dict(fold=self.fold, pool=self.pool, learning_part_ids=list(self.learning_part_ids),
|
||||
node_count=self.node_count, mean=list(self.mean), std=list(self.std),
|
||||
columns=list(BASE_COLUMNS), weighting="equal_learning_nodes", ddof=0,
|
||||
constant_column_scale=1.)
|
||||
|
||||
@property
|
||||
def fingerprint(self):
|
||||
return hashlib.sha256(json.dumps(self.as_dict(), sort_keys=True, allow_nan=False).encode()).hexdigest()
|
||||
|
||||
def transform(self, base, *, fold, pool):
|
||||
if str(fold) != self.fold or str(pool) != self.pool:
|
||||
raise ValueError("standardizer belongs to another fold/pool")
|
||||
a = finite_array(base, name="base input")
|
||||
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
|
||||
raise ValueError("nonempty ten-column base input required")
|
||||
return finite_array((a-np.asarray(self.mean))/np.asarray(self.scale), name="standardized base")
|
||||
|
||||
|
||||
def fit_base_standardizer(part_base, *, learning_part_ids, fold, pool):
|
||||
"""Fit only an exact, explicitly enumerated learning pool; reject extra IDs.
|
||||
|
||||
part_base maps part ID to its full-node base_features output. Validation/test
|
||||
arrays are passed only to the frozen transform, never to this function.
|
||||
"""
|
||||
ids = tuple(learning_part_ids)
|
||||
if not ids or any(not isinstance(p, str) or not p for p in ids) or len(ids) != len(set(ids)):
|
||||
raise ValueError("nonempty unique learning part IDs required")
|
||||
if set(part_base) != set(ids):
|
||||
raise ValueError("base arrays must match learning pool exactly; no held-out or missing parts")
|
||||
count, mean, m2 = 0, np.zeros(len(BASE_COLUMNS)), np.zeros(len(BASE_COLUMNS))
|
||||
for part in sorted(ids):
|
||||
a = finite_array(part_base[part], name=f"learning base {part}")
|
||||
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
|
||||
raise ValueError("nonempty ten-column learning base required")
|
||||
# Shift before summation: an exactly constant column must have exactly
|
||||
# its original mean and zero variance, not roundoff-sized pseudo-variance.
|
||||
local_mean = a[0] + (a-a[0]).mean(axis=0)
|
||||
local_m2 = ((a-local_mean)**2).sum(axis=0)
|
||||
delta = local_mean - mean
|
||||
n = len(a)
|
||||
total = count + n
|
||||
mean += delta * (n/total)
|
||||
m2 += local_m2 + delta*delta*(count*n/total)
|
||||
count = total
|
||||
std = np.sqrt(m2/count)
|
||||
return BaseStandardizer(str(fold), str(pool), tuple(sorted(ids)), count,
|
||||
tuple(float(v) for v in mean), tuple(float(v) for v in std))
|
||||
@@ -238,7 +238,7 @@
|
||||
- return fx
|
||||
--- /dev/null
|
||||
+++ network.py
|
||||
@@ -0,0 +1,109 @@
|
||||
@@ -0,0 +1,119 @@
|
||||
+"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
|
||||
+
|
||||
+No optimiser, training loop, data reader or GPU invocation is provided.
|
||||
@@ -248,6 +248,7 @@
|
||||
+
|
||||
+from .blocks import MLP, Transolver_block
|
||||
+from .features import BASE_COLUMNS
|
||||
+from .normalization import BaseStandardizer
|
||||
+
|
||||
+
|
||||
+TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
@@ -261,10 +262,15 @@
|
||||
+
|
||||
+
|
||||
+class WholePartEncoder(nn.Module):
|
||||
+ def __init__(self, width):
|
||||
+ def __init__(self, width, standardizer):
|
||||
+ super().__init__()
|
||||
+ if width not in (128, 256):
|
||||
+ raise ValueError("S2 permits widths 128 or 256 only")
|
||||
+ if not isinstance(standardizer, BaseStandardizer):
|
||||
+ raise ValueError("frozen learning-pool standardizer required")
|
||||
+ self.standardizer = standardizer
|
||||
+ self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
|
||||
+ self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
|
||||
+ self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
+ self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
+ dropout=0., mlp_ratio=1, slice_num=32,
|
||||
@@ -287,6 +293,10 @@
|
||||
+ if base.ndim != 2 or base.shape[0] == 0:
|
||||
+ raise ValueError("one whole part with all tet nodes required")
|
||||
+ check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
+ # Keep normalisation in FP32 even under future mixed-precision training.
|
||||
+ base = (base.float()-self.base_mean.float()) / self.base_scale.float()
|
||||
+ if not torch.isfinite(base).all():
|
||||
+ raise ValueError("nonfinite standardized base input")
|
||||
+ fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
+ for block in self.blocks:
|
||||
+ fx = block(fx)
|
||||
@@ -296,12 +306,12 @@
|
||||
+
|
||||
+
|
||||
+class QueryModel(nn.Module):
|
||||
+ def __init__(self, arm, width=128):
|
||||
+ def __init__(self, arm, width=128, *, standardizer):
|
||||
+ super().__init__()
|
||||
+ if arm not in ("N", "E"):
|
||||
+ raise ValueError("arm must be N or E")
|
||||
+ self.arm, self.width = arm, width
|
||||
+ self.encoder = WholePartEncoder(width)
|
||||
+ self.encoder = WholePartEncoder(width, standardizer)
|
||||
+ query_dim = 2 if arm == "N" else 6
|
||||
+ self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
+ nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
|
||||
Reference in new issue
Block a user