Normalize neural base inputs with frozen learning-pool statistics

This commit is contained in:
Frank14f committed 2026-11-14 22:59:46 +08:00
1 parent bb09e8e164
commit f3532962b0
10 files changed
+326 -15

No files matched your search

+3 -2
View File
@@ -145,10 +145,11 @@ def r2_features(points, tets, wall_distance, gate, t_set):
BASE_COLUMNS = ("x_star", "y_star", "z_star", "d_over_t", "gate_distance_over_Rg",
"geodesic_over_Rg", "Rg", "volume", "t_part", "t_set")
"geodesic_over_Rg", "log_Rg", "log_volume", "log_t_part", "log_t_set")
def base_features(points, wall_distance, geodesic_distance, gate, center, rg, volume, t_part, t_set):
"""Ten columns before learning-pool standardisation; scales use natural log."""
x = finite_array(points, name="points")
if x.ndim != 2 or x.shape[1] != 3 or not len(x):
raise ValueError("invalid node coordinates")
@@ -159,7 +160,7 @@ def base_features(points, wall_distance, geodesic_distance, gate, center, rg, vo
if np.any(scales <= 0) or np.any(dw < 0) or np.any(dg < 0):
raise ValueError("nonpositive scale or negative distance")
return np.column_stack([(x-center)/rg, dw/t_part, np.linalg.norm(x-gate, axis=1)/rg,
dg/rg, np.tile(scales, (len(x), 1))])
dg/rg, np.tile(np.log(scales), (len(x), 1))])
def histgbrt_grid():
+13 -3
View File
@@ -7,6 +7,7 @@ from torch import nn
from .blocks import MLP, Transolver_block
from .features import BASE_COLUMNS
from .normalization import BaseStandardizer
TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
@@ -20,10 +21,15 @@ def check_tensor(tensor, shape, name):
class WholePartEncoder(nn.Module):
def __init__(self, width):
def __init__(self, width, standardizer):
super().__init__()
if width not in (128, 256):
raise ValueError("S2 permits widths 128 or 256 only")
if not isinstance(standardizer, BaseStandardizer):
raise ValueError("frozen learning-pool standardizer required")
self.standardizer = standardizer
self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
dropout=0., mlp_ratio=1, slice_num=32,
@@ -46,6 +52,10 @@ class WholePartEncoder(nn.Module):
if base.ndim != 2 or base.shape[0] == 0:
raise ValueError("one whole part with all tet nodes required")
check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
# Keep normalisation in FP32 even under future mixed-precision training.
base = (base.float()-self.base_mean.float()) / self.base_scale.float()
if not torch.isfinite(base).all():
raise ValueError("nonfinite standardized base input")
fx = self.preprocess(base[None]) + self.placeholder[None, None]
for block in self.blocks:
fx = block(fx)
@@ -55,12 +65,12 @@ class WholePartEncoder(nn.Module):
class QueryModel(nn.Module):
def __init__(self, arm, width=128):
def __init__(self, arm, width=128, *, standardizer):
super().__init__()
if arm not in ("N", "E"):
raise ValueError("arm must be N or E")
self.arm, self.width = arm, width
self.encoder = WholePartEncoder(width)
self.encoder = WholePartEncoder(width, standardizer)
query_dim = 2 if arm == "N" else 6
self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
+91
View File
@@ -0,0 +1,91 @@
"""Frozen per-fold/pool base-input statistics. Caller supplies the learning manifest.
No I/O or dataset selection occurs here. Statistics use all learning tet nodes
equally, population std (ddof=0), accumulated by part without concatenating the
whole pool. This is input preprocessing, not fitting model parameters.
"""
from dataclasses import dataclass
import hashlib
import json
import numpy as np
from .features import BASE_COLUMNS, finite_array
@dataclass(frozen=True)
class BaseStandardizer:
fold: str
pool: str
learning_part_ids: tuple
node_count: int
mean: tuple
std: tuple
def __post_init__(self):
if not isinstance(self.fold, str) or not self.fold or not isinstance(self.pool, str) or not self.pool:
raise ValueError("fold and pool identifiers required")
if not isinstance(self.learning_part_ids, tuple) or not self.learning_part_ids or any(not isinstance(p, str) or not p for p in self.learning_part_ids) or len(set(self.learning_part_ids)) != len(self.learning_part_ids):
raise ValueError("unique immutable learning part IDs required")
if not isinstance(self.node_count, int) or self.node_count <= 0:
raise ValueError("positive learning node count required")
if not isinstance(self.mean, tuple) or not isinstance(self.std, tuple):
raise ValueError("statistics must be immutable tuples")
finite_array(self.mean, (len(BASE_COLUMNS),), "learning mean")
std = finite_array(self.std, (len(BASE_COLUMNS),), "learning std")
if np.any(std < 0):
raise ValueError("negative std")
@property
def scale(self):
# Constant learning columns become zero. Unseen values are not imputed.
return tuple(s if s > 0 else 1. for s in self.std)
def as_dict(self):
return dict(fold=self.fold, pool=self.pool, learning_part_ids=list(self.learning_part_ids),
node_count=self.node_count, mean=list(self.mean), std=list(self.std),
columns=list(BASE_COLUMNS), weighting="equal_learning_nodes", ddof=0,
constant_column_scale=1.)
@property
def fingerprint(self):
return hashlib.sha256(json.dumps(self.as_dict(), sort_keys=True, allow_nan=False).encode()).hexdigest()
def transform(self, base, *, fold, pool):
if str(fold) != self.fold or str(pool) != self.pool:
raise ValueError("standardizer belongs to another fold/pool")
a = finite_array(base, name="base input")
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
raise ValueError("nonempty ten-column base input required")
return finite_array((a-np.asarray(self.mean))/np.asarray(self.scale), name="standardized base")
def fit_base_standardizer(part_base, *, learning_part_ids, fold, pool):
"""Fit only an exact, explicitly enumerated learning pool; reject extra IDs.
part_base maps part ID to its full-node base_features output. Validation/test
arrays are passed only to the frozen transform, never to this function.
"""
ids = tuple(learning_part_ids)
if not ids or any(not isinstance(p, str) or not p for p in ids) or len(ids) != len(set(ids)):
raise ValueError("nonempty unique learning part IDs required")
if set(part_base) != set(ids):
raise ValueError("base arrays must match learning pool exactly; no held-out or missing parts")
count, mean, m2 = 0, np.zeros(len(BASE_COLUMNS)), np.zeros(len(BASE_COLUMNS))
for part in sorted(ids):
a = finite_array(part_base[part], name=f"learning base {part}")
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
raise ValueError("nonempty ten-column learning base required")
# Shift before summation: an exactly constant column must have exactly
# its original mean and zero variance, not roundoff-sized pseudo-variance.
local_mean = a[0] + (a-a[0]).mean(axis=0)
local_m2 = ((a-local_mean)**2).sum(axis=0)
delta = local_mean - mean
n = len(a)
total = count + n
mean += delta * (n/total)
m2 += local_m2 + delta*delta*(count*n/total)
count = total
std = np.sqrt(m2/count)
return BaseStandardizer(str(fold), str(pool), tuple(sorted(ids)), count,
tuple(float(v) for v in mean), tuple(float(v) for v in std))
+14 -4
View File
@@ -238,7 +238,7 @@
- return fx
--- /dev/null
+++ network.py
@@ -0,0 +1,109 @@
@@ -0,0 +1,119 @@
+"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
+
+No optimiser, training loop, data reader or GPU invocation is provided.
@@ -248,6 +248,7 @@
+
+from .blocks import MLP, Transolver_block
+from .features import BASE_COLUMNS
+from .normalization import BaseStandardizer
+
+
+TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
@@ -261,10 +262,15 @@
+
+
+class WholePartEncoder(nn.Module):
+ def __init__(self, width):
+ def __init__(self, width, standardizer):
+ super().__init__()
+ if width not in (128, 256):
+ raise ValueError("S2 permits widths 128 or 256 only")
+ if not isinstance(standardizer, BaseStandardizer):
+ raise ValueError("frozen learning-pool standardizer required")
+ self.standardizer = standardizer
+ self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
+ self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
+ self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
+ self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
+ dropout=0., mlp_ratio=1, slice_num=32,
@@ -287,6 +293,10 @@
+ if base.ndim != 2 or base.shape[0] == 0:
+ raise ValueError("one whole part with all tet nodes required")
+ check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
+ # Keep normalisation in FP32 even under future mixed-precision training.
+ base = (base.float()-self.base_mean.float()) / self.base_scale.float()
+ if not torch.isfinite(base).all():
+ raise ValueError("nonfinite standardized base input")
+ fx = self.preprocess(base[None]) + self.placeholder[None, None]
+ for block in self.blocks:
+ fx = block(fx)
@@ -296,12 +306,12 @@
+
+
+class QueryModel(nn.Module):
+ def __init__(self, arm, width=128):
+ def __init__(self, arm, width=128, *, standardizer):
+ super().__init__()
+ if arm not in ("N", "E"):
+ raise ValueError("arm must be N or E")
+ self.arm, self.width = arm, width
+ self.encoder = WholePartEncoder(width)
+ self.encoder = WholePartEncoder(width, standardizer)
+ query_dim = 2 if arm == "N" else 6
+ self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
+ nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
+79 -3
View File
@@ -14,6 +14,7 @@ from sklearn.ensemble import HistGradientBoostingRegressor
from s2_models import features as f, objectives as o
from s2_models.attention import Physics_Attention_Irregular_Mesh
from s2_models.network import QueryModel, weighted_mse
from s2_models.normalization import fit_base_standardizer
torch.set_num_threads(1)
@@ -25,6 +26,11 @@ def synthetic_star():
return x, t, np.array([0., 0, 0, 0, .25])
def synthetic_standardizer():
return fit_base_standardizer({"synthetic-a": np.arange(70, dtype=float).reshape(7, 10)/10},
learning_part_ids=["synthetic-a"], fold="test-fold", pool="test-pool")
class GeometryTests(unittest.TestCase):
def test_volume_lumping(self):
x, t, _ = synthetic_star()
@@ -184,6 +190,76 @@ class ObjectiveTests(unittest.TestCase):
self.assertFalse(hasattr(model, "n_iter_"))
class NormalizationTests(unittest.TestCase):
def test_unit_conversion_only_shifts_log_scale_columns(self):
x, t, dw = synthetic_star()
dg, _ = f.graph_distances(x, t, dw, x[0], exclude_wall_edges=False)
center = np.array([.2, .2, .2])
a = f.base_features(x, dw, dg, x[0], center, 2., 1/6, .5, .4)
length_scale, time_scale = 1000., 60.
b = f.base_features(x*length_scale, dw*length_scale, dg*length_scale,
x[0]*length_scale, center*length_scale,
2*length_scale, 1/6*length_scale**3, .5*length_scale, .4*time_scale)
expected = np.r_[np.zeros(6), np.log([length_scale, length_scale**3, length_scale, time_scale])]
np.testing.assert_allclose(b-a, np.tile(expected, (len(x), 1)), atol=2e-15)
aa = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
bb = fit_base_standardizer({"a": b}, learning_part_ids=["a"], fold=0, pool=25)
np.testing.assert_allclose(aa.transform(a, fold=0, pool=25), bb.transform(b, fold=0, pool=25), atol=1e-13)
def test_streaming_population_stats_match_pooled_nodes(self):
rng = np.random.default_rng(42)
a, b = rng.normal(size=(3, 10)), rng.normal(size=(7, 10))+2
a[:, 9] = b[:, 9] = 4.
stats = fit_base_standardizer({"a": a, "b": b}, learning_part_ids=["b", "a"], fold=2, pool=50)
joined = np.vstack([a, b])
np.testing.assert_allclose(stats.mean, joined.mean(axis=0), atol=1e-15)
np.testing.assert_allclose(stats.std, joined.std(axis=0, ddof=0), atol=1e-15)
z = stats.transform(joined, fold=2, pool=50)
np.testing.assert_allclose(z.mean(axis=0), 0., atol=1e-15)
np.testing.assert_allclose(z[:, :9].std(axis=0), 1., atol=1e-15)
np.testing.assert_array_equal(z[:, 9], 0.)
self.assertEqual(stats.scale[9], 1.)
self.assertEqual(stats.node_count, 10)
def test_held_out_data_cannot_enter_fit_or_change_frozen_statistics(self):
from dataclasses import FrozenInstanceError
a, held = np.zeros((2, 10)), np.full((3, 10), 1e6)
with self.assertRaisesRegex(ValueError, "learning pool exactly"):
fit_base_standardizer({"a": a, "validation": held}, learning_part_ids=["a"], fold=0, pool=25)
with self.assertRaises(ValueError):
fit_base_standardizer({}, learning_part_ids=["a"], fold=0, pool=25)
stats = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
fingerprint = stats.fingerprint
np.testing.assert_array_equal(stats.transform(held, fold=0, pool=25), held)
self.assertEqual(fingerprint, stats.fingerprint)
with self.assertRaises(FrozenInstanceError):
stats.mean = tuple(np.ones(10))
with self.assertRaisesRegex(ValueError, "another fold/pool"):
stats.transform(held, fold=1, pool=25)
with self.assertRaisesRegex(ValueError, "another fold/pool"):
stats.transform(held, fold=0, pool=100)
with self.assertRaises(ValueError):
fit_base_standardizer({"a": np.full((2, 10), np.nan)}, learning_part_ids=["a"], fold=0, pool=25)
def test_ne_share_frozen_transform_before_preprocess(self):
stats = synthetic_standardizer()
models = [QueryModel(arm, standardizer=stats).eval() for arm in ("N", "E")]
base = torch.linspace(-1, 1, 50).reshape(5, 10)
expected = stats.transform(base.numpy(), fold="test-fold", pool="test-pool")
captured = []
for model in models:
self.assertIs(model.encoder.standardizer, stats)
self.assertNotIn("base_mean", dict(model.encoder.named_parameters()))
handle = model.encoder.preprocess.register_forward_pre_hook(lambda module, args: captured.append(args[0].clone()))
with torch.inference_mode():
model.encoder(base)
handle.remove()
torch.testing.assert_close(captured[0], captured[1], rtol=0, atol=0)
np.testing.assert_allclose(captured[0][0].numpy(), expected, atol=2e-7)
with self.assertRaises(ValueError):
QueryModel("N", standardizer=None)
class NetworkTests(unittest.TestCase):
def test_original_attention_numerical_parity(self):
path = Path(__file__).parent/'s2_models/upstream/Physics_Attention.py'
@@ -212,7 +288,7 @@ class NetworkTests(unittest.TestCase):
models = {}
for arm in ("N", "E"):
torch.manual_seed(0)
model = QueryModel(arm, width).eval()
model = QueryModel(arm, width, standardizer=synthetic_standardizer()).eval()
before = {k: v.clone() for k, v in model.state_dict().items()}
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4, 6]), torch.rand(3, 2)
edge = torch.rand(3, 4) if arm == "E" else None
@@ -229,7 +305,7 @@ class NetworkTests(unittest.TestCase):
def test_query_order_global_context_and_input_rejection(self):
torch.manual_seed(7)
model = QueryModel("N").eval()
model = QueryModel("N", standardizer=synthetic_standardizer()).eval()
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4]), torch.rand(2, 2)
with torch.inference_mode():
a = model(base, ids, gaps)
@@ -246,7 +322,7 @@ class NetworkTests(unittest.TestCase):
with self.assertRaises(ValueError):
model(changed, ids, gaps)
with self.assertRaises(ValueError):
QueryModel("E")(base, ids, gaps)
QueryModel("E", standardizer=synthetic_standardizer())(base, ids, gaps)
if __name__ == '__main__':