Normalize neural base inputs with frozen learning-pool statistics
This commit is contained in:
1 parent
bb09e8e164
commit
f3532962b0
10 files changed
+326
-15
No files matched your search
@@ -145,10 +145,11 @@ def r2_features(points, tets, wall_distance, gate, t_set):
|
||||
|
||||
|
||||
BASE_COLUMNS = ("x_star", "y_star", "z_star", "d_over_t", "gate_distance_over_Rg",
|
||||
"geodesic_over_Rg", "Rg", "volume", "t_part", "t_set")
|
||||
"geodesic_over_Rg", "log_Rg", "log_volume", "log_t_part", "log_t_set")
|
||||
|
||||
|
||||
def base_features(points, wall_distance, geodesic_distance, gate, center, rg, volume, t_part, t_set):
|
||||
"""Ten columns before learning-pool standardisation; scales use natural log."""
|
||||
x = finite_array(points, name="points")
|
||||
if x.ndim != 2 or x.shape[1] != 3 or not len(x):
|
||||
raise ValueError("invalid node coordinates")
|
||||
@@ -159,7 +160,7 @@ def base_features(points, wall_distance, geodesic_distance, gate, center, rg, vo
|
||||
if np.any(scales <= 0) or np.any(dw < 0) or np.any(dg < 0):
|
||||
raise ValueError("nonpositive scale or negative distance")
|
||||
return np.column_stack([(x-center)/rg, dw/t_part, np.linalg.norm(x-gate, axis=1)/rg,
|
||||
dg/rg, np.tile(scales, (len(x), 1))])
|
||||
dg/rg, np.tile(np.log(scales), (len(x), 1))])
|
||||
|
||||
|
||||
def histgbrt_grid():
|
||||
|
||||
@@ -7,6 +7,7 @@ from torch import nn
|
||||
|
||||
from .blocks import MLP, Transolver_block
|
||||
from .features import BASE_COLUMNS
|
||||
from .normalization import BaseStandardizer
|
||||
|
||||
|
||||
TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
@@ -20,10 +21,15 @@ def check_tensor(tensor, shape, name):
|
||||
|
||||
|
||||
class WholePartEncoder(nn.Module):
|
||||
def __init__(self, width):
|
||||
def __init__(self, width, standardizer):
|
||||
super().__init__()
|
||||
if width not in (128, 256):
|
||||
raise ValueError("S2 permits widths 128 or 256 only")
|
||||
if not isinstance(standardizer, BaseStandardizer):
|
||||
raise ValueError("frozen learning-pool standardizer required")
|
||||
self.standardizer = standardizer
|
||||
self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
|
||||
self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
|
||||
self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
dropout=0., mlp_ratio=1, slice_num=32,
|
||||
@@ -46,6 +52,10 @@ class WholePartEncoder(nn.Module):
|
||||
if base.ndim != 2 or base.shape[0] == 0:
|
||||
raise ValueError("one whole part with all tet nodes required")
|
||||
check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
# Keep normalisation in FP32 even under future mixed-precision training.
|
||||
base = (base.float()-self.base_mean.float()) / self.base_scale.float()
|
||||
if not torch.isfinite(base).all():
|
||||
raise ValueError("nonfinite standardized base input")
|
||||
fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
for block in self.blocks:
|
||||
fx = block(fx)
|
||||
@@ -55,12 +65,12 @@ class WholePartEncoder(nn.Module):
|
||||
|
||||
|
||||
class QueryModel(nn.Module):
|
||||
def __init__(self, arm, width=128):
|
||||
def __init__(self, arm, width=128, *, standardizer):
|
||||
super().__init__()
|
||||
if arm not in ("N", "E"):
|
||||
raise ValueError("arm must be N or E")
|
||||
self.arm, self.width = arm, width
|
||||
self.encoder = WholePartEncoder(width)
|
||||
self.encoder = WholePartEncoder(width, standardizer)
|
||||
query_dim = 2 if arm == "N" else 6
|
||||
self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Frozen per-fold/pool base-input statistics. Caller supplies the learning manifest.
|
||||
|
||||
No I/O or dataset selection occurs here. Statistics use all learning tet nodes
|
||||
equally, population std (ddof=0), accumulated by part without concatenating the
|
||||
whole pool. This is input preprocessing, not fitting model parameters.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .features import BASE_COLUMNS, finite_array
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BaseStandardizer:
|
||||
fold: str
|
||||
pool: str
|
||||
learning_part_ids: tuple
|
||||
node_count: int
|
||||
mean: tuple
|
||||
std: tuple
|
||||
|
||||
def __post_init__(self):
|
||||
if not isinstance(self.fold, str) or not self.fold or not isinstance(self.pool, str) or not self.pool:
|
||||
raise ValueError("fold and pool identifiers required")
|
||||
if not isinstance(self.learning_part_ids, tuple) or not self.learning_part_ids or any(not isinstance(p, str) or not p for p in self.learning_part_ids) or len(set(self.learning_part_ids)) != len(self.learning_part_ids):
|
||||
raise ValueError("unique immutable learning part IDs required")
|
||||
if not isinstance(self.node_count, int) or self.node_count <= 0:
|
||||
raise ValueError("positive learning node count required")
|
||||
if not isinstance(self.mean, tuple) or not isinstance(self.std, tuple):
|
||||
raise ValueError("statistics must be immutable tuples")
|
||||
finite_array(self.mean, (len(BASE_COLUMNS),), "learning mean")
|
||||
std = finite_array(self.std, (len(BASE_COLUMNS),), "learning std")
|
||||
if np.any(std < 0):
|
||||
raise ValueError("negative std")
|
||||
|
||||
@property
|
||||
def scale(self):
|
||||
# Constant learning columns become zero. Unseen values are not imputed.
|
||||
return tuple(s if s > 0 else 1. for s in self.std)
|
||||
|
||||
def as_dict(self):
|
||||
return dict(fold=self.fold, pool=self.pool, learning_part_ids=list(self.learning_part_ids),
|
||||
node_count=self.node_count, mean=list(self.mean), std=list(self.std),
|
||||
columns=list(BASE_COLUMNS), weighting="equal_learning_nodes", ddof=0,
|
||||
constant_column_scale=1.)
|
||||
|
||||
@property
|
||||
def fingerprint(self):
|
||||
return hashlib.sha256(json.dumps(self.as_dict(), sort_keys=True, allow_nan=False).encode()).hexdigest()
|
||||
|
||||
def transform(self, base, *, fold, pool):
|
||||
if str(fold) != self.fold or str(pool) != self.pool:
|
||||
raise ValueError("standardizer belongs to another fold/pool")
|
||||
a = finite_array(base, name="base input")
|
||||
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
|
||||
raise ValueError("nonempty ten-column base input required")
|
||||
return finite_array((a-np.asarray(self.mean))/np.asarray(self.scale), name="standardized base")
|
||||
|
||||
|
||||
def fit_base_standardizer(part_base, *, learning_part_ids, fold, pool):
|
||||
"""Fit only an exact, explicitly enumerated learning pool; reject extra IDs.
|
||||
|
||||
part_base maps part ID to its full-node base_features output. Validation/test
|
||||
arrays are passed only to the frozen transform, never to this function.
|
||||
"""
|
||||
ids = tuple(learning_part_ids)
|
||||
if not ids or any(not isinstance(p, str) or not p for p in ids) or len(ids) != len(set(ids)):
|
||||
raise ValueError("nonempty unique learning part IDs required")
|
||||
if set(part_base) != set(ids):
|
||||
raise ValueError("base arrays must match learning pool exactly; no held-out or missing parts")
|
||||
count, mean, m2 = 0, np.zeros(len(BASE_COLUMNS)), np.zeros(len(BASE_COLUMNS))
|
||||
for part in sorted(ids):
|
||||
a = finite_array(part_base[part], name=f"learning base {part}")
|
||||
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
|
||||
raise ValueError("nonempty ten-column learning base required")
|
||||
# Shift before summation: an exactly constant column must have exactly
|
||||
# its original mean and zero variance, not roundoff-sized pseudo-variance.
|
||||
local_mean = a[0] + (a-a[0]).mean(axis=0)
|
||||
local_m2 = ((a-local_mean)**2).sum(axis=0)
|
||||
delta = local_mean - mean
|
||||
n = len(a)
|
||||
total = count + n
|
||||
mean += delta * (n/total)
|
||||
m2 += local_m2 + delta*delta*(count*n/total)
|
||||
count = total
|
||||
std = np.sqrt(m2/count)
|
||||
return BaseStandardizer(str(fold), str(pool), tuple(sorted(ids)), count,
|
||||
tuple(float(v) for v in mean), tuple(float(v) for v in std))
|
||||
@@ -238,7 +238,7 @@
|
||||
- return fx
|
||||
--- /dev/null
|
||||
+++ network.py
|
||||
@@ -0,0 +1,109 @@
|
||||
@@ -0,0 +1,119 @@
|
||||
+"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
|
||||
+
|
||||
+No optimiser, training loop, data reader or GPU invocation is provided.
|
||||
@@ -248,6 +248,7 @@
|
||||
+
|
||||
+from .blocks import MLP, Transolver_block
|
||||
+from .features import BASE_COLUMNS
|
||||
+from .normalization import BaseStandardizer
|
||||
+
|
||||
+
|
||||
+TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
@@ -261,10 +262,15 @@
|
||||
+
|
||||
+
|
||||
+class WholePartEncoder(nn.Module):
|
||||
+ def __init__(self, width):
|
||||
+ def __init__(self, width, standardizer):
|
||||
+ super().__init__()
|
||||
+ if width not in (128, 256):
|
||||
+ raise ValueError("S2 permits widths 128 or 256 only")
|
||||
+ if not isinstance(standardizer, BaseStandardizer):
|
||||
+ raise ValueError("frozen learning-pool standardizer required")
|
||||
+ self.standardizer = standardizer
|
||||
+ self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
|
||||
+ self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
|
||||
+ self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
+ self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
+ dropout=0., mlp_ratio=1, slice_num=32,
|
||||
@@ -287,6 +293,10 @@
|
||||
+ if base.ndim != 2 or base.shape[0] == 0:
|
||||
+ raise ValueError("one whole part with all tet nodes required")
|
||||
+ check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
+ # Keep normalisation in FP32 even under future mixed-precision training.
|
||||
+ base = (base.float()-self.base_mean.float()) / self.base_scale.float()
|
||||
+ if not torch.isfinite(base).all():
|
||||
+ raise ValueError("nonfinite standardized base input")
|
||||
+ fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
+ for block in self.blocks:
|
||||
+ fx = block(fx)
|
||||
@@ -296,12 +306,12 @@
|
||||
+
|
||||
+
|
||||
+class QueryModel(nn.Module):
|
||||
+ def __init__(self, arm, width=128):
|
||||
+ def __init__(self, arm, width=128, *, standardizer):
|
||||
+ super().__init__()
|
||||
+ if arm not in ("N", "E"):
|
||||
+ raise ValueError("arm must be N or E")
|
||||
+ self.arm, self.width = arm, width
|
||||
+ self.encoder = WholePartEncoder(width)
|
||||
+ self.encoder = WholePartEncoder(width, standardizer)
|
||||
+ query_dim = 2 if arm == "N" else 6
|
||||
+ self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
+ nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
|
||||
+79
-3
@@ -14,6 +14,7 @@ from sklearn.ensemble import HistGradientBoostingRegressor
|
||||
from s2_models import features as f, objectives as o
|
||||
from s2_models.attention import Physics_Attention_Irregular_Mesh
|
||||
from s2_models.network import QueryModel, weighted_mse
|
||||
from s2_models.normalization import fit_base_standardizer
|
||||
|
||||
torch.set_num_threads(1)
|
||||
|
||||
@@ -25,6 +26,11 @@ def synthetic_star():
|
||||
return x, t, np.array([0., 0, 0, 0, .25])
|
||||
|
||||
|
||||
def synthetic_standardizer():
|
||||
return fit_base_standardizer({"synthetic-a": np.arange(70, dtype=float).reshape(7, 10)/10},
|
||||
learning_part_ids=["synthetic-a"], fold="test-fold", pool="test-pool")
|
||||
|
||||
|
||||
class GeometryTests(unittest.TestCase):
|
||||
def test_volume_lumping(self):
|
||||
x, t, _ = synthetic_star()
|
||||
@@ -184,6 +190,76 @@ class ObjectiveTests(unittest.TestCase):
|
||||
self.assertFalse(hasattr(model, "n_iter_"))
|
||||
|
||||
|
||||
class NormalizationTests(unittest.TestCase):
|
||||
def test_unit_conversion_only_shifts_log_scale_columns(self):
|
||||
x, t, dw = synthetic_star()
|
||||
dg, _ = f.graph_distances(x, t, dw, x[0], exclude_wall_edges=False)
|
||||
center = np.array([.2, .2, .2])
|
||||
a = f.base_features(x, dw, dg, x[0], center, 2., 1/6, .5, .4)
|
||||
length_scale, time_scale = 1000., 60.
|
||||
b = f.base_features(x*length_scale, dw*length_scale, dg*length_scale,
|
||||
x[0]*length_scale, center*length_scale,
|
||||
2*length_scale, 1/6*length_scale**3, .5*length_scale, .4*time_scale)
|
||||
expected = np.r_[np.zeros(6), np.log([length_scale, length_scale**3, length_scale, time_scale])]
|
||||
np.testing.assert_allclose(b-a, np.tile(expected, (len(x), 1)), atol=2e-15)
|
||||
aa = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
bb = fit_base_standardizer({"a": b}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
np.testing.assert_allclose(aa.transform(a, fold=0, pool=25), bb.transform(b, fold=0, pool=25), atol=1e-13)
|
||||
|
||||
def test_streaming_population_stats_match_pooled_nodes(self):
|
||||
rng = np.random.default_rng(42)
|
||||
a, b = rng.normal(size=(3, 10)), rng.normal(size=(7, 10))+2
|
||||
a[:, 9] = b[:, 9] = 4.
|
||||
stats = fit_base_standardizer({"a": a, "b": b}, learning_part_ids=["b", "a"], fold=2, pool=50)
|
||||
joined = np.vstack([a, b])
|
||||
np.testing.assert_allclose(stats.mean, joined.mean(axis=0), atol=1e-15)
|
||||
np.testing.assert_allclose(stats.std, joined.std(axis=0, ddof=0), atol=1e-15)
|
||||
z = stats.transform(joined, fold=2, pool=50)
|
||||
np.testing.assert_allclose(z.mean(axis=0), 0., atol=1e-15)
|
||||
np.testing.assert_allclose(z[:, :9].std(axis=0), 1., atol=1e-15)
|
||||
np.testing.assert_array_equal(z[:, 9], 0.)
|
||||
self.assertEqual(stats.scale[9], 1.)
|
||||
self.assertEqual(stats.node_count, 10)
|
||||
|
||||
def test_held_out_data_cannot_enter_fit_or_change_frozen_statistics(self):
|
||||
from dataclasses import FrozenInstanceError
|
||||
a, held = np.zeros((2, 10)), np.full((3, 10), 1e6)
|
||||
with self.assertRaisesRegex(ValueError, "learning pool exactly"):
|
||||
fit_base_standardizer({"a": a, "validation": held}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
with self.assertRaises(ValueError):
|
||||
fit_base_standardizer({}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
stats = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
fingerprint = stats.fingerprint
|
||||
np.testing.assert_array_equal(stats.transform(held, fold=0, pool=25), held)
|
||||
self.assertEqual(fingerprint, stats.fingerprint)
|
||||
with self.assertRaises(FrozenInstanceError):
|
||||
stats.mean = tuple(np.ones(10))
|
||||
with self.assertRaisesRegex(ValueError, "another fold/pool"):
|
||||
stats.transform(held, fold=1, pool=25)
|
||||
with self.assertRaisesRegex(ValueError, "another fold/pool"):
|
||||
stats.transform(held, fold=0, pool=100)
|
||||
with self.assertRaises(ValueError):
|
||||
fit_base_standardizer({"a": np.full((2, 10), np.nan)}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
|
||||
def test_ne_share_frozen_transform_before_preprocess(self):
|
||||
stats = synthetic_standardizer()
|
||||
models = [QueryModel(arm, standardizer=stats).eval() for arm in ("N", "E")]
|
||||
base = torch.linspace(-1, 1, 50).reshape(5, 10)
|
||||
expected = stats.transform(base.numpy(), fold="test-fold", pool="test-pool")
|
||||
captured = []
|
||||
for model in models:
|
||||
self.assertIs(model.encoder.standardizer, stats)
|
||||
self.assertNotIn("base_mean", dict(model.encoder.named_parameters()))
|
||||
handle = model.encoder.preprocess.register_forward_pre_hook(lambda module, args: captured.append(args[0].clone()))
|
||||
with torch.inference_mode():
|
||||
model.encoder(base)
|
||||
handle.remove()
|
||||
torch.testing.assert_close(captured[0], captured[1], rtol=0, atol=0)
|
||||
np.testing.assert_allclose(captured[0][0].numpy(), expected, atol=2e-7)
|
||||
with self.assertRaises(ValueError):
|
||||
QueryModel("N", standardizer=None)
|
||||
|
||||
|
||||
class NetworkTests(unittest.TestCase):
|
||||
def test_original_attention_numerical_parity(self):
|
||||
path = Path(__file__).parent/'s2_models/upstream/Physics_Attention.py'
|
||||
@@ -212,7 +288,7 @@ class NetworkTests(unittest.TestCase):
|
||||
models = {}
|
||||
for arm in ("N", "E"):
|
||||
torch.manual_seed(0)
|
||||
model = QueryModel(arm, width).eval()
|
||||
model = QueryModel(arm, width, standardizer=synthetic_standardizer()).eval()
|
||||
before = {k: v.clone() for k, v in model.state_dict().items()}
|
||||
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4, 6]), torch.rand(3, 2)
|
||||
edge = torch.rand(3, 4) if arm == "E" else None
|
||||
@@ -229,7 +305,7 @@ class NetworkTests(unittest.TestCase):
|
||||
|
||||
def test_query_order_global_context_and_input_rejection(self):
|
||||
torch.manual_seed(7)
|
||||
model = QueryModel("N").eval()
|
||||
model = QueryModel("N", standardizer=synthetic_standardizer()).eval()
|
||||
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4]), torch.rand(2, 2)
|
||||
with torch.inference_mode():
|
||||
a = model(base, ids, gaps)
|
||||
@@ -246,7 +322,7 @@ class NetworkTests(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
model(changed, ids, gaps)
|
||||
with self.assertRaises(ValueError):
|
||||
QueryModel("E")(base, ids, gaps)
|
||||
QueryModel("E", standardizer=synthetic_standardizer())(base, ids, gaps)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in new issue
Block a user