Normalize neural base inputs with frozen learning-pool statistics

This commit is contained in:
Frank14f committed 2026-11-14 22:59:46 +08:00
1 parent bb09e8e164
commit f3532962b0
10 files changed
+326 -15

No files matched your search

+13 -3
View File
@@ -7,6 +7,7 @@ from torch import nn
from .blocks import MLP, Transolver_block
from .features import BASE_COLUMNS
from .normalization import BaseStandardizer
TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
@@ -20,10 +21,15 @@ def check_tensor(tensor, shape, name):
class WholePartEncoder(nn.Module):
def __init__(self, width):
def __init__(self, width, standardizer):
super().__init__()
if width not in (128, 256):
raise ValueError("S2 permits widths 128 or 256 only")
if not isinstance(standardizer, BaseStandardizer):
raise ValueError("frozen learning-pool standardizer required")
self.standardizer = standardizer
self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
dropout=0., mlp_ratio=1, slice_num=32,
@@ -46,6 +52,10 @@ class WholePartEncoder(nn.Module):
if base.ndim != 2 or base.shape[0] == 0:
raise ValueError("one whole part with all tet nodes required")
check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
# Keep normalisation in FP32 even under future mixed-precision training.
base = (base.float()-self.base_mean.float()) / self.base_scale.float()
if not torch.isfinite(base).all():
raise ValueError("nonfinite standardized base input")
fx = self.preprocess(base[None]) + self.placeholder[None, None]
for block in self.blocks:
fx = block(fx)
@@ -55,12 +65,12 @@ class WholePartEncoder(nn.Module):
class QueryModel(nn.Module):
def __init__(self, arm, width=128):
def __init__(self, arm, width=128, *, standardizer):
super().__init__()
if arm not in ("N", "E"):
raise ValueError("arm must be N or E")
self.arm, self.width = arm, width
self.encoder = WholePartEncoder(width)
self.encoder = WholePartEncoder(width, standardizer)
query_dim = 2 if arm == "N" else 6
self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))