Normalize neural base inputs with frozen learning-pool statistics
This commit is contained in:
1 parent
bb09e8e164
commit
f3532962b0
10 files changed
+326
-15
No files matched your search
@@ -238,7 +238,7 @@
|
||||
- return fx
|
||||
--- /dev/null
|
||||
+++ network.py
|
||||
@@ -0,0 +1,109 @@
|
||||
@@ -0,0 +1,119 @@
|
||||
+"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
|
||||
+
|
||||
+No optimiser, training loop, data reader or GPU invocation is provided.
|
||||
@@ -248,6 +248,7 @@
|
||||
+
|
||||
+from .blocks import MLP, Transolver_block
|
||||
+from .features import BASE_COLUMNS
|
||||
+from .normalization import BaseStandardizer
|
||||
+
|
||||
+
|
||||
+TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
@@ -261,10 +262,15 @@
|
||||
+
|
||||
+
|
||||
+class WholePartEncoder(nn.Module):
|
||||
+ def __init__(self, width):
|
||||
+ def __init__(self, width, standardizer):
|
||||
+ super().__init__()
|
||||
+ if width not in (128, 256):
|
||||
+ raise ValueError("S2 permits widths 128 or 256 only")
|
||||
+ if not isinstance(standardizer, BaseStandardizer):
|
||||
+ raise ValueError("frozen learning-pool standardizer required")
|
||||
+ self.standardizer = standardizer
|
||||
+ self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
|
||||
+ self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
|
||||
+ self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
+ self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
+ dropout=0., mlp_ratio=1, slice_num=32,
|
||||
@@ -287,6 +293,10 @@
|
||||
+ if base.ndim != 2 or base.shape[0] == 0:
|
||||
+ raise ValueError("one whole part with all tet nodes required")
|
||||
+ check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
+ # Keep normalisation in FP32 even under future mixed-precision training.
|
||||
+ base = (base.float()-self.base_mean.float()) / self.base_scale.float()
|
||||
+ if not torch.isfinite(base).all():
|
||||
+ raise ValueError("nonfinite standardized base input")
|
||||
+ fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
+ for block in self.blocks:
|
||||
+ fx = block(fx)
|
||||
@@ -296,12 +306,12 @@
|
||||
+
|
||||
+
|
||||
+class QueryModel(nn.Module):
|
||||
+ def __init__(self, arm, width=128):
|
||||
+ def __init__(self, arm, width=128, *, standardizer):
|
||||
+ super().__init__()
|
||||
+ if arm not in ("N", "E"):
|
||||
+ raise ValueError("arm must be N or E")
|
||||
+ self.arm, self.width = arm, width
|
||||
+ self.encoder = WholePartEncoder(width)
|
||||
+ self.encoder = WholePartEncoder(width, standardizer)
|
||||
+ query_dim = 2 if arm == "N" else 6
|
||||
+ self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
+ nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
|
||||
Reference in new issue
Block a user