Record reproduced fp16 overflow and freeze isolated FP32-slice pilot v2
This commit is contained in:
1 parent
f502b7b9f8
commit
8a548cc259
28 files changed
+13918
No files matched your search
@@ -0,0 +1,119 @@
|
||||
"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
|
||||
|
||||
No optimiser, training loop, data reader or GPU invocation is provided.
|
||||
"""
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from .blocks import MLP, Transolver_block
|
||||
from s2_models.features import BASE_COLUMNS
|
||||
from s2_models.normalization import BaseStandardizer
|
||||
|
||||
|
||||
TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
schedule="cosine", batch_parts=1, max_epochs=200,
|
||||
patience=20, seeds=[0, 1, 2], widths=[128, 256])
|
||||
|
||||
|
||||
def check_tensor(tensor, shape, name):
|
||||
if tuple(tensor.shape) != tuple(shape) or not torch.isfinite(tensor).all():
|
||||
raise ValueError(f"invalid {name}")
|
||||
|
||||
|
||||
class WholePartEncoder(nn.Module):
|
||||
def __init__(self, width, standardizer):
|
||||
super().__init__()
|
||||
if width not in (128, 256):
|
||||
raise ValueError("S2 permits widths 128 or 256 only")
|
||||
if not isinstance(standardizer, BaseStandardizer):
|
||||
raise ValueError("frozen learning-pool standardizer required")
|
||||
self.standardizer = standardizer
|
||||
self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
|
||||
self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
|
||||
self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
dropout=0., mlp_ratio=1, slice_num=32,
|
||||
last_layer=False) for _ in range(4)])
|
||||
self.apply(self._initialize)
|
||||
self.placeholder = nn.Parameter(torch.rand(width)/width)
|
||||
|
||||
@staticmethod
|
||||
def _initialize(module):
|
||||
# torch.nn.init replaces timm's same trunc_normal_ defaults; no timm dependency.
|
||||
if isinstance(module, nn.Linear):
|
||||
nn.init.trunc_normal_(module.weight, std=.02)
|
||||
if module.bias is not None:
|
||||
nn.init.zeros_(module.bias)
|
||||
elif isinstance(module, nn.LayerNorm):
|
||||
nn.init.zeros_(module.bias)
|
||||
nn.init.ones_(module.weight)
|
||||
|
||||
def forward(self, base):
|
||||
if base.ndim != 2 or base.shape[0] == 0:
|
||||
raise ValueError("one whole part with all tet nodes required")
|
||||
check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
# Keep normalisation in FP32 even under future mixed-precision training.
|
||||
base = (base.float()-self.base_mean.float()) / self.base_scale.float()
|
||||
if not torch.isfinite(base).all():
|
||||
raise ValueError("nonfinite standardized base input")
|
||||
fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
for block in self.blocks:
|
||||
fx = block(fx)
|
||||
if not torch.isfinite(fx).all():
|
||||
raise ValueError("nonfinite encoder output")
|
||||
return fx[0]
|
||||
|
||||
|
||||
class QueryModel(nn.Module):
|
||||
def __init__(self, arm, width=128, *, standardizer):
|
||||
super().__init__()
|
||||
if arm not in ("N", "E"):
|
||||
raise ValueError("arm must be N or E")
|
||||
self.arm, self.width = arm, width
|
||||
self.encoder = WholePartEncoder(width, standardizer)
|
||||
query_dim = 2 if arm == "N" else 6
|
||||
self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
self.decoder.apply(WholePartEncoder._initialize)
|
||||
|
||||
def forward(self, base, query_ids, gaps, edge_q=None):
|
||||
if query_ids.ndim != 1 or query_ids.dtype != torch.long or query_ids.numel() == 0:
|
||||
raise ValueError("nonempty int64 original-node query IDs required")
|
||||
if int(query_ids.min()) < 0 or int(query_ids.max()) >= len(base):
|
||||
raise ValueError("query node outside whole part")
|
||||
check_tensor(gaps, (len(query_ids), 2), "g12/g23")
|
||||
if self.arm == "E":
|
||||
if edge_q is None:
|
||||
raise ValueError("E requires all four edge summaries")
|
||||
check_tensor(edge_q, (len(query_ids), 4), "edge summaries")
|
||||
local = torch.cat([gaps, edge_q], dim=-1)
|
||||
else:
|
||||
if edge_q is not None:
|
||||
raise ValueError("N must not receive edge summaries")
|
||||
local = gaps
|
||||
embedding = self.encoder(base)[query_ids]
|
||||
y = self.decoder(torch.cat([embedding, local], dim=-1)).squeeze(-1)
|
||||
if not torch.isfinite(y).all():
|
||||
raise ValueError("nonfinite decoder output")
|
||||
return y
|
||||
|
||||
def parameter_counts(self):
|
||||
return {"encoder": sum(p.numel() for p in self.encoder.parameters()),
|
||||
"decoder": sum(p.numel() for p in self.decoder.parameters()),
|
||||
"total": sum(p.numel() for p in self.parameters())}
|
||||
|
||||
|
||||
def weighted_mse(prediction, target, global_weights):
|
||||
"""Sum of contributions with already globally-normalised hierarchy weights.
|
||||
|
||||
Do not renormalise a single part's weights: that would erase component balance.
|
||||
"""
|
||||
if prediction.ndim != 1 or prediction.numel() == 0:
|
||||
raise ValueError("nonempty query predictions required")
|
||||
check_tensor(prediction, prediction.shape, "prediction")
|
||||
check_tensor(target, prediction.shape, "target")
|
||||
check_tensor(global_weights, prediction.shape, "weights")
|
||||
if (global_weights <= 0).any():
|
||||
raise ValueError("positive hierarchy weights required")
|
||||
# FP32 arithmetic under future AMP; do not form residuals in float16.
|
||||
return torch.sum(global_weights.float() * (prediction.float()-target.float()).square())
|
||||
Reference in new issue
Block a user