Files
MeltMind/tools/s2_models/upstream-adaptation.diff
T

351 lines
15 KiB
Diff

--- upstream/Physics_Attention.py
+++ attention.py
@@ -1,6 +1,6 @@
+# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
import torch.nn as nn
import torch
-from einops import rearrange, repeat
class Physics_Attention_Irregular_Mesh(nn.Module):
@@ -53,123 +53,7 @@
### (3) Deslice
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
- out_x = rearrange(out_x, 'b h n d -> b n (h d)')
+ out_x = out_x.permute(0, 2, 1, 3).reshape(B, N, -1)
return self.to_out(out_x)
-class Physics_Attention_Structured_Mesh_2D(nn.Module):
- ## for structured mesh in 2D space
- def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64, H=101, W=31, kernel=3): # kernel=3):
- super().__init__()
- inner_dim = dim_head * heads
- self.dim_head = dim_head
- self.heads = heads
- self.scale = dim_head ** -0.5
- self.softmax = nn.Softmax(dim=-1)
- self.dropout = nn.Dropout(dropout)
- self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
- self.H = H
- self.W = W
-
- self.in_project_x = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2)
- self.in_project_fx = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2)
- self.in_project_slice = nn.Linear(dim_head, slice_num)
- for l in [self.in_project_slice]:
- torch.nn.init.orthogonal_(l.weight) # use a principled initialization
- self.to_q = nn.Linear(dim_head, dim_head, bias=False)
- self.to_k = nn.Linear(dim_head, dim_head, bias=False)
- self.to_v = nn.Linear(dim_head, dim_head, bias=False)
-
- self.to_out = nn.Sequential(
- nn.Linear(inner_dim, dim),
- nn.Dropout(dropout)
- )
-
- def forward(self, x):
- # B N C
- B, N, C = x.shape
- x = x.reshape(B, self.H, self.W, C).contiguous().permute(0, 3, 1, 2).contiguous() # B C H W
-
- ### (1) Slice
- fx_mid = self.in_project_fx(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
- .permute(0, 2, 1, 3).contiguous() # B H N C
- x_mid = self.in_project_x(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
- .permute(0, 2, 1, 3).contiguous() # B H N G
- slice_weights = self.softmax(
- self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G
- slice_norm = slice_weights.sum(2) # B H G
- slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
- slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
-
- ### (2) Attention among slice tokens
- q_slice_token = self.to_q(slice_token)
- k_slice_token = self.to_k(slice_token)
- v_slice_token = self.to_v(slice_token)
- dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
- attn = self.softmax(dots)
- attn = self.dropout(attn)
- out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
-
- ### (3) Deslice
- out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
- out_x = rearrange(out_x, 'b h n d -> b n (h d)')
- return self.to_out(out_x)
-
-
-class Physics_Attention_Structured_Mesh_3D(nn.Module):
- ## for structured mesh in 3D space
- def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=32, H=32, W=32, D=32, kernel=3):
- super().__init__()
- inner_dim = dim_head * heads
- self.dim_head = dim_head
- self.heads = heads
- self.scale = dim_head ** -0.5
- self.softmax = nn.Softmax(dim=-1)
- self.dropout = nn.Dropout(dropout)
- self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
- self.H = H
- self.W = W
- self.D = D
-
- self.in_project_x = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2)
- self.in_project_fx = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2)
- self.in_project_slice = nn.Linear(dim_head, slice_num)
- for l in [self.in_project_slice]:
- torch.nn.init.orthogonal_(l.weight) # use a principled initialization
- self.to_q = nn.Linear(dim_head, dim_head, bias=False)
- self.to_k = nn.Linear(dim_head, dim_head, bias=False)
- self.to_v = nn.Linear(dim_head, dim_head, bias=False)
- self.to_out = nn.Sequential(
- nn.Linear(inner_dim, dim),
- nn.Dropout(dropout)
- )
-
- def forward(self, x):
- # B N C
- B, N, C = x.shape
- x = x.reshape(B, self.H, self.W, self.D, C).contiguous().permute(0, 4, 1, 2, 3).contiguous() # B C H W
-
- ### (1) Slice
- fx_mid = self.in_project_fx(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
- .permute(0, 2, 1, 3).contiguous() # B H N C
- x_mid = self.in_project_x(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
- .permute(0, 2, 1, 3).contiguous() # B H N G
- slice_weights = self.softmax(
- self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G
- slice_norm = slice_weights.sum(2) # B H G
- slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
- slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
-
- ### (2) Attention among slice tokens
- q_slice_token = self.to_q(slice_token)
- k_slice_token = self.to_k(slice_token)
- v_slice_token = self.to_v(slice_token)
- dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
- attn = self.softmax(dots)
- attn = self.dropout(attn)
- out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
-
- ### (3) Deslice
- out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
- out_x = rearrange(out_x, 'b h n d -> b n (h d)')
- return self.to_out(out_x)
--- upstream/Transolver_Irregular_Mesh.py
+++ blocks.py
@@ -1,9 +1,7 @@
+# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
import torch
import torch.nn as nn
-from timm.models.layers import trunc_normal_
-from model.Embedding import timestep_embedding
-import numpy as np
-from model.Physics_Attention import Physics_Attention_Irregular_Mesh
+from .attention import Physics_Attention_Irregular_Mesh
ACTIVATION = {'gelu': nn.GELU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid, 'relu': nn.ReLU, 'leaky_relu': nn.LeakyReLU(0.1),
'softplus': nn.Softplus, 'ELU': nn.ELU, 'silu': nn.SiLU}
@@ -71,88 +69,3 @@
return fx
-class Model(nn.Module):
- def __init__(self,
- space_dim=1,
- n_layers=5,
- n_hidden=256,
- dropout=0.0,
- n_head=8,
- Time_Input=False,
- act='gelu',
- mlp_ratio=1,
- fun_dim=1,
- out_dim=1,
- slice_num=32,
- ref=8,
- unified_pos=False
- ):
- super(Model, self).__init__()
- self.__name__ = 'Transolver_1D'
- self.ref = ref
- self.unified_pos = unified_pos
- self.Time_Input = Time_Input
- self.n_hidden = n_hidden
- self.space_dim = space_dim
- if self.unified_pos:
- self.preprocess = MLP(fun_dim + self.ref * self.ref, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act)
- else:
- self.preprocess = MLP(fun_dim + space_dim, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act)
- if Time_Input:
- self.time_fc = nn.Sequential(nn.Linear(n_hidden, n_hidden), nn.SiLU(), nn.Linear(n_hidden, n_hidden))
-
- self.blocks = nn.ModuleList([Transolver_block(num_heads=n_head, hidden_dim=n_hidden,
- dropout=dropout,
- act=act,
- mlp_ratio=mlp_ratio,
- out_dim=out_dim,
- slice_num=slice_num,
- last_layer=(_ == n_layers - 1))
- for _ in range(n_layers)])
- self.initialize_weights()
- self.placeholder = nn.Parameter((1 / (n_hidden)) * torch.rand(n_hidden, dtype=torch.float))
-
- def initialize_weights(self):
- self.apply(self._init_weights)
-
- def _init_weights(self, m):
- if isinstance(m, nn.Linear):
- trunc_normal_(m.weight, std=0.02)
- if isinstance(m, nn.Linear) and m.bias is not None:
- nn.init.constant_(m.bias, 0)
- elif isinstance(m, (nn.LayerNorm, nn.BatchNorm1d)):
- nn.init.constant_(m.bias, 0)
- nn.init.constant_(m.weight, 1.0)
-
- def get_grid(self, x, batchsize=1):
- # x: B N 2
- # grid_ref
- gridx = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float)
- gridx = gridx.reshape(1, self.ref, 1, 1).repeat([batchsize, 1, self.ref, 1])
- gridy = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float)
- gridy = gridy.reshape(1, 1, self.ref, 1).repeat([batchsize, self.ref, 1, 1])
- grid_ref = torch.cat((gridx, gridy), dim=-1).cuda().reshape(batchsize, self.ref * self.ref, 2) # B H W 8 8 2
-
- pos = torch.sqrt(torch.sum((x[:, :, None, :] - grid_ref[:, None, :, :]) ** 2, dim=-1)). \
- reshape(batchsize, x.shape[1], self.ref * self.ref).contiguous()
- return pos
-
- def forward(self, x, fx, T=None):
- if self.unified_pos:
- x = self.get_grid(x, x.shape[0])
- if fx is not None:
- fx = torch.cat((x, fx), -1)
- fx = self.preprocess(fx)
- else:
- fx = self.preprocess(x)
- fx = fx + self.placeholder[None, None, :]
-
- if T is not None:
- Time_emb = timestep_embedding(T, self.n_hidden).repeat(1, x.shape[1], 1)
- Time_emb = self.time_fc(Time_emb)
- fx = fx + Time_emb
-
- for block in self.blocks:
- fx = block(fx)
-
- return fx
--- /dev/null
+++ network.py
@@ -0,0 +1,109 @@
+"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
+
+No optimiser, training loop, data reader or GPU invocation is provided.
+"""
+import torch
+from torch import nn
+
+from .blocks import MLP, Transolver_block
+from .features import BASE_COLUMNS
+
+
+TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
+ schedule="cosine", batch_parts=1, max_epochs=200,
+ patience=20, seeds=[0, 1, 2], widths=[128, 256])
+
+
+def check_tensor(tensor, shape, name):
+ if tuple(tensor.shape) != tuple(shape) or not torch.isfinite(tensor).all():
+ raise ValueError(f"invalid {name}")
+
+
+class WholePartEncoder(nn.Module):
+ def __init__(self, width):
+ super().__init__()
+ if width not in (128, 256):
+ raise ValueError("S2 permits widths 128 or 256 only")
+ self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
+ self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
+ dropout=0., mlp_ratio=1, slice_num=32,
+ last_layer=False) for _ in range(4)])
+ self.apply(self._initialize)
+ self.placeholder = nn.Parameter(torch.rand(width)/width)
+
+ @staticmethod
+ def _initialize(module):
+ # torch.nn.init replaces timm's same trunc_normal_ defaults; no timm dependency.
+ if isinstance(module, nn.Linear):
+ nn.init.trunc_normal_(module.weight, std=.02)
+ if module.bias is not None:
+ nn.init.zeros_(module.bias)
+ elif isinstance(module, nn.LayerNorm):
+ nn.init.zeros_(module.bias)
+ nn.init.ones_(module.weight)
+
+ def forward(self, base):
+ if base.ndim != 2 or base.shape[0] == 0:
+ raise ValueError("one whole part with all tet nodes required")
+ check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
+ fx = self.preprocess(base[None]) + self.placeholder[None, None]
+ for block in self.blocks:
+ fx = block(fx)
+ if not torch.isfinite(fx).all():
+ raise ValueError("nonfinite encoder output")
+ return fx[0]
+
+
+class QueryModel(nn.Module):
+ def __init__(self, arm, width=128):
+ super().__init__()
+ if arm not in ("N", "E"):
+ raise ValueError("arm must be N or E")
+ self.arm, self.width = arm, width
+ self.encoder = WholePartEncoder(width)
+ query_dim = 2 if arm == "N" else 6
+ self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
+ nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
+ self.decoder.apply(WholePartEncoder._initialize)
+
+ def forward(self, base, query_ids, gaps, edge_q=None):
+ if query_ids.ndim != 1 or query_ids.dtype != torch.long or query_ids.numel() == 0:
+ raise ValueError("nonempty int64 original-node query IDs required")
+ if int(query_ids.min()) < 0 or int(query_ids.max()) >= len(base):
+ raise ValueError("query node outside whole part")
+ check_tensor(gaps, (len(query_ids), 2), "g12/g23")
+ if self.arm == "E":
+ if edge_q is None:
+ raise ValueError("E requires all four edge summaries")
+ check_tensor(edge_q, (len(query_ids), 4), "edge summaries")
+ local = torch.cat([gaps, edge_q], dim=-1)
+ else:
+ if edge_q is not None:
+ raise ValueError("N must not receive edge summaries")
+ local = gaps
+ embedding = self.encoder(base)[query_ids]
+ y = self.decoder(torch.cat([embedding, local], dim=-1)).squeeze(-1)
+ if not torch.isfinite(y).all():
+ raise ValueError("nonfinite decoder output")
+ return y
+
+ def parameter_counts(self):
+ return {"encoder": sum(p.numel() for p in self.encoder.parameters()),
+ "decoder": sum(p.numel() for p in self.decoder.parameters()),
+ "total": sum(p.numel() for p in self.parameters())}
+
+
+def weighted_mse(prediction, target, global_weights):
+ """Sum of contributions with already globally-normalised hierarchy weights.
+
+ Do not renormalise a single part's weights: that would erase component balance.
+ """
+ if prediction.ndim != 1 or prediction.numel() == 0:
+ raise ValueError("nonempty query predictions required")
+ check_tensor(prediction, prediction.shape, "prediction")
+ check_tensor(target, prediction.shape, "target")
+ check_tensor(global_weights, prediction.shape, "weights")
+ if (global_weights <= 0).any():
+ raise ValueError("positive hierarchy weights required")
+ # FP32 arithmetic under future AMP; do not form residuals in float16.
+ return torch.sum(global_weights.float() * (prediction.float()-target.float()).square())