351 lines
15 KiB
Diff
351 lines
15 KiB
Diff
--- upstream/Physics_Attention.py
|
|
+++ attention.py
|
|
@@ -1,6 +1,6 @@
|
|
+# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
|
|
import torch.nn as nn
|
|
import torch
|
|
-from einops import rearrange, repeat
|
|
|
|
|
|
class Physics_Attention_Irregular_Mesh(nn.Module):
|
|
@@ -53,123 +53,7 @@
|
|
|
|
### (3) Deslice
|
|
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
|
- out_x = rearrange(out_x, 'b h n d -> b n (h d)')
|
|
+ out_x = out_x.permute(0, 2, 1, 3).reshape(B, N, -1)
|
|
return self.to_out(out_x)
|
|
|
|
|
|
-class Physics_Attention_Structured_Mesh_2D(nn.Module):
|
|
- ## for structured mesh in 2D space
|
|
- def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64, H=101, W=31, kernel=3): # kernel=3):
|
|
- super().__init__()
|
|
- inner_dim = dim_head * heads
|
|
- self.dim_head = dim_head
|
|
- self.heads = heads
|
|
- self.scale = dim_head ** -0.5
|
|
- self.softmax = nn.Softmax(dim=-1)
|
|
- self.dropout = nn.Dropout(dropout)
|
|
- self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
|
- self.H = H
|
|
- self.W = W
|
|
-
|
|
- self.in_project_x = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2)
|
|
- self.in_project_fx = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2)
|
|
- self.in_project_slice = nn.Linear(dim_head, slice_num)
|
|
- for l in [self.in_project_slice]:
|
|
- torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
|
- self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
|
- self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
|
- self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
|
-
|
|
- self.to_out = nn.Sequential(
|
|
- nn.Linear(inner_dim, dim),
|
|
- nn.Dropout(dropout)
|
|
- )
|
|
-
|
|
- def forward(self, x):
|
|
- # B N C
|
|
- B, N, C = x.shape
|
|
- x = x.reshape(B, self.H, self.W, C).contiguous().permute(0, 3, 1, 2).contiguous() # B C H W
|
|
-
|
|
- ### (1) Slice
|
|
- fx_mid = self.in_project_fx(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
|
- .permute(0, 2, 1, 3).contiguous() # B H N C
|
|
- x_mid = self.in_project_x(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
|
- .permute(0, 2, 1, 3).contiguous() # B H N G
|
|
- slice_weights = self.softmax(
|
|
- self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G
|
|
- slice_norm = slice_weights.sum(2) # B H G
|
|
- slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
|
|
- slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
|
-
|
|
- ### (2) Attention among slice tokens
|
|
- q_slice_token = self.to_q(slice_token)
|
|
- k_slice_token = self.to_k(slice_token)
|
|
- v_slice_token = self.to_v(slice_token)
|
|
- dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
|
- attn = self.softmax(dots)
|
|
- attn = self.dropout(attn)
|
|
- out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
|
-
|
|
- ### (3) Deslice
|
|
- out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
|
- out_x = rearrange(out_x, 'b h n d -> b n (h d)')
|
|
- return self.to_out(out_x)
|
|
-
|
|
-
|
|
-class Physics_Attention_Structured_Mesh_3D(nn.Module):
|
|
- ## for structured mesh in 3D space
|
|
- def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=32, H=32, W=32, D=32, kernel=3):
|
|
- super().__init__()
|
|
- inner_dim = dim_head * heads
|
|
- self.dim_head = dim_head
|
|
- self.heads = heads
|
|
- self.scale = dim_head ** -0.5
|
|
- self.softmax = nn.Softmax(dim=-1)
|
|
- self.dropout = nn.Dropout(dropout)
|
|
- self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
|
- self.H = H
|
|
- self.W = W
|
|
- self.D = D
|
|
-
|
|
- self.in_project_x = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2)
|
|
- self.in_project_fx = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2)
|
|
- self.in_project_slice = nn.Linear(dim_head, slice_num)
|
|
- for l in [self.in_project_slice]:
|
|
- torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
|
- self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
|
- self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
|
- self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
|
- self.to_out = nn.Sequential(
|
|
- nn.Linear(inner_dim, dim),
|
|
- nn.Dropout(dropout)
|
|
- )
|
|
-
|
|
- def forward(self, x):
|
|
- # B N C
|
|
- B, N, C = x.shape
|
|
- x = x.reshape(B, self.H, self.W, self.D, C).contiguous().permute(0, 4, 1, 2, 3).contiguous() # B C H W
|
|
-
|
|
- ### (1) Slice
|
|
- fx_mid = self.in_project_fx(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
|
- .permute(0, 2, 1, 3).contiguous() # B H N C
|
|
- x_mid = self.in_project_x(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
|
- .permute(0, 2, 1, 3).contiguous() # B H N G
|
|
- slice_weights = self.softmax(
|
|
- self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G
|
|
- slice_norm = slice_weights.sum(2) # B H G
|
|
- slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
|
|
- slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
|
-
|
|
- ### (2) Attention among slice tokens
|
|
- q_slice_token = self.to_q(slice_token)
|
|
- k_slice_token = self.to_k(slice_token)
|
|
- v_slice_token = self.to_v(slice_token)
|
|
- dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
|
- attn = self.softmax(dots)
|
|
- attn = self.dropout(attn)
|
|
- out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
|
-
|
|
- ### (3) Deslice
|
|
- out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
|
- out_x = rearrange(out_x, 'b h n d -> b n (h d)')
|
|
- return self.to_out(out_x)
|
|
--- upstream/Transolver_Irregular_Mesh.py
|
|
+++ blocks.py
|
|
@@ -1,9 +1,7 @@
|
|
+# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
|
|
import torch
|
|
import torch.nn as nn
|
|
-from timm.models.layers import trunc_normal_
|
|
-from model.Embedding import timestep_embedding
|
|
-import numpy as np
|
|
-from model.Physics_Attention import Physics_Attention_Irregular_Mesh
|
|
+from .attention import Physics_Attention_Irregular_Mesh
|
|
|
|
ACTIVATION = {'gelu': nn.GELU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid, 'relu': nn.ReLU, 'leaky_relu': nn.LeakyReLU(0.1),
|
|
'softplus': nn.Softplus, 'ELU': nn.ELU, 'silu': nn.SiLU}
|
|
@@ -71,88 +69,3 @@
|
|
return fx
|
|
|
|
|
|
-class Model(nn.Module):
|
|
- def __init__(self,
|
|
- space_dim=1,
|
|
- n_layers=5,
|
|
- n_hidden=256,
|
|
- dropout=0.0,
|
|
- n_head=8,
|
|
- Time_Input=False,
|
|
- act='gelu',
|
|
- mlp_ratio=1,
|
|
- fun_dim=1,
|
|
- out_dim=1,
|
|
- slice_num=32,
|
|
- ref=8,
|
|
- unified_pos=False
|
|
- ):
|
|
- super(Model, self).__init__()
|
|
- self.__name__ = 'Transolver_1D'
|
|
- self.ref = ref
|
|
- self.unified_pos = unified_pos
|
|
- self.Time_Input = Time_Input
|
|
- self.n_hidden = n_hidden
|
|
- self.space_dim = space_dim
|
|
- if self.unified_pos:
|
|
- self.preprocess = MLP(fun_dim + self.ref * self.ref, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act)
|
|
- else:
|
|
- self.preprocess = MLP(fun_dim + space_dim, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act)
|
|
- if Time_Input:
|
|
- self.time_fc = nn.Sequential(nn.Linear(n_hidden, n_hidden), nn.SiLU(), nn.Linear(n_hidden, n_hidden))
|
|
-
|
|
- self.blocks = nn.ModuleList([Transolver_block(num_heads=n_head, hidden_dim=n_hidden,
|
|
- dropout=dropout,
|
|
- act=act,
|
|
- mlp_ratio=mlp_ratio,
|
|
- out_dim=out_dim,
|
|
- slice_num=slice_num,
|
|
- last_layer=(_ == n_layers - 1))
|
|
- for _ in range(n_layers)])
|
|
- self.initialize_weights()
|
|
- self.placeholder = nn.Parameter((1 / (n_hidden)) * torch.rand(n_hidden, dtype=torch.float))
|
|
-
|
|
- def initialize_weights(self):
|
|
- self.apply(self._init_weights)
|
|
-
|
|
- def _init_weights(self, m):
|
|
- if isinstance(m, nn.Linear):
|
|
- trunc_normal_(m.weight, std=0.02)
|
|
- if isinstance(m, nn.Linear) and m.bias is not None:
|
|
- nn.init.constant_(m.bias, 0)
|
|
- elif isinstance(m, (nn.LayerNorm, nn.BatchNorm1d)):
|
|
- nn.init.constant_(m.bias, 0)
|
|
- nn.init.constant_(m.weight, 1.0)
|
|
-
|
|
- def get_grid(self, x, batchsize=1):
|
|
- # x: B N 2
|
|
- # grid_ref
|
|
- gridx = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float)
|
|
- gridx = gridx.reshape(1, self.ref, 1, 1).repeat([batchsize, 1, self.ref, 1])
|
|
- gridy = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float)
|
|
- gridy = gridy.reshape(1, 1, self.ref, 1).repeat([batchsize, self.ref, 1, 1])
|
|
- grid_ref = torch.cat((gridx, gridy), dim=-1).cuda().reshape(batchsize, self.ref * self.ref, 2) # B H W 8 8 2
|
|
-
|
|
- pos = torch.sqrt(torch.sum((x[:, :, None, :] - grid_ref[:, None, :, :]) ** 2, dim=-1)). \
|
|
- reshape(batchsize, x.shape[1], self.ref * self.ref).contiguous()
|
|
- return pos
|
|
-
|
|
- def forward(self, x, fx, T=None):
|
|
- if self.unified_pos:
|
|
- x = self.get_grid(x, x.shape[0])
|
|
- if fx is not None:
|
|
- fx = torch.cat((x, fx), -1)
|
|
- fx = self.preprocess(fx)
|
|
- else:
|
|
- fx = self.preprocess(x)
|
|
- fx = fx + self.placeholder[None, None, :]
|
|
-
|
|
- if T is not None:
|
|
- Time_emb = timestep_embedding(T, self.n_hidden).repeat(1, x.shape[1], 1)
|
|
- Time_emb = self.time_fc(Time_emb)
|
|
- fx = fx + Time_emb
|
|
-
|
|
- for block in self.blocks:
|
|
- fx = block(fx)
|
|
-
|
|
- return fx
|
|
--- /dev/null
|
|
+++ network.py
|
|
@@ -0,0 +1,109 @@
|
|
+"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
|
|
+
|
|
+No optimiser, training loop, data reader or GPU invocation is provided.
|
|
+"""
|
|
+import torch
|
|
+from torch import nn
|
|
+
|
|
+from .blocks import MLP, Transolver_block
|
|
+from .features import BASE_COLUMNS
|
|
+
|
|
+
|
|
+TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
|
+ schedule="cosine", batch_parts=1, max_epochs=200,
|
|
+ patience=20, seeds=[0, 1, 2], widths=[128, 256])
|
|
+
|
|
+
|
|
+def check_tensor(tensor, shape, name):
|
|
+ if tuple(tensor.shape) != tuple(shape) or not torch.isfinite(tensor).all():
|
|
+ raise ValueError(f"invalid {name}")
|
|
+
|
|
+
|
|
+class WholePartEncoder(nn.Module):
|
|
+ def __init__(self, width):
|
|
+ super().__init__()
|
|
+ if width not in (128, 256):
|
|
+ raise ValueError("S2 permits widths 128 or 256 only")
|
|
+ self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
|
+ self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
|
+ dropout=0., mlp_ratio=1, slice_num=32,
|
|
+ last_layer=False) for _ in range(4)])
|
|
+ self.apply(self._initialize)
|
|
+ self.placeholder = nn.Parameter(torch.rand(width)/width)
|
|
+
|
|
+ @staticmethod
|
|
+ def _initialize(module):
|
|
+ # torch.nn.init replaces timm's same trunc_normal_ defaults; no timm dependency.
|
|
+ if isinstance(module, nn.Linear):
|
|
+ nn.init.trunc_normal_(module.weight, std=.02)
|
|
+ if module.bias is not None:
|
|
+ nn.init.zeros_(module.bias)
|
|
+ elif isinstance(module, nn.LayerNorm):
|
|
+ nn.init.zeros_(module.bias)
|
|
+ nn.init.ones_(module.weight)
|
|
+
|
|
+ def forward(self, base):
|
|
+ if base.ndim != 2 or base.shape[0] == 0:
|
|
+ raise ValueError("one whole part with all tet nodes required")
|
|
+ check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
|
+ fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
|
+ for block in self.blocks:
|
|
+ fx = block(fx)
|
|
+ if not torch.isfinite(fx).all():
|
|
+ raise ValueError("nonfinite encoder output")
|
|
+ return fx[0]
|
|
+
|
|
+
|
|
+class QueryModel(nn.Module):
|
|
+ def __init__(self, arm, width=128):
|
|
+ super().__init__()
|
|
+ if arm not in ("N", "E"):
|
|
+ raise ValueError("arm must be N or E")
|
|
+ self.arm, self.width = arm, width
|
|
+ self.encoder = WholePartEncoder(width)
|
|
+ query_dim = 2 if arm == "N" else 6
|
|
+ self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
|
+ nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
|
+ self.decoder.apply(WholePartEncoder._initialize)
|
|
+
|
|
+ def forward(self, base, query_ids, gaps, edge_q=None):
|
|
+ if query_ids.ndim != 1 or query_ids.dtype != torch.long or query_ids.numel() == 0:
|
|
+ raise ValueError("nonempty int64 original-node query IDs required")
|
|
+ if int(query_ids.min()) < 0 or int(query_ids.max()) >= len(base):
|
|
+ raise ValueError("query node outside whole part")
|
|
+ check_tensor(gaps, (len(query_ids), 2), "g12/g23")
|
|
+ if self.arm == "E":
|
|
+ if edge_q is None:
|
|
+ raise ValueError("E requires all four edge summaries")
|
|
+ check_tensor(edge_q, (len(query_ids), 4), "edge summaries")
|
|
+ local = torch.cat([gaps, edge_q], dim=-1)
|
|
+ else:
|
|
+ if edge_q is not None:
|
|
+ raise ValueError("N must not receive edge summaries")
|
|
+ local = gaps
|
|
+ embedding = self.encoder(base)[query_ids]
|
|
+ y = self.decoder(torch.cat([embedding, local], dim=-1)).squeeze(-1)
|
|
+ if not torch.isfinite(y).all():
|
|
+ raise ValueError("nonfinite decoder output")
|
|
+ return y
|
|
+
|
|
+ def parameter_counts(self):
|
|
+ return {"encoder": sum(p.numel() for p in self.encoder.parameters()),
|
|
+ "decoder": sum(p.numel() for p in self.decoder.parameters()),
|
|
+ "total": sum(p.numel() for p in self.parameters())}
|
|
+
|
|
+
|
|
+def weighted_mse(prediction, target, global_weights):
|
|
+ """Sum of contributions with already globally-normalised hierarchy weights.
|
|
+
|
|
+ Do not renormalise a single part's weights: that would erase component balance.
|
|
+ """
|
|
+ if prediction.ndim != 1 or prediction.numel() == 0:
|
|
+ raise ValueError("nonempty query predictions required")
|
|
+ check_tensor(prediction, prediction.shape, "prediction")
|
|
+ check_tensor(target, prediction.shape, "target")
|
|
+ check_tensor(global_weights, prediction.shape, "weights")
|
|
+ if (global_weights <= 0).any():
|
|
+ raise ValueError("positive hierarchy weights required")
|
|
+ # FP32 arithmetic under future AMP; do not form residuals in float16.
|
|
+ return torch.sum(global_weights.float() * (prediction.float()-target.float()).square())
|