# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE. import torch import torch.nn as nn from .attention import Physics_Attention_Irregular_Mesh ACTIVATION = {'gelu': nn.GELU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid, 'relu': nn.ReLU, 'leaky_relu': nn.LeakyReLU(0.1), 'softplus': nn.Softplus, 'ELU': nn.ELU, 'silu': nn.SiLU} class MLP(nn.Module): def __init__(self, n_input, n_hidden, n_output, n_layers=1, act='gelu', res=True): super(MLP, self).__init__() if act in ACTIVATION.keys(): act = ACTIVATION[act] else: raise NotImplementedError self.n_input = n_input self.n_hidden = n_hidden self.n_output = n_output self.n_layers = n_layers self.res = res self.linear_pre = nn.Sequential(nn.Linear(n_input, n_hidden), act()) self.linear_post = nn.Linear(n_hidden, n_output) self.linears = nn.ModuleList([nn.Sequential(nn.Linear(n_hidden, n_hidden), act()) for _ in range(n_layers)]) def forward(self, x): x = self.linear_pre(x) for i in range(self.n_layers): if self.res: x = self.linears[i](x) + x else: x = self.linears[i](x) x = self.linear_post(x) return x class Transolver_block(nn.Module): """Transformer encoder block.""" def __init__( self, num_heads: int, hidden_dim: int, dropout: float, act='gelu', mlp_ratio=4, last_layer=False, out_dim=1, slice_num=32, ): super().__init__() self.last_layer = last_layer self.ln_1 = nn.LayerNorm(hidden_dim) self.Attn = Physics_Attention_Irregular_Mesh(hidden_dim, heads=num_heads, dim_head=hidden_dim // num_heads, dropout=dropout, slice_num=slice_num) self.ln_2 = nn.LayerNorm(hidden_dim) self.mlp = MLP(hidden_dim, hidden_dim * mlp_ratio, hidden_dim, n_layers=0, res=False, act=act) if self.last_layer: self.ln_3 = nn.LayerNorm(hidden_dim) self.mlp2 = nn.Linear(hidden_dim, out_dim) def forward(self, fx): fx = self.Attn(self.ln_1(fx)) + fx fx = self.mlp(self.ln_2(fx)) + fx if self.last_layer: return self.mlp2(self.ln_3(fx)) else: return fx