60 lines
2.5 KiB
Python
60 lines
2.5 KiB
Python
# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
|
|
import torch.nn as nn
|
|
import torch
|
|
|
|
|
|
class Physics_Attention_Irregular_Mesh(nn.Module):
|
|
## for irregular meshes in 1D, 2D or 3D space
|
|
def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64):
|
|
super().__init__()
|
|
inner_dim = dim_head * heads
|
|
self.dim_head = dim_head
|
|
self.heads = heads
|
|
self.scale = dim_head ** -0.5
|
|
self.softmax = nn.Softmax(dim=-1)
|
|
self.dropout = nn.Dropout(dropout)
|
|
self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
|
|
|
self.in_project_x = nn.Linear(dim, inner_dim)
|
|
self.in_project_fx = nn.Linear(dim, inner_dim)
|
|
self.in_project_slice = nn.Linear(dim_head, slice_num)
|
|
for l in [self.in_project_slice]:
|
|
torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
|
self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
|
self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
|
self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
|
self.to_out = nn.Sequential(
|
|
nn.Linear(inner_dim, dim),
|
|
nn.Dropout(dropout)
|
|
)
|
|
|
|
def forward(self, x):
|
|
# B N C
|
|
B, N, C = x.shape
|
|
|
|
### (1) Slice
|
|
fx_mid = self.in_project_fx(x).reshape(B, N, self.heads, self.dim_head) \
|
|
.permute(0, 2, 1, 3).contiguous() # B H N C
|
|
x_mid = self.in_project_x(x).reshape(B, N, self.heads, self.dim_head) \
|
|
.permute(0, 2, 1, 3).contiguous() # B H N C
|
|
slice_weights = self.softmax(self.in_project_slice(x_mid) / self.temperature) # B H N G
|
|
slice_norm = slice_weights.sum(2) # B H G
|
|
slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
|
|
slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
|
|
|
### (2) Attention among slice tokens
|
|
q_slice_token = self.to_q(slice_token)
|
|
k_slice_token = self.to_k(slice_token)
|
|
v_slice_token = self.to_v(slice_token)
|
|
dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
|
attn = self.softmax(dots)
|
|
attn = self.dropout(attn)
|
|
out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
|
|
|
### (3) Deslice
|
|
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
|
out_x = out_x.permute(0, 2, 1, 3).reshape(B, N, -1)
|
|
return self.to_out(out_x)
|
|
|
|
|