--- upstream/Physics_Attention.py +++ attention.py @@ -1,6 +1,6 @@ +# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE. import torch.nn as nn import torch -from einops import rearrange, repeat class Physics_Attention_Irregular_Mesh(nn.Module): @@ -53,123 +53,7 @@ ### (3) Deslice out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights) - out_x = rearrange(out_x, 'b h n d -> b n (h d)') + out_x = out_x.permute(0, 2, 1, 3).reshape(B, N, -1) return self.to_out(out_x) -class Physics_Attention_Structured_Mesh_2D(nn.Module): - ## for structured mesh in 2D space - def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64, H=101, W=31, kernel=3): # kernel=3): - super().__init__() - inner_dim = dim_head * heads - self.dim_head = dim_head - self.heads = heads - self.scale = dim_head ** -0.5 - self.softmax = nn.Softmax(dim=-1) - self.dropout = nn.Dropout(dropout) - self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5) - self.H = H - self.W = W - - self.in_project_x = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2) - self.in_project_fx = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2) - self.in_project_slice = nn.Linear(dim_head, slice_num) - for l in [self.in_project_slice]: - torch.nn.init.orthogonal_(l.weight) # use a principled initialization - self.to_q = nn.Linear(dim_head, dim_head, bias=False) - self.to_k = nn.Linear(dim_head, dim_head, bias=False) - self.to_v = nn.Linear(dim_head, dim_head, bias=False) - - self.to_out = nn.Sequential( - nn.Linear(inner_dim, dim), - nn.Dropout(dropout) - ) - - def forward(self, x): - # B N C - B, N, C = x.shape - x = x.reshape(B, self.H, self.W, C).contiguous().permute(0, 3, 1, 2).contiguous() # B C H W - - ### (1) Slice - fx_mid = self.in_project_fx(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \ - .permute(0, 2, 1, 3).contiguous() # B H N C - x_mid = self.in_project_x(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \ - .permute(0, 2, 1, 3).contiguous() # B H N G - slice_weights = self.softmax( - self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G - slice_norm = slice_weights.sum(2) # B H G - slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights) - slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head)) - - ### (2) Attention among slice tokens - q_slice_token = self.to_q(slice_token) - k_slice_token = self.to_k(slice_token) - v_slice_token = self.to_v(slice_token) - dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale - attn = self.softmax(dots) - attn = self.dropout(attn) - out_slice_token = torch.matmul(attn, v_slice_token) # B H G D - - ### (3) Deslice - out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights) - out_x = rearrange(out_x, 'b h n d -> b n (h d)') - return self.to_out(out_x) - - -class Physics_Attention_Structured_Mesh_3D(nn.Module): - ## for structured mesh in 3D space - def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=32, H=32, W=32, D=32, kernel=3): - super().__init__() - inner_dim = dim_head * heads - self.dim_head = dim_head - self.heads = heads - self.scale = dim_head ** -0.5 - self.softmax = nn.Softmax(dim=-1) - self.dropout = nn.Dropout(dropout) - self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5) - self.H = H - self.W = W - self.D = D - - self.in_project_x = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2) - self.in_project_fx = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2) - self.in_project_slice = nn.Linear(dim_head, slice_num) - for l in [self.in_project_slice]: - torch.nn.init.orthogonal_(l.weight) # use a principled initialization - self.to_q = nn.Linear(dim_head, dim_head, bias=False) - self.to_k = nn.Linear(dim_head, dim_head, bias=False) - self.to_v = nn.Linear(dim_head, dim_head, bias=False) - self.to_out = nn.Sequential( - nn.Linear(inner_dim, dim), - nn.Dropout(dropout) - ) - - def forward(self, x): - # B N C - B, N, C = x.shape - x = x.reshape(B, self.H, self.W, self.D, C).contiguous().permute(0, 4, 1, 2, 3).contiguous() # B C H W - - ### (1) Slice - fx_mid = self.in_project_fx(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \ - .permute(0, 2, 1, 3).contiguous() # B H N C - x_mid = self.in_project_x(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \ - .permute(0, 2, 1, 3).contiguous() # B H N G - slice_weights = self.softmax( - self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G - slice_norm = slice_weights.sum(2) # B H G - slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights) - slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head)) - - ### (2) Attention among slice tokens - q_slice_token = self.to_q(slice_token) - k_slice_token = self.to_k(slice_token) - v_slice_token = self.to_v(slice_token) - dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale - attn = self.softmax(dots) - attn = self.dropout(attn) - out_slice_token = torch.matmul(attn, v_slice_token) # B H G D - - ### (3) Deslice - out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights) - out_x = rearrange(out_x, 'b h n d -> b n (h d)') - return self.to_out(out_x) --- upstream/Transolver_Irregular_Mesh.py +++ blocks.py @@ -1,9 +1,7 @@ +# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE. import torch import torch.nn as nn -from timm.models.layers import trunc_normal_ -from model.Embedding import timestep_embedding -import numpy as np -from model.Physics_Attention import Physics_Attention_Irregular_Mesh +from .attention import Physics_Attention_Irregular_Mesh ACTIVATION = {'gelu': nn.GELU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid, 'relu': nn.ReLU, 'leaky_relu': nn.LeakyReLU(0.1), 'softplus': nn.Softplus, 'ELU': nn.ELU, 'silu': nn.SiLU} @@ -71,88 +69,3 @@ return fx -class Model(nn.Module): - def __init__(self, - space_dim=1, - n_layers=5, - n_hidden=256, - dropout=0.0, - n_head=8, - Time_Input=False, - act='gelu', - mlp_ratio=1, - fun_dim=1, - out_dim=1, - slice_num=32, - ref=8, - unified_pos=False - ): - super(Model, self).__init__() - self.__name__ = 'Transolver_1D' - self.ref = ref - self.unified_pos = unified_pos - self.Time_Input = Time_Input - self.n_hidden = n_hidden - self.space_dim = space_dim - if self.unified_pos: - self.preprocess = MLP(fun_dim + self.ref * self.ref, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act) - else: - self.preprocess = MLP(fun_dim + space_dim, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act) - if Time_Input: - self.time_fc = nn.Sequential(nn.Linear(n_hidden, n_hidden), nn.SiLU(), nn.Linear(n_hidden, n_hidden)) - - self.blocks = nn.ModuleList([Transolver_block(num_heads=n_head, hidden_dim=n_hidden, - dropout=dropout, - act=act, - mlp_ratio=mlp_ratio, - out_dim=out_dim, - slice_num=slice_num, - last_layer=(_ == n_layers - 1)) - for _ in range(n_layers)]) - self.initialize_weights() - self.placeholder = nn.Parameter((1 / (n_hidden)) * torch.rand(n_hidden, dtype=torch.float)) - - def initialize_weights(self): - self.apply(self._init_weights) - - def _init_weights(self, m): - if isinstance(m, nn.Linear): - trunc_normal_(m.weight, std=0.02) - if isinstance(m, nn.Linear) and m.bias is not None: - nn.init.constant_(m.bias, 0) - elif isinstance(m, (nn.LayerNorm, nn.BatchNorm1d)): - nn.init.constant_(m.bias, 0) - nn.init.constant_(m.weight, 1.0) - - def get_grid(self, x, batchsize=1): - # x: B N 2 - # grid_ref - gridx = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float) - gridx = gridx.reshape(1, self.ref, 1, 1).repeat([batchsize, 1, self.ref, 1]) - gridy = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float) - gridy = gridy.reshape(1, 1, self.ref, 1).repeat([batchsize, self.ref, 1, 1]) - grid_ref = torch.cat((gridx, gridy), dim=-1).cuda().reshape(batchsize, self.ref * self.ref, 2) # B H W 8 8 2 - - pos = torch.sqrt(torch.sum((x[:, :, None, :] - grid_ref[:, None, :, :]) ** 2, dim=-1)). \ - reshape(batchsize, x.shape[1], self.ref * self.ref).contiguous() - return pos - - def forward(self, x, fx, T=None): - if self.unified_pos: - x = self.get_grid(x, x.shape[0]) - if fx is not None: - fx = torch.cat((x, fx), -1) - fx = self.preprocess(fx) - else: - fx = self.preprocess(x) - fx = fx + self.placeholder[None, None, :] - - if T is not None: - Time_emb = timestep_embedding(T, self.n_hidden).repeat(1, x.shape[1], 1) - Time_emb = self.time_fc(Time_emb) - fx = fx + Time_emb - - for block in self.blocks: - fx = block(fx) - - return fx --- /dev/null +++ network.py @@ -0,0 +1,119 @@ +"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders. + +No optimiser, training loop, data reader or GPU invocation is provided. +""" +import torch +from torch import nn + +from .blocks import MLP, Transolver_block +from .features import BASE_COLUMNS +from .normalization import BaseStandardizer + + +TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5, + schedule="cosine", batch_parts=1, max_epochs=200, + patience=20, seeds=[0, 1, 2], widths=[128, 256]) + + +def check_tensor(tensor, shape, name): + if tuple(tensor.shape) != tuple(shape) or not torch.isfinite(tensor).all(): + raise ValueError(f"invalid {name}") + + +class WholePartEncoder(nn.Module): + def __init__(self, width, standardizer): + super().__init__() + if width not in (128, 256): + raise ValueError("S2 permits widths 128 or 256 only") + if not isinstance(standardizer, BaseStandardizer): + raise ValueError("frozen learning-pool standardizer required") + self.standardizer = standardizer + self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64)) + self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64)) + self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False) + self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width, + dropout=0., mlp_ratio=1, slice_num=32, + last_layer=False) for _ in range(4)]) + self.apply(self._initialize) + self.placeholder = nn.Parameter(torch.rand(width)/width) + + @staticmethod + def _initialize(module): + # torch.nn.init replaces timm's same trunc_normal_ defaults; no timm dependency. + if isinstance(module, nn.Linear): + nn.init.trunc_normal_(module.weight, std=.02) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif isinstance(module, nn.LayerNorm): + nn.init.zeros_(module.bias) + nn.init.ones_(module.weight) + + def forward(self, base): + if base.ndim != 2 or base.shape[0] == 0: + raise ValueError("one whole part with all tet nodes required") + check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs") + # Keep normalisation in FP32 even under future mixed-precision training. + base = (base.float()-self.base_mean.float()) / self.base_scale.float() + if not torch.isfinite(base).all(): + raise ValueError("nonfinite standardized base input") + fx = self.preprocess(base[None]) + self.placeholder[None, None] + for block in self.blocks: + fx = block(fx) + if not torch.isfinite(fx).all(): + raise ValueError("nonfinite encoder output") + return fx[0] + + +class QueryModel(nn.Module): + def __init__(self, arm, width=128, *, standardizer): + super().__init__() + if arm not in ("N", "E"): + raise ValueError("arm must be N or E") + self.arm, self.width = arm, width + self.encoder = WholePartEncoder(width, standardizer) + query_dim = 2 if arm == "N" else 6 + self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(), + nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1)) + self.decoder.apply(WholePartEncoder._initialize) + + def forward(self, base, query_ids, gaps, edge_q=None): + if query_ids.ndim != 1 or query_ids.dtype != torch.long or query_ids.numel() == 0: + raise ValueError("nonempty int64 original-node query IDs required") + if int(query_ids.min()) < 0 or int(query_ids.max()) >= len(base): + raise ValueError("query node outside whole part") + check_tensor(gaps, (len(query_ids), 2), "g12/g23") + if self.arm == "E": + if edge_q is None: + raise ValueError("E requires all four edge summaries") + check_tensor(edge_q, (len(query_ids), 4), "edge summaries") + local = torch.cat([gaps, edge_q], dim=-1) + else: + if edge_q is not None: + raise ValueError("N must not receive edge summaries") + local = gaps + embedding = self.encoder(base)[query_ids] + y = self.decoder(torch.cat([embedding, local], dim=-1)).squeeze(-1) + if not torch.isfinite(y).all(): + raise ValueError("nonfinite decoder output") + return y + + def parameter_counts(self): + return {"encoder": sum(p.numel() for p in self.encoder.parameters()), + "decoder": sum(p.numel() for p in self.decoder.parameters()), + "total": sum(p.numel() for p in self.parameters())} + + +def weighted_mse(prediction, target, global_weights): + """Sum of contributions with already globally-normalised hierarchy weights. + + Do not renormalise a single part's weights: that would erase component balance. + """ + if prediction.ndim != 1 or prediction.numel() == 0: + raise ValueError("nonempty query predictions required") + check_tensor(prediction, prediction.shape, "prediction") + check_tensor(target, prediction.shape, "target") + check_tensor(global_weights, prediction.shape, "weights") + if (global_weights <= 0).any(): + raise ValueError("positive hierarchy weights required") + # FP32 arithmetic under future AMP; do not form residuals in float16. + return torch.sum(global_weights.float() * (prediction.float()-target.float()).square())