205 lines
9.5 KiB
Python
205 lines
9.5 KiB
Python
"""Transolver (Wu et al., ICML 2024) adapter for grouped Greif fill-time comparison.
|
|
|
|
Pure unstructured-node Physics-Attention model matching H2Geo and GreifGINO:
|
|
exact 19 standardized FP32 inputs, positive nodal mass, FP64 volume-clock
|
|
residual target/decode, and zero-initialized readout starting at the c*Vf prior.
|
|
Code only: no real-data I/O, no GPU launch, no external checkpoint access.
|
|
"""
|
|
import math
|
|
import numpy as np
|
|
import torch
|
|
from torch import nn
|
|
from torch.utils.checkpoint import checkpoint
|
|
|
|
from phase2_h2_model import lr_epoch, optimizer, weighted_loss
|
|
from phase2_residual_math import decode_seconds, learning_clock, prior, residual_target
|
|
from track_b_gino_greif_adapter import COLUMNS
|
|
|
|
NODE_CHUNK = 8192
|
|
TEMP_MIN = 0.1
|
|
TEMP_MAX = 5.0
|
|
MASS_EPS = 1e-6
|
|
UNWEIGHTED_EPS = 1e-5
|
|
VARIANTS = {
|
|
'default': dict(dim=64, depth=6, heads=4, slices=64, mlp_ratio=2),
|
|
'wide': dict(dim=96, depth=4, heads=4, slices=64, mlp_ratio=2),
|
|
}
|
|
|
|
|
|
def count_parameters(model):
|
|
return sum(p.numel() for p in model.parameters())
|
|
|
|
|
|
def standardize_learning_arrays(feature_arrays):
|
|
"""FP64 online population mean/std over learning parts only (matches phase2_h2_data)."""
|
|
n = 0
|
|
mean = np.zeros(len(COLUMNS), dtype=np.float64)
|
|
m2 = np.zeros(len(COLUMNS), dtype=np.float64)
|
|
for arr in feature_arrays:
|
|
x = np.asarray(arr, dtype=np.float64)
|
|
if x.ndim != 2 or x.shape[1] != len(COLUMNS) or len(x) == 0 or not np.isfinite(x).all():
|
|
raise ValueError('Invalid learning feature array')
|
|
k = len(x)
|
|
avg = x.mean(axis=0)
|
|
delta = avg - mean
|
|
m2 += np.square(x - avg).sum(axis=0) + delta * delta * (n * k / (n + k))
|
|
mean += delta * (k / (n + k))
|
|
n += k
|
|
if n == 0:
|
|
raise ValueError('Empty learning set')
|
|
std = np.sqrt(m2 / n)
|
|
scale = np.where(std == 0.0, 1.0, std)
|
|
return dict(mean=mean.tolist(), scale=scale.tolist(), population_std=std.tolist(), nodes=n)
|
|
|
|
|
|
def prepare_inputs(features, mass, stats, component_ids=None):
|
|
"""Standardize 19 columns in FP64 -> FP32 and validate positive mass and components."""
|
|
feat = np.asarray(features, dtype=np.float64)
|
|
m = np.asarray(mass, dtype=np.float64)
|
|
if feat.ndim != 2 or feat.shape[1] != len(COLUMNS) or len(feat) == 0:
|
|
raise ValueError('Expected nonempty (N, 19) feature matrix matching COLUMNS')
|
|
if m.shape != (len(feat),) or not np.isfinite(m).all() or np.any(m <= 0):
|
|
raise ValueError('Expected finite strictly positive nodal mass')
|
|
x = ((feat - np.asarray(stats['mean'], np.float64)) / np.asarray(stats['scale'], np.float64)).astype(np.float32)
|
|
if not np.isfinite(x).all():
|
|
raise ValueError('Nonfinite standardized features')
|
|
if component_ids is None:
|
|
comp = np.zeros(len(feat), dtype=np.int64)
|
|
else:
|
|
comp = np.asarray(component_ids, dtype=np.int64)
|
|
if comp.shape != (len(feat),):
|
|
raise ValueError('component_ids shape mismatch')
|
|
return torch.from_numpy(x), torch.from_numpy(m.astype(np.float32)), torch.from_numpy(comp)
|
|
|
|
|
|
class PhysicsAttention(nn.Module):
|
|
"""Chunked multi-head Physics-Attention over learned slices (Wu et al., ICML 2024)."""
|
|
|
|
def __init__(self, dim=64, heads=4, slices=64, mass_weighted=True):
|
|
super().__init__()
|
|
if dim % heads != 0 or slices <= 0:
|
|
raise ValueError('dim must be divisible by heads and slices > 0')
|
|
self.dim = dim
|
|
self.heads = heads
|
|
self.head_dim = dim // heads
|
|
self.slices = slices
|
|
self.mass_weighted = bool(mass_weighted)
|
|
self.scale = self.head_dim ** -0.5
|
|
|
|
self.in_project_x = nn.Linear(dim, dim)
|
|
self.in_project_fx = nn.Linear(dim, dim)
|
|
self.in_project_slice = nn.Linear(self.head_dim, slices)
|
|
nn.init.orthogonal_(self.in_project_slice.weight)
|
|
self.temperature = nn.Parameter(torch.full((1, heads, 1), 0.5))
|
|
self.to_q = nn.Linear(self.head_dim, self.head_dim, bias=False)
|
|
self.to_k = nn.Linear(self.head_dim, self.head_dim, bias=False)
|
|
self.to_v = nn.Linear(self.head_dim, self.head_dim, bias=False)
|
|
self.to_out = nn.Linear(dim, dim)
|
|
|
|
def forward(self, h, m, components, node_chunk):
|
|
out = h.new_zeros(h.shape)
|
|
temp = torch.clamp(self.temperature, min=TEMP_MIN, max=TEMP_MAX)
|
|
eps = MASS_EPS if self.mass_weighted else UNWEIGHTED_EPS
|
|
for ids in components:
|
|
m_comp = m[ids]
|
|
m_sum = m_comp.sum()
|
|
if not bool(torch.isfinite(m_sum)) or float(m_sum) <= 0.0:
|
|
raise ValueError('Invalid component mass sum')
|
|
m_norm = m_comp / m_sum
|
|
num = h.new_zeros((self.heads, self.slices, self.head_dim))
|
|
den = h.new_zeros((self.heads, self.slices))
|
|
chunks = []
|
|
for lo in range(0, len(ids), node_chunk):
|
|
hi = lo + node_chunk
|
|
ix = ids[lo:hi]
|
|
hh = h[ix]
|
|
x_mid = self.in_project_x(hh).view(-1, self.heads, self.head_dim)
|
|
fx_mid = self.in_project_fx(hh).view(-1, self.heads, self.head_dim)
|
|
w = torch.softmax(self.in_project_slice(x_mid) / temp, dim=-1)
|
|
weight = w * m_norm[lo:hi, None, None] if self.mass_weighted else w
|
|
num = num + torch.einsum('nhg,nhd->hgd', weight, fx_mid)
|
|
den = den + weight.sum(dim=0)
|
|
chunks.append((ix, w))
|
|
if not bool(torch.isfinite(den).all()):
|
|
raise ValueError('Nonfinite physics-slice denominator')
|
|
tokens = num / (den.unsqueeze(-1) + eps)
|
|
q, k, v = self.to_q(tokens), self.to_k(tokens), self.to_v(tokens)
|
|
attn = torch.softmax(torch.matmul(q, k.transpose(-2, -1)) * self.scale, dim=-1)
|
|
z = torch.matmul(attn, v)
|
|
for ix, w in chunks:
|
|
desliced = torch.einsum('nhg,hgd->nhd', w, z).reshape(-1, self.dim)
|
|
out.index_copy_(0, ix, self.to_out(desliced))
|
|
return out
|
|
|
|
|
|
class TransolverBlock(nn.Module):
|
|
def __init__(self, dim=64, heads=4, slices=64, mlp_ratio=2, mass_weighted=True):
|
|
super().__init__()
|
|
self.attn_norm = nn.LayerNorm(dim)
|
|
self.attn = PhysicsAttention(dim=dim, heads=heads, slices=slices, mass_weighted=mass_weighted)
|
|
self.ffn_norm = nn.LayerNorm(dim)
|
|
self.ffn = nn.Sequential(nn.Linear(dim, dim * mlp_ratio), nn.GELU(), nn.Linear(dim * mlp_ratio, dim))
|
|
|
|
def forward(self, h, m, components, node_chunk):
|
|
h = h + self.attn(self.attn_norm(h), m, components, node_chunk)
|
|
return h + self.ffn(self.ffn_norm(h))
|
|
|
|
|
|
class GreifTransolver(nn.Module):
|
|
"""Per-part Transolver residual regressor aligned with H2Geo and GreifGINO."""
|
|
|
|
def __init__(self, dim=64, depth=6, heads=4, slices=64, mlp_ratio=2, mass_weighted=True):
|
|
super().__init__()
|
|
self.dim = dim
|
|
self.depth = depth
|
|
self.heads = heads
|
|
self.slices = slices
|
|
self.mlp_ratio = mlp_ratio
|
|
self.mass_weighted = bool(mass_weighted)
|
|
self.encoder = nn.Sequential(nn.Linear(len(COLUMNS), dim), nn.GELU(), nn.Linear(dim, dim))
|
|
self.blocks = nn.ModuleList([
|
|
TransolverBlock(dim=dim, heads=heads, slices=slices, mlp_ratio=mlp_ratio, mass_weighted=mass_weighted)
|
|
for _ in range(depth)
|
|
])
|
|
self.head = nn.Sequential(nn.Linear(dim + len(COLUMNS), 64), nn.GELU(), nn.Linear(64, 1))
|
|
nn.init.zeros_(self.head[-1].weight)
|
|
nn.init.zeros_(self.head[-1].bias)
|
|
|
|
def forward(self, x, m, component_ids=None, *, node_chunk=NODE_CHUNK, checkpoint_blocks=True):
|
|
if x.ndim != 2 or x.shape[1] != len(COLUMNS) or len(x) == 0:
|
|
raise ValueError('Expected nonempty (N, 19) FP32 features')
|
|
if x.dtype != torch.float32 or m.dtype != torch.float32:
|
|
raise ValueError('Only FP32 features and mass are supported')
|
|
if m.shape != (len(x),) or not bool(torch.isfinite(x).all()) or not bool(torch.isfinite(m).all()) or bool((m <= 0).any()):
|
|
raise ValueError('Invalid features or nonpositive nodal mass')
|
|
if component_ids is None:
|
|
components = [torch.arange(len(x), device=x.device, dtype=torch.long)]
|
|
else:
|
|
if component_ids.shape != (len(x),) or component_ids.dtype not in (torch.int32, torch.int64):
|
|
raise ValueError('component_ids must be 1D integer tensor matching nodes')
|
|
components = [torch.nonzero(component_ids == c, as_tuple=True)[0] for c in torch.unique(component_ids)]
|
|
h = self.encoder(x)
|
|
for block in self.blocks:
|
|
def run(v, b=block):
|
|
return b(v, m, components, node_chunk)
|
|
if checkpoint_blocks and self.training and torch.is_grad_enabled():
|
|
h = checkpoint(run, h, use_reentrant=False, preserve_rng_state=False)
|
|
else:
|
|
h = run(h)
|
|
outputs = [self.head(torch.cat((h[lo:lo + node_chunk], x[lo:lo + node_chunk]), dim=-1))[:, 0]
|
|
for lo in range(0, len(x), node_chunk)]
|
|
return torch.cat(outputs)
|
|
|
|
|
|
def make_greif_transolver(variant='default', *, mass_weighted=True):
|
|
if variant not in VARIANTS:
|
|
raise ValueError(f'Unknown variant: {variant}')
|
|
return GreifTransolver(**VARIANTS[variant], mass_weighted=mass_weighted)
|
|
|
|
|
|
__all__ = [
|
|
'COLUMNS', 'NODE_CHUNK', 'VARIANTS', 'GreifTransolver', 'PhysicsAttention', 'TransolverBlock',
|
|
'count_parameters', 'decode_seconds', 'learning_clock', 'lr_epoch', 'make_greif_transolver',
|
|
'optimizer', 'prepare_inputs', 'prior', 'residual_target', 'standardize_learning_arrays', 'weighted_loss',
|
|
]
|