Add Transolver-Greif adapter, CPU synthetic tests, and README (TRANSOLVER-ADAPTER-1122)
This commit is contained in:
1 parent
591299e0b2
commit
727a01bc3f
3 files changed
+389
No files matched your search
@@ -0,0 +1,155 @@
|
||||
"""CPU-only synthetic tests for track_b_transolver_greif_adapter (no real data, no GPU)."""
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import phase2_h2_model as h2
|
||||
import track_b_gino_greif_adapter as gino
|
||||
import track_b_transolver_greif_adapter as tv
|
||||
|
||||
|
||||
def synthetic_tet_part(seed, nx=4, ny=4, nz=3, second_component=False):
|
||||
"""Build a synthetic 3D tet mesh with lumped positive nodal masses and 19 features."""
|
||||
rng = np.random.default_rng(seed)
|
||||
xs, ys, zs = np.meshgrid(np.linspace(0.0, 1.0, nx), np.linspace(0.0, 0.8, ny), np.linspace(0.0, 0.4, nz), indexing='ij')
|
||||
pts = np.column_stack([xs.ravel(), ys.ravel(), zs.ravel()])
|
||||
pts += rng.normal(0.0, 0.01, size=pts.shape)
|
||||
n0 = len(pts)
|
||||
idx = np.arange(n0).reshape(nx, ny, nz)
|
||||
tets = []
|
||||
for i in range(nx - 1):
|
||||
for j in range(ny - 1):
|
||||
for k in range(nz - 1):
|
||||
c = [idx[i + a, j + b, k + c_] for a in (0, 1) for b in (0, 1) for c_ in (0, 1)]
|
||||
for t in ((0, 1, 3, 7), (0, 1, 5, 7), (0, 4, 5, 7), (0, 2, 3, 7), (0, 2, 6, 7), (0, 4, 6, 7)):
|
||||
tets.append([c[p] for p in t])
|
||||
tets = np.asarray(tets, dtype=np.int64)
|
||||
comp = np.zeros(n0, dtype=np.int64)
|
||||
if second_component:
|
||||
pts2 = pts[:12] + np.array([2.0, 0.0, 0.0])
|
||||
pts = np.vstack([pts, pts2])
|
||||
extra = np.array([[n0 + i, n0 + i + 1, n0 + i + 2, n0 + i + 3] for i in range(0, 9, 3)], dtype=np.int64)
|
||||
tets = np.vstack([tets, extra])
|
||||
comp = np.concatenate([comp, np.ones(len(pts2), dtype=np.int64)])
|
||||
n = len(pts)
|
||||
mass = np.zeros(n, dtype=np.float64)
|
||||
v0, v1, v2, v3 = pts[tets[:, 0]], pts[tets[:, 1]], pts[tets[:, 2]], pts[tets[:, 3]]
|
||||
vol = np.abs(np.einsum('ij,ij->i', v1 - v0, np.cross(v2 - v0, v3 - v0))) / 6.0
|
||||
vol = np.maximum(vol, 1e-6)
|
||||
for corner in range(4):
|
||||
np.add.at(mass, tets[:, corner], vol / 4.0)
|
||||
mass = np.maximum(mass, 1e-7)
|
||||
vf = np.clip(np.argsort(np.argsort(pts[:, 0])).astype(np.float64) / max(n - 1, 1), 0.0, 1.0)
|
||||
feats = rng.normal(0.0, 1.0, size=(n, len(tv.COLUMNS))).astype(np.float64)
|
||||
feats[:, 0] = vf
|
||||
target_star = np.clip(0.95 * vf + 0.05 * rng.normal(size=n), 0.0, None)
|
||||
return dict(points=pts, tets=tets, mass=mass, features=feats, Vf=vf, target_star=target_star, component_ids=comp)
|
||||
|
||||
|
||||
class TestTransolverGreifAdapter(unittest.TestCase):
|
||||
def setUp(self):
|
||||
torch.set_num_threads(1)
|
||||
self.parts = [synthetic_tet_part(10 + i, second_component=(i == 1)) for i in range(2)]
|
||||
self.stats = tv.standardize_learning_arrays([p['features'] for p in self.parts])
|
||||
self.clock = tv.learning_clock([0.96, 1.02], ['fam_a', 'fam_b'])
|
||||
|
||||
def test_interface_and_parameter_counts(self):
|
||||
self.assertEqual(len(tv.COLUMNS), 19)
|
||||
self.assertEqual(tv.COLUMNS, gino.COLUMNS)
|
||||
h2_params = tv.count_parameters(h2.H2Geo())
|
||||
default_params = tv.count_parameters(tv.make_greif_transolver('default'))
|
||||
wide_params = tv.count_parameters(tv.make_greif_transolver('wide'))
|
||||
unweighted_params = tv.count_parameters(tv.make_greif_transolver('default', mass_weighted=False))
|
||||
self.assertEqual(h2_params, 195077)
|
||||
self.assertEqual(default_params, 197913)
|
||||
self.assertEqual(wide_params, 293937)
|
||||
self.assertEqual(unweighted_params, default_params)
|
||||
|
||||
def test_zero_init_starts_at_exact_volume_clock_prior(self):
|
||||
torch.manual_seed(0)
|
||||
model = tv.make_greif_transolver('default')
|
||||
p = self.parts[0]
|
||||
x, m, comp = tv.prepare_inputs(p['features'], p['mass'], self.stats, p['component_ids'])
|
||||
pred = model(x, m, comp, checkpoint_blocks=False)
|
||||
self.assertEqual(pred.shape, (len(p['mass']),))
|
||||
self.assertEqual(pred.dtype, torch.float32)
|
||||
self.assertTrue(torch.allclose(pred, torch.zeros_like(pred), atol=0.0))
|
||||
decoded = tv.decode_seconds(pred.detach().numpy(), p['Vf'], self.clock, t_set=2.5)
|
||||
expected = tv.prior(p['Vf'], self.clock) * 2.5
|
||||
np.testing.assert_allclose(decoded, expected, rtol=0.0, atol=1e-12)
|
||||
|
||||
def test_forward_backward_step_and_multicomponent(self):
|
||||
torch.manual_seed(7)
|
||||
model = tv.make_greif_transolver('default')
|
||||
opt = tv.optimizer(model)
|
||||
total_nodes = sum(len(p['mass']) for p in self.parts)
|
||||
for p in self.parts:
|
||||
x, m, comp = tv.prepare_inputs(p['features'], p['mass'], self.stats, p['component_ids'])
|
||||
target = torch.from_numpy(tv.residual_target(p['target_star'], p['Vf'], self.clock, fit_dtype='float32'))
|
||||
opt.zero_grad(set_to_none=True)
|
||||
pred = model(x, m, comp, node_chunk=17, checkpoint_blocks=True)
|
||||
loss = tv.weighted_loss(pred, target, len(self.parts), len(m), total_nodes)
|
||||
self.assertTrue(bool(torch.isfinite(loss)))
|
||||
loss.backward()
|
||||
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0, error_if_nonfinite=True)
|
||||
opt.step()
|
||||
self.assertTrue(all(bool(torch.isfinite(v).all()) for v in model.parameters()))
|
||||
# After 1 step, head output moves away from zero and gradients reached encoder + physics slices.
|
||||
self.assertGreater(float(model.head[-1].weight.detach().abs().max()), 0.0)
|
||||
x, m, comp = tv.prepare_inputs(self.parts[1]['features'], self.parts[1]['mass'], self.stats, self.parts[1]['component_ids'])
|
||||
out = model(x, m, comp, checkpoint_blocks=False)
|
||||
out.sum().backward()
|
||||
self.assertGreater(float(model.encoder[0].weight.grad.abs().max()), 0.0)
|
||||
self.assertGreater(float(model.blocks[0].attn.in_project_slice.weight.grad.abs().max()), 0.0)
|
||||
|
||||
def test_determinism_chunk_invariance_and_mass_scale_invariance(self):
|
||||
p = self.parts[1]
|
||||
x, m, comp = tv.prepare_inputs(p['features'], p['mass'], self.stats, p['component_ids'])
|
||||
target = torch.from_numpy(tv.residual_target(p['target_star'], p['Vf'], self.clock, fit_dtype='float32'))
|
||||
|
||||
def run_once(chunk, mass_scale=1.0):
|
||||
torch.manual_seed(1234)
|
||||
model = tv.make_greif_transolver('default')
|
||||
opt = tv.optimizer(model)
|
||||
for _ in range(2):
|
||||
opt.zero_grad(set_to_none=True)
|
||||
pred = model(x, m * mass_scale, comp, node_chunk=chunk, checkpoint_blocks=False)
|
||||
loss = tv.weighted_loss(pred, target, 1, len(m), len(m))
|
||||
loss.backward()
|
||||
opt.step()
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
return model(x, m * mass_scale, comp, node_chunk=chunk, checkpoint_blocks=False)
|
||||
|
||||
a = run_once(chunk=tv.NODE_CHUNK)
|
||||
b = run_once(chunk=tv.NODE_CHUNK)
|
||||
c = run_once(chunk=13)
|
||||
d = run_once(chunk=tv.NODE_CHUNK, mass_scale=1e-4)
|
||||
self.assertTrue(torch.equal(a, b))
|
||||
self.assertTrue(torch.allclose(a, c, atol=1e-6, rtol=1e-5))
|
||||
self.assertTrue(torch.allclose(a, d, atol=1e-6, rtol=1e-5))
|
||||
|
||||
def test_input_guards(self):
|
||||
model = tv.make_greif_transolver('default')
|
||||
p = self.parts[0]
|
||||
x, m, comp = tv.prepare_inputs(p['features'], p['mass'], self.stats, p['component_ids'])
|
||||
with self.assertRaises(ValueError):
|
||||
model(x[:, :18], m, comp)
|
||||
with self.assertRaises(ValueError):
|
||||
model(x.double(), m, comp)
|
||||
bad_m = m.clone()
|
||||
bad_m[0] = 0.0
|
||||
with self.assertRaises(ValueError):
|
||||
model(x, bad_m, comp)
|
||||
bad_x = x.clone()
|
||||
bad_x[0, 0] = float('nan')
|
||||
with self.assertRaises(ValueError):
|
||||
model(bad_x, m, comp)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Transolver (Wu et al., ICML 2024) adapter for grouped Greif fill-time comparison.
|
||||
|
||||
Pure unstructured-node Physics-Attention model matching H2Geo and GreifGINO:
|
||||
exact 19 standardized FP32 inputs, positive nodal mass, FP64 volume-clock
|
||||
residual target/decode, and zero-initialized readout starting at the c*Vf prior.
|
||||
Code only: no real-data I/O, no GPU launch, no external checkpoint access.
|
||||
"""
|
||||
import math
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.utils.checkpoint import checkpoint
|
||||
|
||||
from phase2_h2_model import lr_epoch, optimizer, weighted_loss
|
||||
from phase2_residual_math import decode_seconds, learning_clock, prior, residual_target
|
||||
from track_b_gino_greif_adapter import COLUMNS
|
||||
|
||||
NODE_CHUNK = 8192
|
||||
TEMP_MIN = 0.1
|
||||
TEMP_MAX = 5.0
|
||||
MASS_EPS = 1e-6
|
||||
UNWEIGHTED_EPS = 1e-5
|
||||
VARIANTS = {
|
||||
'default': dict(dim=64, depth=6, heads=4, slices=64, mlp_ratio=2),
|
||||
'wide': dict(dim=96, depth=4, heads=4, slices=64, mlp_ratio=2),
|
||||
}
|
||||
|
||||
|
||||
def count_parameters(model):
|
||||
return sum(p.numel() for p in model.parameters())
|
||||
|
||||
|
||||
def standardize_learning_arrays(feature_arrays):
|
||||
"""FP64 online population mean/std over learning parts only (matches phase2_h2_data)."""
|
||||
n = 0
|
||||
mean = np.zeros(len(COLUMNS), dtype=np.float64)
|
||||
m2 = np.zeros(len(COLUMNS), dtype=np.float64)
|
||||
for arr in feature_arrays:
|
||||
x = np.asarray(arr, dtype=np.float64)
|
||||
if x.ndim != 2 or x.shape[1] != len(COLUMNS) or len(x) == 0 or not np.isfinite(x).all():
|
||||
raise ValueError('Invalid learning feature array')
|
||||
k = len(x)
|
||||
avg = x.mean(axis=0)
|
||||
delta = avg - mean
|
||||
m2 += np.square(x - avg).sum(axis=0) + delta * delta * (n * k / (n + k))
|
||||
mean += delta * (k / (n + k))
|
||||
n += k
|
||||
if n == 0:
|
||||
raise ValueError('Empty learning set')
|
||||
std = np.sqrt(m2 / n)
|
||||
scale = np.where(std == 0.0, 1.0, std)
|
||||
return dict(mean=mean.tolist(), scale=scale.tolist(), population_std=std.tolist(), nodes=n)
|
||||
|
||||
|
||||
def prepare_inputs(features, mass, stats, component_ids=None):
|
||||
"""Standardize 19 columns in FP64 -> FP32 and validate positive mass and components."""
|
||||
feat = np.asarray(features, dtype=np.float64)
|
||||
m = np.asarray(mass, dtype=np.float64)
|
||||
if feat.ndim != 2 or feat.shape[1] != len(COLUMNS) or len(feat) == 0:
|
||||
raise ValueError('Expected nonempty (N, 19) feature matrix matching COLUMNS')
|
||||
if m.shape != (len(feat),) or not np.isfinite(m).all() or np.any(m <= 0):
|
||||
raise ValueError('Expected finite strictly positive nodal mass')
|
||||
x = ((feat - np.asarray(stats['mean'], np.float64)) / np.asarray(stats['scale'], np.float64)).astype(np.float32)
|
||||
if not np.isfinite(x).all():
|
||||
raise ValueError('Nonfinite standardized features')
|
||||
if component_ids is None:
|
||||
comp = np.zeros(len(feat), dtype=np.int64)
|
||||
else:
|
||||
comp = np.asarray(component_ids, dtype=np.int64)
|
||||
if comp.shape != (len(feat),):
|
||||
raise ValueError('component_ids shape mismatch')
|
||||
return torch.from_numpy(x), torch.from_numpy(m.astype(np.float32)), torch.from_numpy(comp)
|
||||
|
||||
|
||||
class PhysicsAttention(nn.Module):
|
||||
"""Chunked multi-head Physics-Attention over learned slices (Wu et al., ICML 2024)."""
|
||||
|
||||
def __init__(self, dim=64, heads=4, slices=64, mass_weighted=True):
|
||||
super().__init__()
|
||||
if dim % heads != 0 or slices <= 0:
|
||||
raise ValueError('dim must be divisible by heads and slices > 0')
|
||||
self.dim = dim
|
||||
self.heads = heads
|
||||
self.head_dim = dim // heads
|
||||
self.slices = slices
|
||||
self.mass_weighted = bool(mass_weighted)
|
||||
self.scale = self.head_dim ** -0.5
|
||||
|
||||
self.in_project_x = nn.Linear(dim, dim)
|
||||
self.in_project_fx = nn.Linear(dim, dim)
|
||||
self.in_project_slice = nn.Linear(self.head_dim, slices)
|
||||
nn.init.orthogonal_(self.in_project_slice.weight)
|
||||
self.temperature = nn.Parameter(torch.full((1, heads, 1), 0.5))
|
||||
self.to_q = nn.Linear(self.head_dim, self.head_dim, bias=False)
|
||||
self.to_k = nn.Linear(self.head_dim, self.head_dim, bias=False)
|
||||
self.to_v = nn.Linear(self.head_dim, self.head_dim, bias=False)
|
||||
self.to_out = nn.Linear(dim, dim)
|
||||
|
||||
def forward(self, h, m, components, node_chunk):
|
||||
out = h.new_zeros(h.shape)
|
||||
temp = torch.clamp(self.temperature, min=TEMP_MIN, max=TEMP_MAX)
|
||||
eps = MASS_EPS if self.mass_weighted else UNWEIGHTED_EPS
|
||||
for ids in components:
|
||||
m_comp = m[ids]
|
||||
m_sum = m_comp.sum()
|
||||
if not bool(torch.isfinite(m_sum)) or float(m_sum) <= 0.0:
|
||||
raise ValueError('Invalid component mass sum')
|
||||
m_norm = m_comp / m_sum
|
||||
num = h.new_zeros((self.heads, self.slices, self.head_dim))
|
||||
den = h.new_zeros((self.heads, self.slices))
|
||||
chunks = []
|
||||
for lo in range(0, len(ids), node_chunk):
|
||||
hi = lo + node_chunk
|
||||
ix = ids[lo:hi]
|
||||
hh = h[ix]
|
||||
x_mid = self.in_project_x(hh).view(-1, self.heads, self.head_dim)
|
||||
fx_mid = self.in_project_fx(hh).view(-1, self.heads, self.head_dim)
|
||||
w = torch.softmax(self.in_project_slice(x_mid) / temp, dim=-1)
|
||||
weight = w * m_norm[lo:hi, None, None] if self.mass_weighted else w
|
||||
num = num + torch.einsum('nhg,nhd->hgd', weight, fx_mid)
|
||||
den = den + weight.sum(dim=0)
|
||||
chunks.append((ix, w))
|
||||
if not bool(torch.isfinite(den).all()):
|
||||
raise ValueError('Nonfinite physics-slice denominator')
|
||||
tokens = num / (den.unsqueeze(-1) + eps)
|
||||
q, k, v = self.to_q(tokens), self.to_k(tokens), self.to_v(tokens)
|
||||
attn = torch.softmax(torch.matmul(q, k.transpose(-2, -1)) * self.scale, dim=-1)
|
||||
z = torch.matmul(attn, v)
|
||||
for ix, w in chunks:
|
||||
desliced = torch.einsum('nhg,hgd->nhd', w, z).reshape(-1, self.dim)
|
||||
out.index_copy_(0, ix, self.to_out(desliced))
|
||||
return out
|
||||
|
||||
|
||||
class TransolverBlock(nn.Module):
|
||||
def __init__(self, dim=64, heads=4, slices=64, mlp_ratio=2, mass_weighted=True):
|
||||
super().__init__()
|
||||
self.attn_norm = nn.LayerNorm(dim)
|
||||
self.attn = PhysicsAttention(dim=dim, heads=heads, slices=slices, mass_weighted=mass_weighted)
|
||||
self.ffn_norm = nn.LayerNorm(dim)
|
||||
self.ffn = nn.Sequential(nn.Linear(dim, dim * mlp_ratio), nn.GELU(), nn.Linear(dim * mlp_ratio, dim))
|
||||
|
||||
def forward(self, h, m, components, node_chunk):
|
||||
h = h + self.attn(self.attn_norm(h), m, components, node_chunk)
|
||||
return h + self.ffn(self.ffn_norm(h))
|
||||
|
||||
|
||||
class GreifTransolver(nn.Module):
|
||||
"""Per-part Transolver residual regressor aligned with H2Geo and GreifGINO."""
|
||||
|
||||
def __init__(self, dim=64, depth=6, heads=4, slices=64, mlp_ratio=2, mass_weighted=True):
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.depth = depth
|
||||
self.heads = heads
|
||||
self.slices = slices
|
||||
self.mlp_ratio = mlp_ratio
|
||||
self.mass_weighted = bool(mass_weighted)
|
||||
self.encoder = nn.Sequential(nn.Linear(len(COLUMNS), dim), nn.GELU(), nn.Linear(dim, dim))
|
||||
self.blocks = nn.ModuleList([
|
||||
TransolverBlock(dim=dim, heads=heads, slices=slices, mlp_ratio=mlp_ratio, mass_weighted=mass_weighted)
|
||||
for _ in range(depth)
|
||||
])
|
||||
self.head = nn.Sequential(nn.Linear(dim + len(COLUMNS), 64), nn.GELU(), nn.Linear(64, 1))
|
||||
nn.init.zeros_(self.head[-1].weight)
|
||||
nn.init.zeros_(self.head[-1].bias)
|
||||
|
||||
def forward(self, x, m, component_ids=None, *, node_chunk=NODE_CHUNK, checkpoint_blocks=True):
|
||||
if x.ndim != 2 or x.shape[1] != len(COLUMNS) or len(x) == 0:
|
||||
raise ValueError('Expected nonempty (N, 19) FP32 features')
|
||||
if x.dtype != torch.float32 or m.dtype != torch.float32:
|
||||
raise ValueError('Only FP32 features and mass are supported')
|
||||
if m.shape != (len(x),) or not bool(torch.isfinite(x).all()) or not bool(torch.isfinite(m).all()) or bool((m <= 0).any()):
|
||||
raise ValueError('Invalid features or nonpositive nodal mass')
|
||||
if component_ids is None:
|
||||
components = [torch.arange(len(x), device=x.device, dtype=torch.long)]
|
||||
else:
|
||||
if component_ids.shape != (len(x),) or component_ids.dtype not in (torch.int32, torch.int64):
|
||||
raise ValueError('component_ids must be 1D integer tensor matching nodes')
|
||||
components = [torch.nonzero(component_ids == c, as_tuple=True)[0] for c in torch.unique(component_ids)]
|
||||
h = self.encoder(x)
|
||||
for block in self.blocks:
|
||||
def run(v, b=block):
|
||||
return b(v, m, components, node_chunk)
|
||||
if checkpoint_blocks and self.training and torch.is_grad_enabled():
|
||||
h = checkpoint(run, h, use_reentrant=False, preserve_rng_state=False)
|
||||
else:
|
||||
h = run(h)
|
||||
outputs = [self.head(torch.cat((h[lo:lo + node_chunk], x[lo:lo + node_chunk]), dim=-1))[:, 0]
|
||||
for lo in range(0, len(x), node_chunk)]
|
||||
return torch.cat(outputs)
|
||||
|
||||
|
||||
def make_greif_transolver(variant='default', *, mass_weighted=True):
|
||||
if variant not in VARIANTS:
|
||||
raise ValueError(f'Unknown variant: {variant}')
|
||||
return GreifTransolver(**VARIANTS[variant], mass_weighted=mass_weighted)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'COLUMNS', 'NODE_CHUNK', 'VARIANTS', 'GreifTransolver', 'PhysicsAttention', 'TransolverBlock',
|
||||
'count_parameters', 'decode_seconds', 'learning_clock', 'lr_epoch', 'make_greif_transolver',
|
||||
'optimizer', 'prepare_inputs', 'prior', 'residual_target', 'standardize_learning_arrays', 'weighted_loss',
|
||||
]
|
||||
Reference in new issue
Block a user