Files
MeltMind/tools/test_s2_models.py
T

330 lines
17 KiB
Python

"""Synthetic-only acceptance. Run from tools; never loads any corpus geometry/field."""
import os
os.environ["CUDA_VISIBLE_DEVICES"] = ""
import json
from pathlib import Path
import unittest
from unittest.mock import patch
import numpy as np
import torch
from sklearn.ensemble import HistGradientBoostingRegressor
from s2_models import features as f, objectives as o
from s2_models.attention import Physics_Attention_Irregular_Mesh
from s2_models.network import QueryModel, weighted_mse
from s2_models.normalization import fit_base_standardizer
torch.set_num_threads(1)
def synthetic_star():
# One outer tetrahedron with its barycentre inserted: 4 volume tets.
x = np.array([[0., 0, 0], [1., 0, 0], [0., 1, 0], [0., 0, 1], [.25, .25, .25]])
t = np.array([[0, 1, 2, 4], [0, 1, 3, 4], [0, 2, 3, 4], [1, 2, 3, 4]])
return x, t, np.array([0., 0, 0, 0, .25])
def synthetic_standardizer():
return fit_base_standardizer({"synthetic-a": np.arange(70, dtype=float).reshape(7, 10)/10},
learning_part_ids=["synthetic-a"], fold="test-fold", pool="test-pool")
class GeometryTests(unittest.TestCase):
def test_volume_lumping(self):
x, t, _ = synthetic_star()
*_, volumes, mass, edges, lengths = f.mesh_geometry(x, t)
self.assertAlmostEqual(volumes.sum(), 1/6)
self.assertAlmostEqual(mass.sum(), 1/6)
self.assertEqual(len(edges), 10)
def test_wall_edges_removed_both_distances_and_gate_tie(self):
x, t, dw = synthetic_star()
di, df, source = f.graph_distances(x, t, dw, [0., 0, 0])
expected = np.linalg.norm(x[4]-x[0]) + np.linalg.norm(x[1]-x[4])
self.assertAlmostEqual(di[1], expected)
np.testing.assert_allclose(df, di/.125)
self.assertEqual(source, 0)
unrestricted, _ = f.graph_distances(x, t, dw, x[0], exclude_wall_edges=False)
self.assertAlmostEqual(unrestricted[1], 1.)
self.assertGreater(di[1], unrestricted[1])
# Tie between original node 0 and interior 4, with no closer node.
_, _, source = f.graph_distances(x, t, dw, (x[0]+x[4])/2)
self.assertEqual(source, 0)
def test_unreachable_fail_closed(self):
x, t, dw = synthetic_star()
with self.assertRaisesRegex(ValueError, "unreachable"):
f.graph_distances(x, t, np.zeros_like(dw), x[0])
def test_degenerate_and_unused_fail(self):
x, t, _ = synthetic_star()
with self.assertRaisesRegex(ValueError, "degenerate"):
f.mesh_geometry(x, np.array([[0, 0, 1, 2]]))
with self.assertRaisesRegex(ValueError, "unused"):
f.mesh_geometry(np.vstack([x, [2, 2, 2]]), t)
def test_strict_volume_front_ties(self):
actual = f.relative_flow_volume([0, 2, 2, 3], [1, 2, 3, 4])
np.testing.assert_allclose(actual, [0, .1, .1, .6])
def test_affine_p1_and_zero_gradient(self):
x, t, _ = synthetic_star()
a = np.array([2., -3., 4.])
np.testing.assert_allclose(f.p1_nodal_gradient(x, t, x@a+7, normalize=False), np.tile(a, (5, 1)), atol=1e-13)
np.testing.assert_allclose(f.p1_nodal_gradient(x, t, x@a+7), np.tile(a/np.linalg.norm(a), (5, 1)), atol=1e-13)
np.testing.assert_array_equal(f.p1_nodal_gradient(x, t, np.ones(5)), np.zeros((5, 3)))
def test_moment_volume_weight_rotation_and_filter(self):
x, t, _ = synthetic_star()
di = np.zeros(5)
s, radius = f.spatial_moment(x, t, di)
centers = x[t].mean(axis=1)
directions = centers-x[0]
directions /= np.linalg.norm(directions, axis=1)[:, None]
expected = np.mean([np.outer(v, v) for v in directions], axis=0)
np.testing.assert_allclose(s[0], expected)
self.assertAlmostEqual(radius, 3*np.sqrt(2))
q = np.array([[0., -1, 0], [1., 0, 0], [0., 0, 1]])
rotated, _ = f.spatial_moment(x@q.T+3, t, di)
np.testing.assert_allclose(rotated, q@s@q.T, atol=1e-14)
self.assertGreaterEqual(np.linalg.eigvalsh(s).min(), -1e-14)
np.testing.assert_allclose(np.trace(s, axis1=1, axis2=2), 1.)
# D_i exclusion can empty a neighbourhood; no fabricated fallback tensor.
with self.assertRaisesRegex(ValueError, "empty"):
f.spatial_moment(x, t, [100, 0, 0, 0, 0])
def test_features_and_physical_scaling(self):
x, t, dw = synthetic_star()
a = f.r2_features(x, t, dw, x[0], .4)
b = f.r2_features(2*x, t, 2*dw, 2*x[0], .4)
self.assertEqual(a["features"].shape, (5, 20))
np.testing.assert_allclose(b["Df"], a["Df"])
np.testing.assert_allclose(b["Di"], 2*a["Di"])
np.testing.assert_allclose(b["Vf"], a["Vf"])
np.testing.assert_allclose(b["S"], a["S"])
np.testing.assert_allclose(a["features"][:, -1], .4)
def test_moment_unequal_volumes_and_coincident_center(self):
x, t, _ = synthetic_star()
x[4] = [.125, .25, .125]
s, radius = f.spatial_moment(x, t, np.zeros(5))
_, _, _, v, _, _, _ = f.mesh_geometry(x, t)
direction = x[t].mean(axis=1) - x[0]
direction /= np.linalg.norm(direction, axis=1)[:, None]
expected = np.einsum('j,ja,jb->ab', v/v.sum(), direction, direction)
np.testing.assert_allclose(s[0], expected)
# Add a tetrahedron whose centre is exactly node 4. Its volume must
# be excluded from that node's denominator as well as its numerator.
x, t, _ = synthetic_star()
s0, _ = f.spatial_moment(x, t, np.zeros(5))
s1, _ = f.spatial_moment(x, np.vstack([t, [0, 1, 2, 3]]), np.zeros(5))
np.testing.assert_allclose(s1[4], s0[4])
self.assertAlmostEqual(np.trace(s1[4]), 1.)
def test_base_slots_and_nonfinite_fail(self):
x, t, dw = synthetic_star()
dg, _ = f.graph_distances(x, t, dw, x[0], exclude_wall_edges=False)
base = f.base_features(x, dw, dg, x[0], [0, 0, 0], 2, 1/6, .5, .4)
self.assertEqual(base.shape, (5, 10))
np.testing.assert_allclose(base[:, :3], x/2)
np.testing.assert_allclose(base[:, 3], dw/.5)
np.testing.assert_allclose(base[:, 5], dg/2)
with self.assertRaises(ValueError):
f.base_features(x, dw, dg*float("nan"), x[0], [0, 0, 0], 2, 1/6, .5, .4)
class ObjectiveTests(unittest.TestCase):
def test_hierarchy_and_draw_compression(self):
# A has two parts; B one. Their masses must be .25, .25, .5.
c, p, g = [1, 3, 2, 4], [0, 0, 1, 2], ["A", "A", "A", "B"]
w = o.hierarchy_weights(c, p, g)
np.testing.assert_allclose(w, [.0625, .1875, .25, .5])
pred = torch.tensor([1., 3, 2, 4])
loss = weighted_mse(pred, torch.zeros(4), torch.tensor(w))
expanded = o.hierarchy_weights(np.ones(10), np.repeat(p, c), np.repeat(g, c))
expected = np.sum(expanded*np.repeat(pred.numpy(), c)**2)
self.assertAlmostEqual(loss.item(), expected)
# Sum of part contributions equals pooled objective, without local renorm.
subtotal = sum(weighted_mse(pred[np.array(p)==i], torch.zeros(sum(np.array(p)==i)), torch.tensor(w[np.array(p)==i])).item() for i in [0, 1, 2])
self.assertAlmostEqual(subtotal, loss.item())
def test_invalid_hierarchy(self):
with self.assertRaises(ValueError):
o.hierarchy_weights([1, 1], [0, 0], ["A", "B"])
with self.assertRaises(ValueError):
o.hierarchy_weights([1, 0], [0, 1], ["A", "B"])
def test_process_target_roundtrip_and_scale_invariance(self):
seconds = np.array([.1, .2, .6])
target = o.dimensionless_target(seconds, .4)
np.testing.assert_allclose(o.prediction_seconds(target, .4), seconds)
score1 = o.relative_l2(seconds*1.1, seconds, [1, 3, 1], [True]*3)
score2 = o.relative_l2(target*1.1, target, [1, 3, 1], [True]*3)
self.assertAlmostEqual(score1["value"], score2["value"])
np.testing.assert_allclose(o.b0_seconds([0, .5, 1], .4), [0, .2, .4])
def test_zero_missing_scales_no_fallback(self):
for scale in [0., -1., float("nan")]:
with self.assertRaises(ValueError):
o.dimensionless_target([1.], scale)
with self.assertRaises(ValueError):
o.prediction_seconds([float("nan")], .4)
self.assertFalse(hasattr(o, "log_target"))
self.assertFalse(hasattr(o, "weighted_median"))
def test_score_masks_before_arithmetic_and_zero_truth(self):
result = o.relative_l2([2., np.nan], [1., np.nan], [2, 0], [True, False])
self.assertEqual(result["value"], 1.)
self.assertIsNone(o.relative_l2([1.], [0.], [1], [True])["value"])
def test_histgbrt_grid_unfitted(self):
grid = f.histgbrt_grid()
self.assertEqual(len(grid), 12)
self.assertEqual(len({json.dumps(c, sort_keys=True) for c in grid}), 12)
with patch.object(HistGradientBoostingRegressor, "fit", side_effect=AssertionError("fitting forbidden")):
for config in grid:
model = HistGradientBoostingRegressor(**config)
self.assertFalse(model.early_stopping)
self.assertFalse(hasattr(model, "n_iter_"))
class NormalizationTests(unittest.TestCase):
def test_unit_conversion_only_shifts_log_scale_columns(self):
x, t, dw = synthetic_star()
dg, _ = f.graph_distances(x, t, dw, x[0], exclude_wall_edges=False)
center = np.array([.2, .2, .2])
a = f.base_features(x, dw, dg, x[0], center, 2., 1/6, .5, .4)
length_scale, time_scale = 1000., 60.
b = f.base_features(x*length_scale, dw*length_scale, dg*length_scale,
x[0]*length_scale, center*length_scale,
2*length_scale, 1/6*length_scale**3, .5*length_scale, .4*time_scale)
expected = np.r_[np.zeros(6), np.log([length_scale, length_scale**3, length_scale, time_scale])]
np.testing.assert_allclose(b-a, np.tile(expected, (len(x), 1)), atol=2e-15)
aa = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
bb = fit_base_standardizer({"a": b}, learning_part_ids=["a"], fold=0, pool=25)
np.testing.assert_allclose(aa.transform(a, fold=0, pool=25), bb.transform(b, fold=0, pool=25), atol=1e-13)
def test_streaming_population_stats_match_pooled_nodes(self):
rng = np.random.default_rng(42)
a, b = rng.normal(size=(3, 10)), rng.normal(size=(7, 10))+2
a[:, 9] = b[:, 9] = 4.
stats = fit_base_standardizer({"a": a, "b": b}, learning_part_ids=["b", "a"], fold=2, pool=50)
joined = np.vstack([a, b])
np.testing.assert_allclose(stats.mean, joined.mean(axis=0), atol=1e-15)
np.testing.assert_allclose(stats.std, joined.std(axis=0, ddof=0), atol=1e-15)
z = stats.transform(joined, fold=2, pool=50)
np.testing.assert_allclose(z.mean(axis=0), 0., atol=1e-15)
np.testing.assert_allclose(z[:, :9].std(axis=0), 1., atol=1e-15)
np.testing.assert_array_equal(z[:, 9], 0.)
self.assertEqual(stats.scale[9], 1.)
self.assertEqual(stats.node_count, 10)
def test_held_out_data_cannot_enter_fit_or_change_frozen_statistics(self):
from dataclasses import FrozenInstanceError
a, held = np.zeros((2, 10)), np.full((3, 10), 1e6)
with self.assertRaisesRegex(ValueError, "learning pool exactly"):
fit_base_standardizer({"a": a, "validation": held}, learning_part_ids=["a"], fold=0, pool=25)
with self.assertRaises(ValueError):
fit_base_standardizer({}, learning_part_ids=["a"], fold=0, pool=25)
stats = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
fingerprint = stats.fingerprint
np.testing.assert_array_equal(stats.transform(held, fold=0, pool=25), held)
self.assertEqual(fingerprint, stats.fingerprint)
with self.assertRaises(FrozenInstanceError):
stats.mean = tuple(np.ones(10))
with self.assertRaisesRegex(ValueError, "another fold/pool"):
stats.transform(held, fold=1, pool=25)
with self.assertRaisesRegex(ValueError, "another fold/pool"):
stats.transform(held, fold=0, pool=100)
with self.assertRaises(ValueError):
fit_base_standardizer({"a": np.full((2, 10), np.nan)}, learning_part_ids=["a"], fold=0, pool=25)
def test_ne_share_frozen_transform_before_preprocess(self):
stats = synthetic_standardizer()
models = [QueryModel(arm, standardizer=stats).eval() for arm in ("N", "E")]
base = torch.linspace(-1, 1, 50).reshape(5, 10)
expected = stats.transform(base.numpy(), fold="test-fold", pool="test-pool")
captured = []
for model in models:
self.assertIs(model.encoder.standardizer, stats)
self.assertNotIn("base_mean", dict(model.encoder.named_parameters()))
handle = model.encoder.preprocess.register_forward_pre_hook(lambda module, args: captured.append(args[0].clone()))
with torch.inference_mode():
model.encoder(base)
handle.remove()
torch.testing.assert_close(captured[0], captured[1], rtol=0, atol=0)
np.testing.assert_allclose(captured[0][0].numpy(), expected, atol=2e-7)
with self.assertRaises(ValueError):
QueryModel("N", standardizer=None)
class NetworkTests(unittest.TestCase):
def test_original_attention_numerical_parity(self):
path = Path(__file__).parent/'s2_models/upstream/Physics_Attention.py'
source = path.read_text().split('class Physics_Attention_Structured_Mesh_2D')[0]
source = source.replace('from einops import rearrange, repeat', '')
namespace = {"rearrange": lambda x, pattern: x.permute(0, 2, 1, 3).reshape(x.shape[0], x.shape[2], -1)}
exec(compile(source, str(path), "exec"), namespace)
torch.manual_seed(9)
original = namespace['Physics_Attention_Irregular_Mesh'](128, heads=8, dim_head=16, slice_num=32).eval()
adapted = Physics_Attention_Irregular_Mesh(128, heads=8, dim_head=16, slice_num=32).eval()
adapted.load_state_dict(original.state_dict())
x = torch.randn(1, 7, 128)
with torch.inference_mode():
torch.testing.assert_close(original(x), adapted(x), rtol=0, atol=0)
def test_upstream_snapshot_hashes(self):
import hashlib
root = Path(__file__).parent/'s2_models/upstream'
manifest = json.loads((root/'manifest.json').read_text())
self.assertEqual(manifest['revision'], '75e0f67643806a81cd1d3f6adc88dd8c02416fe7')
for name, digest in manifest['files'].items():
self.assertEqual(hashlib.sha256((root/Path(name).name).read_bytes()).hexdigest(), digest)
def test_four_models_forward_only_and_parameter_counts(self):
for width in (128, 256):
models = {}
for arm in ("N", "E"):
torch.manual_seed(0)
model = QueryModel(arm, width, standardizer=synthetic_standardizer()).eval()
before = {k: v.clone() for k, v in model.state_dict().items()}
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4, 6]), torch.rand(3, 2)
edge = torch.rand(3, 4) if arm == "E" else None
with torch.inference_mode():
pred = model(base, ids, gaps, edge)
self.assertEqual(pred.shape, (3,))
self.assertTrue(torch.isfinite(pred).all())
self.assertEqual(next(model.parameters()).device.type, "cpu")
for k, value in model.state_dict().items():
torch.testing.assert_close(value, before[k], rtol=0, atol=0)
models[arm] = model.parameter_counts()
self.assertEqual(models['E']['encoder'], models['N']['encoder'])
self.assertEqual(models['E']['total']-models['N']['total'], 4*width)
def test_query_order_global_context_and_input_rejection(self):
torch.manual_seed(7)
model = QueryModel("N", standardizer=synthetic_standardizer()).eval()
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4]), torch.rand(2, 2)
with torch.inference_mode():
a = model(base, ids, gaps)
b = model(base, ids.flip(0), gaps.flip(0))
torch.testing.assert_close(a, b.flip(0))
changed = base.clone(); changed[6] += 20
c = model(changed, ids, gaps)
self.assertGreater(float(torch.max(torch.abs(a-c))), 1e-9)
with self.assertRaises(ValueError):
model(base, ids, gaps, torch.zeros(2, 4))
with self.assertRaises(ValueError):
model(base, torch.tensor([7]), torch.zeros(1, 2))
changed[6, 0] = float("nan")
with self.assertRaises(ValueError):
model(changed, ids, gaps)
with self.assertRaises(ValueError):
QueryModel("E", standardizer=synthetic_standardizer())(base, ids, gaps)
if __name__ == '__main__':
unittest.main(verbosity=2)