Normalize neural base inputs with frozen learning-pool statistics
This commit is contained in:
1 parent
bb09e8e164
commit
f3532962b0
10 files changed
+326
-15
No files matched your search
+79
-3
@@ -14,6 +14,7 @@ from sklearn.ensemble import HistGradientBoostingRegressor
|
||||
from s2_models import features as f, objectives as o
|
||||
from s2_models.attention import Physics_Attention_Irregular_Mesh
|
||||
from s2_models.network import QueryModel, weighted_mse
|
||||
from s2_models.normalization import fit_base_standardizer
|
||||
|
||||
torch.set_num_threads(1)
|
||||
|
||||
@@ -25,6 +26,11 @@ def synthetic_star():
|
||||
return x, t, np.array([0., 0, 0, 0, .25])
|
||||
|
||||
|
||||
def synthetic_standardizer():
|
||||
return fit_base_standardizer({"synthetic-a": np.arange(70, dtype=float).reshape(7, 10)/10},
|
||||
learning_part_ids=["synthetic-a"], fold="test-fold", pool="test-pool")
|
||||
|
||||
|
||||
class GeometryTests(unittest.TestCase):
|
||||
def test_volume_lumping(self):
|
||||
x, t, _ = synthetic_star()
|
||||
@@ -184,6 +190,76 @@ class ObjectiveTests(unittest.TestCase):
|
||||
self.assertFalse(hasattr(model, "n_iter_"))
|
||||
|
||||
|
||||
class NormalizationTests(unittest.TestCase):
|
||||
def test_unit_conversion_only_shifts_log_scale_columns(self):
|
||||
x, t, dw = synthetic_star()
|
||||
dg, _ = f.graph_distances(x, t, dw, x[0], exclude_wall_edges=False)
|
||||
center = np.array([.2, .2, .2])
|
||||
a = f.base_features(x, dw, dg, x[0], center, 2., 1/6, .5, .4)
|
||||
length_scale, time_scale = 1000., 60.
|
||||
b = f.base_features(x*length_scale, dw*length_scale, dg*length_scale,
|
||||
x[0]*length_scale, center*length_scale,
|
||||
2*length_scale, 1/6*length_scale**3, .5*length_scale, .4*time_scale)
|
||||
expected = np.r_[np.zeros(6), np.log([length_scale, length_scale**3, length_scale, time_scale])]
|
||||
np.testing.assert_allclose(b-a, np.tile(expected, (len(x), 1)), atol=2e-15)
|
||||
aa = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
bb = fit_base_standardizer({"a": b}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
np.testing.assert_allclose(aa.transform(a, fold=0, pool=25), bb.transform(b, fold=0, pool=25), atol=1e-13)
|
||||
|
||||
def test_streaming_population_stats_match_pooled_nodes(self):
|
||||
rng = np.random.default_rng(42)
|
||||
a, b = rng.normal(size=(3, 10)), rng.normal(size=(7, 10))+2
|
||||
a[:, 9] = b[:, 9] = 4.
|
||||
stats = fit_base_standardizer({"a": a, "b": b}, learning_part_ids=["b", "a"], fold=2, pool=50)
|
||||
joined = np.vstack([a, b])
|
||||
np.testing.assert_allclose(stats.mean, joined.mean(axis=0), atol=1e-15)
|
||||
np.testing.assert_allclose(stats.std, joined.std(axis=0, ddof=0), atol=1e-15)
|
||||
z = stats.transform(joined, fold=2, pool=50)
|
||||
np.testing.assert_allclose(z.mean(axis=0), 0., atol=1e-15)
|
||||
np.testing.assert_allclose(z[:, :9].std(axis=0), 1., atol=1e-15)
|
||||
np.testing.assert_array_equal(z[:, 9], 0.)
|
||||
self.assertEqual(stats.scale[9], 1.)
|
||||
self.assertEqual(stats.node_count, 10)
|
||||
|
||||
def test_held_out_data_cannot_enter_fit_or_change_frozen_statistics(self):
|
||||
from dataclasses import FrozenInstanceError
|
||||
a, held = np.zeros((2, 10)), np.full((3, 10), 1e6)
|
||||
with self.assertRaisesRegex(ValueError, "learning pool exactly"):
|
||||
fit_base_standardizer({"a": a, "validation": held}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
with self.assertRaises(ValueError):
|
||||
fit_base_standardizer({}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
stats = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
fingerprint = stats.fingerprint
|
||||
np.testing.assert_array_equal(stats.transform(held, fold=0, pool=25), held)
|
||||
self.assertEqual(fingerprint, stats.fingerprint)
|
||||
with self.assertRaises(FrozenInstanceError):
|
||||
stats.mean = tuple(np.ones(10))
|
||||
with self.assertRaisesRegex(ValueError, "another fold/pool"):
|
||||
stats.transform(held, fold=1, pool=25)
|
||||
with self.assertRaisesRegex(ValueError, "another fold/pool"):
|
||||
stats.transform(held, fold=0, pool=100)
|
||||
with self.assertRaises(ValueError):
|
||||
fit_base_standardizer({"a": np.full((2, 10), np.nan)}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
|
||||
def test_ne_share_frozen_transform_before_preprocess(self):
|
||||
stats = synthetic_standardizer()
|
||||
models = [QueryModel(arm, standardizer=stats).eval() for arm in ("N", "E")]
|
||||
base = torch.linspace(-1, 1, 50).reshape(5, 10)
|
||||
expected = stats.transform(base.numpy(), fold="test-fold", pool="test-pool")
|
||||
captured = []
|
||||
for model in models:
|
||||
self.assertIs(model.encoder.standardizer, stats)
|
||||
self.assertNotIn("base_mean", dict(model.encoder.named_parameters()))
|
||||
handle = model.encoder.preprocess.register_forward_pre_hook(lambda module, args: captured.append(args[0].clone()))
|
||||
with torch.inference_mode():
|
||||
model.encoder(base)
|
||||
handle.remove()
|
||||
torch.testing.assert_close(captured[0], captured[1], rtol=0, atol=0)
|
||||
np.testing.assert_allclose(captured[0][0].numpy(), expected, atol=2e-7)
|
||||
with self.assertRaises(ValueError):
|
||||
QueryModel("N", standardizer=None)
|
||||
|
||||
|
||||
class NetworkTests(unittest.TestCase):
|
||||
def test_original_attention_numerical_parity(self):
|
||||
path = Path(__file__).parent/'s2_models/upstream/Physics_Attention.py'
|
||||
@@ -212,7 +288,7 @@ class NetworkTests(unittest.TestCase):
|
||||
models = {}
|
||||
for arm in ("N", "E"):
|
||||
torch.manual_seed(0)
|
||||
model = QueryModel(arm, width).eval()
|
||||
model = QueryModel(arm, width, standardizer=synthetic_standardizer()).eval()
|
||||
before = {k: v.clone() for k, v in model.state_dict().items()}
|
||||
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4, 6]), torch.rand(3, 2)
|
||||
edge = torch.rand(3, 4) if arm == "E" else None
|
||||
@@ -229,7 +305,7 @@ class NetworkTests(unittest.TestCase):
|
||||
|
||||
def test_query_order_global_context_and_input_rejection(self):
|
||||
torch.manual_seed(7)
|
||||
model = QueryModel("N").eval()
|
||||
model = QueryModel("N", standardizer=synthetic_standardizer()).eval()
|
||||
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4]), torch.rand(2, 2)
|
||||
with torch.inference_mode():
|
||||
a = model(base, ids, gaps)
|
||||
@@ -246,7 +322,7 @@ class NetworkTests(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
model(changed, ids, gaps)
|
||||
with self.assertRaises(ValueError):
|
||||
QueryModel("E")(base, ids, gaps)
|
||||
QueryModel("E", standardizer=synthetic_standardizer())(base, ids, gaps)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in new issue
Block a user