Normalize neural base inputs with frozen learning-pool statistics
This commit is contained in:
1 parent
bb09e8e164
commit
f3532962b0
10 files changed
+326
-15
No files matched your search
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"status": "FAIL",
|
||||
"tests_run": 24,
|
||||
"failures": 1,
|
||||
"failed_test": "test_unit_conversion_only_shifts_log_scale_columns",
|
||||
"assertion": "standardized inputs differ after unit conversion",
|
||||
"max_absolute_difference": 1.0,
|
||||
"mismatched_elements": 5,
|
||||
"total_elements": 50,
|
||||
"cause": "Summing identical floating constants before dividing can move their mean by one ULP; subtracting this mean manufactures a tiny nonzero variance and standardized +/-1 instead of zero.",
|
||||
"normalization_source_sha256": "eeb2caa014eb3cdd4f13e90365894f64115f275200f0f28a37ccf9e41b41d914",
|
||||
"record_type": "Summary of observed first acceptance failure; not a reconstructed raw test log."
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"timestamp": "2026-11-14T14:59:29.743964+00:00",
|
||||
"base_commit": "bb09e8e164df28928d1d74befc582d3c2fd34441",
|
||||
"ruling": "1794667627.081159",
|
||||
"status": "PASS",
|
||||
"test_count": 24,
|
||||
"unittest_seconds": 0.413,
|
||||
"cuda_visible_devices": "",
|
||||
"numerical_threads": 1,
|
||||
"real_data_reads": false,
|
||||
"real_pool_statistics_fitted": false,
|
||||
"synthetic_pool_statistics_computed": true,
|
||||
"estimator_fits": 0,
|
||||
"backward_passes": 0,
|
||||
"parameter_updates": 0,
|
||||
"initial_failure_summary": "initial-failure.json",
|
||||
"unchanged": [
|
||||
"R2-G24 features/grid",
|
||||
"direct t_star target",
|
||||
"model parameter counts",
|
||||
"registration source and process"
|
||||
],
|
||||
"sha256": {
|
||||
"tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"tools/s2_models/requirements-tested.txt": "d0397e13dc138c303e4760eb93c52a28d2d260fc2eb91b12f119bf1da1f0463c",
|
||||
"tools/s2_models/upstream/LICENSE": "2c919cd03fa823bf7eefc00a957ff8324c865cd22aa5285e563dc4b558084f25",
|
||||
"tools/s2_models/upstream/Physics_Attention.py": "4fc710f6f96788fb59cb19fab8d64873d3d16e9e182f12a544d451c74c548cc7",
|
||||
"tools/s2_models/upstream/Transolver_Irregular_Mesh.py": "5259759f7ecc6a954c8442ac8359990b9f242f1437c452c69e5e0f03549b5311",
|
||||
"tools/s2_models/upstream/manifest.json": "fbcaa6673dbc82031e110ea18680574a9d9edcbf33ca6abd537129be4d860eee",
|
||||
"tools/s2_models/upstream-adaptation.diff": "ba74e70b720d127d40609da1ed619713c880287ea9c7b9d035096d7f8327fd3c",
|
||||
"tools/test_s2_models.py": "490e0dd7cb447d7086d9fa31d9cabea1d129af03b64c0ba34f81fe8fd9a00f76"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"timestamp": "2026-11-14T14:59:29.743964+00:00",
|
||||
"progress": {
|
||||
"completed_jobs": 600,
|
||||
"active_jobs": [
|
||||
592,
|
||||
594,
|
||||
595,
|
||||
596,
|
||||
597,
|
||||
600,
|
||||
601,
|
||||
602,
|
||||
608,
|
||||
609,
|
||||
610,
|
||||
611,
|
||||
612,
|
||||
613,
|
||||
614,
|
||||
615
|
||||
],
|
||||
"next_job": 616,
|
||||
"completed_worker_cpu_seconds": 49182.74786200006,
|
||||
"live_tree_cpu_seconds": 880.6200000000001,
|
||||
"active_tree_rss_bytes": 2357587968,
|
||||
"stop_reason": null,
|
||||
"elapsed_seconds": 3271.3069857549854
|
||||
},
|
||||
"terminal_counts": {
|
||||
"PASS": 600
|
||||
},
|
||||
"completion_flags": []
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
test_affine_p1_and_zero_gradient (__main__.GeometryTests) ... ok
|
||||
test_base_slots_and_nonfinite_fail (__main__.GeometryTests) ... ok
|
||||
test_degenerate_and_unused_fail (__main__.GeometryTests) ... ok
|
||||
test_features_and_physical_scaling (__main__.GeometryTests) ... ok
|
||||
test_moment_unequal_volumes_and_coincident_center (__main__.GeometryTests) ... ok
|
||||
test_moment_volume_weight_rotation_and_filter (__main__.GeometryTests) ... ok
|
||||
test_strict_volume_front_ties (__main__.GeometryTests) ... ok
|
||||
test_unreachable_fail_closed (__main__.GeometryTests) ... ok
|
||||
test_volume_lumping (__main__.GeometryTests) ... ok
|
||||
test_wall_edges_removed_both_distances_and_gate_tie (__main__.GeometryTests) ... ok
|
||||
test_four_models_forward_only_and_parameter_counts (__main__.NetworkTests) ... ok
|
||||
test_original_attention_numerical_parity (__main__.NetworkTests) ... ok
|
||||
test_query_order_global_context_and_input_rejection (__main__.NetworkTests) ... ok
|
||||
test_upstream_snapshot_hashes (__main__.NetworkTests) ... ok
|
||||
test_held_out_data_cannot_enter_fit_or_change_frozen_statistics (__main__.NormalizationTests) ... ok
|
||||
test_ne_share_frozen_transform_before_preprocess (__main__.NormalizationTests) ... ok
|
||||
test_streaming_population_stats_match_pooled_nodes (__main__.NormalizationTests) ... ok
|
||||
test_unit_conversion_only_shifts_log_scale_columns (__main__.NormalizationTests) ... ok
|
||||
test_hierarchy_and_draw_compression (__main__.ObjectiveTests) ... ok
|
||||
test_histgbrt_grid_unfitted (__main__.ObjectiveTests) ... ok
|
||||
test_invalid_hierarchy (__main__.ObjectiveTests) ... ok
|
||||
test_process_target_roundtrip_and_scale_invariance (__main__.ObjectiveTests) ... ok
|
||||
test_score_masks_before_arithmetic_and_zero_truth (__main__.ObjectiveTests) ... ok
|
||||
test_zero_missing_scales_no_fallback (__main__.ObjectiveTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 24 tests in 0.413s
|
||||
|
||||
OK
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
## Scope and decisions
|
||||
|
||||
This implements Claude's [22:18:01 ruling](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794665881295239), the v0.5 Gre24 mapping and v0.6 process-conditioned target, with the [22:37:54 correction](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794667074550589). It is a synthetic-only code delivery for Claude's primary review, not a fitted model, real-corpus preprocessing receipt, or approval of predictive accuracy.
|
||||
This implements Claude's [22:18:01 ruling](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794665881295239), the v0.5 Gre24 mapping and v0.6 process-conditioned target, with the [22:37:54 correction](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794667074550589). Claude's [22:47:07 full-code main review](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794667627081159) approved `bb09e8e` with one required fix: log/standardise encoder base inputs. That fix is implemented below; R2 and the direct prediction target are unchanged. This remains a synthetic-only code delivery, not a fitted model, real-corpus preprocessing receipt, or approval of predictive accuracy.
|
||||
|
||||
No real geometry, LOG or simulation field is loaded by these modules/tests. The existing registration queue and its pinned sources are untouched. Test execution has empty CUDA visibility, one numerical thread and CPU tensors. No estimator `.fit`, backward pass, optimiser or parameter update is executed. Model initialisation is random, fixed by test seeds.
|
||||
|
||||
@@ -27,7 +27,13 @@ Upstream is [thuml/Transolver at 75e0f67643806a81cd1d3f6adc88dd8c02416fe7](https
|
||||
|
||||
The encoder uses all tet nodes, four irregular Transolver blocks, eight heads, 32 slices, widths 128 or 256, GELU, dropout 0 and upstream Model default mlp_ratio 1. It retains the upstream preprocessing MLP, learned placeholder, initialisation and residual attention/MLP blocks. The final scalar head is removed so each node yields an embedding. No time embedding or unified-position grid is used. Pure torch transpose/reshape replaces einops; torch trunc_normal_ replaces timm; structured-grid classes are omitted. An identical-state attention forward comparison against the original source is tested.
|
||||
|
||||
Base columns (10): `(x-center)/Rg` (3), `d/t_part`, Euclidean gate distance/Rg, unrestricted tet-edge geodesic/Rg, `Rg`, volume, `t_part`, `t_set`. The geodesic helper without R2 wall-edge exclusion is a separate input path. Physical scale and process setting are explicit. `t_part` and geodesic/Rg follow the earlier executable feature conventions. No O2 quantity enters the whole-part encoder.
|
||||
Base columns (10), before standardisation: `(x-center)/Rg` (3), `d/t_part`, Euclidean gate distance/Rg, unrestricted tet-edge geodesic/Rg, `log(Rg)`, `log(volume)`, `log(t_part)`, `log(t_set)` (natural logs). The geodesic helper without R2 wall-edge exclusion is a separate input path. Physical scale and process setting are explicit. `t_part` and geodesic/Rg follow the earlier executable feature conventions. No O2 quantity enters the whole-part encoder.
|
||||
|
||||
`normalization.py` fits mean/population std (`ddof=0`) for all ten columns from **the current fold/pool learning parts only**, taking all their tet nodes with equal node weight. It never uses query draw weights or labels. The caller supplies the frozen learning-part manifest; the array mapping must match exactly, so extra validation/test parts or missing learning parts are rejected. Statistics accumulate by part in float64 without concatenating the corpus. Each part's mean is computed after shifting by its first row: this preserves exact constant columns instead of manufacturing tiny variance through summation roundoff. A zero-std learning column uses scale 1 and maps its learning values to zero; held-out deviations remain observable, not imputed.
|
||||
|
||||
The frozen `BaseStandardizer` stores immutable tuples, fold/pool IDs, learning-part IDs and node count, and exports a reproducible fingerprint plus full statistics. An array transform rejects mismatched fold/pool IDs. N/E must be constructed with the **same** standardizer (`QueryModel(arm, width, standardizer=shared_stats)`); there is no unnormalised default. The encoder applies its mean/scale buffers before preprocessing in FP32, including under future AMP. These buffers are not learned parameters and are retained by `state_dict`; the exported statistics/provenance must accompany the run manifest. No validation/test transform refits statistics. Synthetic tests inspect the actual preprocess input to confirm identical N/E transforms. Parameter counts are unchanged.
|
||||
|
||||
Unit conversion test: multiplying all lengths (including gate/centre/distances/Rg/t_part) by L, volume by L³ and time setting by H leaves the first six columns unchanged and shifts the last four by `(log L,3 log L,log L,log H)` for every node. Applying correspondingly converted learning statistics gives the same standardised input. The first test attempt exposed the constant-column roundoff issue; its observed failure summary is preserved in `artifacts/s2-model-normalization-2026-11-14/initial-failure.json`, followed by the corrected 24/24 PASS log and source hashes. No real learning-pool statistics have been computed in this task.
|
||||
|
||||
The query decoder uses the original node index directly. N receives `[embedding,g12,g23]`; E additionally receives `[q1_mean,q1_max,q2_mean,q2_max]`. Two hidden GELU layers have the encoder width and a linear scalar output predicts `t*`. Missing E inputs, E inputs passed to N, nonfinite base/query values, empty/out-of-range queries fail closed. Qualification (positive distance, valid spectrum, N_eff≥64, complete finite edge block) must be enforced by the common-query producer; this module neither invents eligibility nor reads stored draws. All arms must receive identical eligible query IDs and counts.
|
||||
|
||||
@@ -41,7 +47,9 @@ Per-part relative L2 masks ineligible rows **before** arithmetic, counts draws w
|
||||
|
||||
## Acceptance and limitations
|
||||
|
||||
Final synthetic acceptance: **20/20 PASS** (2026-11-14 22:44:43 UTC+8).
|
||||
Latest synthetic acceptance after the required normalisation fix: **24/24 PASS**, cases took 0.413 s. See `artifacts/s2-model-normalization-2026-11-14/receipt.json` and `unittest.txt`. This adds unit conversion, learning-only/frozen statistics, population mean/std/constant handling, and actual shared N/E preprocessing checks. Only synthetic statistics were calculated; no estimator fit, training update, true-data read or GPU operation occurred.
|
||||
|
||||
Historical pre-review acceptance at `bb09e8e`: **20/20 PASS** (2026-11-14 22:44:43 UTC+8). Its receipt/source hashes remain a record of that fixed commit rather than the subsequently modified source.
|
||||
The test subprocess consumed 5.040 CPU-seconds, peak RSS 554.32 MiB including imports; unittest cases themselves took 0.214 s. The 8.753 s wall receipt includes parent imports and parameter-count construction as explicitly noted there. No benchmark claim is made from these tiny cases.
|
||||
|
||||
| Width | N parameters | E parameters | Shared encoder count |
|
||||
|
||||
@@ -145,10 +145,11 @@ def r2_features(points, tets, wall_distance, gate, t_set):
|
||||
|
||||
|
||||
BASE_COLUMNS = ("x_star", "y_star", "z_star", "d_over_t", "gate_distance_over_Rg",
|
||||
"geodesic_over_Rg", "Rg", "volume", "t_part", "t_set")
|
||||
"geodesic_over_Rg", "log_Rg", "log_volume", "log_t_part", "log_t_set")
|
||||
|
||||
|
||||
def base_features(points, wall_distance, geodesic_distance, gate, center, rg, volume, t_part, t_set):
|
||||
"""Ten columns before learning-pool standardisation; scales use natural log."""
|
||||
x = finite_array(points, name="points")
|
||||
if x.ndim != 2 or x.shape[1] != 3 or not len(x):
|
||||
raise ValueError("invalid node coordinates")
|
||||
@@ -159,7 +160,7 @@ def base_features(points, wall_distance, geodesic_distance, gate, center, rg, vo
|
||||
if np.any(scales <= 0) or np.any(dw < 0) or np.any(dg < 0):
|
||||
raise ValueError("nonpositive scale or negative distance")
|
||||
return np.column_stack([(x-center)/rg, dw/t_part, np.linalg.norm(x-gate, axis=1)/rg,
|
||||
dg/rg, np.tile(scales, (len(x), 1))])
|
||||
dg/rg, np.tile(np.log(scales), (len(x), 1))])
|
||||
|
||||
|
||||
def histgbrt_grid():
|
||||
|
||||
@@ -7,6 +7,7 @@ from torch import nn
|
||||
|
||||
from .blocks import MLP, Transolver_block
|
||||
from .features import BASE_COLUMNS
|
||||
from .normalization import BaseStandardizer
|
||||
|
||||
|
||||
TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
@@ -20,10 +21,15 @@ def check_tensor(tensor, shape, name):
|
||||
|
||||
|
||||
class WholePartEncoder(nn.Module):
|
||||
def __init__(self, width):
|
||||
def __init__(self, width, standardizer):
|
||||
super().__init__()
|
||||
if width not in (128, 256):
|
||||
raise ValueError("S2 permits widths 128 or 256 only")
|
||||
if not isinstance(standardizer, BaseStandardizer):
|
||||
raise ValueError("frozen learning-pool standardizer required")
|
||||
self.standardizer = standardizer
|
||||
self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
|
||||
self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
|
||||
self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
dropout=0., mlp_ratio=1, slice_num=32,
|
||||
@@ -46,6 +52,10 @@ class WholePartEncoder(nn.Module):
|
||||
if base.ndim != 2 or base.shape[0] == 0:
|
||||
raise ValueError("one whole part with all tet nodes required")
|
||||
check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
# Keep normalisation in FP32 even under future mixed-precision training.
|
||||
base = (base.float()-self.base_mean.float()) / self.base_scale.float()
|
||||
if not torch.isfinite(base).all():
|
||||
raise ValueError("nonfinite standardized base input")
|
||||
fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
for block in self.blocks:
|
||||
fx = block(fx)
|
||||
@@ -55,12 +65,12 @@ class WholePartEncoder(nn.Module):
|
||||
|
||||
|
||||
class QueryModel(nn.Module):
|
||||
def __init__(self, arm, width=128):
|
||||
def __init__(self, arm, width=128, *, standardizer):
|
||||
super().__init__()
|
||||
if arm not in ("N", "E"):
|
||||
raise ValueError("arm must be N or E")
|
||||
self.arm, self.width = arm, width
|
||||
self.encoder = WholePartEncoder(width)
|
||||
self.encoder = WholePartEncoder(width, standardizer)
|
||||
query_dim = 2 if arm == "N" else 6
|
||||
self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Frozen per-fold/pool base-input statistics. Caller supplies the learning manifest.
|
||||
|
||||
No I/O or dataset selection occurs here. Statistics use all learning tet nodes
|
||||
equally, population std (ddof=0), accumulated by part without concatenating the
|
||||
whole pool. This is input preprocessing, not fitting model parameters.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .features import BASE_COLUMNS, finite_array
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BaseStandardizer:
|
||||
fold: str
|
||||
pool: str
|
||||
learning_part_ids: tuple
|
||||
node_count: int
|
||||
mean: tuple
|
||||
std: tuple
|
||||
|
||||
def __post_init__(self):
|
||||
if not isinstance(self.fold, str) or not self.fold or not isinstance(self.pool, str) or not self.pool:
|
||||
raise ValueError("fold and pool identifiers required")
|
||||
if not isinstance(self.learning_part_ids, tuple) or not self.learning_part_ids or any(not isinstance(p, str) or not p for p in self.learning_part_ids) or len(set(self.learning_part_ids)) != len(self.learning_part_ids):
|
||||
raise ValueError("unique immutable learning part IDs required")
|
||||
if not isinstance(self.node_count, int) or self.node_count <= 0:
|
||||
raise ValueError("positive learning node count required")
|
||||
if not isinstance(self.mean, tuple) or not isinstance(self.std, tuple):
|
||||
raise ValueError("statistics must be immutable tuples")
|
||||
finite_array(self.mean, (len(BASE_COLUMNS),), "learning mean")
|
||||
std = finite_array(self.std, (len(BASE_COLUMNS),), "learning std")
|
||||
if np.any(std < 0):
|
||||
raise ValueError("negative std")
|
||||
|
||||
@property
|
||||
def scale(self):
|
||||
# Constant learning columns become zero. Unseen values are not imputed.
|
||||
return tuple(s if s > 0 else 1. for s in self.std)
|
||||
|
||||
def as_dict(self):
|
||||
return dict(fold=self.fold, pool=self.pool, learning_part_ids=list(self.learning_part_ids),
|
||||
node_count=self.node_count, mean=list(self.mean), std=list(self.std),
|
||||
columns=list(BASE_COLUMNS), weighting="equal_learning_nodes", ddof=0,
|
||||
constant_column_scale=1.)
|
||||
|
||||
@property
|
||||
def fingerprint(self):
|
||||
return hashlib.sha256(json.dumps(self.as_dict(), sort_keys=True, allow_nan=False).encode()).hexdigest()
|
||||
|
||||
def transform(self, base, *, fold, pool):
|
||||
if str(fold) != self.fold or str(pool) != self.pool:
|
||||
raise ValueError("standardizer belongs to another fold/pool")
|
||||
a = finite_array(base, name="base input")
|
||||
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
|
||||
raise ValueError("nonempty ten-column base input required")
|
||||
return finite_array((a-np.asarray(self.mean))/np.asarray(self.scale), name="standardized base")
|
||||
|
||||
|
||||
def fit_base_standardizer(part_base, *, learning_part_ids, fold, pool):
|
||||
"""Fit only an exact, explicitly enumerated learning pool; reject extra IDs.
|
||||
|
||||
part_base maps part ID to its full-node base_features output. Validation/test
|
||||
arrays are passed only to the frozen transform, never to this function.
|
||||
"""
|
||||
ids = tuple(learning_part_ids)
|
||||
if not ids or any(not isinstance(p, str) or not p for p in ids) or len(ids) != len(set(ids)):
|
||||
raise ValueError("nonempty unique learning part IDs required")
|
||||
if set(part_base) != set(ids):
|
||||
raise ValueError("base arrays must match learning pool exactly; no held-out or missing parts")
|
||||
count, mean, m2 = 0, np.zeros(len(BASE_COLUMNS)), np.zeros(len(BASE_COLUMNS))
|
||||
for part in sorted(ids):
|
||||
a = finite_array(part_base[part], name=f"learning base {part}")
|
||||
if a.ndim != 2 or not len(a) or a.shape[1] != len(BASE_COLUMNS):
|
||||
raise ValueError("nonempty ten-column learning base required")
|
||||
# Shift before summation: an exactly constant column must have exactly
|
||||
# its original mean and zero variance, not roundoff-sized pseudo-variance.
|
||||
local_mean = a[0] + (a-a[0]).mean(axis=0)
|
||||
local_m2 = ((a-local_mean)**2).sum(axis=0)
|
||||
delta = local_mean - mean
|
||||
n = len(a)
|
||||
total = count + n
|
||||
mean += delta * (n/total)
|
||||
m2 += local_m2 + delta*delta*(count*n/total)
|
||||
count = total
|
||||
std = np.sqrt(m2/count)
|
||||
return BaseStandardizer(str(fold), str(pool), tuple(sorted(ids)), count,
|
||||
tuple(float(v) for v in mean), tuple(float(v) for v in std))
|
||||
@@ -238,7 +238,7 @@
|
||||
- return fx
|
||||
--- /dev/null
|
||||
+++ network.py
|
||||
@@ -0,0 +1,109 @@
|
||||
@@ -0,0 +1,119 @@
|
||||
+"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
|
||||
+
|
||||
+No optimiser, training loop, data reader or GPU invocation is provided.
|
||||
@@ -248,6 +248,7 @@
|
||||
+
|
||||
+from .blocks import MLP, Transolver_block
|
||||
+from .features import BASE_COLUMNS
|
||||
+from .normalization import BaseStandardizer
|
||||
+
|
||||
+
|
||||
+TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
@@ -261,10 +262,15 @@
|
||||
+
|
||||
+
|
||||
+class WholePartEncoder(nn.Module):
|
||||
+ def __init__(self, width):
|
||||
+ def __init__(self, width, standardizer):
|
||||
+ super().__init__()
|
||||
+ if width not in (128, 256):
|
||||
+ raise ValueError("S2 permits widths 128 or 256 only")
|
||||
+ if not isinstance(standardizer, BaseStandardizer):
|
||||
+ raise ValueError("frozen learning-pool standardizer required")
|
||||
+ self.standardizer = standardizer
|
||||
+ self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
|
||||
+ self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
|
||||
+ self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
+ self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
+ dropout=0., mlp_ratio=1, slice_num=32,
|
||||
@@ -287,6 +293,10 @@
|
||||
+ if base.ndim != 2 or base.shape[0] == 0:
|
||||
+ raise ValueError("one whole part with all tet nodes required")
|
||||
+ check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
+ # Keep normalisation in FP32 even under future mixed-precision training.
|
||||
+ base = (base.float()-self.base_mean.float()) / self.base_scale.float()
|
||||
+ if not torch.isfinite(base).all():
|
||||
+ raise ValueError("nonfinite standardized base input")
|
||||
+ fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
+ for block in self.blocks:
|
||||
+ fx = block(fx)
|
||||
@@ -296,12 +306,12 @@
|
||||
+
|
||||
+
|
||||
+class QueryModel(nn.Module):
|
||||
+ def __init__(self, arm, width=128):
|
||||
+ def __init__(self, arm, width=128, *, standardizer):
|
||||
+ super().__init__()
|
||||
+ if arm not in ("N", "E"):
|
||||
+ raise ValueError("arm must be N or E")
|
||||
+ self.arm, self.width = arm, width
|
||||
+ self.encoder = WholePartEncoder(width)
|
||||
+ self.encoder = WholePartEncoder(width, standardizer)
|
||||
+ query_dim = 2 if arm == "N" else 6
|
||||
+ self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
+ nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
|
||||
+79
-3
@@ -14,6 +14,7 @@ from sklearn.ensemble import HistGradientBoostingRegressor
|
||||
from s2_models import features as f, objectives as o
|
||||
from s2_models.attention import Physics_Attention_Irregular_Mesh
|
||||
from s2_models.network import QueryModel, weighted_mse
|
||||
from s2_models.normalization import fit_base_standardizer
|
||||
|
||||
torch.set_num_threads(1)
|
||||
|
||||
@@ -25,6 +26,11 @@ def synthetic_star():
|
||||
return x, t, np.array([0., 0, 0, 0, .25])
|
||||
|
||||
|
||||
def synthetic_standardizer():
|
||||
return fit_base_standardizer({"synthetic-a": np.arange(70, dtype=float).reshape(7, 10)/10},
|
||||
learning_part_ids=["synthetic-a"], fold="test-fold", pool="test-pool")
|
||||
|
||||
|
||||
class GeometryTests(unittest.TestCase):
|
||||
def test_volume_lumping(self):
|
||||
x, t, _ = synthetic_star()
|
||||
@@ -184,6 +190,76 @@ class ObjectiveTests(unittest.TestCase):
|
||||
self.assertFalse(hasattr(model, "n_iter_"))
|
||||
|
||||
|
||||
class NormalizationTests(unittest.TestCase):
|
||||
def test_unit_conversion_only_shifts_log_scale_columns(self):
|
||||
x, t, dw = synthetic_star()
|
||||
dg, _ = f.graph_distances(x, t, dw, x[0], exclude_wall_edges=False)
|
||||
center = np.array([.2, .2, .2])
|
||||
a = f.base_features(x, dw, dg, x[0], center, 2., 1/6, .5, .4)
|
||||
length_scale, time_scale = 1000., 60.
|
||||
b = f.base_features(x*length_scale, dw*length_scale, dg*length_scale,
|
||||
x[0]*length_scale, center*length_scale,
|
||||
2*length_scale, 1/6*length_scale**3, .5*length_scale, .4*time_scale)
|
||||
expected = np.r_[np.zeros(6), np.log([length_scale, length_scale**3, length_scale, time_scale])]
|
||||
np.testing.assert_allclose(b-a, np.tile(expected, (len(x), 1)), atol=2e-15)
|
||||
aa = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
bb = fit_base_standardizer({"a": b}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
np.testing.assert_allclose(aa.transform(a, fold=0, pool=25), bb.transform(b, fold=0, pool=25), atol=1e-13)
|
||||
|
||||
def test_streaming_population_stats_match_pooled_nodes(self):
|
||||
rng = np.random.default_rng(42)
|
||||
a, b = rng.normal(size=(3, 10)), rng.normal(size=(7, 10))+2
|
||||
a[:, 9] = b[:, 9] = 4.
|
||||
stats = fit_base_standardizer({"a": a, "b": b}, learning_part_ids=["b", "a"], fold=2, pool=50)
|
||||
joined = np.vstack([a, b])
|
||||
np.testing.assert_allclose(stats.mean, joined.mean(axis=0), atol=1e-15)
|
||||
np.testing.assert_allclose(stats.std, joined.std(axis=0, ddof=0), atol=1e-15)
|
||||
z = stats.transform(joined, fold=2, pool=50)
|
||||
np.testing.assert_allclose(z.mean(axis=0), 0., atol=1e-15)
|
||||
np.testing.assert_allclose(z[:, :9].std(axis=0), 1., atol=1e-15)
|
||||
np.testing.assert_array_equal(z[:, 9], 0.)
|
||||
self.assertEqual(stats.scale[9], 1.)
|
||||
self.assertEqual(stats.node_count, 10)
|
||||
|
||||
def test_held_out_data_cannot_enter_fit_or_change_frozen_statistics(self):
|
||||
from dataclasses import FrozenInstanceError
|
||||
a, held = np.zeros((2, 10)), np.full((3, 10), 1e6)
|
||||
with self.assertRaisesRegex(ValueError, "learning pool exactly"):
|
||||
fit_base_standardizer({"a": a, "validation": held}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
with self.assertRaises(ValueError):
|
||||
fit_base_standardizer({}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
stats = fit_base_standardizer({"a": a}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
fingerprint = stats.fingerprint
|
||||
np.testing.assert_array_equal(stats.transform(held, fold=0, pool=25), held)
|
||||
self.assertEqual(fingerprint, stats.fingerprint)
|
||||
with self.assertRaises(FrozenInstanceError):
|
||||
stats.mean = tuple(np.ones(10))
|
||||
with self.assertRaisesRegex(ValueError, "another fold/pool"):
|
||||
stats.transform(held, fold=1, pool=25)
|
||||
with self.assertRaisesRegex(ValueError, "another fold/pool"):
|
||||
stats.transform(held, fold=0, pool=100)
|
||||
with self.assertRaises(ValueError):
|
||||
fit_base_standardizer({"a": np.full((2, 10), np.nan)}, learning_part_ids=["a"], fold=0, pool=25)
|
||||
|
||||
def test_ne_share_frozen_transform_before_preprocess(self):
|
||||
stats = synthetic_standardizer()
|
||||
models = [QueryModel(arm, standardizer=stats).eval() for arm in ("N", "E")]
|
||||
base = torch.linspace(-1, 1, 50).reshape(5, 10)
|
||||
expected = stats.transform(base.numpy(), fold="test-fold", pool="test-pool")
|
||||
captured = []
|
||||
for model in models:
|
||||
self.assertIs(model.encoder.standardizer, stats)
|
||||
self.assertNotIn("base_mean", dict(model.encoder.named_parameters()))
|
||||
handle = model.encoder.preprocess.register_forward_pre_hook(lambda module, args: captured.append(args[0].clone()))
|
||||
with torch.inference_mode():
|
||||
model.encoder(base)
|
||||
handle.remove()
|
||||
torch.testing.assert_close(captured[0], captured[1], rtol=0, atol=0)
|
||||
np.testing.assert_allclose(captured[0][0].numpy(), expected, atol=2e-7)
|
||||
with self.assertRaises(ValueError):
|
||||
QueryModel("N", standardizer=None)
|
||||
|
||||
|
||||
class NetworkTests(unittest.TestCase):
|
||||
def test_original_attention_numerical_parity(self):
|
||||
path = Path(__file__).parent/'s2_models/upstream/Physics_Attention.py'
|
||||
@@ -212,7 +288,7 @@ class NetworkTests(unittest.TestCase):
|
||||
models = {}
|
||||
for arm in ("N", "E"):
|
||||
torch.manual_seed(0)
|
||||
model = QueryModel(arm, width).eval()
|
||||
model = QueryModel(arm, width, standardizer=synthetic_standardizer()).eval()
|
||||
before = {k: v.clone() for k, v in model.state_dict().items()}
|
||||
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4, 6]), torch.rand(3, 2)
|
||||
edge = torch.rand(3, 4) if arm == "E" else None
|
||||
@@ -229,7 +305,7 @@ class NetworkTests(unittest.TestCase):
|
||||
|
||||
def test_query_order_global_context_and_input_rejection(self):
|
||||
torch.manual_seed(7)
|
||||
model = QueryModel("N").eval()
|
||||
model = QueryModel("N", standardizer=synthetic_standardizer()).eval()
|
||||
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4]), torch.rand(2, 2)
|
||||
with torch.inference_mode():
|
||||
a = model(base, ids, gaps)
|
||||
@@ -246,7 +322,7 @@ class NetworkTests(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
model(changed, ids, gaps)
|
||||
with self.assertRaises(ValueError):
|
||||
QueryModel("E")(base, ids, gaps)
|
||||
QueryModel("E", standardizer=synthetic_standardizer())(base, ids, gaps)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in new issue
Block a user