Implement S2 R2-G24 and Transolver query models with synthetic acceptance
This commit is contained in:
1 parent
d4e26d48d0
commit
bb09e8e164
17 files changed
+1734
No files matched your search
@@ -0,0 +1,157 @@
|
||||
{
|
||||
"timestamp": "2026-11-14T14:44:43.272858+00:00",
|
||||
"status": "PASS",
|
||||
"returncode": 0,
|
||||
"test_count": 20,
|
||||
"test_child_cpu_seconds": 5.0396279999999996,
|
||||
"test_child_peak_rss_kib": 567628,
|
||||
"versions": {
|
||||
"numpy": "1.26.4",
|
||||
"scipy": "1.15.3",
|
||||
"sklearn": "1.7.2",
|
||||
"torch": "2.5.1+cu124"
|
||||
},
|
||||
"cuda_visible_devices": "",
|
||||
"torch_threads": 1,
|
||||
"real_dataset_reads": false,
|
||||
"fits": 0,
|
||||
"backward_passes": 0,
|
||||
"parameter_updates": 0,
|
||||
"parameter_counts": {
|
||||
"N-128": {
|
||||
"encoder": 373408,
|
||||
"decoder": 33409,
|
||||
"total": 406817
|
||||
},
|
||||
"E-128": {
|
||||
"encoder": 373408,
|
||||
"decoder": 33921,
|
||||
"total": 407329
|
||||
},
|
||||
"N-256": {
|
||||
"encoder": 1473696,
|
||||
"decoder": 132353,
|
||||
"total": 1606049
|
||||
},
|
||||
"E-256": {
|
||||
"encoder": 1473696,
|
||||
"decoder": 133377,
|
||||
"total": 1607073
|
||||
}
|
||||
},
|
||||
"grid": [
|
||||
{
|
||||
"learning_rate": 0.05,
|
||||
"max_leaf_nodes": 31,
|
||||
"max_iter": 200,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.05,
|
||||
"max_leaf_nodes": 31,
|
||||
"max_iter": 500,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.05,
|
||||
"max_leaf_nodes": 31,
|
||||
"max_iter": 1000,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.05,
|
||||
"max_leaf_nodes": 63,
|
||||
"max_iter": 200,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.05,
|
||||
"max_leaf_nodes": 63,
|
||||
"max_iter": 500,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.05,
|
||||
"max_leaf_nodes": 63,
|
||||
"max_iter": 1000,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.1,
|
||||
"max_leaf_nodes": 31,
|
||||
"max_iter": 200,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.1,
|
||||
"max_leaf_nodes": 31,
|
||||
"max_iter": 500,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.1,
|
||||
"max_leaf_nodes": 31,
|
||||
"max_iter": 1000,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.1,
|
||||
"max_leaf_nodes": 63,
|
||||
"max_iter": 200,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.1,
|
||||
"max_leaf_nodes": 63,
|
||||
"max_iter": 500,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
},
|
||||
{
|
||||
"learning_rate": 0.1,
|
||||
"max_leaf_nodes": 63,
|
||||
"max_iter": 1000,
|
||||
"min_samples_leaf": 20,
|
||||
"random_state": 0,
|
||||
"early_stopping": false
|
||||
}
|
||||
],
|
||||
"sha256": {
|
||||
"tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"tools/s2_models/features.py": "760e580c811284876e592138931563ddddb7e0ca7c4a2fe8151705510d35e035",
|
||||
"tools/s2_models/network.py": "cfff6c519b6d71f6c71b3bdf330e0fe63691f8d1a308c0e5e46ce7307def2697",
|
||||
"tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"tools/s2_models/requirements-tested.txt": "d0397e13dc138c303e4760eb93c52a28d2d260fc2eb91b12f119bf1da1f0463c",
|
||||
"tools/s2_models/upstream/LICENSE": "2c919cd03fa823bf7eefc00a957ff8324c865cd22aa5285e563dc4b558084f25",
|
||||
"tools/s2_models/upstream/Physics_Attention.py": "4fc710f6f96788fb59cb19fab8d64873d3d16e9e182f12a544d451c74c548cc7",
|
||||
"tools/s2_models/upstream/Transolver_Irregular_Mesh.py": "5259759f7ecc6a954c8442ac8359990b9f242f1437c452c69e5e0f03549b5311",
|
||||
"tools/s2_models/upstream/manifest.json": "fbcaa6673dbc82031e110ea18680574a9d9edcbf33ca6abd537129be4d860eee",
|
||||
"tools/s2_models/upstream-adaptation.diff": "e62a4a7c24bfbc318365a84f4638f875bce587abb082738360bc7f12187c46c3",
|
||||
"tools/test_s2_models.py": "f1c40626ee5e8f1b210ca7f7b665237f5809da920c1c0148e35b499ecf21ca6a"
|
||||
},
|
||||
"acceptance_and_count_wall_seconds": 8.752674713963643,
|
||||
"measurement_note": "Wall includes the test subprocess, parent library import and four model parameter-count constructions. Child CPU/RSS cover the unittest subprocess including imports; unittest internal duration is in unittest.txt. Data/training flags describe audited test/code scope, not an OS file-access trace."
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"timestamp": "2026-11-14T14:44:43.272858+00:00",
|
||||
"progress": {
|
||||
"completed_jobs": 425,
|
||||
"active_jobs": [
|
||||
417,
|
||||
418,
|
||||
422,
|
||||
423,
|
||||
426,
|
||||
430,
|
||||
431,
|
||||
432,
|
||||
433,
|
||||
434,
|
||||
435,
|
||||
436,
|
||||
437,
|
||||
438,
|
||||
439,
|
||||
440
|
||||
],
|
||||
"next_job": 441,
|
||||
"completed_worker_cpu_seconds": 35366.68382500002,
|
||||
"live_tree_cpu_seconds": 801.2499999999997,
|
||||
"active_tree_rss_bytes": 2344103936,
|
||||
"stop_reason": null,
|
||||
"elapsed_seconds": 2384.1496831209515
|
||||
},
|
||||
"terminal_receipts": 425,
|
||||
"failed_receipts": 0,
|
||||
"completion_flags": []
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
test_affine_p1_and_zero_gradient (__main__.GeometryTests) ... ok
|
||||
test_base_slots_and_nonfinite_fail (__main__.GeometryTests) ... ok
|
||||
test_degenerate_and_unused_fail (__main__.GeometryTests) ... ok
|
||||
test_features_and_physical_scaling (__main__.GeometryTests) ... ok
|
||||
test_moment_unequal_volumes_and_coincident_center (__main__.GeometryTests) ... ok
|
||||
test_moment_volume_weight_rotation_and_filter (__main__.GeometryTests) ... ok
|
||||
test_strict_volume_front_ties (__main__.GeometryTests) ... ok
|
||||
test_unreachable_fail_closed (__main__.GeometryTests) ... ok
|
||||
test_volume_lumping (__main__.GeometryTests) ... ok
|
||||
test_wall_edges_removed_both_distances_and_gate_tie (__main__.GeometryTests) ... ok
|
||||
test_four_models_forward_only_and_parameter_counts (__main__.NetworkTests) ... ok
|
||||
test_original_attention_numerical_parity (__main__.NetworkTests) ... ok
|
||||
test_query_order_global_context_and_input_rejection (__main__.NetworkTests) ... ok
|
||||
test_upstream_snapshot_hashes (__main__.NetworkTests) ... ok
|
||||
test_hierarchy_and_draw_compression (__main__.ObjectiveTests) ... ok
|
||||
test_histgbrt_grid_unfitted (__main__.ObjectiveTests) ... ok
|
||||
test_invalid_hierarchy (__main__.ObjectiveTests) ... ok
|
||||
test_process_target_roundtrip_and_scale_invariance (__main__.ObjectiveTests) ... ok
|
||||
test_score_masks_before_arithmetic_and_zero_truth (__main__.ObjectiveTests) ... ok
|
||||
test_zero_missing_scales_no_fallback (__main__.ObjectiveTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 20 tests in 0.214s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,58 @@
|
||||
# S2 R2-G24 / N / E synthetic code acceptance
|
||||
|
||||
## Scope and decisions
|
||||
|
||||
This implements Claude's [22:18:01 ruling](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794665881295239), the v0.5 Gre24 mapping and v0.6 process-conditioned target, with the [22:37:54 correction](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794667074550589). It is a synthetic-only code delivery for Claude's primary review, not a fitted model, real-corpus preprocessing receipt, or approval of predictive accuracy.
|
||||
|
||||
No real geometry, LOG or simulation field is loaded by these modules/tests. The existing registration queue and its pinned sources are untouched. Test execution has empty CUDA visibility, one numerical thread and CPU tensors. No estimator `.fit`, backward pass, optimiser or parameter update is executed. Model initialisation is random, fixed by test seeds.
|
||||
|
||||
**The latest target ruling replaces v0.3's transform entirely:** predict `t* = T/t_set` directly, loss on `t*`, seconds = prediction × `t_set`. There is no log transform, tau, fitted scale, oracle end-of-fill time or fallback. The initial pre-ruling tests were superseded; the receipt in this delivery tests the corrected implementation. `t_set` must be a finite positive process setting supplied equally to all arms. `B0 = V*_f` in normalised units; multiply by `t_set` for seconds. Predictions are not silently clipped.
|
||||
|
||||
## R2-G24 computational contract
|
||||
|
||||
`tools/s2_models/features.py` takes in-memory arrays; it has no data-loading entrypoint.
|
||||
|
||||
- Coordinates are physical 3D positions; tetrahedra index all tet nodes in ascending original-ID order. Invalid indices, unused nodes, degenerate cells and nonfinite inputs fail closed. Exact `D_w=d(x)` is supplied by the existing distance preparation, not recomputed or inferred here.
|
||||
- Tet edges are unique unordered pairs. Gate source is the nearest node to the supplied gate, with exact-distance ties going to the first/smaller original ID.
|
||||
- Both R2 `D_i` and `D_f` exclude edges whose endpoints both have `D_w=0`. Remaining costs are length and length/mean(endpoint `D_w`), respectively. Unreachable nodes fail the part; no epsilon speed or imputation. This is a graph approximation of the eikonal, not tetFIM.
|
||||
- Lumped volume is `Σ incident tet volume/4`. `V*_f(i)` is the strict weighted CDF `Σ_{D_f(j)<D_f(i)} m_j / Σm_j`, with equal values sharing the same CDF. Consequently the maximum-distance node may have `V*_f<1`; the code does not change `<` into `≤` or add half-tie mass. Normalising `D_f` first gives the same ordering.
|
||||
- `D_i*` and `D_w*` divide by their positive maxima. Each scalar's constant P1 cell gradient is solved from the four nodal values; raw gradients are cell-volume averaged to nodes and then unit-normalised. Exactly zero gradients stay zero. **This averaging order and volume weighting are explicit implementation conventions submitted for review.**
|
||||
- Spatial moment: `R=3*h_max`, with `h_max` the longest tet edge. Cell centres are arithmetic vertex means; `D_i(center)` is their P1 mean. Both `||center-node||<R` and `|D_i(center)-D_i(node)|<R` are strict. Coincident centres are excluded before computing neighbourhood volume. `S=Σ(v_j/V_i) u_ij u_ij^T` uses unit directions; trace is one and the tensor is positive semidefinite. Empty neighbourhoods fail closed. A spatial tree avoids storing every node/cell neighbourhood at once.
|
||||
- Columns, in order: `Vf`, its 3 normalised gradient components; `Di*`, its 3 gradient components; raw `Dw`, `Dw*`, its 3 gradient components; `Sxx,Sxy,Sxz,Syy,Syz,Szz`; `t_set` (20 columns). Six independent symmetric entries preserve the full tensor. The common process setting is appended to the Gre24 geometry tuple as required by v0.6.
|
||||
- HistGradientBoostingRegressor is pinned to sklearn 1.7.2; twelve configs: learning rate `{.05,.1}` × leaves `{31,63}` × iterations `{200,500,1000}`, min_samples_leaf 20, random_state 0, early_stopping false. Only configurations and unfitted estimators are tested; no HPO is run.
|
||||
|
||||
## N/E architecture and input contract
|
||||
|
||||
Upstream is [thuml/Transolver at 75e0f67643806a81cd1d3f6adc88dd8c02416fe7](https://github.com/thuml/Transolver/tree/75e0f67643806a81cd1d3f6adc88dd8c02416fe7), MIT. Exact irregular-mesh model, Physics_Attention source, licence and SHA256 manifest are preserved in `tools/s2_models/upstream/`. `upstream-adaptation.diff` records source changes. This is an adaptation, not a claimed reproduction.
|
||||
|
||||
The encoder uses all tet nodes, four irregular Transolver blocks, eight heads, 32 slices, widths 128 or 256, GELU, dropout 0 and upstream Model default mlp_ratio 1. It retains the upstream preprocessing MLP, learned placeholder, initialisation and residual attention/MLP blocks. The final scalar head is removed so each node yields an embedding. No time embedding or unified-position grid is used. Pure torch transpose/reshape replaces einops; torch trunc_normal_ replaces timm; structured-grid classes are omitted. An identical-state attention forward comparison against the original source is tested.
|
||||
|
||||
Base columns (10): `(x-center)/Rg` (3), `d/t_part`, Euclidean gate distance/Rg, unrestricted tet-edge geodesic/Rg, `Rg`, volume, `t_part`, `t_set`. The geodesic helper without R2 wall-edge exclusion is a separate input path. Physical scale and process setting are explicit. `t_part` and geodesic/Rg follow the earlier executable feature conventions. No O2 quantity enters the whole-part encoder.
|
||||
|
||||
The query decoder uses the original node index directly. N receives `[embedding,g12,g23]`; E additionally receives `[q1_mean,q1_max,q2_mean,q2_max]`. Two hidden GELU layers have the encoder width and a linear scalar output predicts `t*`. Missing E inputs, E inputs passed to N, nonfinite base/query values, empty/out-of-range queries fail closed. Qualification (positive distance, valid spectrum, N_eff≥64, complete finite edge block) must be enforced by the common-query producer; this module neither invents eligibility nor reads stored draws. All arms must receive identical eligible query IDs and counts.
|
||||
|
||||
Parameter counts for both tiers/arms are in the test receipt; E adds exactly `4*width` decoder parameters. Capacity selection remains the already-defined shared N/E validation choice; this delivery makes no choice. Optimisation settings are recorded as inert configuration (AdamW 1e-3, weight decay 1e-5, cosine, batch one part, ≤200 epochs, patience 20, seeds 0/1/2). There is no training loop or optimiser construction.
|
||||
|
||||
## Objective and score
|
||||
|
||||
For node i in part p in component g, `w_i = c_i / Σ_{j in p}c_j / n_parts(g) / n_components`. The full learning-pool weights sum to one. Neural loss is `Σ w_i(predicted_t*_i−t*_i)^2`, computed in float32; HistGBRT must use the same weights. Do not renormalise a one-part minibatch, which would erase group balance. An epoch objective sums part contributions. A future uniformly sampled part step multiplies its contribution by the number of learning parts to estimate that full objective without bias.
|
||||
|
||||
Per-part relative L2 masks ineligible rows **before** arithmetic, counts draws without another mass factor, and marks zero truth norm NA. It is invariant to the common positive per-part seconds scale. Aggregation, splits, inner validation, HPO, the training loop and corpus adapters remain later integration work, not represented as implemented execution by this delivery.
|
||||
|
||||
## Acceptance and limitations
|
||||
|
||||
Final synthetic acceptance: **20/20 PASS** (2026-11-14 22:44:43 UTC+8).
|
||||
The test subprocess consumed 5.040 CPU-seconds, peak RSS 554.32 MiB including imports; unittest cases themselves took 0.214 s. The 8.753 s wall receipt includes parent imports and parameter-count construction as explicitly noted there. No benchmark claim is made from these tiny cases.
|
||||
|
||||
| Width | N parameters | E parameters | Shared encoder count |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| 128 | 406,817 | 407,329 | 373,408 |
|
||||
| 256 | 1,606,049 | 1,607,073 | 1,473,696 |
|
||||
|
||||
The registration monitoring snapshot at delivery recorded 425 terminal PASS receipts, 0 failures, 16 active jobs, `stop_reason=null` and no completion flag. This is a partial status, not completion of the 2,012-pair run.
|
||||
|
||||
Run `env OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1 CUDA_VISIBLE_DEVICES='' /home/frank14f/anaconda3/envs/pycuda_3_10/bin/python tools/test_s2_models.py`.
|
||||
|
||||
The synthetic suite checks analytic graph paths (including removed wall shortcuts and gate ties), unreachable/degenerate cases, lumped volume, strict CDF ties, affine P1 recovery, zero gradients, moment volume weighting/rotation/trace/scaling/coincident-centre exclusion, process-scale roundtrip, no missing-scale fallback, draw compression/hierarchy weighting, score masks, 12 unfitted configs, preserved upstream hashes and numerical attention parity, all four architecture forwards, query order/global context and unchanged parameters. Versions and measured CPU/wall/RSS are in `artifacts/s2-model-synthetic-2026-11-14/receipt.json`.
|
||||
|
||||
This does not establish real-corpus reachability after removing wall-wall edges, feature runtime/memory at corpus scale, neural accuracy, trainability, pilot fitness or GPU fit. Those are distinct later receipts. No completed blind review is repeated. The registration process continues independently; its status snapshot is monitoring evidence only.
|
||||
@@ -0,0 +1 @@
|
||||
"""S2 synthetic-validated model and feature primitives; no data loaders or training entrypoint."""
|
||||
@@ -0,0 +1,59 @@
|
||||
# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
|
||||
import torch.nn as nn
|
||||
import torch
|
||||
|
||||
|
||||
class Physics_Attention_Irregular_Mesh(nn.Module):
|
||||
## for irregular meshes in 1D, 2D or 3D space
|
||||
def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64):
|
||||
super().__init__()
|
||||
inner_dim = dim_head * heads
|
||||
self.dim_head = dim_head
|
||||
self.heads = heads
|
||||
self.scale = dim_head ** -0.5
|
||||
self.softmax = nn.Softmax(dim=-1)
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
||||
|
||||
self.in_project_x = nn.Linear(dim, inner_dim)
|
||||
self.in_project_fx = nn.Linear(dim, inner_dim)
|
||||
self.in_project_slice = nn.Linear(dim_head, slice_num)
|
||||
for l in [self.in_project_slice]:
|
||||
torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
||||
self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_out = nn.Sequential(
|
||||
nn.Linear(inner_dim, dim),
|
||||
nn.Dropout(dropout)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
# B N C
|
||||
B, N, C = x.shape
|
||||
|
||||
### (1) Slice
|
||||
fx_mid = self.in_project_fx(x).reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
x_mid = self.in_project_x(x).reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
slice_weights = self.softmax(self.in_project_slice(x_mid) / self.temperature) # B H N G
|
||||
slice_norm = slice_weights.sum(2) # B H G
|
||||
slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
|
||||
slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
||||
|
||||
### (2) Attention among slice tokens
|
||||
q_slice_token = self.to_q(slice_token)
|
||||
k_slice_token = self.to_k(slice_token)
|
||||
v_slice_token = self.to_v(slice_token)
|
||||
dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
||||
attn = self.softmax(dots)
|
||||
attn = self.dropout(attn)
|
||||
out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
||||
|
||||
### (3) Deslice
|
||||
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
||||
out_x = out_x.permute(0, 2, 1, 3).reshape(B, N, -1)
|
||||
return self.to_out(out_x)
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from .attention import Physics_Attention_Irregular_Mesh
|
||||
|
||||
ACTIVATION = {'gelu': nn.GELU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid, 'relu': nn.ReLU, 'leaky_relu': nn.LeakyReLU(0.1),
|
||||
'softplus': nn.Softplus, 'ELU': nn.ELU, 'silu': nn.SiLU}
|
||||
|
||||
|
||||
class MLP(nn.Module):
|
||||
def __init__(self, n_input, n_hidden, n_output, n_layers=1, act='gelu', res=True):
|
||||
super(MLP, self).__init__()
|
||||
|
||||
if act in ACTIVATION.keys():
|
||||
act = ACTIVATION[act]
|
||||
else:
|
||||
raise NotImplementedError
|
||||
self.n_input = n_input
|
||||
self.n_hidden = n_hidden
|
||||
self.n_output = n_output
|
||||
self.n_layers = n_layers
|
||||
self.res = res
|
||||
self.linear_pre = nn.Sequential(nn.Linear(n_input, n_hidden), act())
|
||||
self.linear_post = nn.Linear(n_hidden, n_output)
|
||||
self.linears = nn.ModuleList([nn.Sequential(nn.Linear(n_hidden, n_hidden), act()) for _ in range(n_layers)])
|
||||
|
||||
def forward(self, x):
|
||||
x = self.linear_pre(x)
|
||||
for i in range(self.n_layers):
|
||||
if self.res:
|
||||
x = self.linears[i](x) + x
|
||||
else:
|
||||
x = self.linears[i](x)
|
||||
x = self.linear_post(x)
|
||||
return x
|
||||
|
||||
|
||||
class Transolver_block(nn.Module):
|
||||
"""Transformer encoder block."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
hidden_dim: int,
|
||||
dropout: float,
|
||||
act='gelu',
|
||||
mlp_ratio=4,
|
||||
last_layer=False,
|
||||
out_dim=1,
|
||||
slice_num=32,
|
||||
):
|
||||
super().__init__()
|
||||
self.last_layer = last_layer
|
||||
self.ln_1 = nn.LayerNorm(hidden_dim)
|
||||
self.Attn = Physics_Attention_Irregular_Mesh(hidden_dim, heads=num_heads, dim_head=hidden_dim // num_heads,
|
||||
dropout=dropout, slice_num=slice_num)
|
||||
self.ln_2 = nn.LayerNorm(hidden_dim)
|
||||
self.mlp = MLP(hidden_dim, hidden_dim * mlp_ratio, hidden_dim, n_layers=0, res=False, act=act)
|
||||
if self.last_layer:
|
||||
self.ln_3 = nn.LayerNorm(hidden_dim)
|
||||
self.mlp2 = nn.Linear(hidden_dim, out_dim)
|
||||
|
||||
def forward(self, fx):
|
||||
fx = self.Attn(self.ln_1(fx)) + fx
|
||||
fx = self.mlp(self.ln_2(fx)) + fx
|
||||
if self.last_layer:
|
||||
return self.mlp2(self.ln_3(fx))
|
||||
else:
|
||||
return fx
|
||||
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Array-only R2-G24 graph approximation. No dataset access or learned state."""
|
||||
from itertools import combinations, product
|
||||
|
||||
import numpy as np
|
||||
from scipy.sparse import coo_matrix
|
||||
from scipy.sparse.csgraph import dijkstra
|
||||
from scipy.spatial import cKDTree
|
||||
|
||||
|
||||
def finite_array(value, shape=None, name="array"):
|
||||
a = np.asarray(value, dtype=np.float64)
|
||||
if not np.isfinite(a).all() or (shape is not None and a.shape != shape):
|
||||
raise ValueError(f"invalid {name}: expected finite {shape}, got {a.shape}")
|
||||
return a
|
||||
|
||||
|
||||
def mesh_geometry(points, tets):
|
||||
x = finite_array(points, name="points")
|
||||
t = np.asarray(tets)
|
||||
if x.ndim != 2 or x.shape[1] != 3 or not len(x):
|
||||
raise ValueError("points must be nonempty (N,3)")
|
||||
if t.ndim != 2 or t.shape[1] != 4 or not len(t) or t.dtype.kind not in "iu":
|
||||
raise ValueError("tets must be nonempty integer (M,4)")
|
||||
if t.min() < 0 or t.max() >= len(x):
|
||||
raise ValueError("tet node outside points")
|
||||
cells = x[t]
|
||||
matrices = cells[:, 1:] - cells[:, :1]
|
||||
volumes = np.abs(np.linalg.det(matrices)) / 6
|
||||
if not np.isfinite(volumes).all() or np.any(volumes <= 0):
|
||||
raise ValueError("degenerate tetrahedron")
|
||||
mass = np.bincount(t.ravel(), weights=np.repeat(volumes / 4, 4), minlength=len(x))
|
||||
if np.any(mass <= 0):
|
||||
raise ValueError("unused mesh node")
|
||||
edges = np.unique(np.sort(np.concatenate([t[:, [i, j]] for i, j in combinations(range(4), 2)]), axis=1), axis=0)
|
||||
lengths = np.linalg.norm(x[edges[:, 1]] - x[edges[:, 0]], axis=1)
|
||||
return x, t, matrices, volumes, mass, edges, lengths
|
||||
|
||||
|
||||
def graph_distances(points, tets, wall_distance, gate, *, exclude_wall_edges=True):
|
||||
x, t, matrices, vol, mass, edges, lengths = mesh_geometry(points, tets)
|
||||
dw = finite_array(wall_distance, (len(x),), "wall distance")
|
||||
gate = finite_array(gate, (3,), "gate")
|
||||
if np.any(dw < 0):
|
||||
raise ValueError("negative wall distance")
|
||||
# argmin returns the first node in original ascending node-ID order.
|
||||
source = int(np.argmin(np.linalg.norm(x - gate, axis=1)))
|
||||
keep = (dw[edges].sum(axis=1) > 0) if exclude_wall_edges else np.ones(len(edges), dtype=bool)
|
||||
e, length = edges[keep], lengths[keep]
|
||||
|
||||
def solve(cost):
|
||||
graph = coo_matrix((np.tile(cost, 2), (np.r_[e[:, 0], e[:, 1]], np.r_[e[:, 1], e[:, 0]])), shape=(len(x), len(x))).tocsr()
|
||||
out = dijkstra(graph, directed=False, indices=source)
|
||||
if not np.isfinite(out).all():
|
||||
raise ValueError("unreachable tet node after graph-edge rule")
|
||||
return out
|
||||
|
||||
di = solve(length)
|
||||
# Unrestricted geodesic is a separate N/E base input, not flow distance.
|
||||
if not exclude_wall_edges:
|
||||
return di, source
|
||||
df = solve(length / dw[e].mean(axis=1))
|
||||
return di, df, source
|
||||
|
||||
|
||||
def relative_flow_volume(flow_distance, lumped_mass):
|
||||
"""Strict front CDF: sum masses with D_f[j] < D_f[i]; ties share a value."""
|
||||
d = finite_array(flow_distance, name="flow distance")
|
||||
m = finite_array(lumped_mass, d.shape, "lumped mass")
|
||||
if d.ndim != 1 or not len(d) or np.any(m <= 0):
|
||||
raise ValueError("invalid volume CDF inputs")
|
||||
order = np.argsort(d, kind="stable")
|
||||
prefix = np.r_[0., np.cumsum(m[order])]
|
||||
return prefix[np.searchsorted(d[order], d, side="left")] / m.sum()
|
||||
|
||||
|
||||
def p1_nodal_gradient(points, tets, values, *, normalize=True):
|
||||
"""Volume-average constant cell P1 gradients, then unit-normalise at nodes.
|
||||
|
||||
An exactly zero averaged gradient stays zero; no epsilon direction is invented.
|
||||
"""
|
||||
x, t, matrices, volumes, mass, _, _ = mesh_geometry(points, tets)
|
||||
v = finite_array(values, (len(x),), "P1 nodal values")
|
||||
cell_grad = np.linalg.solve(matrices, (v[t[:, 1:]] - v[t[:, :1]])[..., None])[..., 0]
|
||||
grad = np.zeros((len(x), 3))
|
||||
for k in range(4):
|
||||
np.add.at(grad, t[:, k], cell_grad * volumes[:, None])
|
||||
grad /= (4 * mass[:, None])
|
||||
if normalize:
|
||||
norm = np.linalg.norm(grad, axis=1)
|
||||
np.divide(grad, norm[:, None], out=grad, where=norm[:, None] > 0)
|
||||
return grad
|
||||
|
||||
|
||||
def spatial_moment(points, tets, injection_distance):
|
||||
"""Volume-weighted unit-direction moment with two strict neighbourhood tests.
|
||||
|
||||
Claude 1794667074.550589: exclude coincident cell centres before normalising
|
||||
volumes. Trace is one. Empty neighbourhoods fail closed.
|
||||
"""
|
||||
x, t, _, volumes, _, _, lengths = mesh_geometry(points, tets)
|
||||
di = finite_array(injection_distance, (len(x),), "injection distance")
|
||||
radius = 3 * lengths.max()
|
||||
centers = x[t].mean(axis=1)
|
||||
cell_di = di[t].mean(axis=1)
|
||||
tree = cKDTree(centers)
|
||||
result = np.empty((len(x), 3, 3))
|
||||
# One query at a time avoids storing all node/cell neighbourhoods together.
|
||||
for i, point in enumerate(x):
|
||||
ids = np.asarray(tree.query_ball_point(point, radius), dtype=np.int64)
|
||||
distance = np.linalg.norm(centers[ids] - point, axis=1)
|
||||
ids = ids[(np.abs(cell_di[ids] - di[i]) < radius) & (distance < radius) & (distance > 0)]
|
||||
if not len(ids):
|
||||
raise ValueError(f"empty spatial-moment neighbourhood at node {i}")
|
||||
delta = centers[ids] - point
|
||||
delta /= np.linalg.norm(delta, axis=1)[:, None]
|
||||
weight = volumes[ids] / volumes[ids].sum()
|
||||
result[i] = np.einsum("j,ja,jb->ab", weight, delta, delta)
|
||||
return result, float(radius)
|
||||
|
||||
|
||||
R2_COLUMNS = ("Vf", "grad_Vf_x", "grad_Vf_y", "grad_Vf_z", "Di_star",
|
||||
"grad_Di_x", "grad_Di_y", "grad_Di_z", "Dw", "Dw_star",
|
||||
"grad_Dw_x", "grad_Dw_y", "grad_Dw_z", "Sxx", "Sxy", "Sxz", "Syy", "Syz", "Szz", "t_set")
|
||||
|
||||
|
||||
def r2_features(points, tets, wall_distance, gate, t_set):
|
||||
x, t, _, _, mass, _, _ = mesh_geometry(points, tets)
|
||||
dw = finite_array(wall_distance, (len(x),), "wall distance")
|
||||
ts = float(t_set)
|
||||
if not np.isfinite(ts) or ts <= 0:
|
||||
raise ValueError("missing/nonpositive t_set")
|
||||
di, df, source = graph_distances(x, t, dw, gate)
|
||||
if di.max() <= 0 or dw.max() <= 0:
|
||||
raise ValueError("zero normalisation scale")
|
||||
di_star, dw_star = di / di.max(), dw / dw.max()
|
||||
vf = relative_flow_volume(df, mass)
|
||||
moment, radius = spatial_moment(x, t, di)
|
||||
sym = moment[:, [0, 0, 0, 1, 1, 2], [0, 1, 2, 1, 2, 2]]
|
||||
features = np.column_stack([vf, p1_nodal_gradient(x, t, vf), di_star,
|
||||
p1_nodal_gradient(x, t, di_star), dw, dw_star,
|
||||
p1_nodal_gradient(x, t, dw_star), sym, np.full(len(x), ts)])
|
||||
finite_array(features, (len(x), len(R2_COLUMNS)), "R2 inputs")
|
||||
return {"features": features, "columns": R2_COLUMNS, "Di": di, "Df": df,
|
||||
"Vf": vf, "S": moment, "radius": radius, "gate_node": source}
|
||||
|
||||
|
||||
BASE_COLUMNS = ("x_star", "y_star", "z_star", "d_over_t", "gate_distance_over_Rg",
|
||||
"geodesic_over_Rg", "Rg", "volume", "t_part", "t_set")
|
||||
|
||||
|
||||
def base_features(points, wall_distance, geodesic_distance, gate, center, rg, volume, t_part, t_set):
|
||||
x = finite_array(points, name="points")
|
||||
if x.ndim != 2 or x.shape[1] != 3 or not len(x):
|
||||
raise ValueError("invalid node coordinates")
|
||||
dw = finite_array(wall_distance, (len(x),), "wall distance")
|
||||
dg = finite_array(geodesic_distance, (len(x),), "geodesic")
|
||||
gate, center = finite_array(gate, (3,), "gate"), finite_array(center, (3,), "center")
|
||||
scales = finite_array([rg, volume, t_part, t_set], (4,), "physical scales")
|
||||
if np.any(scales <= 0) or np.any(dw < 0) or np.any(dg < 0):
|
||||
raise ValueError("nonpositive scale or negative distance")
|
||||
return np.column_stack([(x-center)/rg, dw/t_part, np.linalg.norm(x-gate, axis=1)/rg,
|
||||
dg/rg, np.tile(scales, (len(x), 1))])
|
||||
|
||||
|
||||
def histgbrt_grid():
|
||||
"""Twelve unfitted estimators' configurations; never launches fits."""
|
||||
return [dict(learning_rate=lr, max_leaf_nodes=leaf, max_iter=n, min_samples_leaf=20,
|
||||
random_state=0, early_stopping=False)
|
||||
for lr, leaf, n in product((.05, .1), (31, 63), (200, 500, 1000))]
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
|
||||
|
||||
No optimiser, training loop, data reader or GPU invocation is provided.
|
||||
"""
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from .blocks import MLP, Transolver_block
|
||||
from .features import BASE_COLUMNS
|
||||
|
||||
|
||||
TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
schedule="cosine", batch_parts=1, max_epochs=200,
|
||||
patience=20, seeds=[0, 1, 2], widths=[128, 256])
|
||||
|
||||
|
||||
def check_tensor(tensor, shape, name):
|
||||
if tuple(tensor.shape) != tuple(shape) or not torch.isfinite(tensor).all():
|
||||
raise ValueError(f"invalid {name}")
|
||||
|
||||
|
||||
class WholePartEncoder(nn.Module):
|
||||
def __init__(self, width):
|
||||
super().__init__()
|
||||
if width not in (128, 256):
|
||||
raise ValueError("S2 permits widths 128 or 256 only")
|
||||
self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
dropout=0., mlp_ratio=1, slice_num=32,
|
||||
last_layer=False) for _ in range(4)])
|
||||
self.apply(self._initialize)
|
||||
self.placeholder = nn.Parameter(torch.rand(width)/width)
|
||||
|
||||
@staticmethod
|
||||
def _initialize(module):
|
||||
# torch.nn.init replaces timm's same trunc_normal_ defaults; no timm dependency.
|
||||
if isinstance(module, nn.Linear):
|
||||
nn.init.trunc_normal_(module.weight, std=.02)
|
||||
if module.bias is not None:
|
||||
nn.init.zeros_(module.bias)
|
||||
elif isinstance(module, nn.LayerNorm):
|
||||
nn.init.zeros_(module.bias)
|
||||
nn.init.ones_(module.weight)
|
||||
|
||||
def forward(self, base):
|
||||
if base.ndim != 2 or base.shape[0] == 0:
|
||||
raise ValueError("one whole part with all tet nodes required")
|
||||
check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
for block in self.blocks:
|
||||
fx = block(fx)
|
||||
if not torch.isfinite(fx).all():
|
||||
raise ValueError("nonfinite encoder output")
|
||||
return fx[0]
|
||||
|
||||
|
||||
class QueryModel(nn.Module):
|
||||
def __init__(self, arm, width=128):
|
||||
super().__init__()
|
||||
if arm not in ("N", "E"):
|
||||
raise ValueError("arm must be N or E")
|
||||
self.arm, self.width = arm, width
|
||||
self.encoder = WholePartEncoder(width)
|
||||
query_dim = 2 if arm == "N" else 6
|
||||
self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
self.decoder.apply(WholePartEncoder._initialize)
|
||||
|
||||
def forward(self, base, query_ids, gaps, edge_q=None):
|
||||
if query_ids.ndim != 1 or query_ids.dtype != torch.long or query_ids.numel() == 0:
|
||||
raise ValueError("nonempty int64 original-node query IDs required")
|
||||
if int(query_ids.min()) < 0 or int(query_ids.max()) >= len(base):
|
||||
raise ValueError("query node outside whole part")
|
||||
check_tensor(gaps, (len(query_ids), 2), "g12/g23")
|
||||
if self.arm == "E":
|
||||
if edge_q is None:
|
||||
raise ValueError("E requires all four edge summaries")
|
||||
check_tensor(edge_q, (len(query_ids), 4), "edge summaries")
|
||||
local = torch.cat([gaps, edge_q], dim=-1)
|
||||
else:
|
||||
if edge_q is not None:
|
||||
raise ValueError("N must not receive edge summaries")
|
||||
local = gaps
|
||||
embedding = self.encoder(base)[query_ids]
|
||||
y = self.decoder(torch.cat([embedding, local], dim=-1)).squeeze(-1)
|
||||
if not torch.isfinite(y).all():
|
||||
raise ValueError("nonfinite decoder output")
|
||||
return y
|
||||
|
||||
def parameter_counts(self):
|
||||
return {"encoder": sum(p.numel() for p in self.encoder.parameters()),
|
||||
"decoder": sum(p.numel() for p in self.decoder.parameters()),
|
||||
"total": sum(p.numel() for p in self.parameters())}
|
||||
|
||||
|
||||
def weighted_mse(prediction, target, global_weights):
|
||||
"""Sum of contributions with already globally-normalised hierarchy weights.
|
||||
|
||||
Do not renormalise a single part's weights: that would erase component balance.
|
||||
"""
|
||||
if prediction.ndim != 1 or prediction.numel() == 0:
|
||||
raise ValueError("nonempty query predictions required")
|
||||
check_tensor(prediction, prediction.shape, "prediction")
|
||||
check_tensor(target, prediction.shape, "target")
|
||||
check_tensor(global_weights, prediction.shape, "weights")
|
||||
if (global_weights <= 0).any():
|
||||
raise ValueError("positive hierarchy weights required")
|
||||
# FP32 arithmetic under future AMP; do not form residuals in float16.
|
||||
return torch.sum(global_weights.float() * (prediction.float()-target.float()).square())
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Process-conditioned target, hierarchy weights and scoring; no fitting I/O."""
|
||||
import numpy as np
|
||||
|
||||
from .features import finite_array
|
||||
|
||||
|
||||
def positive_scale(value, name):
|
||||
v = float(value)
|
||||
if not np.isfinite(v) or v <= 0:
|
||||
raise ValueError(f"{name} must be finite and positive")
|
||||
return v
|
||||
|
||||
|
||||
def dimensionless_target(seconds, t_set):
|
||||
t = finite_array(seconds, name="seconds")
|
||||
if np.any(t < 0):
|
||||
raise ValueError("negative arrival time")
|
||||
return t / positive_scale(t_set, "t_set")
|
||||
|
||||
|
||||
def prediction_seconds(t_star, t_set):
|
||||
"""Direct target per Claude 1794667074.550589: no log transform or tau.
|
||||
|
||||
Predictions are not clipped; invalid output fails closed and is reported.
|
||||
"""
|
||||
t = finite_array(t_star, name="prediction t_star")
|
||||
return finite_array(t * positive_scale(t_set, "t_set"), name="predicted seconds")
|
||||
|
||||
|
||||
def b0_seconds(relative_volume, t_set):
|
||||
vf = finite_array(relative_volume, name="relative flow volume")
|
||||
if np.any((vf < 0) | (vf > 1)):
|
||||
raise ValueError("relative volume outside [0,1]")
|
||||
return vf * positive_scale(t_set, "t_set")
|
||||
|
||||
|
||||
def hierarchy_weights(draw_counts, part_ids, component_ids):
|
||||
"""Sum=1; equal components, equal parts in each, count-weighted nodes.
|
||||
|
||||
Call on the full learning pool, not on one minibatch. A batch-1-part
|
||||
objective sums each part's weighted contribution over the epoch; if
|
||||
sampling parts uniformly with replacement multiply each contribution by
|
||||
the number of learning parts for an unbiased per-step objective.
|
||||
"""
|
||||
c = finite_array(draw_counts, name="draw counts")
|
||||
p, g = np.asarray(part_ids), np.asarray(component_ids)
|
||||
if c.ndim != 1 or not len(c) or p.shape != c.shape or g.shape != c.shape:
|
||||
raise ValueError("invalid hierarchy shape")
|
||||
if np.any(c <= 0) or np.any(c != np.floor(c)):
|
||||
raise ValueError("only eligible positive draw counts accepted")
|
||||
groups = np.unique(g)
|
||||
w = np.empty_like(c)
|
||||
for part in np.unique(p):
|
||||
mask = p == part
|
||||
if len(np.unique(g[mask])) != 1:
|
||||
raise ValueError("one part belongs to multiple components")
|
||||
for group in groups:
|
||||
mask_g = g == group
|
||||
parts = np.unique(p[mask_g])
|
||||
for part in parts:
|
||||
mask = mask_g & (p == part)
|
||||
w[mask] = c[mask] / c[mask].sum() / len(parts) / len(groups)
|
||||
return w
|
||||
|
||||
|
||||
def relative_l2(prediction, truth, draw_counts, eligible):
|
||||
p, t, c = np.asarray(prediction), np.asarray(truth), np.asarray(draw_counts)
|
||||
e = np.asarray(eligible)
|
||||
if p.ndim != 1 or t.shape != p.shape or c.shape != p.shape or e.shape != p.shape or e.dtype.kind != "b":
|
||||
raise ValueError("invalid score arrays")
|
||||
# Mask before any arithmetic: ineligible NaNs must never enter 0*NaN.
|
||||
p, t, c = (finite_array(v[e], name="eligible score input") for v in (p, t, c))
|
||||
if not len(c) or np.any(c <= 0) or np.any(c != np.floor(c)):
|
||||
raise ValueError("no eligible positive integer draw counts")
|
||||
denominator = np.sum(c*t*t)
|
||||
if denominator <= 0:
|
||||
return {"value": None, "reason": "zero truth norm"}
|
||||
value = float(np.sqrt(np.sum(c*(p-t)**2)/denominator))
|
||||
if not np.isfinite(value):
|
||||
raise ValueError("nonfinite relative L2")
|
||||
return {"value": value, "reason": None}
|
||||
@@ -0,0 +1,5 @@
|
||||
# Tested existing CPU execution environment. No installation was performed.
|
||||
numpy==1.26.4
|
||||
scipy==1.15.3
|
||||
scikit-learn==1.7.2
|
||||
torch==2.5.1+cu124
|
||||
@@ -0,0 +1,350 @@
|
||||
--- upstream/Physics_Attention.py
|
||||
+++ attention.py
|
||||
@@ -1,6 +1,6 @@
|
||||
+# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
|
||||
import torch.nn as nn
|
||||
import torch
|
||||
-from einops import rearrange, repeat
|
||||
|
||||
|
||||
class Physics_Attention_Irregular_Mesh(nn.Module):
|
||||
@@ -53,123 +53,7 @@
|
||||
|
||||
### (3) Deslice
|
||||
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
||||
- out_x = rearrange(out_x, 'b h n d -> b n (h d)')
|
||||
+ out_x = out_x.permute(0, 2, 1, 3).reshape(B, N, -1)
|
||||
return self.to_out(out_x)
|
||||
|
||||
|
||||
-class Physics_Attention_Structured_Mesh_2D(nn.Module):
|
||||
- ## for structured mesh in 2D space
|
||||
- def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64, H=101, W=31, kernel=3): # kernel=3):
|
||||
- super().__init__()
|
||||
- inner_dim = dim_head * heads
|
||||
- self.dim_head = dim_head
|
||||
- self.heads = heads
|
||||
- self.scale = dim_head ** -0.5
|
||||
- self.softmax = nn.Softmax(dim=-1)
|
||||
- self.dropout = nn.Dropout(dropout)
|
||||
- self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
||||
- self.H = H
|
||||
- self.W = W
|
||||
-
|
||||
- self.in_project_x = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2)
|
||||
- self.in_project_fx = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2)
|
||||
- self.in_project_slice = nn.Linear(dim_head, slice_num)
|
||||
- for l in [self.in_project_slice]:
|
||||
- torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
||||
- self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
||||
- self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
||||
- self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
||||
-
|
||||
- self.to_out = nn.Sequential(
|
||||
- nn.Linear(inner_dim, dim),
|
||||
- nn.Dropout(dropout)
|
||||
- )
|
||||
-
|
||||
- def forward(self, x):
|
||||
- # B N C
|
||||
- B, N, C = x.shape
|
||||
- x = x.reshape(B, self.H, self.W, C).contiguous().permute(0, 3, 1, 2).contiguous() # B C H W
|
||||
-
|
||||
- ### (1) Slice
|
||||
- fx_mid = self.in_project_fx(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
||||
- .permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
- x_mid = self.in_project_x(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
||||
- .permute(0, 2, 1, 3).contiguous() # B H N G
|
||||
- slice_weights = self.softmax(
|
||||
- self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G
|
||||
- slice_norm = slice_weights.sum(2) # B H G
|
||||
- slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
|
||||
- slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
||||
-
|
||||
- ### (2) Attention among slice tokens
|
||||
- q_slice_token = self.to_q(slice_token)
|
||||
- k_slice_token = self.to_k(slice_token)
|
||||
- v_slice_token = self.to_v(slice_token)
|
||||
- dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
||||
- attn = self.softmax(dots)
|
||||
- attn = self.dropout(attn)
|
||||
- out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
||||
-
|
||||
- ### (3) Deslice
|
||||
- out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
||||
- out_x = rearrange(out_x, 'b h n d -> b n (h d)')
|
||||
- return self.to_out(out_x)
|
||||
-
|
||||
-
|
||||
-class Physics_Attention_Structured_Mesh_3D(nn.Module):
|
||||
- ## for structured mesh in 3D space
|
||||
- def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=32, H=32, W=32, D=32, kernel=3):
|
||||
- super().__init__()
|
||||
- inner_dim = dim_head * heads
|
||||
- self.dim_head = dim_head
|
||||
- self.heads = heads
|
||||
- self.scale = dim_head ** -0.5
|
||||
- self.softmax = nn.Softmax(dim=-1)
|
||||
- self.dropout = nn.Dropout(dropout)
|
||||
- self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
||||
- self.H = H
|
||||
- self.W = W
|
||||
- self.D = D
|
||||
-
|
||||
- self.in_project_x = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2)
|
||||
- self.in_project_fx = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2)
|
||||
- self.in_project_slice = nn.Linear(dim_head, slice_num)
|
||||
- for l in [self.in_project_slice]:
|
||||
- torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
||||
- self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
||||
- self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
||||
- self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
||||
- self.to_out = nn.Sequential(
|
||||
- nn.Linear(inner_dim, dim),
|
||||
- nn.Dropout(dropout)
|
||||
- )
|
||||
-
|
||||
- def forward(self, x):
|
||||
- # B N C
|
||||
- B, N, C = x.shape
|
||||
- x = x.reshape(B, self.H, self.W, self.D, C).contiguous().permute(0, 4, 1, 2, 3).contiguous() # B C H W
|
||||
-
|
||||
- ### (1) Slice
|
||||
- fx_mid = self.in_project_fx(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
||||
- .permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
- x_mid = self.in_project_x(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
||||
- .permute(0, 2, 1, 3).contiguous() # B H N G
|
||||
- slice_weights = self.softmax(
|
||||
- self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G
|
||||
- slice_norm = slice_weights.sum(2) # B H G
|
||||
- slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
|
||||
- slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
||||
-
|
||||
- ### (2) Attention among slice tokens
|
||||
- q_slice_token = self.to_q(slice_token)
|
||||
- k_slice_token = self.to_k(slice_token)
|
||||
- v_slice_token = self.to_v(slice_token)
|
||||
- dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
||||
- attn = self.softmax(dots)
|
||||
- attn = self.dropout(attn)
|
||||
- out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
||||
-
|
||||
- ### (3) Deslice
|
||||
- out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
||||
- out_x = rearrange(out_x, 'b h n d -> b n (h d)')
|
||||
- return self.to_out(out_x)
|
||||
--- upstream/Transolver_Irregular_Mesh.py
|
||||
+++ blocks.py
|
||||
@@ -1,9 +1,7 @@
|
||||
+# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
-from timm.models.layers import trunc_normal_
|
||||
-from model.Embedding import timestep_embedding
|
||||
-import numpy as np
|
||||
-from model.Physics_Attention import Physics_Attention_Irregular_Mesh
|
||||
+from .attention import Physics_Attention_Irregular_Mesh
|
||||
|
||||
ACTIVATION = {'gelu': nn.GELU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid, 'relu': nn.ReLU, 'leaky_relu': nn.LeakyReLU(0.1),
|
||||
'softplus': nn.Softplus, 'ELU': nn.ELU, 'silu': nn.SiLU}
|
||||
@@ -71,88 +69,3 @@
|
||||
return fx
|
||||
|
||||
|
||||
-class Model(nn.Module):
|
||||
- def __init__(self,
|
||||
- space_dim=1,
|
||||
- n_layers=5,
|
||||
- n_hidden=256,
|
||||
- dropout=0.0,
|
||||
- n_head=8,
|
||||
- Time_Input=False,
|
||||
- act='gelu',
|
||||
- mlp_ratio=1,
|
||||
- fun_dim=1,
|
||||
- out_dim=1,
|
||||
- slice_num=32,
|
||||
- ref=8,
|
||||
- unified_pos=False
|
||||
- ):
|
||||
- super(Model, self).__init__()
|
||||
- self.__name__ = 'Transolver_1D'
|
||||
- self.ref = ref
|
||||
- self.unified_pos = unified_pos
|
||||
- self.Time_Input = Time_Input
|
||||
- self.n_hidden = n_hidden
|
||||
- self.space_dim = space_dim
|
||||
- if self.unified_pos:
|
||||
- self.preprocess = MLP(fun_dim + self.ref * self.ref, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act)
|
||||
- else:
|
||||
- self.preprocess = MLP(fun_dim + space_dim, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act)
|
||||
- if Time_Input:
|
||||
- self.time_fc = nn.Sequential(nn.Linear(n_hidden, n_hidden), nn.SiLU(), nn.Linear(n_hidden, n_hidden))
|
||||
-
|
||||
- self.blocks = nn.ModuleList([Transolver_block(num_heads=n_head, hidden_dim=n_hidden,
|
||||
- dropout=dropout,
|
||||
- act=act,
|
||||
- mlp_ratio=mlp_ratio,
|
||||
- out_dim=out_dim,
|
||||
- slice_num=slice_num,
|
||||
- last_layer=(_ == n_layers - 1))
|
||||
- for _ in range(n_layers)])
|
||||
- self.initialize_weights()
|
||||
- self.placeholder = nn.Parameter((1 / (n_hidden)) * torch.rand(n_hidden, dtype=torch.float))
|
||||
-
|
||||
- def initialize_weights(self):
|
||||
- self.apply(self._init_weights)
|
||||
-
|
||||
- def _init_weights(self, m):
|
||||
- if isinstance(m, nn.Linear):
|
||||
- trunc_normal_(m.weight, std=0.02)
|
||||
- if isinstance(m, nn.Linear) and m.bias is not None:
|
||||
- nn.init.constant_(m.bias, 0)
|
||||
- elif isinstance(m, (nn.LayerNorm, nn.BatchNorm1d)):
|
||||
- nn.init.constant_(m.bias, 0)
|
||||
- nn.init.constant_(m.weight, 1.0)
|
||||
-
|
||||
- def get_grid(self, x, batchsize=1):
|
||||
- # x: B N 2
|
||||
- # grid_ref
|
||||
- gridx = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float)
|
||||
- gridx = gridx.reshape(1, self.ref, 1, 1).repeat([batchsize, 1, self.ref, 1])
|
||||
- gridy = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float)
|
||||
- gridy = gridy.reshape(1, 1, self.ref, 1).repeat([batchsize, self.ref, 1, 1])
|
||||
- grid_ref = torch.cat((gridx, gridy), dim=-1).cuda().reshape(batchsize, self.ref * self.ref, 2) # B H W 8 8 2
|
||||
-
|
||||
- pos = torch.sqrt(torch.sum((x[:, :, None, :] - grid_ref[:, None, :, :]) ** 2, dim=-1)). \
|
||||
- reshape(batchsize, x.shape[1], self.ref * self.ref).contiguous()
|
||||
- return pos
|
||||
-
|
||||
- def forward(self, x, fx, T=None):
|
||||
- if self.unified_pos:
|
||||
- x = self.get_grid(x, x.shape[0])
|
||||
- if fx is not None:
|
||||
- fx = torch.cat((x, fx), -1)
|
||||
- fx = self.preprocess(fx)
|
||||
- else:
|
||||
- fx = self.preprocess(x)
|
||||
- fx = fx + self.placeholder[None, None, :]
|
||||
-
|
||||
- if T is not None:
|
||||
- Time_emb = timestep_embedding(T, self.n_hidden).repeat(1, x.shape[1], 1)
|
||||
- Time_emb = self.time_fc(Time_emb)
|
||||
- fx = fx + Time_emb
|
||||
-
|
||||
- for block in self.blocks:
|
||||
- fx = block(fx)
|
||||
-
|
||||
- return fx
|
||||
--- /dev/null
|
||||
+++ network.py
|
||||
@@ -0,0 +1,109 @@
|
||||
+"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
|
||||
+
|
||||
+No optimiser, training loop, data reader or GPU invocation is provided.
|
||||
+"""
|
||||
+import torch
|
||||
+from torch import nn
|
||||
+
|
||||
+from .blocks import MLP, Transolver_block
|
||||
+from .features import BASE_COLUMNS
|
||||
+
|
||||
+
|
||||
+TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
+ schedule="cosine", batch_parts=1, max_epochs=200,
|
||||
+ patience=20, seeds=[0, 1, 2], widths=[128, 256])
|
||||
+
|
||||
+
|
||||
+def check_tensor(tensor, shape, name):
|
||||
+ if tuple(tensor.shape) != tuple(shape) or not torch.isfinite(tensor).all():
|
||||
+ raise ValueError(f"invalid {name}")
|
||||
+
|
||||
+
|
||||
+class WholePartEncoder(nn.Module):
|
||||
+ def __init__(self, width):
|
||||
+ super().__init__()
|
||||
+ if width not in (128, 256):
|
||||
+ raise ValueError("S2 permits widths 128 or 256 only")
|
||||
+ self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
+ self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
+ dropout=0., mlp_ratio=1, slice_num=32,
|
||||
+ last_layer=False) for _ in range(4)])
|
||||
+ self.apply(self._initialize)
|
||||
+ self.placeholder = nn.Parameter(torch.rand(width)/width)
|
||||
+
|
||||
+ @staticmethod
|
||||
+ def _initialize(module):
|
||||
+ # torch.nn.init replaces timm's same trunc_normal_ defaults; no timm dependency.
|
||||
+ if isinstance(module, nn.Linear):
|
||||
+ nn.init.trunc_normal_(module.weight, std=.02)
|
||||
+ if module.bias is not None:
|
||||
+ nn.init.zeros_(module.bias)
|
||||
+ elif isinstance(module, nn.LayerNorm):
|
||||
+ nn.init.zeros_(module.bias)
|
||||
+ nn.init.ones_(module.weight)
|
||||
+
|
||||
+ def forward(self, base):
|
||||
+ if base.ndim != 2 or base.shape[0] == 0:
|
||||
+ raise ValueError("one whole part with all tet nodes required")
|
||||
+ check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
+ fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
+ for block in self.blocks:
|
||||
+ fx = block(fx)
|
||||
+ if not torch.isfinite(fx).all():
|
||||
+ raise ValueError("nonfinite encoder output")
|
||||
+ return fx[0]
|
||||
+
|
||||
+
|
||||
+class QueryModel(nn.Module):
|
||||
+ def __init__(self, arm, width=128):
|
||||
+ super().__init__()
|
||||
+ if arm not in ("N", "E"):
|
||||
+ raise ValueError("arm must be N or E")
|
||||
+ self.arm, self.width = arm, width
|
||||
+ self.encoder = WholePartEncoder(width)
|
||||
+ query_dim = 2 if arm == "N" else 6
|
||||
+ self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
+ nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
+ self.decoder.apply(WholePartEncoder._initialize)
|
||||
+
|
||||
+ def forward(self, base, query_ids, gaps, edge_q=None):
|
||||
+ if query_ids.ndim != 1 or query_ids.dtype != torch.long or query_ids.numel() == 0:
|
||||
+ raise ValueError("nonempty int64 original-node query IDs required")
|
||||
+ if int(query_ids.min()) < 0 or int(query_ids.max()) >= len(base):
|
||||
+ raise ValueError("query node outside whole part")
|
||||
+ check_tensor(gaps, (len(query_ids), 2), "g12/g23")
|
||||
+ if self.arm == "E":
|
||||
+ if edge_q is None:
|
||||
+ raise ValueError("E requires all four edge summaries")
|
||||
+ check_tensor(edge_q, (len(query_ids), 4), "edge summaries")
|
||||
+ local = torch.cat([gaps, edge_q], dim=-1)
|
||||
+ else:
|
||||
+ if edge_q is not None:
|
||||
+ raise ValueError("N must not receive edge summaries")
|
||||
+ local = gaps
|
||||
+ embedding = self.encoder(base)[query_ids]
|
||||
+ y = self.decoder(torch.cat([embedding, local], dim=-1)).squeeze(-1)
|
||||
+ if not torch.isfinite(y).all():
|
||||
+ raise ValueError("nonfinite decoder output")
|
||||
+ return y
|
||||
+
|
||||
+ def parameter_counts(self):
|
||||
+ return {"encoder": sum(p.numel() for p in self.encoder.parameters()),
|
||||
+ "decoder": sum(p.numel() for p in self.decoder.parameters()),
|
||||
+ "total": sum(p.numel() for p in self.parameters())}
|
||||
+
|
||||
+
|
||||
+def weighted_mse(prediction, target, global_weights):
|
||||
+ """Sum of contributions with already globally-normalised hierarchy weights.
|
||||
+
|
||||
+ Do not renormalise a single part's weights: that would erase component balance.
|
||||
+ """
|
||||
+ if prediction.ndim != 1 or prediction.numel() == 0:
|
||||
+ raise ValueError("nonempty query predictions required")
|
||||
+ check_tensor(prediction, prediction.shape, "prediction")
|
||||
+ check_tensor(target, prediction.shape, "target")
|
||||
+ check_tensor(global_weights, prediction.shape, "weights")
|
||||
+ if (global_weights <= 0).any():
|
||||
+ raise ValueError("positive hierarchy weights required")
|
||||
+ # FP32 arithmetic under future AMP; do not form residuals in float16.
|
||||
+ return torch.sum(global_weights.float() * (prediction.float()-target.float()).square())
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 THUML @ Tsinghua University
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,175 @@
|
||||
import torch.nn as nn
|
||||
import torch
|
||||
from einops import rearrange, repeat
|
||||
|
||||
|
||||
class Physics_Attention_Irregular_Mesh(nn.Module):
|
||||
## for irregular meshes in 1D, 2D or 3D space
|
||||
def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64):
|
||||
super().__init__()
|
||||
inner_dim = dim_head * heads
|
||||
self.dim_head = dim_head
|
||||
self.heads = heads
|
||||
self.scale = dim_head ** -0.5
|
||||
self.softmax = nn.Softmax(dim=-1)
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
||||
|
||||
self.in_project_x = nn.Linear(dim, inner_dim)
|
||||
self.in_project_fx = nn.Linear(dim, inner_dim)
|
||||
self.in_project_slice = nn.Linear(dim_head, slice_num)
|
||||
for l in [self.in_project_slice]:
|
||||
torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
||||
self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_out = nn.Sequential(
|
||||
nn.Linear(inner_dim, dim),
|
||||
nn.Dropout(dropout)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
# B N C
|
||||
B, N, C = x.shape
|
||||
|
||||
### (1) Slice
|
||||
fx_mid = self.in_project_fx(x).reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
x_mid = self.in_project_x(x).reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
slice_weights = self.softmax(self.in_project_slice(x_mid) / self.temperature) # B H N G
|
||||
slice_norm = slice_weights.sum(2) # B H G
|
||||
slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
|
||||
slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
||||
|
||||
### (2) Attention among slice tokens
|
||||
q_slice_token = self.to_q(slice_token)
|
||||
k_slice_token = self.to_k(slice_token)
|
||||
v_slice_token = self.to_v(slice_token)
|
||||
dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
||||
attn = self.softmax(dots)
|
||||
attn = self.dropout(attn)
|
||||
out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
||||
|
||||
### (3) Deslice
|
||||
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
||||
out_x = rearrange(out_x, 'b h n d -> b n (h d)')
|
||||
return self.to_out(out_x)
|
||||
|
||||
|
||||
class Physics_Attention_Structured_Mesh_2D(nn.Module):
|
||||
## for structured mesh in 2D space
|
||||
def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64, H=101, W=31, kernel=3): # kernel=3):
|
||||
super().__init__()
|
||||
inner_dim = dim_head * heads
|
||||
self.dim_head = dim_head
|
||||
self.heads = heads
|
||||
self.scale = dim_head ** -0.5
|
||||
self.softmax = nn.Softmax(dim=-1)
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
||||
self.H = H
|
||||
self.W = W
|
||||
|
||||
self.in_project_x = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2)
|
||||
self.in_project_fx = nn.Conv2d(dim, inner_dim, kernel, 1, kernel // 2)
|
||||
self.in_project_slice = nn.Linear(dim_head, slice_num)
|
||||
for l in [self.in_project_slice]:
|
||||
torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
||||
self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
||||
|
||||
self.to_out = nn.Sequential(
|
||||
nn.Linear(inner_dim, dim),
|
||||
nn.Dropout(dropout)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
# B N C
|
||||
B, N, C = x.shape
|
||||
x = x.reshape(B, self.H, self.W, C).contiguous().permute(0, 3, 1, 2).contiguous() # B C H W
|
||||
|
||||
### (1) Slice
|
||||
fx_mid = self.in_project_fx(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
x_mid = self.in_project_x(x).permute(0, 2, 3, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N G
|
||||
slice_weights = self.softmax(
|
||||
self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G
|
||||
slice_norm = slice_weights.sum(2) # B H G
|
||||
slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
|
||||
slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
||||
|
||||
### (2) Attention among slice tokens
|
||||
q_slice_token = self.to_q(slice_token)
|
||||
k_slice_token = self.to_k(slice_token)
|
||||
v_slice_token = self.to_v(slice_token)
|
||||
dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
||||
attn = self.softmax(dots)
|
||||
attn = self.dropout(attn)
|
||||
out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
||||
|
||||
### (3) Deslice
|
||||
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
||||
out_x = rearrange(out_x, 'b h n d -> b n (h d)')
|
||||
return self.to_out(out_x)
|
||||
|
||||
|
||||
class Physics_Attention_Structured_Mesh_3D(nn.Module):
|
||||
## for structured mesh in 3D space
|
||||
def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=32, H=32, W=32, D=32, kernel=3):
|
||||
super().__init__()
|
||||
inner_dim = dim_head * heads
|
||||
self.dim_head = dim_head
|
||||
self.heads = heads
|
||||
self.scale = dim_head ** -0.5
|
||||
self.softmax = nn.Softmax(dim=-1)
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
||||
self.H = H
|
||||
self.W = W
|
||||
self.D = D
|
||||
|
||||
self.in_project_x = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2)
|
||||
self.in_project_fx = nn.Conv3d(dim, inner_dim, kernel, 1, kernel // 2)
|
||||
self.in_project_slice = nn.Linear(dim_head, slice_num)
|
||||
for l in [self.in_project_slice]:
|
||||
torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
||||
self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_out = nn.Sequential(
|
||||
nn.Linear(inner_dim, dim),
|
||||
nn.Dropout(dropout)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
# B N C
|
||||
B, N, C = x.shape
|
||||
x = x.reshape(B, self.H, self.W, self.D, C).contiguous().permute(0, 4, 1, 2, 3).contiguous() # B C H W
|
||||
|
||||
### (1) Slice
|
||||
fx_mid = self.in_project_fx(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
x_mid = self.in_project_x(x).permute(0, 2, 3, 4, 1).contiguous().reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N G
|
||||
slice_weights = self.softmax(
|
||||
self.in_project_slice(x_mid) / torch.clamp(self.temperature, min=0.1, max=5)) # B H N G
|
||||
slice_norm = slice_weights.sum(2) # B H G
|
||||
slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid, slice_weights)
|
||||
slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
||||
|
||||
### (2) Attention among slice tokens
|
||||
q_slice_token = self.to_q(slice_token)
|
||||
k_slice_token = self.to_k(slice_token)
|
||||
v_slice_token = self.to_v(slice_token)
|
||||
dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
||||
attn = self.softmax(dots)
|
||||
attn = self.dropout(attn)
|
||||
out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
||||
|
||||
### (3) Deslice
|
||||
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
||||
out_x = rearrange(out_x, 'b h n d -> b n (h d)')
|
||||
return self.to_out(out_x)
|
||||
@@ -0,0 +1,158 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from timm.models.layers import trunc_normal_
|
||||
from model.Embedding import timestep_embedding
|
||||
import numpy as np
|
||||
from model.Physics_Attention import Physics_Attention_Irregular_Mesh
|
||||
|
||||
ACTIVATION = {'gelu': nn.GELU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid, 'relu': nn.ReLU, 'leaky_relu': nn.LeakyReLU(0.1),
|
||||
'softplus': nn.Softplus, 'ELU': nn.ELU, 'silu': nn.SiLU}
|
||||
|
||||
|
||||
class MLP(nn.Module):
|
||||
def __init__(self, n_input, n_hidden, n_output, n_layers=1, act='gelu', res=True):
|
||||
super(MLP, self).__init__()
|
||||
|
||||
if act in ACTIVATION.keys():
|
||||
act = ACTIVATION[act]
|
||||
else:
|
||||
raise NotImplementedError
|
||||
self.n_input = n_input
|
||||
self.n_hidden = n_hidden
|
||||
self.n_output = n_output
|
||||
self.n_layers = n_layers
|
||||
self.res = res
|
||||
self.linear_pre = nn.Sequential(nn.Linear(n_input, n_hidden), act())
|
||||
self.linear_post = nn.Linear(n_hidden, n_output)
|
||||
self.linears = nn.ModuleList([nn.Sequential(nn.Linear(n_hidden, n_hidden), act()) for _ in range(n_layers)])
|
||||
|
||||
def forward(self, x):
|
||||
x = self.linear_pre(x)
|
||||
for i in range(self.n_layers):
|
||||
if self.res:
|
||||
x = self.linears[i](x) + x
|
||||
else:
|
||||
x = self.linears[i](x)
|
||||
x = self.linear_post(x)
|
||||
return x
|
||||
|
||||
|
||||
class Transolver_block(nn.Module):
|
||||
"""Transformer encoder block."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
hidden_dim: int,
|
||||
dropout: float,
|
||||
act='gelu',
|
||||
mlp_ratio=4,
|
||||
last_layer=False,
|
||||
out_dim=1,
|
||||
slice_num=32,
|
||||
):
|
||||
super().__init__()
|
||||
self.last_layer = last_layer
|
||||
self.ln_1 = nn.LayerNorm(hidden_dim)
|
||||
self.Attn = Physics_Attention_Irregular_Mesh(hidden_dim, heads=num_heads, dim_head=hidden_dim // num_heads,
|
||||
dropout=dropout, slice_num=slice_num)
|
||||
self.ln_2 = nn.LayerNorm(hidden_dim)
|
||||
self.mlp = MLP(hidden_dim, hidden_dim * mlp_ratio, hidden_dim, n_layers=0, res=False, act=act)
|
||||
if self.last_layer:
|
||||
self.ln_3 = nn.LayerNorm(hidden_dim)
|
||||
self.mlp2 = nn.Linear(hidden_dim, out_dim)
|
||||
|
||||
def forward(self, fx):
|
||||
fx = self.Attn(self.ln_1(fx)) + fx
|
||||
fx = self.mlp(self.ln_2(fx)) + fx
|
||||
if self.last_layer:
|
||||
return self.mlp2(self.ln_3(fx))
|
||||
else:
|
||||
return fx
|
||||
|
||||
|
||||
class Model(nn.Module):
|
||||
def __init__(self,
|
||||
space_dim=1,
|
||||
n_layers=5,
|
||||
n_hidden=256,
|
||||
dropout=0.0,
|
||||
n_head=8,
|
||||
Time_Input=False,
|
||||
act='gelu',
|
||||
mlp_ratio=1,
|
||||
fun_dim=1,
|
||||
out_dim=1,
|
||||
slice_num=32,
|
||||
ref=8,
|
||||
unified_pos=False
|
||||
):
|
||||
super(Model, self).__init__()
|
||||
self.__name__ = 'Transolver_1D'
|
||||
self.ref = ref
|
||||
self.unified_pos = unified_pos
|
||||
self.Time_Input = Time_Input
|
||||
self.n_hidden = n_hidden
|
||||
self.space_dim = space_dim
|
||||
if self.unified_pos:
|
||||
self.preprocess = MLP(fun_dim + self.ref * self.ref, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act)
|
||||
else:
|
||||
self.preprocess = MLP(fun_dim + space_dim, n_hidden * 2, n_hidden, n_layers=0, res=False, act=act)
|
||||
if Time_Input:
|
||||
self.time_fc = nn.Sequential(nn.Linear(n_hidden, n_hidden), nn.SiLU(), nn.Linear(n_hidden, n_hidden))
|
||||
|
||||
self.blocks = nn.ModuleList([Transolver_block(num_heads=n_head, hidden_dim=n_hidden,
|
||||
dropout=dropout,
|
||||
act=act,
|
||||
mlp_ratio=mlp_ratio,
|
||||
out_dim=out_dim,
|
||||
slice_num=slice_num,
|
||||
last_layer=(_ == n_layers - 1))
|
||||
for _ in range(n_layers)])
|
||||
self.initialize_weights()
|
||||
self.placeholder = nn.Parameter((1 / (n_hidden)) * torch.rand(n_hidden, dtype=torch.float))
|
||||
|
||||
def initialize_weights(self):
|
||||
self.apply(self._init_weights)
|
||||
|
||||
def _init_weights(self, m):
|
||||
if isinstance(m, nn.Linear):
|
||||
trunc_normal_(m.weight, std=0.02)
|
||||
if isinstance(m, nn.Linear) and m.bias is not None:
|
||||
nn.init.constant_(m.bias, 0)
|
||||
elif isinstance(m, (nn.LayerNorm, nn.BatchNorm1d)):
|
||||
nn.init.constant_(m.bias, 0)
|
||||
nn.init.constant_(m.weight, 1.0)
|
||||
|
||||
def get_grid(self, x, batchsize=1):
|
||||
# x: B N 2
|
||||
# grid_ref
|
||||
gridx = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float)
|
||||
gridx = gridx.reshape(1, self.ref, 1, 1).repeat([batchsize, 1, self.ref, 1])
|
||||
gridy = torch.tensor(np.linspace(0, 1, self.ref), dtype=torch.float)
|
||||
gridy = gridy.reshape(1, 1, self.ref, 1).repeat([batchsize, self.ref, 1, 1])
|
||||
grid_ref = torch.cat((gridx, gridy), dim=-1).cuda().reshape(batchsize, self.ref * self.ref, 2) # B H W 8 8 2
|
||||
|
||||
pos = torch.sqrt(torch.sum((x[:, :, None, :] - grid_ref[:, None, :, :]) ** 2, dim=-1)). \
|
||||
reshape(batchsize, x.shape[1], self.ref * self.ref).contiguous()
|
||||
return pos
|
||||
|
||||
def forward(self, x, fx, T=None):
|
||||
if self.unified_pos:
|
||||
x = self.get_grid(x, x.shape[0])
|
||||
if fx is not None:
|
||||
fx = torch.cat((x, fx), -1)
|
||||
fx = self.preprocess(fx)
|
||||
else:
|
||||
fx = self.preprocess(x)
|
||||
fx = fx + self.placeholder[None, None, :]
|
||||
|
||||
if T is not None:
|
||||
Time_emb = timestep_embedding(T, self.n_hidden).repeat(1, x.shape[1], 1)
|
||||
Time_emb = self.time_fc(Time_emb)
|
||||
fx = fx + Time_emb
|
||||
|
||||
for block in self.blocks:
|
||||
fx = block(fx)
|
||||
|
||||
return fx
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"revision": "75e0f67643806a81cd1d3f6adc88dd8c02416fe7",
|
||||
"repository": "https://github.com/thuml/Transolver",
|
||||
"files": {
|
||||
"LICENSE": "2c919cd03fa823bf7eefc00a957ff8324c865cd22aa5285e563dc4b558084f25",
|
||||
"PDE-Solving-StandardBenchmark/model/Transolver_Irregular_Mesh.py": "5259759f7ecc6a954c8442ac8359990b9f242f1437c452c69e5e0f03549b5311",
|
||||
"PDE-Solving-StandardBenchmark/model/Physics_Attention.py": "4fc710f6f96788fb59cb19fab8d64873d3d16e9e182f12a544d451c74c548cc7"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Synthetic-only acceptance. Run from tools; never loads any corpus geometry/field."""
|
||||
import os
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = ""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from sklearn.ensemble import HistGradientBoostingRegressor
|
||||
|
||||
from s2_models import features as f, objectives as o
|
||||
from s2_models.attention import Physics_Attention_Irregular_Mesh
|
||||
from s2_models.network import QueryModel, weighted_mse
|
||||
|
||||
torch.set_num_threads(1)
|
||||
|
||||
|
||||
def synthetic_star():
|
||||
# One outer tetrahedron with its barycentre inserted: 4 volume tets.
|
||||
x = np.array([[0., 0, 0], [1., 0, 0], [0., 1, 0], [0., 0, 1], [.25, .25, .25]])
|
||||
t = np.array([[0, 1, 2, 4], [0, 1, 3, 4], [0, 2, 3, 4], [1, 2, 3, 4]])
|
||||
return x, t, np.array([0., 0, 0, 0, .25])
|
||||
|
||||
|
||||
class GeometryTests(unittest.TestCase):
|
||||
def test_volume_lumping(self):
|
||||
x, t, _ = synthetic_star()
|
||||
*_, volumes, mass, edges, lengths = f.mesh_geometry(x, t)
|
||||
self.assertAlmostEqual(volumes.sum(), 1/6)
|
||||
self.assertAlmostEqual(mass.sum(), 1/6)
|
||||
self.assertEqual(len(edges), 10)
|
||||
|
||||
def test_wall_edges_removed_both_distances_and_gate_tie(self):
|
||||
x, t, dw = synthetic_star()
|
||||
di, df, source = f.graph_distances(x, t, dw, [0., 0, 0])
|
||||
expected = np.linalg.norm(x[4]-x[0]) + np.linalg.norm(x[1]-x[4])
|
||||
self.assertAlmostEqual(di[1], expected)
|
||||
np.testing.assert_allclose(df, di/.125)
|
||||
self.assertEqual(source, 0)
|
||||
unrestricted, _ = f.graph_distances(x, t, dw, x[0], exclude_wall_edges=False)
|
||||
self.assertAlmostEqual(unrestricted[1], 1.)
|
||||
self.assertGreater(di[1], unrestricted[1])
|
||||
# Tie between original node 0 and interior 4, with no closer node.
|
||||
_, _, source = f.graph_distances(x, t, dw, (x[0]+x[4])/2)
|
||||
self.assertEqual(source, 0)
|
||||
|
||||
def test_unreachable_fail_closed(self):
|
||||
x, t, dw = synthetic_star()
|
||||
with self.assertRaisesRegex(ValueError, "unreachable"):
|
||||
f.graph_distances(x, t, np.zeros_like(dw), x[0])
|
||||
|
||||
def test_degenerate_and_unused_fail(self):
|
||||
x, t, _ = synthetic_star()
|
||||
with self.assertRaisesRegex(ValueError, "degenerate"):
|
||||
f.mesh_geometry(x, np.array([[0, 0, 1, 2]]))
|
||||
with self.assertRaisesRegex(ValueError, "unused"):
|
||||
f.mesh_geometry(np.vstack([x, [2, 2, 2]]), t)
|
||||
|
||||
def test_strict_volume_front_ties(self):
|
||||
actual = f.relative_flow_volume([0, 2, 2, 3], [1, 2, 3, 4])
|
||||
np.testing.assert_allclose(actual, [0, .1, .1, .6])
|
||||
|
||||
def test_affine_p1_and_zero_gradient(self):
|
||||
x, t, _ = synthetic_star()
|
||||
a = np.array([2., -3., 4.])
|
||||
np.testing.assert_allclose(f.p1_nodal_gradient(x, t, x@a+7, normalize=False), np.tile(a, (5, 1)), atol=1e-13)
|
||||
np.testing.assert_allclose(f.p1_nodal_gradient(x, t, x@a+7), np.tile(a/np.linalg.norm(a), (5, 1)), atol=1e-13)
|
||||
np.testing.assert_array_equal(f.p1_nodal_gradient(x, t, np.ones(5)), np.zeros((5, 3)))
|
||||
|
||||
def test_moment_volume_weight_rotation_and_filter(self):
|
||||
x, t, _ = synthetic_star()
|
||||
di = np.zeros(5)
|
||||
s, radius = f.spatial_moment(x, t, di)
|
||||
centers = x[t].mean(axis=1)
|
||||
directions = centers-x[0]
|
||||
directions /= np.linalg.norm(directions, axis=1)[:, None]
|
||||
expected = np.mean([np.outer(v, v) for v in directions], axis=0)
|
||||
np.testing.assert_allclose(s[0], expected)
|
||||
self.assertAlmostEqual(radius, 3*np.sqrt(2))
|
||||
q = np.array([[0., -1, 0], [1., 0, 0], [0., 0, 1]])
|
||||
rotated, _ = f.spatial_moment(x@q.T+3, t, di)
|
||||
np.testing.assert_allclose(rotated, q@s@q.T, atol=1e-14)
|
||||
self.assertGreaterEqual(np.linalg.eigvalsh(s).min(), -1e-14)
|
||||
np.testing.assert_allclose(np.trace(s, axis1=1, axis2=2), 1.)
|
||||
# D_i exclusion can empty a neighbourhood; no fabricated fallback tensor.
|
||||
with self.assertRaisesRegex(ValueError, "empty"):
|
||||
f.spatial_moment(x, t, [100, 0, 0, 0, 0])
|
||||
|
||||
def test_features_and_physical_scaling(self):
|
||||
x, t, dw = synthetic_star()
|
||||
a = f.r2_features(x, t, dw, x[0], .4)
|
||||
b = f.r2_features(2*x, t, 2*dw, 2*x[0], .4)
|
||||
self.assertEqual(a["features"].shape, (5, 20))
|
||||
np.testing.assert_allclose(b["Df"], a["Df"])
|
||||
np.testing.assert_allclose(b["Di"], 2*a["Di"])
|
||||
np.testing.assert_allclose(b["Vf"], a["Vf"])
|
||||
np.testing.assert_allclose(b["S"], a["S"])
|
||||
np.testing.assert_allclose(a["features"][:, -1], .4)
|
||||
|
||||
def test_moment_unequal_volumes_and_coincident_center(self):
|
||||
x, t, _ = synthetic_star()
|
||||
x[4] = [.125, .25, .125]
|
||||
s, radius = f.spatial_moment(x, t, np.zeros(5))
|
||||
_, _, _, v, _, _, _ = f.mesh_geometry(x, t)
|
||||
direction = x[t].mean(axis=1) - x[0]
|
||||
direction /= np.linalg.norm(direction, axis=1)[:, None]
|
||||
expected = np.einsum('j,ja,jb->ab', v/v.sum(), direction, direction)
|
||||
np.testing.assert_allclose(s[0], expected)
|
||||
# Add a tetrahedron whose centre is exactly node 4. Its volume must
|
||||
# be excluded from that node's denominator as well as its numerator.
|
||||
x, t, _ = synthetic_star()
|
||||
s0, _ = f.spatial_moment(x, t, np.zeros(5))
|
||||
s1, _ = f.spatial_moment(x, np.vstack([t, [0, 1, 2, 3]]), np.zeros(5))
|
||||
np.testing.assert_allclose(s1[4], s0[4])
|
||||
self.assertAlmostEqual(np.trace(s1[4]), 1.)
|
||||
|
||||
def test_base_slots_and_nonfinite_fail(self):
|
||||
x, t, dw = synthetic_star()
|
||||
dg, _ = f.graph_distances(x, t, dw, x[0], exclude_wall_edges=False)
|
||||
base = f.base_features(x, dw, dg, x[0], [0, 0, 0], 2, 1/6, .5, .4)
|
||||
self.assertEqual(base.shape, (5, 10))
|
||||
np.testing.assert_allclose(base[:, :3], x/2)
|
||||
np.testing.assert_allclose(base[:, 3], dw/.5)
|
||||
np.testing.assert_allclose(base[:, 5], dg/2)
|
||||
with self.assertRaises(ValueError):
|
||||
f.base_features(x, dw, dg*float("nan"), x[0], [0, 0, 0], 2, 1/6, .5, .4)
|
||||
|
||||
|
||||
class ObjectiveTests(unittest.TestCase):
|
||||
def test_hierarchy_and_draw_compression(self):
|
||||
# A has two parts; B one. Their masses must be .25, .25, .5.
|
||||
c, p, g = [1, 3, 2, 4], [0, 0, 1, 2], ["A", "A", "A", "B"]
|
||||
w = o.hierarchy_weights(c, p, g)
|
||||
np.testing.assert_allclose(w, [.0625, .1875, .25, .5])
|
||||
pred = torch.tensor([1., 3, 2, 4])
|
||||
loss = weighted_mse(pred, torch.zeros(4), torch.tensor(w))
|
||||
expanded = o.hierarchy_weights(np.ones(10), np.repeat(p, c), np.repeat(g, c))
|
||||
expected = np.sum(expanded*np.repeat(pred.numpy(), c)**2)
|
||||
self.assertAlmostEqual(loss.item(), expected)
|
||||
# Sum of part contributions equals pooled objective, without local renorm.
|
||||
subtotal = sum(weighted_mse(pred[np.array(p)==i], torch.zeros(sum(np.array(p)==i)), torch.tensor(w[np.array(p)==i])).item() for i in [0, 1, 2])
|
||||
self.assertAlmostEqual(subtotal, loss.item())
|
||||
|
||||
def test_invalid_hierarchy(self):
|
||||
with self.assertRaises(ValueError):
|
||||
o.hierarchy_weights([1, 1], [0, 0], ["A", "B"])
|
||||
with self.assertRaises(ValueError):
|
||||
o.hierarchy_weights([1, 0], [0, 1], ["A", "B"])
|
||||
|
||||
def test_process_target_roundtrip_and_scale_invariance(self):
|
||||
seconds = np.array([.1, .2, .6])
|
||||
target = o.dimensionless_target(seconds, .4)
|
||||
np.testing.assert_allclose(o.prediction_seconds(target, .4), seconds)
|
||||
score1 = o.relative_l2(seconds*1.1, seconds, [1, 3, 1], [True]*3)
|
||||
score2 = o.relative_l2(target*1.1, target, [1, 3, 1], [True]*3)
|
||||
self.assertAlmostEqual(score1["value"], score2["value"])
|
||||
np.testing.assert_allclose(o.b0_seconds([0, .5, 1], .4), [0, .2, .4])
|
||||
|
||||
def test_zero_missing_scales_no_fallback(self):
|
||||
for scale in [0., -1., float("nan")]:
|
||||
with self.assertRaises(ValueError):
|
||||
o.dimensionless_target([1.], scale)
|
||||
with self.assertRaises(ValueError):
|
||||
o.prediction_seconds([float("nan")], .4)
|
||||
self.assertFalse(hasattr(o, "log_target"))
|
||||
self.assertFalse(hasattr(o, "weighted_median"))
|
||||
|
||||
def test_score_masks_before_arithmetic_and_zero_truth(self):
|
||||
result = o.relative_l2([2., np.nan], [1., np.nan], [2, 0], [True, False])
|
||||
self.assertEqual(result["value"], 1.)
|
||||
self.assertIsNone(o.relative_l2([1.], [0.], [1], [True])["value"])
|
||||
|
||||
def test_histgbrt_grid_unfitted(self):
|
||||
grid = f.histgbrt_grid()
|
||||
self.assertEqual(len(grid), 12)
|
||||
self.assertEqual(len({json.dumps(c, sort_keys=True) for c in grid}), 12)
|
||||
with patch.object(HistGradientBoostingRegressor, "fit", side_effect=AssertionError("fitting forbidden")):
|
||||
for config in grid:
|
||||
model = HistGradientBoostingRegressor(**config)
|
||||
self.assertFalse(model.early_stopping)
|
||||
self.assertFalse(hasattr(model, "n_iter_"))
|
||||
|
||||
|
||||
class NetworkTests(unittest.TestCase):
|
||||
def test_original_attention_numerical_parity(self):
|
||||
path = Path(__file__).parent/'s2_models/upstream/Physics_Attention.py'
|
||||
source = path.read_text().split('class Physics_Attention_Structured_Mesh_2D')[0]
|
||||
source = source.replace('from einops import rearrange, repeat', '')
|
||||
namespace = {"rearrange": lambda x, pattern: x.permute(0, 2, 1, 3).reshape(x.shape[0], x.shape[2], -1)}
|
||||
exec(compile(source, str(path), "exec"), namespace)
|
||||
torch.manual_seed(9)
|
||||
original = namespace['Physics_Attention_Irregular_Mesh'](128, heads=8, dim_head=16, slice_num=32).eval()
|
||||
adapted = Physics_Attention_Irregular_Mesh(128, heads=8, dim_head=16, slice_num=32).eval()
|
||||
adapted.load_state_dict(original.state_dict())
|
||||
x = torch.randn(1, 7, 128)
|
||||
with torch.inference_mode():
|
||||
torch.testing.assert_close(original(x), adapted(x), rtol=0, atol=0)
|
||||
|
||||
def test_upstream_snapshot_hashes(self):
|
||||
import hashlib
|
||||
root = Path(__file__).parent/'s2_models/upstream'
|
||||
manifest = json.loads((root/'manifest.json').read_text())
|
||||
self.assertEqual(manifest['revision'], '75e0f67643806a81cd1d3f6adc88dd8c02416fe7')
|
||||
for name, digest in manifest['files'].items():
|
||||
self.assertEqual(hashlib.sha256((root/Path(name).name).read_bytes()).hexdigest(), digest)
|
||||
|
||||
def test_four_models_forward_only_and_parameter_counts(self):
|
||||
for width in (128, 256):
|
||||
models = {}
|
||||
for arm in ("N", "E"):
|
||||
torch.manual_seed(0)
|
||||
model = QueryModel(arm, width).eval()
|
||||
before = {k: v.clone() for k, v in model.state_dict().items()}
|
||||
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4, 6]), torch.rand(3, 2)
|
||||
edge = torch.rand(3, 4) if arm == "E" else None
|
||||
with torch.inference_mode():
|
||||
pred = model(base, ids, gaps, edge)
|
||||
self.assertEqual(pred.shape, (3,))
|
||||
self.assertTrue(torch.isfinite(pred).all())
|
||||
self.assertEqual(next(model.parameters()).device.type, "cpu")
|
||||
for k, value in model.state_dict().items():
|
||||
torch.testing.assert_close(value, before[k], rtol=0, atol=0)
|
||||
models[arm] = model.parameter_counts()
|
||||
self.assertEqual(models['E']['encoder'], models['N']['encoder'])
|
||||
self.assertEqual(models['E']['total']-models['N']['total'], 4*width)
|
||||
|
||||
def test_query_order_global_context_and_input_rejection(self):
|
||||
torch.manual_seed(7)
|
||||
model = QueryModel("N").eval()
|
||||
base, ids, gaps = torch.randn(7, 10), torch.tensor([0, 4]), torch.rand(2, 2)
|
||||
with torch.inference_mode():
|
||||
a = model(base, ids, gaps)
|
||||
b = model(base, ids.flip(0), gaps.flip(0))
|
||||
torch.testing.assert_close(a, b.flip(0))
|
||||
changed = base.clone(); changed[6] += 20
|
||||
c = model(changed, ids, gaps)
|
||||
self.assertGreater(float(torch.max(torch.abs(a-c))), 1e-9)
|
||||
with self.assertRaises(ValueError):
|
||||
model(base, ids, gaps, torch.zeros(2, 4))
|
||||
with self.assertRaises(ValueError):
|
||||
model(base, torch.tensor([7]), torch.zeros(1, 2))
|
||||
changed[6, 0] = float("nan")
|
||||
with self.assertRaises(ValueError):
|
||||
model(changed, ids, gaps)
|
||||
with self.assertRaises(ValueError):
|
||||
QueryModel("E")(base, ids, gaps)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user