Files
MeltMind/tools/phase3_h2_shared_miss_attributes.py
T

214 lines
11 KiB
Python

"""Use frozen scalar receipts for the registered common-miss description."""
from collections import defaultdict
import csv
import hashlib
from io import StringIO
import json
import os
from pathlib import Path
import numpy as np
BASE = Path(__file__).resolve().parents[1]
OUT = BASE / "artifacts/phase3-h2-shared-miss-attributes-2026-11-19"
PREREG_COMMIT = "d62b919e95b9f774e3283b503aa699c23f86c104"
SEED = 2026111408
REPLICATES = 2000
FILES = {
"prereg": ("docs/moldastra-h2-shared-miss-attributes-prereg-2026-11-19.md",
"d42c3a809234bed4e21287a08d4c405d37ee812b2e9cf23742df81652896879d"),
"s2_tail": ("artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18/output/result.json",
"01b5c3de72ed708b2c4eb8ed1da6efe705aa1809ebd88621dfbbf7c932b25c7c"),
"seed_comparison": ("artifacts/phase3-h2-family-diagnostic-2026-11-19/by-part.csv",
"c6def5706acf8cf5b55ad5bd137e69665856b0f3bc254c639292366c58e195ec"),
"pilot": ("artifacts/phase2-h2-pilot-2026-11-17/contract.json",
"9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b"),
"v5_preparation": ("artifacts/gre24-reproduction-2026-11-15/v5-preparation-continuation-2026-11-16/complete.json",
"8f3799fccfc61271d0c52194ff41838f644560024f8545fee17b72bd7597a15f"),
}
AVAILABLE = {
"gate_count": "count",
"volume_raw_cubed": "raw coordinate cubed",
"Rg_raw": "raw coordinate",
"tet_count": "count",
"true_peak_node_count": "count",
"true_peak_volume_fraction": "whole retained volume fraction",
"true_peak_volume_raw_cubed": "raw coordinate cubed",
"A_miss": "proportion",
"A_field_relative_l2": "fraction",
}
UNAVAILABLE = {
"physical_branch_count_and_length": "No independent branch or streamline annotations",
"late_tail_component_count": "Frozen scalars omit the original per-node membership and induced tet edge graph",
"late_tail_major_component_count": "Frozen scalars omit component masses",
"late_tail_component_depth": "Frozen scalars omit per-node membership, edges and entrance sets",
"late_tail_component_depth_mass_coverage": "Frozen scalars omit per-component membership and mass",
"near_tie": "Major component peaks are unavailable from the frozen scalars",
"major_component_peak_gap": "Major component peaks are unavailable from the frozen scalars",
"physical_flow_length_to_wall_thickness": "No validated streamline length or general physical wall thickness",
"L_tail_to_h_g": "No frozen retained-boundary area or true-tail geodesic moment",
"L_tail_raw": "No frozen true-tail weighted full tet edge distance from independent gates",
"L_tail_over_Rg": "No frozen true-tail weighted full tet edge distance from independent gates",
"tail_unreachable_mass_fraction": "No frozen per-node reachability joined to true-tail membership",
"surface_area_raw_squared": "No matching frozen retained-tet exterior boundary area scalar was located",
"sqrt_lambda2_over_lambda1": "No matching frozen lumped-node covariance scalar was located",
"sqrt_lambda3_over_lambda1": "No matching frozen lumped-node covariance scalar was located",
"semantic_family_type": "No existing semantic type mapping covers the registered part support",
}
def checked_bytes(path, digest):
raw = path.read_bytes()
if hashlib.sha256(raw).hexdigest() != digest:
raise ValueError(f"Frozen data changed: {path}")
return raw
def inputs():
return {key: checked_bytes(BASE / path, digest) for key, (path, digest) in FILES.items()}
def unique_index(rows):
by_case = {r["case"]: r for r in rows}
if len(by_case) != len(rows):
raise ValueError("Duplicate case identities")
return by_case
def typed_rows(raw):
comparison = list(csv.DictReader(StringIO(raw["seed_comparison"].decode("utf-8"))))
tail = unique_index(json.loads(raw["s2_tail"])["rows"])
pilot = json.loads(raw["pilot"])
receipts = unique_index(json.loads(raw["v5_preparation"])["records"])
sources = {}
out = []
for row in comparison:
case = row["case"]
component, fold = row["component_id"], int(row["fold"])
r = tail[case]
if (component, fold) != (r["component_id"], r["fold"]) or component != pilot["parts"][case]["clock_group"]:
raise ValueError(f"Group or fold differs: {case}")
if row["constant_truth"] != "False" or r["truth_tail"]["constant_field"]:
raise ValueError(f"Unexpected constant reference tail: {case}")
a = r["arms"]["A"]["full"]
if float(row["A.d_tail_over_Rg"]) != a["d_tail_over_Rg"]:
raise ValueError(f"A tail scalar changed: {case}")
rng = r["Rg"]
if rng != pilot["geometry"][case]["Rg"] or a["true_peak_volume"] is None:
raise ValueError(f"Frozen geometry/peak identity differs: {case}")
record = receipts[case]
target = pilot["parts"][case]
path = BASE / record["result_path"]
if path.resolve() != (BASE / target["output"] / "result.json").resolve():
raise ValueError(f"Preparation result identity differs: {case}")
prepared = json.loads(checked_bytes(path, record["result_sha256"]))
sources[case] = dict(path=str(path), sha256=record["result_sha256"])
if (prepared["status"] != "PASS" or prepared["case"] != case
or prepared["output_sha256"] != target["output_sha256"]):
raise ValueError(f"Frozen preparation differs: {case}")
geometry, features = prepared["geometry"], prepared["features"]
config = geometry.get("inlet_config", {})
inlet = config.get("inlet_location", [])
gate_ids = features.get("gate_raw_ids", [])
gate_known = (geometry.get("gate_source", "").startswith("YAML inlet_location")
and np.shape(inlet) == (3,) and np.isfinite(inlet).all() and len(gate_ids) >= 1)
tets = geometry["raw_cell_types"]["10"]
volume = features["volume"]
if not (geometry["tet_nodes"] == a["nodes"] == r["nodes"] and tets > 0 and volume > 0
and np.isfinite(volume) and 0 < a["true_peak_volume"] <= 1):
raise ValueError(f"Invalid fixed retained-domain scalars: {case}")
values = dict(
gate_count=1 if gate_known else None, volume_raw_cubed=volume, Rg_raw=rng, tet_count=tets,
true_peak_node_count=a["true_peak_count"], true_peak_volume_fraction=a["true_peak_volume"],
true_peak_volume_raw_cubed=a["true_peak_volume"] * volume,
A_miss=int(a["d_tail_over_Rg"] > .10), A_field_relative_l2=float(row["A.relative_l2"]))
values.update(dict.fromkeys(UNAVAILABLE))
out.append(dict(case=case, component_id=component, fold=fold,
shared_miss=all(float(row[f"{s}.d_tail_over_Rg"]) > .10 for s in ("seed0", "seed1")),
gate_inlet_xyz=inlet if gate_known else None,
gate_mapping_raw_ids=gate_ids if gate_known else None,
gate_source=geometry.get("gate_source") if gate_known else None, **values))
if len(out) != 552 or len(unique_index(out)) != 552 or len({r["component_id"] for r in out}) != 204:
raise ValueError("Registered support changed")
if sum(r["shared_miss"] for r in out) != 34:
raise ValueError("Registered grouping changed")
return out, sources
def grouped(rows):
out = defaultdict(list)
for r in rows:
out[r["component_id"]].append(r)
return out
def attribute_rows(family_ids, groups, metric):
expected = grouped(groups)
valid = grouped([r for r in groups if r[metric] is not None])
member_means = {key: np.mean([r[metric] for r in members]) for key, members in valid.items()}
array = np.array([member_means.get(key, np.nan) for key in family_ids], dtype=np.float64)
series = array[np.isfinite(array)]
available_mass = sum(r["volume_raw_cubed"] for members in valid.values() for r in members)
total_mass = sum(r["volume_raw_cubed"] for r in groups)
detail = dict(expected_parts=len(groups), expected_families=len(expected),
available_parts=sum(map(len, valid.values())), available_families=len(valid),
missing_parts=len(groups)-sum(map(len, valid.values())),
families_all_missing=len(set(expected)-set(valid)),
available_mesh_mass_fraction=available_mass/total_mass,
family_equal_mean=float(series.mean()) if len(series) else None,
quartiles=np.quantile(series, [.25, .5, .75]).tolist() if len(series) else None)
return array, detail
def sample_means(values, draws):
samples = values[draws]
counts = np.sum(np.isfinite(samples), axis=1)
return np.divide(np.nansum(samples, axis=1), counts, where=counts > 0,
out=np.full(len(samples), np.nan))
def comparisons(rows):
families = sorted(grouped(rows))
miss = [r for r in rows if r["shared_miss"]]
other = [r for r in rows if not r["shared_miss"]]
# The same complete-family draw is used for both groups and all properties.
draws = np.random.default_rng(SEED).integers(len(families), size=(REPLICATES, len(families)))
results = {}
for metric in (*AVAILABLE, *UNAVAILABLE):
x, shared = attribute_rows(families, miss, metric)
y, control = attribute_rows(families, other, metric)
_, all_parts = attribute_rows(families, rows, metric)
diff = sample_means(x, draws) - sample_means(y, draws)
observed = diff[np.isfinite(diff)]
left, right = shared["family_equal_mean"], control["family_equal_mean"]
results[metric] = dict(units=AVAILABLE.get(metric), unavailable_reason=UNAVAILABLE.get(metric),
all_parts=all_parts, shared_miss=shared, other=control,
effect=left-right if left is not None and right is not None else None,
interval_95=np.quantile(observed, [.025, .975]).tolist() if len(observed) else None,
undefined_replicates=REPLICATES-len(observed))
return results
def main():
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
raise RuntimeError("Use an empty CUDA device mask for this CPU-only description")
data = inputs()
rows, sources = typed_rows(data)
metrics = comparisons(rows)
result = dict(prereg_commit=PREREG_COMMIT, inputs=FILES, per_case_preparation_sources=sources,
bootstrap=dict(replicates=REPLICATES, seed=SEED, unit="original family, common draw"),
metrics=metrics, gpu_seconds=0, target_arrays_read=0, full_geometry_arrays_read=0,
prediction_arrays_read=0, raw_targets_decoded=0, new_fits=0)
OUT.mkdir(exist_ok=False)
(OUT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
with (OUT / "by-part.jsonl").open("x") as stream:
for r in rows:
stream.write(json.dumps(r, ensure_ascii=False, allow_nan=False) + "\n")
if __name__ == "__main__":
main()