Files
MeltMind/tools/phase3_h2_family_diagnostic.py
T

221 lines
11 KiB
Python

"""Aggregate four frozen score files. No target or prediction arrays are opened."""
from collections import defaultdict
import csv
import hashlib
import json
import math
from pathlib import Path
BASE = Path(__file__).resolve().parents[1]
RUNS = BASE.parents[1] / "MeltMind-runs"
TAIL = BASE / "artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18"
OUT = BASE / "artifacts/phase3-h2-family-diagnostic-2026-11-19"
PREREG_COMMIT = "97f29e5357f3efe6c96ecba93bbf0aabc0acb84b"
PREREG_SHA256 = "1e7fe092da2e4cfe7235cb441bf2c45a08a1ad76ebf6e5f2814b2069c8af1bf1"
INPUTS = {
"roster": (TAIL / "plan.json",
"023baa1be5b0216ca9936d0ad12a96ea11242697699c57402cd24c358214fbd4", None),
"baseline": (RUNS / "phase2-h2-s2-postfit-2026-11-17/score/result.json",
"c43a6ab2460affdcafd3d51522e7b5e2d60a684b2b1a79a90e06042b0e19bc45",
"4b2a710de434aad2d1591d8e379a7c89e718bd55"),
"seed1": (RUNS / "phase3-seed1-postfit-2026-11-18/score/result.json",
"d259c156216f274f26ee8ba04fe56a763cb72e7750fe6d71df6e15ba7419f6d4",
"ea8f2a6276ed1db756f3c8bdfebf502673e5fe6f"),
"original_tail": (TAIL / "output/result.json",
"01b5c3de72ed708b2c4eb8ed1da6efe705aa1809ebd88621dfbbf7c932b25c7c",
"98519a6c365dd873dc76dfe2d393027592ba1675"),
"seed_tail": (RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/score/result.json",
"e269695a1ce6953b5148b16036f003e01ca5f6b040fc146771f2382d838b089c",
"520a85772a3b4f9a411818091fcc2b71cf4586b8"),
}
ARMS = ("A", "seed0", "seed1")
METRICS = ("relative_l2", "last_location", "E_tail", "Miss", "d_tail_over_Rg")
def checked_json(path, digest):
payload = path.read_bytes()
if hashlib.sha256(payload).hexdigest() != digest:
raise ValueError(f"Frozen evidence differs: {path}")
return json.loads(payload)
def by_case(rows):
index = {row["case"]: row for row in rows}
if len(index) != len(rows):
raise ValueError("Duplicate case identities")
return index
def mean(values):
values = list(values)
return math.fsum(values) / len(values) if values else None
def groups(rows):
out = defaultdict(list)
for row in rows:
out[row["component_id"]].append(row)
return out
def support(rows):
return dict(parts=len(rows), families=len(groups(rows)))
def query_metrics(arm):
return {metric: arm[metric]["value"] for metric in METRICS[:2]} if arm else {}
def tail_metrics(arm, overlap_key):
if not arm:
return {}
return {key: arm[overlap_key][key] for key in METRICS[2:4]} | {
"d_tail_over_Rg": arm["full"]["d_tail_over_Rg"]}
def paired_rows():
prereg = BASE / "docs/phase3-h2-family-diagnostic-prereg-2026-11-19.md"
if hashlib.sha256(prereg.read_bytes()).hexdigest() != PREREG_SHA256:
raise ValueError("Pre-registered scope differs")
data = {name: checked_json(path, digest) for name, (path, digest, _) in INPUTS.items()}
roster = by_case(data.pop("roster")["parts"])
if len(roster) != 552 or len(groups(roster.values())) != 204:
raise ValueError("Frozen roster differs")
if any(len({r["fold"] for r in family}) != 1 for family in groups(roster.values()).values()):
raise ValueError("Family crosses held-out folds")
sources = {}
for key, item in data.items():
if item["status"] != "COMPLETE":
raise ValueError(f"{key} is not a frozen complete score")
sources[key] = by_case(item["rows"])
for case, row in sources[key].items():
if case not in roster or any(roster[case][k] != row[k] for k in ("component_id", "fold")):
raise ValueError(f"Cross-source identity differs: {key} {case}")
rows = []
for case, ident in sorted(roster.items()):
base, seed, tail, c = (sources[key].get(case, {}) for key in
("baseline", "seed1", "original_tail", "seed_tail"))
if base and seed and base["H2"] != seed["seed0"]:
raise ValueError(f"Reused seed0 query values differ: {case}")
if base and c and base["H2"] != c["seed0"]:
raise ValueError(f"Phase 3(c) seed0 query values differ: {case}")
old_arms = tail.get("arms", {})
new_tail = c.get("tail", {})
new_arms = new_tail.get("arms", {})
original_seed0_tail = tail_metrics(old_arms.get("H2"), "tail")
repeated_seed0_tail = tail_metrics(new_arms.get("seed0"), "overlap")
if original_seed0_tail and repeated_seed0_tail:
for metric in original_seed0_tail:
if not math.isclose(original_seed0_tail[metric], repeated_seed0_tail[metric],
rel_tol=1e-12, abs_tol=1e-12):
raise ValueError(f"Repeated seed0 tail {metric} differs: {case}")
truth = tail.get("truth_tail")
repeated_truth = new_tail.get("truth")
if truth is not None and repeated_truth is not None and truth != repeated_truth:
raise ValueError(f"True volume tail identities differ: {case}")
truth = truth if truth is not None else repeated_truth
arms = {
"A": query_metrics(base.get("A")) | tail_metrics(old_arms.get("A"), "tail"),
"seed0": query_metrics(base.get("H2") or seed.get("seed0") or c.get("seed0"))
| (original_seed0_tail or repeated_seed0_tail),
"seed1": query_metrics(seed.get("seed1")) | tail_metrics(new_arms.get("seed1"), "overlap"),
}
for arm in ARMS:
for key, value in arms[arm].items():
if value is not None and (not math.isfinite(value) or value < 0):
raise ValueError(f"Invalid frozen {arm} {key}: {case}")
rows.append(dict(case=case, component_id=ident["component_id"], fold=ident["fold"],
constant_truth=truth["constant_field"] if truth is not None else None,
arms=arms, sources_missing=[key for key, value in sources.items() if case not in value]))
return rows
def metric_summary(rows, metric):
shared = [r for r in rows if all(r["arms"][a].get(metric) is not None for a in ARMS)]
grouped = groups(shared)
return dict(paired=support(shared), family_equal={
a: mean(mean(r["arms"][a][metric] for r in family) for family in grouped.values()) for a in ARMS
}, by_arm={a: support([r for r in rows if r["arms"][a].get(metric) is not None]) for a in ARMS},
missing_cases={a: [r["case"] for r in rows if r["arms"][a].get(metric) is None] for a in ARMS})
def field_gain(rows, arm):
paired = [r for r in rows if all(r["arms"][a].get("relative_l2") is not None for a in ("A", arm))]
gains = {key: mean(r["arms"]["A"]["relative_l2"] - r["arms"][arm]["relative_l2"] for r in family)
for key, family in sorted(groups(paired).items())}
avg = mean(gains.values())
leave_one = {key: mean(x for other, x in gains.items() if other != key) for key in gains}
leave_one = {key: value for key, value in leave_one.items() if value is not None}
positives = sorted(((key, x) for key, x in gains.items() if x > 0), key=lambda item: (-item[1], item[0]))
top = positives[:10]
return dict(support=support(paired), mean_gain=avg, families_positive=len(positives),
families_negative=sum(x < 0 for x in gains.values()), families_zero=sum(x == 0 for x in gains.values()),
family_gains=gains, leave_one_family_out=leave_one,
leave_one_min=min(leave_one.values(), default=None), leave_one_max=max(leave_one.values(), default=None),
sign_flips=[key for key, x in leave_one.items() if avg * x < 0],
reaches_zero=[key for key, x in leave_one.items() if x == 0],
top_positive_families=[key for key, _ in top],
top_ten_positive_share=math.fsum(x for _, x in top) / math.fsum(x for _, x in positives)
if positives else None)
def same_tail_miss(rows):
complete = [r for r in rows if all(r["arms"][a].get("d_tail_over_Rg") is not None
for a in ("seed0", "seed1"))]
categories = {(True, True): "both", (True, False): "seed0_only",
(False, True): "seed1_only", (False, False): "neither"}
per_family = defaultdict(list)
by_category = {label: [] for label in categories.values()}
for r in complete:
if r["constant_truth"] is not False:
continue
key = tuple(r["arms"][a]["d_tail_over_Rg"] > .10 for a in ("seed0", "seed1"))
label = categories[key]
by_category[label].append(r)
per_family[r["component_id"]].append(label)
counts = {key: len(value) for key, value in by_category.items()}
total = sum(counts.values())
expected = ((counts["both"] + counts["seed0_only"]) * (counts["both"] + counts["seed1_only"]) / total
if total else None)
return dict(support=dict(parts=total, families=len(per_family)), counts=counts,
family_equal={k: mean(f.count(k) / len(f) for f in per_family.values()) for k in by_category},
independence_expected_parts=expected,
observed_over_expected=counts["both"] / expected if expected else None,
both_cases=[dict(case=r["case"], component_id=r["component_id"], fold=r["fold"])
for r in by_category["both"]],
shared_constant_truth=[r["case"] for r in complete if r["constant_truth"] is True],
shared_unknown_truth=[r["case"] for r in complete if r["constant_truth"] is None],
missing_pair_cases=sorted(set(r["case"] for r in rows) - set(r["case"] for r in complete)))
def summarize(rows):
return dict(expected=support(rows), source_missing={
key: [r["case"] for r in rows if key in r["sources_missing"]] for key in INPUTS if key != "roster"},
metrics={metric: metric_summary(rows, metric) for metric in METRICS},
field_gain={arm: field_gain(rows, arm) for arm in ARMS[1:]}, same_tail_miss=same_tail_miss(rows))
def main():
rows = paired_rows()
cohorts = {"pooled": rows} | {f"fold{fold}": [r for r in rows if r["fold"] == fold] for fold in range(5)}
result = dict(prereg_commit=PREREG_COMMIT, prereg_sha256=PREREG_SHA256,
inputs={k: dict(path=str(p), sha256=h, commit=c) for k, (p, h, c) in INPUTS.items()},
gpu_seconds=0, target_arrays_read=0, prediction_arrays_read=0, new_fits=0,
cohorts={label: summarize(sample) for label, sample in cohorts.items()})
OUT.mkdir(exist_ok=False)
(OUT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
with (OUT / "by-part.csv").open("x", newline="") as stream:
writer = csv.writer(stream)
keys = ("case", "component_id", "fold", "constant_truth")
writer.writerow([*keys, "sources_missing", *(f"{a}.{m}" for a in ARMS for m in METRICS)])
for r in rows:
writer.writerow([*(r[key] for key in keys), " ".join(r["sources_missing"]),
*(r["arms"][a].get(m) for a in ARMS for m in METRICS)])
if __name__ == "__main__":
main()