Publish frozen-pair H2 family and two-seed diagnostics

This commit is contained in:
NewMeltAssist committed 2026-11-19 14:04:21 +08:00
1 parent 97f29e5357
commit 2f8d87872a
4 files changed
+4333

No files matched your search

+220
View File
@@ -0,0 +1,220 @@
"""Aggregate four frozen score files. No target or prediction arrays are opened."""
from collections import defaultdict
import csv
import hashlib
import json
import math
from pathlib import Path
BASE = Path(__file__).resolve().parents[1]
RUNS = BASE.parents[1] / "MeltMind-runs"
TAIL = BASE / "artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18"
OUT = BASE / "artifacts/phase3-h2-family-diagnostic-2026-11-19"
PREREG_COMMIT = "97f29e5357f3efe6c96ecba93bbf0aabc0acb84b"
PREREG_SHA256 = "1e7fe092da2e4cfe7235cb441bf2c45a08a1ad76ebf6e5f2814b2069c8af1bf1"
INPUTS = {
"roster": (TAIL / "plan.json",
"023baa1be5b0216ca9936d0ad12a96ea11242697699c57402cd24c358214fbd4", None),
"baseline": (RUNS / "phase2-h2-s2-postfit-2026-11-17/score/result.json",
"c43a6ab2460affdcafd3d51522e7b5e2d60a684b2b1a79a90e06042b0e19bc45",
"4b2a710de434aad2d1591d8e379a7c89e718bd55"),
"seed1": (RUNS / "phase3-seed1-postfit-2026-11-18/score/result.json",
"d259c156216f274f26ee8ba04fe56a763cb72e7750fe6d71df6e15ba7419f6d4",
"ea8f2a6276ed1db756f3c8bdfebf502673e5fe6f"),
"original_tail": (TAIL / "output/result.json",
"01b5c3de72ed708b2c4eb8ed1da6efe705aa1809ebd88621dfbbf7c932b25c7c",
"98519a6c365dd873dc76dfe2d393027592ba1675"),
"seed_tail": (RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/score/result.json",
"e269695a1ce6953b5148b16036f003e01ca5f6b040fc146771f2382d838b089c",
"520a85772a3b4f9a411818091fcc2b71cf4586b8"),
}
ARMS = ("A", "seed0", "seed1")
METRICS = ("relative_l2", "last_location", "E_tail", "Miss", "d_tail_over_Rg")
def checked_json(path, digest):
payload = path.read_bytes()
if hashlib.sha256(payload).hexdigest() != digest:
raise ValueError(f"Frozen evidence differs: {path}")
return json.loads(payload)
def by_case(rows):
index = {row["case"]: row for row in rows}
if len(index) != len(rows):
raise ValueError("Duplicate case identities")
return index
def mean(values):
values = list(values)
return math.fsum(values) / len(values) if values else None
def groups(rows):
out = defaultdict(list)
for row in rows:
out[row["component_id"]].append(row)
return out
def support(rows):
return dict(parts=len(rows), families=len(groups(rows)))
def query_metrics(arm):
return {metric: arm[metric]["value"] for metric in METRICS[:2]} if arm else {}
def tail_metrics(arm, overlap_key):
if not arm:
return {}
return {key: arm[overlap_key][key] for key in METRICS[2:4]} | {
"d_tail_over_Rg": arm["full"]["d_tail_over_Rg"]}
def paired_rows():
prereg = BASE / "docs/phase3-h2-family-diagnostic-prereg-2026-11-19.md"
if hashlib.sha256(prereg.read_bytes()).hexdigest() != PREREG_SHA256:
raise ValueError("Pre-registered scope differs")
data = {name: checked_json(path, digest) for name, (path, digest, _) in INPUTS.items()}
roster = by_case(data.pop("roster")["parts"])
if len(roster) != 552 or len(groups(roster.values())) != 204:
raise ValueError("Frozen roster differs")
if any(len({r["fold"] for r in family}) != 1 for family in groups(roster.values()).values()):
raise ValueError("Family crosses held-out folds")
sources = {}
for key, item in data.items():
if item["status"] != "COMPLETE":
raise ValueError(f"{key} is not a frozen complete score")
sources[key] = by_case(item["rows"])
for case, row in sources[key].items():
if case not in roster or any(roster[case][k] != row[k] for k in ("component_id", "fold")):
raise ValueError(f"Cross-source identity differs: {key} {case}")
rows = []
for case, ident in sorted(roster.items()):
base, seed, tail, c = (sources[key].get(case, {}) for key in
("baseline", "seed1", "original_tail", "seed_tail"))
if base and seed and base["H2"] != seed["seed0"]:
raise ValueError(f"Reused seed0 query values differ: {case}")
if base and c and base["H2"] != c["seed0"]:
raise ValueError(f"Phase 3(c) seed0 query values differ: {case}")
old_arms = tail.get("arms", {})
new_tail = c.get("tail", {})
new_arms = new_tail.get("arms", {})
original_seed0_tail = tail_metrics(old_arms.get("H2"), "tail")
repeated_seed0_tail = tail_metrics(new_arms.get("seed0"), "overlap")
if original_seed0_tail and repeated_seed0_tail:
for metric in original_seed0_tail:
if not math.isclose(original_seed0_tail[metric], repeated_seed0_tail[metric],
rel_tol=1e-12, abs_tol=1e-12):
raise ValueError(f"Repeated seed0 tail {metric} differs: {case}")
truth = tail.get("truth_tail")
repeated_truth = new_tail.get("truth")
if truth is not None and repeated_truth is not None and truth != repeated_truth:
raise ValueError(f"True volume tail identities differ: {case}")
truth = truth if truth is not None else repeated_truth
arms = {
"A": query_metrics(base.get("A")) | tail_metrics(old_arms.get("A"), "tail"),
"seed0": query_metrics(base.get("H2") or seed.get("seed0") or c.get("seed0"))
| (original_seed0_tail or repeated_seed0_tail),
"seed1": query_metrics(seed.get("seed1")) | tail_metrics(new_arms.get("seed1"), "overlap"),
}
for arm in ARMS:
for key, value in arms[arm].items():
if value is not None and (not math.isfinite(value) or value < 0):
raise ValueError(f"Invalid frozen {arm} {key}: {case}")
rows.append(dict(case=case, component_id=ident["component_id"], fold=ident["fold"],
constant_truth=truth["constant_field"] if truth is not None else None,
arms=arms, sources_missing=[key for key, value in sources.items() if case not in value]))
return rows
def metric_summary(rows, metric):
shared = [r for r in rows if all(r["arms"][a].get(metric) is not None for a in ARMS)]
grouped = groups(shared)
return dict(paired=support(shared), family_equal={
a: mean(mean(r["arms"][a][metric] for r in family) for family in grouped.values()) for a in ARMS
}, by_arm={a: support([r for r in rows if r["arms"][a].get(metric) is not None]) for a in ARMS},
missing_cases={a: [r["case"] for r in rows if r["arms"][a].get(metric) is None] for a in ARMS})
def field_gain(rows, arm):
paired = [r for r in rows if all(r["arms"][a].get("relative_l2") is not None for a in ("A", arm))]
gains = {key: mean(r["arms"]["A"]["relative_l2"] - r["arms"][arm]["relative_l2"] for r in family)
for key, family in sorted(groups(paired).items())}
avg = mean(gains.values())
leave_one = {key: mean(x for other, x in gains.items() if other != key) for key in gains}
leave_one = {key: value for key, value in leave_one.items() if value is not None}
positives = sorted(((key, x) for key, x in gains.items() if x > 0), key=lambda item: (-item[1], item[0]))
top = positives[:10]
return dict(support=support(paired), mean_gain=avg, families_positive=len(positives),
families_negative=sum(x < 0 for x in gains.values()), families_zero=sum(x == 0 for x in gains.values()),
family_gains=gains, leave_one_family_out=leave_one,
leave_one_min=min(leave_one.values(), default=None), leave_one_max=max(leave_one.values(), default=None),
sign_flips=[key for key, x in leave_one.items() if avg * x < 0],
reaches_zero=[key for key, x in leave_one.items() if x == 0],
top_positive_families=[key for key, _ in top],
top_ten_positive_share=math.fsum(x for _, x in top) / math.fsum(x for _, x in positives)
if positives else None)
def same_tail_miss(rows):
complete = [r for r in rows if all(r["arms"][a].get("d_tail_over_Rg") is not None
for a in ("seed0", "seed1"))]
categories = {(True, True): "both", (True, False): "seed0_only",
(False, True): "seed1_only", (False, False): "neither"}
per_family = defaultdict(list)
by_category = {label: [] for label in categories.values()}
for r in complete:
if r["constant_truth"] is not False:
continue
key = tuple(r["arms"][a]["d_tail_over_Rg"] > .10 for a in ("seed0", "seed1"))
label = categories[key]
by_category[label].append(r)
per_family[r["component_id"]].append(label)
counts = {key: len(value) for key, value in by_category.items()}
total = sum(counts.values())
expected = ((counts["both"] + counts["seed0_only"]) * (counts["both"] + counts["seed1_only"]) / total
if total else None)
return dict(support=dict(parts=total, families=len(per_family)), counts=counts,
family_equal={k: mean(f.count(k) / len(f) for f in per_family.values()) for k in by_category},
independence_expected_parts=expected,
observed_over_expected=counts["both"] / expected if expected else None,
both_cases=[dict(case=r["case"], component_id=r["component_id"], fold=r["fold"])
for r in by_category["both"]],
shared_constant_truth=[r["case"] for r in complete if r["constant_truth"] is True],
shared_unknown_truth=[r["case"] for r in complete if r["constant_truth"] is None],
missing_pair_cases=sorted(set(r["case"] for r in rows) - set(r["case"] for r in complete)))
def summarize(rows):
return dict(expected=support(rows), source_missing={
key: [r["case"] for r in rows if key in r["sources_missing"]] for key in INPUTS if key != "roster"},
metrics={metric: metric_summary(rows, metric) for metric in METRICS},
field_gain={arm: field_gain(rows, arm) for arm in ARMS[1:]}, same_tail_miss=same_tail_miss(rows))
def main():
rows = paired_rows()
cohorts = {"pooled": rows} | {f"fold{fold}": [r for r in rows if r["fold"] == fold] for fold in range(5)}
result = dict(prereg_commit=PREREG_COMMIT, prereg_sha256=PREREG_SHA256,
inputs={k: dict(path=str(p), sha256=h, commit=c) for k, (p, h, c) in INPUTS.items()},
gpu_seconds=0, target_arrays_read=0, prediction_arrays_read=0, new_fits=0,
cohorts={label: summarize(sample) for label, sample in cohorts.items()})
OUT.mkdir(exist_ok=False)
(OUT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
with (OUT / "by-part.csv").open("x", newline="") as stream:
writer = csv.writer(stream)
keys = ("case", "component_id", "fold", "constant_truth")
writer.writerow([*keys, "sources_missing", *(f"{a}.{m}" for a in ARMS for m in METRICS)])
for r in rows:
writer.writerow([*(r[key] for key in keys), " ".join(r["sources_missing"]),
*(r["arms"][a].get(m) for a in ARMS for m in METRICS)])
if __name__ == "__main__":
main()