Publish frozen-pair H2 family and two-seed diagnostics
This commit is contained in:
1 parent
97f29e5357
commit
2f8d87872a
4 files changed
+4333
No files matched your search
@@ -0,0 +1,220 @@
|
||||
"""Aggregate four frozen score files. No target or prediction arrays are opened."""
|
||||
|
||||
from collections import defaultdict
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
BASE = Path(__file__).resolve().parents[1]
|
||||
RUNS = BASE.parents[1] / "MeltMind-runs"
|
||||
TAIL = BASE / "artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18"
|
||||
OUT = BASE / "artifacts/phase3-h2-family-diagnostic-2026-11-19"
|
||||
PREREG_COMMIT = "97f29e5357f3efe6c96ecba93bbf0aabc0acb84b"
|
||||
PREREG_SHA256 = "1e7fe092da2e4cfe7235cb441bf2c45a08a1ad76ebf6e5f2814b2069c8af1bf1"
|
||||
INPUTS = {
|
||||
"roster": (TAIL / "plan.json",
|
||||
"023baa1be5b0216ca9936d0ad12a96ea11242697699c57402cd24c358214fbd4", None),
|
||||
"baseline": (RUNS / "phase2-h2-s2-postfit-2026-11-17/score/result.json",
|
||||
"c43a6ab2460affdcafd3d51522e7b5e2d60a684b2b1a79a90e06042b0e19bc45",
|
||||
"4b2a710de434aad2d1591d8e379a7c89e718bd55"),
|
||||
"seed1": (RUNS / "phase3-seed1-postfit-2026-11-18/score/result.json",
|
||||
"d259c156216f274f26ee8ba04fe56a763cb72e7750fe6d71df6e15ba7419f6d4",
|
||||
"ea8f2a6276ed1db756f3c8bdfebf502673e5fe6f"),
|
||||
"original_tail": (TAIL / "output/result.json",
|
||||
"01b5c3de72ed708b2c4eb8ed1da6efe705aa1809ebd88621dfbbf7c932b25c7c",
|
||||
"98519a6c365dd873dc76dfe2d393027592ba1675"),
|
||||
"seed_tail": (RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/score/result.json",
|
||||
"e269695a1ce6953b5148b16036f003e01ca5f6b040fc146771f2382d838b089c",
|
||||
"520a85772a3b4f9a411818091fcc2b71cf4586b8"),
|
||||
}
|
||||
ARMS = ("A", "seed0", "seed1")
|
||||
METRICS = ("relative_l2", "last_location", "E_tail", "Miss", "d_tail_over_Rg")
|
||||
|
||||
|
||||
def checked_json(path, digest):
|
||||
payload = path.read_bytes()
|
||||
if hashlib.sha256(payload).hexdigest() != digest:
|
||||
raise ValueError(f"Frozen evidence differs: {path}")
|
||||
return json.loads(payload)
|
||||
|
||||
|
||||
def by_case(rows):
|
||||
index = {row["case"]: row for row in rows}
|
||||
if len(index) != len(rows):
|
||||
raise ValueError("Duplicate case identities")
|
||||
return index
|
||||
|
||||
|
||||
def mean(values):
|
||||
values = list(values)
|
||||
return math.fsum(values) / len(values) if values else None
|
||||
|
||||
|
||||
def groups(rows):
|
||||
out = defaultdict(list)
|
||||
for row in rows:
|
||||
out[row["component_id"]].append(row)
|
||||
return out
|
||||
|
||||
|
||||
def support(rows):
|
||||
return dict(parts=len(rows), families=len(groups(rows)))
|
||||
|
||||
|
||||
def query_metrics(arm):
|
||||
return {metric: arm[metric]["value"] for metric in METRICS[:2]} if arm else {}
|
||||
|
||||
|
||||
def tail_metrics(arm, overlap_key):
|
||||
if not arm:
|
||||
return {}
|
||||
return {key: arm[overlap_key][key] for key in METRICS[2:4]} | {
|
||||
"d_tail_over_Rg": arm["full"]["d_tail_over_Rg"]}
|
||||
|
||||
|
||||
def paired_rows():
|
||||
prereg = BASE / "docs/phase3-h2-family-diagnostic-prereg-2026-11-19.md"
|
||||
if hashlib.sha256(prereg.read_bytes()).hexdigest() != PREREG_SHA256:
|
||||
raise ValueError("Pre-registered scope differs")
|
||||
data = {name: checked_json(path, digest) for name, (path, digest, _) in INPUTS.items()}
|
||||
roster = by_case(data.pop("roster")["parts"])
|
||||
if len(roster) != 552 or len(groups(roster.values())) != 204:
|
||||
raise ValueError("Frozen roster differs")
|
||||
if any(len({r["fold"] for r in family}) != 1 for family in groups(roster.values()).values()):
|
||||
raise ValueError("Family crosses held-out folds")
|
||||
sources = {}
|
||||
for key, item in data.items():
|
||||
if item["status"] != "COMPLETE":
|
||||
raise ValueError(f"{key} is not a frozen complete score")
|
||||
sources[key] = by_case(item["rows"])
|
||||
for case, row in sources[key].items():
|
||||
if case not in roster or any(roster[case][k] != row[k] for k in ("component_id", "fold")):
|
||||
raise ValueError(f"Cross-source identity differs: {key} {case}")
|
||||
|
||||
rows = []
|
||||
for case, ident in sorted(roster.items()):
|
||||
base, seed, tail, c = (sources[key].get(case, {}) for key in
|
||||
("baseline", "seed1", "original_tail", "seed_tail"))
|
||||
if base and seed and base["H2"] != seed["seed0"]:
|
||||
raise ValueError(f"Reused seed0 query values differ: {case}")
|
||||
if base and c and base["H2"] != c["seed0"]:
|
||||
raise ValueError(f"Phase 3(c) seed0 query values differ: {case}")
|
||||
old_arms = tail.get("arms", {})
|
||||
new_tail = c.get("tail", {})
|
||||
new_arms = new_tail.get("arms", {})
|
||||
original_seed0_tail = tail_metrics(old_arms.get("H2"), "tail")
|
||||
repeated_seed0_tail = tail_metrics(new_arms.get("seed0"), "overlap")
|
||||
if original_seed0_tail and repeated_seed0_tail:
|
||||
for metric in original_seed0_tail:
|
||||
if not math.isclose(original_seed0_tail[metric], repeated_seed0_tail[metric],
|
||||
rel_tol=1e-12, abs_tol=1e-12):
|
||||
raise ValueError(f"Repeated seed0 tail {metric} differs: {case}")
|
||||
truth = tail.get("truth_tail")
|
||||
repeated_truth = new_tail.get("truth")
|
||||
if truth is not None and repeated_truth is not None and truth != repeated_truth:
|
||||
raise ValueError(f"True volume tail identities differ: {case}")
|
||||
truth = truth if truth is not None else repeated_truth
|
||||
arms = {
|
||||
"A": query_metrics(base.get("A")) | tail_metrics(old_arms.get("A"), "tail"),
|
||||
"seed0": query_metrics(base.get("H2") or seed.get("seed0") or c.get("seed0"))
|
||||
| (original_seed0_tail or repeated_seed0_tail),
|
||||
"seed1": query_metrics(seed.get("seed1")) | tail_metrics(new_arms.get("seed1"), "overlap"),
|
||||
}
|
||||
for arm in ARMS:
|
||||
for key, value in arms[arm].items():
|
||||
if value is not None and (not math.isfinite(value) or value < 0):
|
||||
raise ValueError(f"Invalid frozen {arm} {key}: {case}")
|
||||
rows.append(dict(case=case, component_id=ident["component_id"], fold=ident["fold"],
|
||||
constant_truth=truth["constant_field"] if truth is not None else None,
|
||||
arms=arms, sources_missing=[key for key, value in sources.items() if case not in value]))
|
||||
return rows
|
||||
|
||||
|
||||
def metric_summary(rows, metric):
|
||||
shared = [r for r in rows if all(r["arms"][a].get(metric) is not None for a in ARMS)]
|
||||
grouped = groups(shared)
|
||||
return dict(paired=support(shared), family_equal={
|
||||
a: mean(mean(r["arms"][a][metric] for r in family) for family in grouped.values()) for a in ARMS
|
||||
}, by_arm={a: support([r for r in rows if r["arms"][a].get(metric) is not None]) for a in ARMS},
|
||||
missing_cases={a: [r["case"] for r in rows if r["arms"][a].get(metric) is None] for a in ARMS})
|
||||
|
||||
|
||||
def field_gain(rows, arm):
|
||||
paired = [r for r in rows if all(r["arms"][a].get("relative_l2") is not None for a in ("A", arm))]
|
||||
gains = {key: mean(r["arms"]["A"]["relative_l2"] - r["arms"][arm]["relative_l2"] for r in family)
|
||||
for key, family in sorted(groups(paired).items())}
|
||||
avg = mean(gains.values())
|
||||
leave_one = {key: mean(x for other, x in gains.items() if other != key) for key in gains}
|
||||
leave_one = {key: value for key, value in leave_one.items() if value is not None}
|
||||
positives = sorted(((key, x) for key, x in gains.items() if x > 0), key=lambda item: (-item[1], item[0]))
|
||||
top = positives[:10]
|
||||
return dict(support=support(paired), mean_gain=avg, families_positive=len(positives),
|
||||
families_negative=sum(x < 0 for x in gains.values()), families_zero=sum(x == 0 for x in gains.values()),
|
||||
family_gains=gains, leave_one_family_out=leave_one,
|
||||
leave_one_min=min(leave_one.values(), default=None), leave_one_max=max(leave_one.values(), default=None),
|
||||
sign_flips=[key for key, x in leave_one.items() if avg * x < 0],
|
||||
reaches_zero=[key for key, x in leave_one.items() if x == 0],
|
||||
top_positive_families=[key for key, _ in top],
|
||||
top_ten_positive_share=math.fsum(x for _, x in top) / math.fsum(x for _, x in positives)
|
||||
if positives else None)
|
||||
|
||||
|
||||
def same_tail_miss(rows):
|
||||
complete = [r for r in rows if all(r["arms"][a].get("d_tail_over_Rg") is not None
|
||||
for a in ("seed0", "seed1"))]
|
||||
categories = {(True, True): "both", (True, False): "seed0_only",
|
||||
(False, True): "seed1_only", (False, False): "neither"}
|
||||
per_family = defaultdict(list)
|
||||
by_category = {label: [] for label in categories.values()}
|
||||
for r in complete:
|
||||
if r["constant_truth"] is not False:
|
||||
continue
|
||||
key = tuple(r["arms"][a]["d_tail_over_Rg"] > .10 for a in ("seed0", "seed1"))
|
||||
label = categories[key]
|
||||
by_category[label].append(r)
|
||||
per_family[r["component_id"]].append(label)
|
||||
counts = {key: len(value) for key, value in by_category.items()}
|
||||
total = sum(counts.values())
|
||||
expected = ((counts["both"] + counts["seed0_only"]) * (counts["both"] + counts["seed1_only"]) / total
|
||||
if total else None)
|
||||
return dict(support=dict(parts=total, families=len(per_family)), counts=counts,
|
||||
family_equal={k: mean(f.count(k) / len(f) for f in per_family.values()) for k in by_category},
|
||||
independence_expected_parts=expected,
|
||||
observed_over_expected=counts["both"] / expected if expected else None,
|
||||
both_cases=[dict(case=r["case"], component_id=r["component_id"], fold=r["fold"])
|
||||
for r in by_category["both"]],
|
||||
shared_constant_truth=[r["case"] for r in complete if r["constant_truth"] is True],
|
||||
shared_unknown_truth=[r["case"] for r in complete if r["constant_truth"] is None],
|
||||
missing_pair_cases=sorted(set(r["case"] for r in rows) - set(r["case"] for r in complete)))
|
||||
|
||||
|
||||
def summarize(rows):
|
||||
return dict(expected=support(rows), source_missing={
|
||||
key: [r["case"] for r in rows if key in r["sources_missing"]] for key in INPUTS if key != "roster"},
|
||||
metrics={metric: metric_summary(rows, metric) for metric in METRICS},
|
||||
field_gain={arm: field_gain(rows, arm) for arm in ARMS[1:]}, same_tail_miss=same_tail_miss(rows))
|
||||
|
||||
|
||||
def main():
|
||||
rows = paired_rows()
|
||||
cohorts = {"pooled": rows} | {f"fold{fold}": [r for r in rows if r["fold"] == fold] for fold in range(5)}
|
||||
result = dict(prereg_commit=PREREG_COMMIT, prereg_sha256=PREREG_SHA256,
|
||||
inputs={k: dict(path=str(p), sha256=h, commit=c) for k, (p, h, c) in INPUTS.items()},
|
||||
gpu_seconds=0, target_arrays_read=0, prediction_arrays_read=0, new_fits=0,
|
||||
cohorts={label: summarize(sample) for label, sample in cohorts.items()})
|
||||
OUT.mkdir(exist_ok=False)
|
||||
(OUT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
|
||||
with (OUT / "by-part.csv").open("x", newline="") as stream:
|
||||
writer = csv.writer(stream)
|
||||
keys = ("case", "component_id", "fold", "constant_truth")
|
||||
writer.writerow([*keys, "sources_missing", *(f"{a}.{m}" for a in ARMS for m in METRICS)])
|
||||
for r in rows:
|
||||
writer.writerow([*(r[key] for key in keys), " ".join(r["sources_missing"]),
|
||||
*(r["arms"][a].get(m) for a in ARMS for m in METRICS)])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user