"""Aggregate four frozen score files. No target or prediction arrays are opened.""" from collections import defaultdict import csv import hashlib import json import math from pathlib import Path BASE = Path(__file__).resolve().parents[1] RUNS = BASE.parents[1] / "MeltMind-runs" TAIL = BASE / "artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18" OUT = BASE / "artifacts/phase3-h2-family-diagnostic-2026-11-19" PREREG_COMMIT = "97f29e5357f3efe6c96ecba93bbf0aabc0acb84b" PREREG_SHA256 = "1e7fe092da2e4cfe7235cb441bf2c45a08a1ad76ebf6e5f2814b2069c8af1bf1" INPUTS = { "roster": (TAIL / "plan.json", "023baa1be5b0216ca9936d0ad12a96ea11242697699c57402cd24c358214fbd4", None), "baseline": (RUNS / "phase2-h2-s2-postfit-2026-11-17/score/result.json", "c43a6ab2460affdcafd3d51522e7b5e2d60a684b2b1a79a90e06042b0e19bc45", "4b2a710de434aad2d1591d8e379a7c89e718bd55"), "seed1": (RUNS / "phase3-seed1-postfit-2026-11-18/score/result.json", "d259c156216f274f26ee8ba04fe56a763cb72e7750fe6d71df6e15ba7419f6d4", "ea8f2a6276ed1db756f3c8bdfebf502673e5fe6f"), "original_tail": (TAIL / "output/result.json", "01b5c3de72ed708b2c4eb8ed1da6efe705aa1809ebd88621dfbbf7c932b25c7c", "98519a6c365dd873dc76dfe2d393027592ba1675"), "seed_tail": (RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/score/result.json", "e269695a1ce6953b5148b16036f003e01ca5f6b040fc146771f2382d838b089c", "520a85772a3b4f9a411818091fcc2b71cf4586b8"), } ARMS = ("A", "seed0", "seed1") METRICS = ("relative_l2", "last_location", "E_tail", "Miss", "d_tail_over_Rg") def checked_json(path, digest): payload = path.read_bytes() if hashlib.sha256(payload).hexdigest() != digest: raise ValueError(f"Frozen evidence differs: {path}") return json.loads(payload) def by_case(rows): index = {row["case"]: row for row in rows} if len(index) != len(rows): raise ValueError("Duplicate case identities") return index def mean(values): values = list(values) return math.fsum(values) / len(values) if values else None def groups(rows): out = defaultdict(list) for row in rows: out[row["component_id"]].append(row) return out def support(rows): return dict(parts=len(rows), families=len(groups(rows))) def query_metrics(arm): return {metric: arm[metric]["value"] for metric in METRICS[:2]} if arm else {} def tail_metrics(arm, overlap_key): if not arm: return {} return {key: arm[overlap_key][key] for key in METRICS[2:4]} | { "d_tail_over_Rg": arm["full"]["d_tail_over_Rg"]} def paired_rows(): prereg = BASE / "docs/phase3-h2-family-diagnostic-prereg-2026-11-19.md" if hashlib.sha256(prereg.read_bytes()).hexdigest() != PREREG_SHA256: raise ValueError("Pre-registered scope differs") data = {name: checked_json(path, digest) for name, (path, digest, _) in INPUTS.items()} roster = by_case(data.pop("roster")["parts"]) if len(roster) != 552 or len(groups(roster.values())) != 204: raise ValueError("Frozen roster differs") if any(len({r["fold"] for r in family}) != 1 for family in groups(roster.values()).values()): raise ValueError("Family crosses held-out folds") sources = {} for key, item in data.items(): if item["status"] != "COMPLETE": raise ValueError(f"{key} is not a frozen complete score") sources[key] = by_case(item["rows"]) for case, row in sources[key].items(): if case not in roster or any(roster[case][k] != row[k] for k in ("component_id", "fold")): raise ValueError(f"Cross-source identity differs: {key} {case}") rows = [] for case, ident in sorted(roster.items()): base, seed, tail, c = (sources[key].get(case, {}) for key in ("baseline", "seed1", "original_tail", "seed_tail")) if base and seed and base["H2"] != seed["seed0"]: raise ValueError(f"Reused seed0 query values differ: {case}") if base and c and base["H2"] != c["seed0"]: raise ValueError(f"Phase 3(c) seed0 query values differ: {case}") old_arms = tail.get("arms", {}) new_tail = c.get("tail", {}) new_arms = new_tail.get("arms", {}) original_seed0_tail = tail_metrics(old_arms.get("H2"), "tail") repeated_seed0_tail = tail_metrics(new_arms.get("seed0"), "overlap") if original_seed0_tail and repeated_seed0_tail: for metric in original_seed0_tail: if not math.isclose(original_seed0_tail[metric], repeated_seed0_tail[metric], rel_tol=1e-12, abs_tol=1e-12): raise ValueError(f"Repeated seed0 tail {metric} differs: {case}") truth = tail.get("truth_tail") repeated_truth = new_tail.get("truth") if truth is not None and repeated_truth is not None and truth != repeated_truth: raise ValueError(f"True volume tail identities differ: {case}") truth = truth if truth is not None else repeated_truth arms = { "A": query_metrics(base.get("A")) | tail_metrics(old_arms.get("A"), "tail"), "seed0": query_metrics(base.get("H2") or seed.get("seed0") or c.get("seed0")) | (original_seed0_tail or repeated_seed0_tail), "seed1": query_metrics(seed.get("seed1")) | tail_metrics(new_arms.get("seed1"), "overlap"), } for arm in ARMS: for key, value in arms[arm].items(): if value is not None and (not math.isfinite(value) or value < 0): raise ValueError(f"Invalid frozen {arm} {key}: {case}") rows.append(dict(case=case, component_id=ident["component_id"], fold=ident["fold"], constant_truth=truth["constant_field"] if truth is not None else None, arms=arms, sources_missing=[key for key, value in sources.items() if case not in value])) return rows def metric_summary(rows, metric): shared = [r for r in rows if all(r["arms"][a].get(metric) is not None for a in ARMS)] grouped = groups(shared) return dict(paired=support(shared), family_equal={ a: mean(mean(r["arms"][a][metric] for r in family) for family in grouped.values()) for a in ARMS }, by_arm={a: support([r for r in rows if r["arms"][a].get(metric) is not None]) for a in ARMS}, missing_cases={a: [r["case"] for r in rows if r["arms"][a].get(metric) is None] for a in ARMS}) def field_gain(rows, arm): paired = [r for r in rows if all(r["arms"][a].get("relative_l2") is not None for a in ("A", arm))] gains = {key: mean(r["arms"]["A"]["relative_l2"] - r["arms"][arm]["relative_l2"] for r in family) for key, family in sorted(groups(paired).items())} avg = mean(gains.values()) leave_one = {key: mean(x for other, x in gains.items() if other != key) for key in gains} leave_one = {key: value for key, value in leave_one.items() if value is not None} positives = sorted(((key, x) for key, x in gains.items() if x > 0), key=lambda item: (-item[1], item[0])) top = positives[:10] return dict(support=support(paired), mean_gain=avg, families_positive=len(positives), families_negative=sum(x < 0 for x in gains.values()), families_zero=sum(x == 0 for x in gains.values()), family_gains=gains, leave_one_family_out=leave_one, leave_one_min=min(leave_one.values(), default=None), leave_one_max=max(leave_one.values(), default=None), sign_flips=[key for key, x in leave_one.items() if avg * x < 0], reaches_zero=[key for key, x in leave_one.items() if x == 0], top_positive_families=[key for key, _ in top], top_ten_positive_share=math.fsum(x for _, x in top) / math.fsum(x for _, x in positives) if positives else None) def same_tail_miss(rows): complete = [r for r in rows if all(r["arms"][a].get("d_tail_over_Rg") is not None for a in ("seed0", "seed1"))] categories = {(True, True): "both", (True, False): "seed0_only", (False, True): "seed1_only", (False, False): "neither"} per_family = defaultdict(list) by_category = {label: [] for label in categories.values()} for r in complete: if r["constant_truth"] is not False: continue key = tuple(r["arms"][a]["d_tail_over_Rg"] > .10 for a in ("seed0", "seed1")) label = categories[key] by_category[label].append(r) per_family[r["component_id"]].append(label) counts = {key: len(value) for key, value in by_category.items()} total = sum(counts.values()) expected = ((counts["both"] + counts["seed0_only"]) * (counts["both"] + counts["seed1_only"]) / total if total else None) return dict(support=dict(parts=total, families=len(per_family)), counts=counts, family_equal={k: mean(f.count(k) / len(f) for f in per_family.values()) for k in by_category}, independence_expected_parts=expected, observed_over_expected=counts["both"] / expected if expected else None, both_cases=[dict(case=r["case"], component_id=r["component_id"], fold=r["fold"]) for r in by_category["both"]], shared_constant_truth=[r["case"] for r in complete if r["constant_truth"] is True], shared_unknown_truth=[r["case"] for r in complete if r["constant_truth"] is None], missing_pair_cases=sorted(set(r["case"] for r in rows) - set(r["case"] for r in complete))) def summarize(rows): return dict(expected=support(rows), source_missing={ key: [r["case"] for r in rows if key in r["sources_missing"]] for key in INPUTS if key != "roster"}, metrics={metric: metric_summary(rows, metric) for metric in METRICS}, field_gain={arm: field_gain(rows, arm) for arm in ARMS[1:]}, same_tail_miss=same_tail_miss(rows)) def main(): rows = paired_rows() cohorts = {"pooled": rows} | {f"fold{fold}": [r for r in rows if r["fold"] == fold] for fold in range(5)} result = dict(prereg_commit=PREREG_COMMIT, prereg_sha256=PREREG_SHA256, inputs={k: dict(path=str(p), sha256=h, commit=c) for k, (p, h, c) in INPUTS.items()}, gpu_seconds=0, target_arrays_read=0, prediction_arrays_read=0, new_fits=0, cohorts={label: summarize(sample) for label, sample in cohorts.items()}) OUT.mkdir(exist_ok=False) (OUT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") with (OUT / "by-part.csv").open("x", newline="") as stream: writer = csv.writer(stream) keys = ("case", "component_id", "fold", "constant_truth") writer.writerow([*keys, "sources_missing", *(f"{a}.{m}" for a in ARMS for m in METRICS)]) for r in rows: writer.writerow([*(r[key] for key in keys), " ".join(r["sources_missing"]), *(r["arms"][a].get(m) for a in ARMS for m in METRICS)]) if __name__ == "__main__": main()