221 lines
11 KiB
Python
221 lines
11 KiB
Python
"""Aggregate four frozen score files. No target or prediction arrays are opened."""
|
|
|
|
from collections import defaultdict
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
|
|
|
|
BASE = Path(__file__).resolve().parents[1]
|
|
RUNS = BASE.parents[1] / "MeltMind-runs"
|
|
TAIL = BASE / "artifacts/phase2-h2-s2-tail-diagnostic-2026-11-18"
|
|
OUT = BASE / "artifacts/phase3-h2-family-diagnostic-2026-11-19"
|
|
PREREG_COMMIT = "97f29e5357f3efe6c96ecba93bbf0aabc0acb84b"
|
|
PREREG_SHA256 = "1e7fe092da2e4cfe7235cb441bf2c45a08a1ad76ebf6e5f2814b2069c8af1bf1"
|
|
INPUTS = {
|
|
"roster": (TAIL / "plan.json",
|
|
"023baa1be5b0216ca9936d0ad12a96ea11242697699c57402cd24c358214fbd4", None),
|
|
"baseline": (RUNS / "phase2-h2-s2-postfit-2026-11-17/score/result.json",
|
|
"c43a6ab2460affdcafd3d51522e7b5e2d60a684b2b1a79a90e06042b0e19bc45",
|
|
"4b2a710de434aad2d1591d8e379a7c89e718bd55"),
|
|
"seed1": (RUNS / "phase3-seed1-postfit-2026-11-18/score/result.json",
|
|
"d259c156216f274f26ee8ba04fe56a763cb72e7750fe6d71df6e15ba7419f6d4",
|
|
"ea8f2a6276ed1db756f3c8bdfebf502673e5fe6f"),
|
|
"original_tail": (TAIL / "output/result.json",
|
|
"01b5c3de72ed708b2c4eb8ed1da6efe705aa1809ebd88621dfbbf7c932b25c7c",
|
|
"98519a6c365dd873dc76dfe2d393027592ba1675"),
|
|
"seed_tail": (RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/score/result.json",
|
|
"e269695a1ce6953b5148b16036f003e01ca5f6b040fc146771f2382d838b089c",
|
|
"520a85772a3b4f9a411818091fcc2b71cf4586b8"),
|
|
}
|
|
ARMS = ("A", "seed0", "seed1")
|
|
METRICS = ("relative_l2", "last_location", "E_tail", "Miss", "d_tail_over_Rg")
|
|
|
|
|
|
def checked_json(path, digest):
|
|
payload = path.read_bytes()
|
|
if hashlib.sha256(payload).hexdigest() != digest:
|
|
raise ValueError(f"Frozen evidence differs: {path}")
|
|
return json.loads(payload)
|
|
|
|
|
|
def by_case(rows):
|
|
index = {row["case"]: row for row in rows}
|
|
if len(index) != len(rows):
|
|
raise ValueError("Duplicate case identities")
|
|
return index
|
|
|
|
|
|
def mean(values):
|
|
values = list(values)
|
|
return math.fsum(values) / len(values) if values else None
|
|
|
|
|
|
def groups(rows):
|
|
out = defaultdict(list)
|
|
for row in rows:
|
|
out[row["component_id"]].append(row)
|
|
return out
|
|
|
|
|
|
def support(rows):
|
|
return dict(parts=len(rows), families=len(groups(rows)))
|
|
|
|
|
|
def query_metrics(arm):
|
|
return {metric: arm[metric]["value"] for metric in METRICS[:2]} if arm else {}
|
|
|
|
|
|
def tail_metrics(arm, overlap_key):
|
|
if not arm:
|
|
return {}
|
|
return {key: arm[overlap_key][key] for key in METRICS[2:4]} | {
|
|
"d_tail_over_Rg": arm["full"]["d_tail_over_Rg"]}
|
|
|
|
|
|
def paired_rows():
|
|
prereg = BASE / "docs/phase3-h2-family-diagnostic-prereg-2026-11-19.md"
|
|
if hashlib.sha256(prereg.read_bytes()).hexdigest() != PREREG_SHA256:
|
|
raise ValueError("Pre-registered scope differs")
|
|
data = {name: checked_json(path, digest) for name, (path, digest, _) in INPUTS.items()}
|
|
roster = by_case(data.pop("roster")["parts"])
|
|
if len(roster) != 552 or len(groups(roster.values())) != 204:
|
|
raise ValueError("Frozen roster differs")
|
|
if any(len({r["fold"] for r in family}) != 1 for family in groups(roster.values()).values()):
|
|
raise ValueError("Family crosses held-out folds")
|
|
sources = {}
|
|
for key, item in data.items():
|
|
if item["status"] != "COMPLETE":
|
|
raise ValueError(f"{key} is not a frozen complete score")
|
|
sources[key] = by_case(item["rows"])
|
|
for case, row in sources[key].items():
|
|
if case not in roster or any(roster[case][k] != row[k] for k in ("component_id", "fold")):
|
|
raise ValueError(f"Cross-source identity differs: {key} {case}")
|
|
|
|
rows = []
|
|
for case, ident in sorted(roster.items()):
|
|
base, seed, tail, c = (sources[key].get(case, {}) for key in
|
|
("baseline", "seed1", "original_tail", "seed_tail"))
|
|
if base and seed and base["H2"] != seed["seed0"]:
|
|
raise ValueError(f"Reused seed0 query values differ: {case}")
|
|
if base and c and base["H2"] != c["seed0"]:
|
|
raise ValueError(f"Phase 3(c) seed0 query values differ: {case}")
|
|
old_arms = tail.get("arms", {})
|
|
new_tail = c.get("tail", {})
|
|
new_arms = new_tail.get("arms", {})
|
|
original_seed0_tail = tail_metrics(old_arms.get("H2"), "tail")
|
|
repeated_seed0_tail = tail_metrics(new_arms.get("seed0"), "overlap")
|
|
if original_seed0_tail and repeated_seed0_tail:
|
|
for metric in original_seed0_tail:
|
|
if not math.isclose(original_seed0_tail[metric], repeated_seed0_tail[metric],
|
|
rel_tol=1e-12, abs_tol=1e-12):
|
|
raise ValueError(f"Repeated seed0 tail {metric} differs: {case}")
|
|
truth = tail.get("truth_tail")
|
|
repeated_truth = new_tail.get("truth")
|
|
if truth is not None and repeated_truth is not None and truth != repeated_truth:
|
|
raise ValueError(f"True volume tail identities differ: {case}")
|
|
truth = truth if truth is not None else repeated_truth
|
|
arms = {
|
|
"A": query_metrics(base.get("A")) | tail_metrics(old_arms.get("A"), "tail"),
|
|
"seed0": query_metrics(base.get("H2") or seed.get("seed0") or c.get("seed0"))
|
|
| (original_seed0_tail or repeated_seed0_tail),
|
|
"seed1": query_metrics(seed.get("seed1")) | tail_metrics(new_arms.get("seed1"), "overlap"),
|
|
}
|
|
for arm in ARMS:
|
|
for key, value in arms[arm].items():
|
|
if value is not None and (not math.isfinite(value) or value < 0):
|
|
raise ValueError(f"Invalid frozen {arm} {key}: {case}")
|
|
rows.append(dict(case=case, component_id=ident["component_id"], fold=ident["fold"],
|
|
constant_truth=truth["constant_field"] if truth is not None else None,
|
|
arms=arms, sources_missing=[key for key, value in sources.items() if case not in value]))
|
|
return rows
|
|
|
|
|
|
def metric_summary(rows, metric):
|
|
shared = [r for r in rows if all(r["arms"][a].get(metric) is not None for a in ARMS)]
|
|
grouped = groups(shared)
|
|
return dict(paired=support(shared), family_equal={
|
|
a: mean(mean(r["arms"][a][metric] for r in family) for family in grouped.values()) for a in ARMS
|
|
}, by_arm={a: support([r for r in rows if r["arms"][a].get(metric) is not None]) for a in ARMS},
|
|
missing_cases={a: [r["case"] for r in rows if r["arms"][a].get(metric) is None] for a in ARMS})
|
|
|
|
|
|
def field_gain(rows, arm):
|
|
paired = [r for r in rows if all(r["arms"][a].get("relative_l2") is not None for a in ("A", arm))]
|
|
gains = {key: mean(r["arms"]["A"]["relative_l2"] - r["arms"][arm]["relative_l2"] for r in family)
|
|
for key, family in sorted(groups(paired).items())}
|
|
avg = mean(gains.values())
|
|
leave_one = {key: mean(x for other, x in gains.items() if other != key) for key in gains}
|
|
leave_one = {key: value for key, value in leave_one.items() if value is not None}
|
|
positives = sorted(((key, x) for key, x in gains.items() if x > 0), key=lambda item: (-item[1], item[0]))
|
|
top = positives[:10]
|
|
return dict(support=support(paired), mean_gain=avg, families_positive=len(positives),
|
|
families_negative=sum(x < 0 for x in gains.values()), families_zero=sum(x == 0 for x in gains.values()),
|
|
family_gains=gains, leave_one_family_out=leave_one,
|
|
leave_one_min=min(leave_one.values(), default=None), leave_one_max=max(leave_one.values(), default=None),
|
|
sign_flips=[key for key, x in leave_one.items() if avg * x < 0],
|
|
reaches_zero=[key for key, x in leave_one.items() if x == 0],
|
|
top_positive_families=[key for key, _ in top],
|
|
top_ten_positive_share=math.fsum(x for _, x in top) / math.fsum(x for _, x in positives)
|
|
if positives else None)
|
|
|
|
|
|
def same_tail_miss(rows):
|
|
complete = [r for r in rows if all(r["arms"][a].get("d_tail_over_Rg") is not None
|
|
for a in ("seed0", "seed1"))]
|
|
categories = {(True, True): "both", (True, False): "seed0_only",
|
|
(False, True): "seed1_only", (False, False): "neither"}
|
|
per_family = defaultdict(list)
|
|
by_category = {label: [] for label in categories.values()}
|
|
for r in complete:
|
|
if r["constant_truth"] is not False:
|
|
continue
|
|
key = tuple(r["arms"][a]["d_tail_over_Rg"] > .10 for a in ("seed0", "seed1"))
|
|
label = categories[key]
|
|
by_category[label].append(r)
|
|
per_family[r["component_id"]].append(label)
|
|
counts = {key: len(value) for key, value in by_category.items()}
|
|
total = sum(counts.values())
|
|
expected = ((counts["both"] + counts["seed0_only"]) * (counts["both"] + counts["seed1_only"]) / total
|
|
if total else None)
|
|
return dict(support=dict(parts=total, families=len(per_family)), counts=counts,
|
|
family_equal={k: mean(f.count(k) / len(f) for f in per_family.values()) for k in by_category},
|
|
independence_expected_parts=expected,
|
|
observed_over_expected=counts["both"] / expected if expected else None,
|
|
both_cases=[dict(case=r["case"], component_id=r["component_id"], fold=r["fold"])
|
|
for r in by_category["both"]],
|
|
shared_constant_truth=[r["case"] for r in complete if r["constant_truth"] is True],
|
|
shared_unknown_truth=[r["case"] for r in complete if r["constant_truth"] is None],
|
|
missing_pair_cases=sorted(set(r["case"] for r in rows) - set(r["case"] for r in complete)))
|
|
|
|
|
|
def summarize(rows):
|
|
return dict(expected=support(rows), source_missing={
|
|
key: [r["case"] for r in rows if key in r["sources_missing"]] for key in INPUTS if key != "roster"},
|
|
metrics={metric: metric_summary(rows, metric) for metric in METRICS},
|
|
field_gain={arm: field_gain(rows, arm) for arm in ARMS[1:]}, same_tail_miss=same_tail_miss(rows))
|
|
|
|
|
|
def main():
|
|
rows = paired_rows()
|
|
cohorts = {"pooled": rows} | {f"fold{fold}": [r for r in rows if r["fold"] == fold] for fold in range(5)}
|
|
result = dict(prereg_commit=PREREG_COMMIT, prereg_sha256=PREREG_SHA256,
|
|
inputs={k: dict(path=str(p), sha256=h, commit=c) for k, (p, h, c) in INPUTS.items()},
|
|
gpu_seconds=0, target_arrays_read=0, prediction_arrays_read=0, new_fits=0,
|
|
cohorts={label: summarize(sample) for label, sample in cohorts.items()})
|
|
OUT.mkdir(exist_ok=False)
|
|
(OUT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
|
|
with (OUT / "by-part.csv").open("x", newline="") as stream:
|
|
writer = csv.writer(stream)
|
|
keys = ("case", "component_id", "fold", "constant_truth")
|
|
writer.writerow([*keys, "sources_missing", *(f"{a}.{m}" for a in ARMS for m in METRICS)])
|
|
for r in rows:
|
|
writer.writerow([*(r[key] for key in keys), " ".join(r["sources_missing"]),
|
|
*(r["arms"][a].get(m) for a in ARMS for m in METRICS)])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|