From d437faa5012e943c9ecef99075ff8499f8685f28 Mon Sep 17 00:00:00 2001 From: Grok Date: Wed, 21 Oct 2026 16:47:22 +0800 Subject: [PATCH] test: check historical source manifests against their recorded commits Manifests and receipts unchanged. Each of the 8 group-A identity tests now extracts its manifest's recorded commit and verifies hashes there instead of against the current checkout. Tamper assertions and runner source refusals are kept; missing recorded history is an explicit failure, not a skip or repin. --- tests/unit/historical_tree.py | 46 +++++++++++++++++++ tests/unit/test_jet_circle_cv.py | 10 +++- tests/unit/test_les_flow_runner.py | 40 +++++++++------- tests/unit/test_re100.py | 8 +++- tests/unit/test_refined_dfg.py | 18 +++++++- .../unit/test_validation_source_provenance.py | 12 +++-- 6 files changed, 109 insertions(+), 25 deletions(-) create mode 100644 tests/unit/historical_tree.py diff --git a/tests/unit/historical_tree.py b/tests/unit/historical_tree.py new file mode 100644 index 0000000..8b87709 --- /dev/null +++ b/tests/unit/historical_tree.py @@ -0,0 +1,46 @@ +"""Check historical source manifests against their own recorded commit, not today's checkout.""" + +import io +from pathlib import Path +import subprocess +import tarfile + +ROOT = Path(__file__).resolve().parents[2] + + +def _require(commit): + """Missing recorded history is an explicit failure, never a skip or a repin.""" + found = subprocess.run( + ["git", "-C", str(ROOT), "cat-file", "-e", commit + "^{commit}"], + capture_output=True, + ) + if found.returncode: + raise AssertionError( + f"Required history missing: commit {commit} is not in this clone; " + "fetch it instead of repinning or substituting" + ) + + +def tracked(commit, prefix="src"): + """Tracked paths under prefix at the recorded commit.""" + _require(commit) + return set( + subprocess.check_output( + ["git", "-C", str(ROOT), "ls-tree", "-r", "--name-only", commit, prefix], + text=True, + ).splitlines() + ) + + +def historical_tree(commit, dest): + """Extract the tracked tree of the recorded commit into dest.""" + _require(commit) + archive = subprocess.check_output( + ["git", "-C", str(ROOT), "archive", "--format=tar", commit] + ) + dest = Path(dest) + dest.mkdir(parents=True, exist_ok=True) + extra = {"filter": "data"} if hasattr(tarfile, "data_filter") else {} + with tarfile.open(fileobj=io.BytesIO(archive)) as tar: + tar.extractall(dest, **extra) + return dest diff --git a/tests/unit/test_jet_circle_cv.py b/tests/unit/test_jet_circle_cv.py index 03d4377..e97b8eb 100644 --- a/tests/unit/test_jet_circle_cv.py +++ b/tests/unit/test_jet_circle_cv.py @@ -8,6 +8,7 @@ import numpy as np ROOT = Path(__file__).resolve().parents[2] REPO = ROOT sys.path.insert(0, str(ROOT / "tests/validation")) +sys.path.insert(0, str(ROOT / "tests/unit")) sys.path.insert(0, str(REPO / "src")) from run_jet_circle_cv import ( CX, @@ -19,6 +20,7 @@ from run_jet_circle_cv import ( cv_from_macros, aperture_probe, measured_aperture_flux, + PIN, verify, ) @@ -166,7 +168,13 @@ class Host(unittest.TestCase): self.assertEqual(a["steps"] * 4, b["steps"]) def test_source_pin(self): - verify(REPO) + from tempfile import TemporaryDirectory + from historical_tree import historical_tree + + with TemporaryDirectory() as directory: + verify(historical_tree(PIN, directory)) + with self.assertRaisesRegex(RuntimeError, "Source hash mismatch"): + verify(REPO) if __name__ == "__main__": diff --git a/tests/unit/test_les_flow_runner.py b/tests/unit/test_les_flow_runner.py index 5063f60..3aeca64 100644 --- a/tests/unit/test_les_flow_runner.py +++ b/tests/unit/test_les_flow_runner.py @@ -15,6 +15,10 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "validation")) from run_uniform_les_flow import source_guard, decode_macros, HERE, HEADER ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT / "tests/unit")) +from historical_tree import historical_tree, tracked + +A135 = "a135252ae78b47b57d7e77d19001700b3bedac8b" class GuardTests(unittest.TestCase): @@ -42,19 +46,20 @@ class GuardTests(unittest.TestCase): decode_macros(invalid, w, cx, cy) def test_current_corrected_pins(self): - self.assertEqual( - source_guard(ROOT, "corrected", "a135252")["base_commit"], - "a135252ae78b47b57d7e77d19001700b3bedac8b", - ) + with TemporaryDirectory() as directory: + root = historical_tree(A135, directory) + with patch("run_uniform_les_flow.subprocess.check_output", return_value=A135 + "\n"): + self.assertEqual(source_guard(root, "corrected", "a135252")["base_commit"], A135) def test_corrected_and_tamper_guard(self): pins = json.loads((HERE / "data/les_flow/source_pins_a135252.json").read_text()) with TemporaryDirectory() as directory: - root = Path(directory) + origin = historical_tree(A135, Path(directory) / "origin") + root = Path(directory) / "fixture" for name in pins["sources"]: dest = root / name dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(ROOT / name, dest) + shutil.copyfile(origin / name, dest) with patch( "run_uniform_les_flow.subprocess.check_output", return_value="isolated-fixture\n", @@ -87,23 +92,26 @@ class GuardTests(unittest.TestCase): def test_current_manifest_pins_actual_corrected_tree(self): pins = json.loads((HERE / "data/les_flow/source_pins_a135252.json").read_text()) - tracked = subprocess.check_output(["git", "-C", str(ROOT), "ls-files", "src"], text=True) - self.assertEqual(set(pins["sources"]), set(tracked.splitlines())) - for name, expected in pins["sources"].items(): - self.assertEqual(hashlib.sha256((ROOT / name).read_bytes()).hexdigest(), expected) - self.assertEqual(pins["sources"][HEADER], pins["candidate_header_sha256"]) - self.assertNotEqual(pins["sources"][HEADER], pins["baseline_header_sha256"]) - with self.assertRaisesRegex(RuntimeError, "Source pin mismatch"): - source_guard(ROOT, "baseline", "a135252") + self.assertEqual(pins["base_commit"], A135) + self.assertEqual(set(pins["sources"]), tracked(A135)) + with TemporaryDirectory() as directory: + root = historical_tree(A135, directory) + for name, expected in pins["sources"].items(): + self.assertEqual(hashlib.sha256((root / name).read_bytes()).hexdigest(), expected) + self.assertEqual(pins["sources"][HEADER], pins["candidate_header_sha256"]) + self.assertNotEqual(pins["sources"][HEADER], pins["baseline_header_sha256"]) + with self.assertRaisesRegex(RuntimeError, "Source pin mismatch"): + source_guard(root, "baseline", "a135252") def test_named_baseline_accepts_only_reversed_header(self): pins = json.loads((HERE / "data/les_flow/source_pins_a135252.json").read_text()) with TemporaryDirectory() as directory: - root = Path(directory) + origin = historical_tree(A135, Path(directory) / "origin") + root = Path(directory) / "fixture" for name in pins["sources"]: dest = root / name dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(ROOT / name, dest) + shutil.copyfile(origin / name, dest) subprocess.run(["git", "apply", "--reverse", str(HERE / "les/data/smagorinsky_normalization.patch")], cwd=root, check=True) with patch("run_uniform_les_flow.subprocess.check_output", return_value="baseline-fixture\n"): source_guard(root, "baseline", "a135252") diff --git a/tests/unit/test_re100.py b/tests/unit/test_re100.py index 2d0dc09..bc50b3d 100644 --- a/tests/unit/test_re100.py +++ b/tests/unit/test_re100.py @@ -5,6 +5,7 @@ import pytest ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "tests/validation")) +sys.path.insert(0, str(ROOT / "tests/unit")) import re100_helpers as h import periodic as p @@ -62,13 +63,16 @@ def test_periodic_synthetic(): assert not p.assess(rows()[:100])["periodic"] -def test_production_unmodified(): +def test_production_unmodified(tmp_path): + from historical_tree import historical_tree + m = json.loads( (ROOT / "tests/validation/data/dfg_re100/source_manifest.json").read_text() ) assert m["commit"] == h.PIN + origin = historical_tree(m["commit"], tmp_path) for name, digest in m["production_sha256"].items(): - assert hashlib.sha256((ROOT / name).read_bytes()).hexdigest() == digest, name + assert hashlib.sha256((origin / name).read_bytes()).hexdigest() == digest, name def test_reference(): diff --git a/tests/unit/test_refined_dfg.py b/tests/unit/test_refined_dfg.py index fff8f2f..661a99d 100644 --- a/tests/unit/test_refined_dfg.py +++ b/tests/unit/test_refined_dfg.py @@ -7,7 +7,10 @@ import numpy as np import unittest sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "validation")) +sys.path.insert(0, str(Path(__file__).resolve().parent)) import run_refined_dfg as r +from historical_tree import historical_tree +from tempfile import TemporaryDirectory h, periodic = r.helpers(r.DEFAULT_SOURCE) @@ -17,7 +20,20 @@ def test_source_pin(): assert historical["commit"].startswith("cd3a657") mrt_history = json.loads((r.HERE / "data/refined_dfg/source_manifest_mrt.json").read_text()) assert mrt_history["commit"].startswith("3ea131e") - assert r.verify_source(r.DEFAULT_SOURCE, "MRT", "window")["commit"].startswith("a233c90") + with TemporaryDirectory() as directory: + for collision, mode, prefix in ( + ("SRT", "last_base_step", "cd3a657"), + ("MRT", "last_base_step", "3ea131e"), + ("MRT", "window", "a233c90"), + ): + name = {"SRT": "source_manifest.json", "MRT": "source_manifest_mrt.json"}[collision] + name = "source_manifest_window.json" if mode == "window" else name + commit = json.loads((r.HERE / "data/refined_dfg" / name).read_text())["commit"] + assert commit.startswith(prefix) + root = historical_tree(commit, Path(directory) / commit) + assert r.verify_source(root, collision, mode)["commit"] == commit + with unittest.TestCase().assertRaisesRegex(RuntimeError, "Source mismatch"): + r.verify_source(r.DEFAULT_SOURCE, "MRT", "window") with unittest.TestCase().assertRaisesRegex(RuntimeError, "Source mismatch"): r.verify_source(r.DEFAULT_SOURCE, "MRT") with unittest.TestCase().assertRaisesRegex(RuntimeError, "Source mismatch"): diff --git a/tests/unit/test_validation_source_provenance.py b/tests/unit/test_validation_source_provenance.py index d184e1a..f721384 100644 --- a/tests/unit/test_validation_source_provenance.py +++ b/tests/unit/test_validation_source_provenance.py @@ -12,13 +12,15 @@ from source_provenance import checkout_provenance import re100_helpers import run_jet_circle_cv +sys.path.insert(0, str(ROOT / "tests/unit")) +from historical_tree import historical_tree, tracked + ORIGIN = "47b05bc8abfc957fe876b0c63bdb386e3c6cbaa5" -def test_manifests_cover_current_source_and_preserve_origin(): - names = subprocess.check_output( - ["git", "-C", str(ROOT), "ls-files", "src"], text=True - ).splitlines() +def test_manifests_cover_current_source_and_preserve_origin(tmp_path): + names = tracked(ORIGIN) + origin = historical_tree(ORIGIN, tmp_path) for group, hashes_key, revision_key in ( ("dfg_re100", "production_sha256", "commit"), ("jet_circle_cv", "files", "revision"), @@ -38,7 +40,7 @@ def test_manifests_cover_current_source_and_preserve_origin(): assert "not inherited from d79" in manifest["experiment"] for name, expected in manifest[hashes_key].items(): assert ( - hashlib.sha256((ROOT / name).read_bytes()).hexdigest() == expected + hashlib.sha256((origin / name).read_bytes()).hexdigest() == expected ), name assert re100_helpers.PIN == run_jet_circle_cv.PIN == ORIGIN