test: current-checkout mode for the projected-pair TG runner and assessor
run_case --current-checkout records the clean checkout (HEAD, src fingerprint, harness hashes, pinned build-capture helper) labelled current_checkout instead of requiring the frozena135252tree; receipts carry numerical_revision=None. assess --current-checkout applies the same gates to such receipts. Frozen mode is unchanged and now rejects any receipt with source_mode, so the two never cross-qualify. A missinga135252object is reported as unavailable history instead of a source diff. check_projected_pairs.py is untouched, so its pinned hash and existing receipts keep their meaning. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aceea70e5f
commit
db3d098f43
3 files changed
+94
-7
No files matched your search
@@ -73,7 +73,10 @@ def valid_builds(record, n, depth):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def assess(records, expected_revision=None):
|
def assess(records, expected_revision=None, current=False):
|
||||||
|
"""current=True assesses current-checkout receipts (run_case --current-checkout) with
|
||||||
|
the same gates; it never accepts them as the frozen a135252 qualification, and the
|
||||||
|
frozen mode never accepts current-checkout receipts."""
|
||||||
indexed, duplicates = {}, []
|
indexed, duplicates = {}, []
|
||||||
for record in records:
|
for record in records:
|
||||||
key = tuple(record.get(k) for k in ("re_L", "n", "depth", "explosion"))
|
key = tuple(record.get(k) for k in ("re_L", "n", "depth", "explosion"))
|
||||||
@@ -87,8 +90,16 @@ def assess(records, expected_revision=None):
|
|||||||
sources = [r.get("source", {}) for r in records]
|
sources = [r.get("source", {}) for r in records]
|
||||||
checks["same_source_and_harness"] = bool(sources) and len({
|
checks["same_source_and_harness"] = bool(sources) and len({
|
||||||
json.dumps(s, sort_keys=True) for s in sources}) == 1
|
json.dumps(s, sort_keys=True) for s in sources}) == 1
|
||||||
|
if current:
|
||||||
|
checks["clean_current_checkout_source"] = bool(sources) and expected_revision is not None and all(
|
||||||
|
s.get("dirty") is False and s.get("source_mode") == "current_checkout"
|
||||||
|
and bool(s.get("source_tree_sha256")) and bool(s.get("harness_sha256"))
|
||||||
|
and s.get("build_capture_sha256") == BUILD_CAPTURE_SHA256
|
||||||
|
and s.get("revision") == expected_revision for s in sources)
|
||||||
|
else:
|
||||||
checks["clean_a135252_numerical_source"] = bool(sources) and all(
|
checks["clean_a135252_numerical_source"] = bool(sources) and all(
|
||||||
s.get("dirty") is False and s.get("source_tree_sha256") == SOURCE_SHA256
|
s.get("dirty") is False and s.get("source_tree_sha256") == SOURCE_SHA256
|
||||||
|
and "source_mode" not in s
|
||||||
and bool(s.get("revision")) and bool(s.get("harness_sha256"))
|
and bool(s.get("revision")) and bool(s.get("harness_sha256"))
|
||||||
and s.get("build_capture_sha256") == BUILD_CAPTURE_SHA256
|
and s.get("build_capture_sha256") == BUILD_CAPTURE_SHA256
|
||||||
and (expected_revision is None or s.get("revision") == expected_revision) for s in sources)
|
and (expected_revision is None or s.get("revision") == expected_revision) for s in sources)
|
||||||
@@ -125,7 +136,13 @@ def assess(records, expected_revision=None):
|
|||||||
and r.get("mach") == U * math.sqrt(3)
|
and r.get("mach") == U * math.sqrt(3)
|
||||||
)
|
)
|
||||||
checks[label + " actual_builds"] = valid_builds(r, n, depth)
|
checks[label + " actual_builds"] = valid_builds(r, n, depth)
|
||||||
|
if current:
|
||||||
|
checks[label + " reference"] = (r.get("numerical_revision") is None
|
||||||
|
and r.get("source_mode") == "current_checkout"
|
||||||
|
and r.get("reference_harness") == REFERENCE_HARNESS)
|
||||||
|
else:
|
||||||
checks[label + " reference"] = (r.get("numerical_revision") == NUMERICAL_REVISION
|
checks[label + " reference"] = (r.get("numerical_revision") == NUMERICAL_REVISION
|
||||||
|
and "source_mode" not in r
|
||||||
and r.get("reference_harness") == REFERENCE_HARNESS)
|
and r.get("reference_harness") == REFERENCE_HARNESS)
|
||||||
checks[label + " numerics"] = r.get("numerics") == NUMERICS and r.get("initialization") == INITIALIZATION
|
checks[label + " numerics"] = r.get("numerics") == NUMERICS and r.get("initialization") == INITIALIZATION
|
||||||
runtime = r.get("runtime_source", {})
|
runtime = r.get("runtime_source", {})
|
||||||
@@ -185,7 +202,10 @@ def assess(records, expected_revision=None):
|
|||||||
valid_case_and_controls=valid, baseline_passed=baseline_passed)
|
valid_case_and_controls=valid, baseline_passed=baseline_passed)
|
||||||
return dict(passed=all(checks.values()), missing=missing, duplicates=duplicates, checks=checks,
|
return dict(passed=all(checks.values()), missing=missing, duplicates=duplicates, checks=checks,
|
||||||
comparisons=comparisons, scheme_results=scheme_results, early_pressure=early,
|
comparisons=comparisons, scheme_results=scheme_results, early_pressure=early,
|
||||||
numerical_revision=NUMERICAL_REVISION, source_tree_sha256=SOURCE_SHA256,
|
source_mode="current_checkout" if current else "frozen",
|
||||||
|
numerical_revision=None if current else NUMERICAL_REVISION,
|
||||||
|
source_tree_sha256=(sources[0].get("source_tree_sha256") if current and sources
|
||||||
|
else None if current else SOURCE_SHA256),
|
||||||
reference_harness=REFERENCE_HARNESS, expected_revision=expected_revision,
|
reference_harness=REFERENCE_HARNESS, expected_revision=expected_revision,
|
||||||
floors=FLOORS, baseline_factor=BASELINE_FACTOR,
|
floors=FLOORS, baseline_factor=BASELINE_FACTOR,
|
||||||
main_statistic="maximum of four post-initial samples at t*=.12,.24,.36,.48",
|
main_statistic="maximum of four post-initial samples at t*=.12,.24,.36,.48",
|
||||||
@@ -199,10 +219,13 @@ if __name__ == "__main__":
|
|||||||
p.add_argument("files", type=Path, nargs="+")
|
p.add_argument("files", type=Path, nargs="+")
|
||||||
p.add_argument("--output", type=Path, required=True)
|
p.add_argument("--output", type=Path, required=True)
|
||||||
p.add_argument("--expected-revision", required=True)
|
p.add_argument("--expected-revision", required=True)
|
||||||
|
p.add_argument("--current-checkout", action="store_true",
|
||||||
|
help="assess current-checkout receipts; never the frozen qualification")
|
||||||
args = p.parse_args()
|
args = p.parse_args()
|
||||||
if args.output.exists():
|
if args.output.exists():
|
||||||
p.error("Output already exists; choose a new path to retain prior evidence")
|
p.error("Output already exists; choose a new path to retain prior evidence")
|
||||||
result = assess([json.loads(path.read_text()) for path in args.files], args.expected_revision)
|
result = assess([json.loads(path.read_text()) for path in args.files], args.expected_revision,
|
||||||
|
current=args.current_checkout)
|
||||||
args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
|
args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
|
||||||
print(json.dumps(result, indent=2, allow_nan=False))
|
print(json.dumps(result, indent=2, allow_nan=False))
|
||||||
raise SystemExit(0 if result["passed"] else 1)
|
raise SystemExit(0 if result["passed"] else 1)
|
||||||
@@ -64,8 +64,16 @@ def provenance(repo):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def verify_source(repo, revision):
|
CURRENT_CAPTURE = "current checkout, not the frozen a135252 qualification"
|
||||||
"""Exact clean harness commit, fixed production tree, and reviewed capture helper."""
|
|
||||||
|
|
||||||
|
def verify_source(repo, revision, current=False):
|
||||||
|
"""Exact clean harness commit, fixed production tree, and reviewed capture helper.
|
||||||
|
|
||||||
|
current=True records the clean checkout itself instead of requiring the frozen
|
||||||
|
numerical tree; such receipts carry source_mode="current_checkout" and are only
|
||||||
|
assessed with assess(..., current=True), never as the frozen qualification.
|
||||||
|
"""
|
||||||
import oracle
|
import oracle
|
||||||
from tests.validation.multigrid import check_projected_pairs as gate
|
from tests.validation.multigrid import check_projected_pairs as gate
|
||||||
|
|
||||||
@@ -77,6 +85,17 @@ def verify_source(repo, revision):
|
|||||||
if (Path(gate.__file__).resolve() != helper
|
if (Path(gate.__file__).resolve() != helper
|
||||||
or hashlib.sha256(helper.read_bytes()).hexdigest() != BUILD_CAPTURE_SHA256):
|
or hashlib.sha256(helper.read_bytes()).hexdigest() != BUILD_CAPTURE_SHA256):
|
||||||
raise RuntimeError("Wrong build-capture helper")
|
raise RuntimeError("Wrong build-capture helper")
|
||||||
|
if current:
|
||||||
|
result = provenance(repo)
|
||||||
|
if result["revision"] != revision or result["dirty"]:
|
||||||
|
raise ValueError("requires the exact expected clean checkout")
|
||||||
|
result.update(source_mode="current_checkout", source_capture=CURRENT_CAPTURE,
|
||||||
|
build_capture_sha256=BUILD_CAPTURE_SHA256)
|
||||||
|
return result
|
||||||
|
if subprocess.run(["git", "-C", str(repo), "cat-file", "-e", f"{NUMERICAL_REVISION}^{{commit}}"],
|
||||||
|
capture_output=True).returncode:
|
||||||
|
raise RuntimeError(f"frozen numerical revision {NUMERICAL_REVISION} is not available "
|
||||||
|
"in this clone; the frozen qualification cannot be checked")
|
||||||
if gate.preflight(repo, revision, "candidate") != NUMERICAL_REVISION:
|
if gate.preflight(repo, revision, "candidate") != NUMERICAL_REVISION:
|
||||||
raise RuntimeError("Wrong production revision")
|
raise RuntimeError("Wrong production revision")
|
||||||
result = provenance(repo)
|
result = provenance(repo)
|
||||||
@@ -149,6 +168,8 @@ def main():
|
|||||||
p.add_argument("--depth", type=int, choices=[1, 2, 3], required=True)
|
p.add_argument("--depth", type=int, choices=[1, 2, 3], required=True)
|
||||||
p.add_argument("--output", type=Path, required=True)
|
p.add_argument("--output", type=Path, required=True)
|
||||||
p.add_argument("--explosion", choices=["constant", "projected_linear"], default="constant")
|
p.add_argument("--explosion", choices=["constant", "projected_linear"], default="constant")
|
||||||
|
p.add_argument("--current-checkout", action="store_true",
|
||||||
|
help="record the clean current checkout instead of the frozen a135252 tree")
|
||||||
p.add_argument("--early-pressure", action="store_true",
|
p.add_argument("--early-pressure", action="store_true",
|
||||||
help="also save 65 scalar pressure samples at t*=k*.001875, k=0..64")
|
help="also save 65 scalar pressure samples at t*=k*.001875, k=0..64")
|
||||||
args = p.parse_args()
|
args = p.parse_args()
|
||||||
@@ -161,7 +182,7 @@ def main():
|
|||||||
p.error("--run-gpu and exactly one explicit CUDA_VISIBLE_DEVICES are required")
|
p.error("--run-gpu and exactly one explicit CUDA_VISIBLE_DEVICES are required")
|
||||||
repo = args.repo.resolve()
|
repo = args.repo.resolve()
|
||||||
sys.path[:0] = [str(repo / "src"), str(repo)]
|
sys.path[:0] = [str(repo / "src"), str(repo)]
|
||||||
source = verify_source(repo, args.expected_revision)
|
source = verify_source(repo, args.expected_revision, current=args.current_checkout)
|
||||||
runtime = runtime_source(repo)
|
runtime = runtime_source(repo)
|
||||||
if args.dry_run:
|
if args.dry_run:
|
||||||
print(json.dumps(dict(source=source, runtime_source=runtime, dry_run=True)))
|
print(json.dumps(dict(source=source, runtime_source=runtime, dry_run=True)))
|
||||||
@@ -182,7 +203,8 @@ def main():
|
|||||||
report = dict(
|
report = dict(
|
||||||
schema=1,
|
schema=1,
|
||||||
source=source,
|
source=source,
|
||||||
numerical_revision=NUMERICAL_REVISION,
|
numerical_revision=None if args.current_checkout else NUMERICAL_REVISION,
|
||||||
|
**({"source_mode": "current_checkout"} if args.current_checkout else {}),
|
||||||
reference_harness=REFERENCE_HARNESS,
|
reference_harness=REFERENCE_HARNESS,
|
||||||
actual_build_sidecars=[],
|
actual_build_sidecars=[],
|
||||||
runtime_identity=None,
|
runtime_identity=None,
|
||||||
|
|||||||
@@ -291,6 +291,48 @@ class ProjectedPairsTests(unittest.TestCase):
|
|||||||
else:
|
else:
|
||||||
self.assertEqual(run_case.verify_source(REPO, expected), source())
|
self.assertEqual(run_case.verify_source(REPO, expected), source())
|
||||||
|
|
||||||
|
def test_frozen_mode_reports_missing_history_not_a_diff(self):
|
||||||
|
from tests.validation.multigrid import check_projected_pairs as gate
|
||||||
|
missing = SimpleNamespace(returncode=128)
|
||||||
|
with patch.object(run_case.subprocess, "run", return_value=missing), \
|
||||||
|
patch.object(gate, "preflight") as preflight:
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "not available in this clone"):
|
||||||
|
run_case.verify_source(REPO, MOCK_REVISION)
|
||||||
|
preflight.assert_not_called()
|
||||||
|
|
||||||
|
def test_current_checkout_source_is_recorded_not_frozen(self):
|
||||||
|
from tests.validation.multigrid import check_projected_pairs as gate
|
||||||
|
current = dict(source(), source_tree_sha256="c" * 64)
|
||||||
|
with patch.object(run_case, "provenance", return_value=dict(current)), \
|
||||||
|
patch.object(gate, "preflight") as preflight:
|
||||||
|
result = run_case.verify_source(REPO, MOCK_REVISION, current=True)
|
||||||
|
preflight.assert_not_called()
|
||||||
|
self.assertEqual(result["source_mode"], "current_checkout")
|
||||||
|
self.assertEqual(result["source_capture"], run_case.CURRENT_CAPTURE)
|
||||||
|
for bad in (dict(current, dirty=True), dict(current, revision=OLD_REVISION)):
|
||||||
|
with self.subTest(bad=bad), patch.object(run_case, "provenance", return_value=bad):
|
||||||
|
with self.assertRaisesRegex(ValueError, "exact expected clean checkout"):
|
||||||
|
run_case.verify_source(REPO, MOCK_REVISION, current=True)
|
||||||
|
|
||||||
|
def test_current_and_frozen_receipts_never_cross_assess(self):
|
||||||
|
frozen = records()
|
||||||
|
current = deepcopy(frozen)
|
||||||
|
for r in current:
|
||||||
|
r["source"] = dict(r["source"], source_tree_sha256="c" * 64,
|
||||||
|
source_mode="current_checkout",
|
||||||
|
source_capture=run_case.CURRENT_CAPTURE)
|
||||||
|
r["numerical_revision"], r["source_mode"] = None, "current_checkout"
|
||||||
|
ok = assess(current, MOCK_REVISION, current=True)
|
||||||
|
self.assertTrue(ok["checks"]["clean_current_checkout_source"])
|
||||||
|
self.assertTrue(all(v for k, v in ok["checks"].items() if k.endswith(" reference")))
|
||||||
|
self.assertEqual((ok["source_mode"], ok["numerical_revision"]), ("current_checkout", None))
|
||||||
|
self.assertEqual(ok["source_tree_sha256"], "c" * 64)
|
||||||
|
self.assertFalse(assess(current, MOCK_REVISION)["checks"]["clean_a135252_numerical_source"])
|
||||||
|
self.assertFalse(assess(current, MOCK_REVISION)["passed"])
|
||||||
|
self.assertFalse(assess(frozen, MOCK_REVISION, current=True)["checks"]["clean_current_checkout_source"])
|
||||||
|
self.assertFalse(assess(current, None, current=True)["checks"]["clean_current_checkout_source"])
|
||||||
|
self.assertTrue(assess(frozen, MOCK_REVISION)["checks"]["clean_a135252_numerical_source"])
|
||||||
|
|
||||||
def test_explicit_opt_in_selector_and_dry_run_without_cuda(self):
|
def test_explicit_opt_in_selector_and_dry_run_without_cuda(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
output = Path(tmp) / "case.json"
|
output = Path(tmp) / "case.json"
|
||||||
|
|||||||
Reference in new issue
Block a user