Files
MeltMind/tools/prepare_track_b_gino_import_checked.py
T

195 lines
10 KiB
Python

"""CPU accept the pinned paper environment, then bind the measurement prerequisite."""
import argparse
from copy import deepcopy
import hashlib
from importlib import metadata
from pathlib import Path
import os
import sys
from tempfile import TemporaryDirectory
from unittest.mock import patch
from zipfile import ZipFile
import track_b_gino_import_checked_entry as entry
import track_b_gino_paper_keyword_patch as compatibility
import track_b_gino_shapenet_entry as legacy
from track_b_gino_shapenet_worker import paper_model, test_environment
BASE_CONTRACT = legacy.MEASUREMENT_ROOT / "contract.json"
ACCEPTANCE = entry.ROOT / "cpu-environment-acceptance.json"
def accept():
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
raise PermissionError("The acceptance interpreter must have an empty CUDA mask")
c = legacy.contract(legacy.checked(BASE_CONTRACT, entry.ORIGINAL_CONTRACT))
c["paper_compatibility"] = compatibility.patch_binding()
patch = compatibility.patch_identity(c)
python = Path(sys.executable).absolute()
prefix = python.parent.parent
if str(python) != c["environment"]["python"] or Path(sys.prefix).absolute() != prefix:
raise PermissionError("Use the exact isolated interpreter in the measurement contract")
cfg = prefix / "pyvenv.cfg"
if "include-system-site-packages = false" not in cfg.read_text():
raise PermissionError("System Python packages must be excluded")
test_environment(c)
import numpy as np
import torch
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
torch.manual_seed(0)
learning_case = c["split"]["train"][0]
if learning_case in c["split"]["test"]:
raise PermissionError("Acceptance must only open a learning case")
audit = legacy.read(c["mapping_audit"]["path"])
row = next(r for r in audit["rows"] if r["case"] == learning_case)
if row["split"] != "train":
raise PermissionError("Source audit must identify the same learning role")
with TemporaryDirectory(prefix="track-b-paper-import-") as temporary:
root = Path(temporary)
compatibility.extract_patched_paper_code(c, root)
from neuralop.utils import UnitGaussianNormalizer
from src.data.cfd_datamodule import CFDSDFDataModule
from src.losses import LpLoss
# Exercise the original loader on exactly one audited learning case.
local = root / "data"
local.mkdir()
with ZipFile(c["paper_data"]["path"]) as archive:
for field, name in (("mesh", f"mesh_{learning_case:03d}.ply"),
("pressure", f"press_{learning_case:03d}.npy")):
payload = archive.read(f"car-pressure-data/data/{name}")
if hashlib.sha256(payload).hexdigest() != row[field + "_sha256"]:
raise ValueError("Learning file differs from the paper mapping audit")
(local / name).write_bytes(payload)
loader = object.__new__(CFDSDFDataModule)
pressure = loader.load_pressure(root, learning_case)
mesh_path = local / f"mesh_{learning_case:03d}.ply"
axis = [np.linspace(a, b, 2) for a, b in zip(*c["split"]["raw_global_bounds"])]
grid = np.stack(np.meshgrid(*axis, indexing="ij"), axis=-1).astype(np.float32)
sdf, vertices, _ = loader.sdf_vertices_closest_from_mesh(mesh_path, grid, False)
if (pressure.shape != (3586,) or vertices.shape != (3586, 3) or sdf.shape != (2, 2, 2)
or not np.isfinite(sdf).all()
or hashlib.sha256(np.asarray(pressure, dtype="<f4").tobytes()).hexdigest() != row["retained_pressure_f32_sha256"]):
raise ValueError("Original loader support mismatch on the learning case")
exact = paper_model(c)
if any(parameter.device.type != "cpu" for parameter in exact.parameters()):
raise ValueError("Full protocol decoder must instantiate on CPU")
parameters = sum(parameter.numel() for parameter in exact.parameters())
del exact
# A reduced, synthetic sample exercises the same decoder implementation
# and native radius/segment extensions without timing or fitting it.
smoke = deepcopy(c)
smoke["protocol"].update(grid=[8, 8, 8], modes=[4, 4, 4])
model = paper_model(smoke).eval()
coordinates = torch.linspace(-1, 1, 8)
positions = torch.stack(torch.meshgrid(coordinates, coordinates, coordinates, indexing="ij"))
inputs = torch.cat((positions, torch.zeros(1, 8, 8, 8))).unsqueeze(0)
points = torch.tensor([[[-1., -1., -1.], [1., 1., 1.], [-1., 1., -1.]]])
with torch.no_grad():
output = model(inputs, points)
if tuple(output.shape) != (1, 1, 3) or output.device.type != "cpu" or not bool(torch.isfinite(output).all()):
raise ValueError(f"Original CPU decoder forward shape or values: {tuple(output.shape)}")
reference = torch.tensor([[3., 4.]])
estimate = torch.tensor([[0., 4.]])
encoder = UnitGaussianNormalizer(torch.tensor([[1., 3.], [2., 4.]]),
eps=c["protocol"]["pressure_epsilon"], reduce_dim=[0, 1], verbose=False)
true_decoded = encoder.decode(encoder.encode(reference.clone()))
pred_decoded = encoder.decode(encoder.encode(estimate.clone()))
original_metric = float(LpLoss(size_average=True)(pred_decoded, true_decoded))
entry_metric = float(torch.linalg.vector_norm(pred_decoded - true_decoded) / torch.linalg.vector_norm(true_decoded))
if abs(original_metric - .6) > 1e-6 or abs(entry_metric - original_metric) > 1e-6:
raise ValueError("Decoded relative L2 differs from the original paper metric")
entry.ROOT.mkdir(exist_ok=True)
if ACCEPTANCE.exists():
raise FileExistsError("The acceptance receipt is immutable")
receipt = dict(
status="PASS", environment_ruling="h-0d21f3bd",
original_contract_sha256=entry.ORIGINAL_CONTRACT,
comparison_contract_sha256=c["comparison_contract_sha256"],
interpreter_path=str(python), interpreter_resolved_path=str(python.resolve(strict=True)),
interpreter_sha256=legacy.sha(python), pyvenv_cfg_sha256=legacy.sha(cfg),
package_versions=c["environment"]["package_versions"], neuralop_commit=c["environment"]["neuralop_commit"],
installed_distributions=sorted((d.metadata["Name"], d.version) for d in metadata.distributions()),
torch_cuda_build=torch.version.cuda, cuda_visible_devices="", scientific_training=False,
paper_compatibility=c["paper_compatibility"], patched_decoder_file_sha256=patch["patched_file_sha256"],
paper_decoder_cpu_forward=True, paper_loader_learning_only=True, paper_metric_cpu=True,
audited_learning_cases=[learning_case], heldout_target_reads=0,
full_decoder_cpu_parameters=parameters, synthetic_decoder_output_shape=list(output.shape),
cpu_metric=entry_metric)
legacy.save(ACCEPTANCE, receipt)
print(f"PASS {legacy.sha(ACCEPTANCE)}")
def gate_replay(path):
c = entry.contract(path)
receipt = entry.accepted_environment(c)
outcomes = {}
for case in ("missing", "bad_receipt_sha", "bad_interpreter_path", "bad_interpreter_sha",
"accepted_receipt_before_review"):
with TemporaryDirectory(prefix="track-b-runtime-gate-") as directory:
temp = Path(directory)
probe = deepcopy(c)
if case == "missing":
probe["environment_acceptance"]["path"] = str(temp / "absent.json")
elif case == "bad_receipt_sha":
probe["environment_acceptance"]["sha256"] = "0" * 64
elif case in ("bad_interpreter_path", "bad_interpreter_sha"):
fake = temp / "wrong-interpreter.json"
field, value = (("interpreter_path", str(temp / "unaccepted-python"))
if case == "bad_interpreter_path" else ("interpreter_sha256", "0" * 64))
legacy.save(fake, dict(receipt, **{field: value}))
probe["environment_acceptance"] = dict(path=str(fake), sha256=legacy.sha(fake))
legacy.save(temp / "contract.json", probe)
with (patch.object(entry, "ROOT", temp),
patch.object(legacy, "validate_review", side_effect=PermissionError("Independent review absent")) as review,
patch.object(entry, "dispatch_guard", side_effect=AssertionError("Can't dispatch without prerequisites")) as dispatch):
try:
entry.run(temp)
except (FileNotFoundError, PermissionError, ValueError) as exc:
outcomes[case] = dict(error=str(exc), passed_to_review=review.call_count)
else:
raise AssertionError(f"Expected the entry to stop: {case}")
if dispatch.call_count or (temp / "started.json").exists():
raise AssertionError("Unverified environment consumed the sole attempt")
if review.call_count != (1 if case == "accepted_receipt_before_review" else 0):
raise AssertionError("CPU acceptance must precede runtime review and start")
return dict(status="PASS", actual_contract_sha256=legacy.sha(path),
cpu_receipt_sha256=c["environment_acceptance"]["sha256"], cases=outcomes,
real_gpu_queries=0, real_workers=0, started_records_written=0)
def bind():
if not ACCEPTANCE.is_file():
raise FileNotFoundError("Actual isolated CPU acceptance must finish first")
old = legacy.contract(legacy.checked(BASE_CONTRACT, entry.ORIGINAL_CONTRACT))
new = deepcopy(old)
new["environment_ruling"] = "h-0d21f3bd"
new["environment_acceptance"] = dict(path=str(ACCEPTANCE), sha256=legacy.sha(ACCEPTANCE))
new["paper_compatibility"] = compatibility.patch_binding()
for p in (entry.__file__, entry.WORKER, __file__, compatibility.__file__):
new["code_sha256"][str(Path(p).resolve())] = legacy.sha(p)
entry.accepted_environment(new)
path = entry.ROOT / "contract.json"
if path.exists():
raise FileExistsError("Contract is already frozen")
legacy.save(path, new)
proof = gate_replay(path)
legacy.save(entry.ROOT / "environment-prerequisite-cpu-replay.json", proof)
print(proof)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("accept", "bind"))
args = parser.parse_args()
{"accept": accept, "bind": bind}[args.mode]()