Prepare original ShapeNet GINO measurement entry and record 1b gate blocker

This commit is contained in:
NewMeltAssist committed 2026-11-19 12:43:08 +08:00
1 parent 8d5c498e3b
commit cc19b47ea9
17 files changed
+2439

No files matched your search

+182
View File
@@ -0,0 +1,182 @@
"""Freeze the paper protocol and CPU-test the measurement dispatch contract."""
from functools import partial
from pathlib import Path
from tempfile import TemporaryDirectory
from types import SimpleNamespace
from unittest.mock import patch
from phase2_h1_worker_v2 import checked, save, sha
from track_a_h2_prepare import local_code_closure
import track_b_gino_shapenet_entry as entry
from s2_gpu_idle_gate import wait_idle
BASE = entry.BASE
DATA = BASE / "data/track-b-gino-shapenet611-2026-11-19"
AUDIT = BASE / "artifacts/track-b-gino-shapenet611-2026-11-19"
UPSTREAM = Path("/tmp/meltmind-track-b-shapenet611-20261119")
NEURALOP = "fbdc7f872792b75b84545c2f374e7a881c72a718"
GPU = "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36"
def prepare():
root = entry.MEASUREMENT_ROOT
if root.exists():
raise FileExistsError("The Track B measurement preparation already exists")
split_path = AUDIT / "paper-split.json"
split = entry.read(split_path)
mapping = AUDIT / "paper-mapping-audit.json"
audit = entry.read(mapping)
if (audit["status"] != "PAPER_ARCHIVE_SPLIT_AND_MAPPING_AUDITED"
or audit["manifest_sha256"] != split["manifest_sha256"]
or audit["global_bounds_sha256"] != split["raw_global_bounds_sha256"]):
raise ValueError("Paper data not verified")
for partition in ("train", "test"):
if [r["case"] for r in audit["rows"] if r["split"] == partition] != split[partition]:
raise ValueError("Audit sample order differs")
pressure = checked(DATA / "paper-car-pressure-data.zip", audit["archive_sha256"])
supplement = checked(DATA / "paper-neurips-supplement.zip", split["paper_supplement_zip_sha256"])
paper_code = split["paper_code_zip_sha256"]
pin_names = {"utils.py": "legacy-utils.py",
"models/__init__.py": "legacy-__init__.py",
"models/tfno.py": "legacy-tfno.py",
"models/spectral_convolution.py": "legacy-spectral_convolution.py"}
upstream_hashes = {key: sha(UPSTREAM / name) for key, name in pin_names.items()}
environment = dict(
python=str(BASE / "data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python"),
python_version="3.10", torch_cuda="11.6", neuralop_commit=NEURALOP, neuralop_file_sha256=upstream_hashes,
package_versions={
"torch": "1.13.1+cu116", "open3d": "0.17.0", "numpy": "1.24.3",
"torch-scatter": "2.1.1+pt113cu116", "torch-sparse": "0.6.17+pt113cu116",
"torch-geometric": "2.3.1", "tensorly": "0.8.1", "tensorly-torch": "0.4.0",
})
protocol = dict(
model="NeurIPS 2023 supplement SDFFNOGNO decoder", seeds=[0, 1, 2],
grid=[64, 64, 64], modes=[24, 24, 24], hidden_channels=32, radius=.05,
epochs_per_seed=100, batch_size=1, learning_rate=.001, bounds_expansion=.01,
pressure_epsilon=1.e-6, optimizer="Adam", weight_decay=1.e-4,
lr_step_epochs=50, lr_gamma=.5, pressure_training_transform="scalar Gaussian on the 500 learning cases",
metric="per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)",
report="equal mean over 111 cases for each seed, then equal mean over the three seeds",
acceptance_percent=[6.62, 7.62], paper_reference_percent=7.12,
checkpoint_epoch=100, heldout_access_after_all_checkpoints_frozen=True)
root.mkdir(exist_ok=False)
comparison = root / "comparison-contract.json"
save(comparison, dict(
ruling=entry.RULING, protocol=protocol, paper_code_zip_sha256=paper_code,
source_protocol_commit="8d5c498e3be25580cf022890add6b32d7d6b8a93",
split_sha256=sha(split_path), audited_mapping_sha256=sha(mapping),
environment=environment, compute_regime="one V100, one serial process, no other GPU occupants",
opens_only_original_paper_data=True, acceptance_interval_provenance="team preregistered tolerance",
exact_original_published_cli_overrides_known=False))
code_paths = [Path(__file__), Path(entry.__file__), BASE / "tools/track_b_gino_shapenet_worker.py"]
codes = local_code_closure(code_paths)
limit = dict(gpu_wall_seconds=3600, cleanup_seconds=60, idle_wait_seconds=60, cpu_seconds=3500,
rss_bytes=12 * 2**30, recorded_vram_bytes=14 * 2**30, device_vram_bytes=15 * 2**30,
disk_bytes=2 * 2**30, torch_threads=1, max_processes=1)
path = root / "contract.json"
def pin(p):
return dict(path=str(p), sha256=sha(p))
save(path, dict(mode="measurement", ruling=entry.RULING, retry=False, evaluate_other_models=False,
measurement_allowed_heldout_labels=False, gpu_uuid=GPU, limits=limit,
protocol=protocol, environment=environment, comparison_contract=str(comparison),
comparison_contract_sha256=sha(comparison), code_sha256=codes,
paper_data=pin(pressure), paper_supplement=pin(supplement),
paper_code_zip_sha256=paper_code, split_receipt=pin(split_path), mapping_audit=pin(mapping),
split={key: split[key] for key in ("train", "test", "raw_global_bounds")}))
save(root / "measurement-gate.json", dict(
status="NOT_AUTHORIZED_FOR_EXECUTION", candidate_contract_sha256=sha(path),
runtime_installed_and_import_verified=False, measured_epoch_exists=False,
requested_candidate_limit=dict(gpu_wall_seconds=3600, cpu_seconds=3500, device_count=1),
requires=["install and CPU import-verify the isolated pinned environment",
"Claude stage/budget authorization binding the measurement contract",
"independent exact PASS on the measurement contract and CPU mock receipt"],
measurement="one complete 500-case seed-0 training epoch plus preproc and a 111-case training-only forward proxy",
heldout_labels_read=False, training_weights_reused=False,
benchmark_contract_created=False,
benchmark_cap_formula="ceil(1.25*(prep_seconds+300*epoch_seconds+proxy_prep_seconds+3*proxy_inference_seconds)) + wait + cleanup",
benchmark_requires="fixed same-regime measurement receipts, new exact cap and separate review"))
return path
def preflight(path):
c = entry.contract(path)
gpu = c["gpu_uuid"]
clock = 0.
mock_cases = []
for scenario in ("idle", "busy", "late_final_check", "total_cap"):
clock = 0.
polls = 0
def sleep(seconds):
nonlocal clock
clock += seconds
def sample(uuid):
nonlocal polls
if uuid != gpu:
raise ValueError("Actual contract GPU identity lost")
polls += 1
idle = scenario != "busy"
if scenario == "late_final_check":
early_busy = polls <= 3
duration = 2. if early_busy else 1. if polls <= 6 else 3.
idle = not early_busy
sleep(duration)
elif scenario == "busy":
sleep(2.)
return dict(uuid=uuid, idle=idle, memory_MiB=5 if idle else 1600, occupants=[])
def worker(command, actual_root, name, limits, *, poll, gpu_uuid):
if (actual_root != root or name != "gino" or command[0] != c["environment"]["python"]
or Path(command[1]).name != "track_b_gino_shapenet_worker.py" or command[2:] != [str(root)]
or gpu_uuid != gpu or limits["total_cpu_seconds"] != c["limits"]["cpu_seconds"]
or limits["tree_rss_bytes"] != c["limits"]["rss_bytes"]
or limits["disk_bytes"] != c["limits"]["disk_bytes"]
or limits["max_processes"] != c["limits"]["max_processes"]
or not 0 < limits["job_wall_seconds"] < c["limits"]["gpu_wall_seconds"]):
raise ValueError("Modified fields at actual dispatch")
records = root / "worker-records"
records.mkdir()
save(records / "result.json", dict(
status="COMPLETE", mode="measurement", heldout_labels_read=False, epochs=1,
training_examples=500, proxy_examples=111, reuse_measurement_weights=False,
preparation_seconds=1., epoch_seconds=2., proxy_preparation_seconds=1.,
proxy_inference_seconds=1., peak_allocated_bytes=100, peak_reserved_bytes=200))
sleep(c["limits"]["gpu_wall_seconds"] - clock + .8 if scenario == "total_cap" else 2.)
return dict(status="PASS", returncode=0, monitor_errors=0)
expected = {
"idle": ("COMPLETE", None, 1),
"busy": ("STOPPED", "IDLE_DEADLINE", 0),
"late_final_check": ("STOPPED", "IDLE_DEADLINE_AFTER_FINAL_CHECK", 0),
"total_cap": ("STOPPED", "TOTAL_GPU_WALL_CAP", 1),
}[scenario]
with TemporaryDirectory(prefix="track-b-gino-cpu-guard-") as scratch:
root = Path(scratch)
with (patch.object(entry, "snapshot", side_effect=sample),
patch.object(entry, "time", SimpleNamespace(monotonic=lambda: clock)),
patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: clock, sleep=sleep)),
patch.object(entry, "supervise", side_effect=worker) as dispatch):
code = entry.dispatch_guard(root, path)
terminal = entry.read(root / "complete.json")
if (terminal["status"], terminal["trigger"], dispatch.call_count) != expected:
raise RuntimeError(f"Actual guard differed for {scenario}: {terminal}")
if code != (0 if scenario == "idle" else 1):
raise RuntimeError("Terminal return code differs")
mock_cases.append(dict(scenario=scenario, synthetic_worker_dispatches=dispatch.call_count,
status=terminal["status"], trigger=terminal["trigger"],
booked_idle_seconds=terminal["booked_idle_seconds"],
charged_gpu_wall_seconds=terminal["cumulative_gpu_wall_seconds"]))
return dict(status="PASS", actual_contract_sha256=sha(path), cases=mock_cases,
real_gpu_queries=0, real_workers=0, scientific_arrays_read=False,
does_not_supply_measured_epoch_seconds=True)
if __name__ == "__main__":
path = prepare()
proof = preflight(path)
save(path.parent / "actual-guard-cpu-preflight.json", proof)
print(proof)
+262
View File
@@ -0,0 +1,262 @@
"""Review-gated dispatch for the original ShapeNet GINO reproduction."""
import argparse
import ctypes
import fcntl
import math
import os
from pathlib import Path
import shutil
import signal
import threading
import time
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase3_item1b_common import read
from phase3_shared_capacity import process_identity
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords, active_command_pids
from s2_recovery_gpu_supervisor import stop_group, supervise
BASE = Path(__file__).resolve().parents[1]
MEASUREMENT_ROOT = BASE / "artifacts/track-b-gino-shapenet611-measurement-2026-11-19"
BENCHMARK_ROOT = BASE / "artifacts/track-b-gino-shapenet611-benchmark-2026-11-19"
RULING = "h-f84547d7"
def projected_budget(measurement, limits):
prep = measurement["preparation_seconds"]
train = measurement["epoch_seconds"]
test_prep = measurement["proxy_preparation_seconds"]
inference = measurement["proxy_inference_seconds"]
if any(not math.isfinite(x) or x <= 0 for x in (prep, train, test_prep, inference)):
raise ValueError("Incomplete empirical timing components")
if (measurement["epochs"], measurement["training_examples"], measurement["proxy_examples"],
measurement["heldout_labels_read"], measurement["reuse_measurement_weights"]) != (1, 500, 111, False, False):
raise ValueError("Timing does not represent the reviewed learning-only measurement")
# One serial worker, one preprocessing pass per domain, and three fresh
# serial 100-epoch fits. The fixed wait and cleanup are charged separately.
return math.ceil(1.25 * (prep + 3 * 100 * train + test_prep + 3 * inference)) + (
limits["idle_wait_seconds"] + limits["cleanup_seconds"])
def contract(path):
c = read(path)
if c["ruling"] != RULING or c["mode"] not in ("measurement", "benchmark"):
raise PermissionError("Only the paper GINO execution scope is supported")
if c["retry"] is not False or c["evaluate_other_models"] is not False:
raise PermissionError("Single reviewed GINO attempt required")
for path, digest in c["code_sha256"].items():
checked(path, digest)
sources = ("paper_data", "paper_supplement", "split_receipt", "mapping_audit")
for name in sources:
pin = c[name]
checked(pin["path"], pin["sha256"])
comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"]))
if c["protocol"] != comparison["protocol"]:
raise ValueError("Model/score protocol differs from the comparison contract")
split = c["split"]
original = read(c["split_receipt"]["path"])
if any(split[key] != original[key] for key in ("train", "test", "raw_global_bounds")):
raise ValueError("Original ShapeNet split or bounds changed")
if len(set(split["train"])) != 500 or len(set(split["test"])) != 111 or set(split["train"]) & set(split["test"]):
raise ValueError("Expected disjoint original 500/111 split")
p = c["protocol"]
if p["grid"] != [64, 64, 64] or p["epochs_per_seed"] != 100 or p["seeds"] != [0, 1, 2]:
raise ValueError("Fixed decoder resolution, epochs or seeds changed")
if p["acceptance_percent"] != [6.62, 7.62]:
raise ValueError("Original pre-registered team interval changed")
limit = c["limits"]
caps = ("gpu_wall_seconds", "cleanup_seconds", "idle_wait_seconds", "cpu_seconds", "rss_bytes",
"recorded_vram_bytes", "device_vram_bytes", "disk_bytes", "torch_threads", "max_processes")
if any(type(limit[k]) is not int or limit[k] <= 0 for k in caps):
raise ValueError("All runtime limits must be positive integers")
if limit["idle_wait_seconds"] + limit["cleanup_seconds"] >= limit["gpu_wall_seconds"]:
raise ValueError("No runtime after booking admission and cleanup")
if c["mode"] == "measurement":
if c["measurement_allowed_heldout_labels"] is not False:
raise PermissionError("Measurement may only use the learning split")
else:
timed = c["measured_timing"]
expected_paths = {
"path": MEASUREMENT_ROOT / "worker-records/result.json",
"guard_path": MEASUREMENT_ROOT / "complete.json",
"contract_path": MEASUREMENT_ROOT / "contract.json",
}
if any(Path(timed[key]).resolve() != expected for key, expected in expected_paths.items()):
raise ValueError("Timing must come from the sole reviewed measurement root")
timing = read(checked(timed["path"], timed["sha256"]))
witness = read(checked(timed["guard_path"], timed["guard_sha256"]))
if (witness["status"] != "COMPLETE" or witness["contract_sha256"] != timed["contract_sha256"]
or witness["result_sha256"] != timed["sha256"] or timing["mode"] != "measurement"
or timing["status"] != "COMPLETE"):
raise ValueError("Benchmark requires the authentic measured timing and guard result")
measured = contract(Path(checked(timed["contract_path"], timed["contract_sha256"])))
for key in ("gpu_uuid", "paper_data", "paper_code_zip_sha256", "comparison_contract_sha256", "environment"):
if measured[key] != c[key]:
raise ValueError(f"Timing and benchmark execution regime differs: {key}")
if measured["mode"] != "measurement" or measured["limits"]["torch_threads"] != limit["torch_threads"]:
raise ValueError("Timing is not from the same worker/CPU configuration")
validate_review(MEASUREMENT_ROOT)
if limit["gpu_wall_seconds"] != projected_budget(timing, limit):
raise ValueError("Benchmark cap must come from the measured +25% projection")
return c
def dispatch_guard(root, contract_path):
started = time.monotonic()
c = contract(contract_path)
cap = c["limits"]
deadline = started + cap["gpu_wall_seconds"]
wait_deadline = started + cap["idle_wait_seconds"]
gpu = c["gpu_uuid"]
status, reason = "STOPPED", None
receipt = None
waiting = 0.
output = None
tracker = PeakRecords()
command = [c["environment"]["python"], str(Path(__file__).with_name("track_b_gino_shapenet_worker.py")), str(root)]
try:
if shutil.disk_usage(root).free < cap["disk_bytes"]:
raise RuntimeError("DISK_HEADROOM")
# Use the shared poller, then look at wall time again after the last
# independent check and the dispatch receipt write.
begin_wait = time.monotonic()
remaining_window = max(0., wait_deadline - begin_wait)
idle = wait_idle(gpu, remaining_window, lambda _: None, sample=snapshot)
waiting = time.monotonic() - begin_wait
save(root / "idle-gate.json", idle)
if idle["status"] != "IDLE_CONFIRMED" or time.monotonic() >= wait_deadline:
raise RuntimeError("IDLE_DEADLINE")
if not snapshot(gpu)["idle"]:
raise RuntimeError("SELECTED_GPU_BECAME_BUSY")
save(root / "dispatch.json", dict(contract_sha256=sha(contract_path),
coordinator=process_identity(os.getpid()),
start_by_monotonic=wait_deadline))
now = time.monotonic()
waiting = now - begin_wait
if now >= wait_deadline:
raise RuntimeError("IDLE_DEADLINE_AFTER_FINAL_CHECK")
remaining = deadline - now - cap["cleanup_seconds"]
if remaining <= 0:
raise RuntimeError("INSUFFICIENT_REMAINING_BUDGET")
reasons = []
done = threading.Event()
def monitor():
while not done.is_set():
try:
tracker.scan(root / "worker-records")
state = snapshot(gpu)
allowed = set(active_command_pids(command))
if max(tracker.maxima.values()) > cap["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if state["memory_MiB"] * 2**20 > cap["device_vram_bytes"]:
reasons.append("DEVICE_VRAM_LIMIT")
if any(p["pid"] not in allowed for p in state["occupants"]):
reasons.append("FOREIGN_GPU_OCCUPANT")
except Exception as exc:
reasons.append("MONITOR_FAILURE: " + str(exc))
if reasons:
for pid in active_command_pids(command):
stop_group(pid)
done.wait(.5)
watcher = threading.Thread(target=monitor, daemon=True)
watcher.start()
try:
receipt = supervise(command, root, "gino", dict(
total_cpu_seconds=cap["cpu_seconds"], job_cpu_seconds=cap["cpu_seconds"],
job_wall_seconds=remaining, tree_rss_bytes=cap["rss_bytes"],
disk_bytes=cap["disk_bytes"], max_processes=cap["max_processes"]), poll=.2, gpu_uuid=gpu)
finally:
done.set()
watcher.join(timeout=8)
if watcher.is_alive():
raise RuntimeError("MONITOR_DID_NOT_EXIT")
tracker.scan(root / "worker-records")
if max(tracker.maxima.values()) > cap["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0 or receipt["monitor_errors"] != 0:
raise RuntimeError(reasons[0] if reasons else receipt.get("stop_reason") or "WORKER_FAILED")
output = root / "worker-records/result.json"
result = read(output)
if (result["status"] != "COMPLETE" or result["mode"] != c["mode"]
or set(tracker.witnesses) != set(tracker.maxima)):
raise RuntimeError("INCOMPLETE_WORKER_RESULT")
if c["mode"] == "measurement" and (result["heldout_labels_read"] or result["epochs"] != 1):
raise RuntimeError("MEASUREMENT_SCOPE_MISMATCH")
if c["mode"] == "benchmark" and (not result["heldout_labels_read"]
or set(result["seeds"]) != {str(x) for x in c["protocol"]["seeds"]}
or any([item["case"] for item in seed["cases"]] != c["split"]["test"] for seed in result["seeds"].values())):
raise RuntimeError("BENCHMARK_SUPPORT_MISMATCH")
status = "COMPLETE"
except (Exception, KeyboardInterrupt) as exc:
reason = str(exc) or type(exc).__name__
result_hash = sha(output) if output is not None and output.is_file() else None
contract_hash = sha(contract_path)
measured = time.monotonic() - started
if measured > cap["gpu_wall_seconds"]:
status, reason = "STOPPED", "TOTAL_GPU_WALL_CAP"
save(root / "complete.json", dict(
status=status, trigger=reason, contract_sha256=contract_hash, result_sha256=result_hash,
worker=receipt, booked_idle_seconds=waiting, cumulative_gpu_wall_seconds=measured,
gpu_wall_cap_seconds=cap["gpu_wall_seconds"], recorded_peaks=tracker.maxima, no_retry=True))
return 0 if status == "COMPLETE" else 1
def authorized_child(root):
root = Path(root).resolve()
if root not in (MEASUREMENT_ROOT, BENCHMARK_ROOT):
raise PermissionError("Unrecognized Track B run root")
ticket = read(root / "dispatch.json")
parent = os.getppid()
if (ticket["coordinator"] != process_identity(parent)
or ticket["contract_sha256"] != sha(root / "contract.json")
or time.monotonic() >= ticket["start_by_monotonic"]):
raise PermissionError("Child is not attached to this reviewed guard")
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0 or os.getppid() != parent:
raise RuntimeError("Could not attach child lifetime to its coordinator")
validate_review(root)
return contract(root / "contract.json")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", type=Path)
args = parser.parse_args()
root = args.root.resolve()
if root not in (MEASUREMENT_ROOT, BENCHMARK_ROOT):
raise PermissionError("Unrecognized Track B run root")
validate_review(root)
path = root / "contract.json"
c = contract(path)
if (root == MEASUREMENT_ROOT) != (c["mode"] == "measurement"):
raise PermissionError("Execution root is tied to its reviewed stage")
# Separate authority for the costed measurement or benchmark.
authorization = read(root / "stage-authorization.json")
if (authorization["contract_sha256"] != sha(path)
or authorization["decision"] != "APPROVED"
or authorization["mode"] != c["mode"]):
raise PermissionError("Exact stage authorization is required")
if not os.access(c["environment"]["python"], os.X_OK):
raise FileNotFoundError("The isolated pinned GINO runtime is not available")
with (root / "entry.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / "started.json").exists():
raise FileExistsError("One attempt already recorded")
save(root / "started.json", dict(coordinator=process_identity(os.getpid()), contract_sha256=sha(path)))
def request_stop(signum, _frame):
raise KeyboardInterrupt(signal.Signals(signum).name)
handler = signal.signal(signal.SIGTERM, request_stop)
try:
return dispatch_guard(root, path)
finally:
signal.signal(signal.SIGTERM, handler)
if __name__ == "__main__":
raise SystemExit(main())
+315
View File
@@ -0,0 +1,315 @@
"""Paper GINO-decoder worker with a measurement-only stage and held-out scoring."""
import argparse
import io
import json
import math
import os
from pathlib import Path, PurePosixPath
import sys
from tempfile import TemporaryDirectory
import time
from zipfile import ZipFile
from track_b_gino_shapenet_entry import authorized_child, checked, save, sha
def extract_paper_code(c, directory):
with ZipFile(c["paper_supplement"]["path"]) as outer:
archive = outer.read("GINO-supp/code.zip")
import hashlib
if hashlib.sha256(archive).hexdigest() != c["paper_code_zip_sha256"]:
raise ValueError("NeurIPS code archive identity mismatch")
directory = Path(directory)
with ZipFile(io.BytesIO(archive)) as code:
for member in code.infolist():
path = PurePosixPath(member.filename)
if path.is_absolute() or ".." in path.parts:
raise ValueError("Invalid archive member path")
if not member.is_dir() and len(path.parts) >= 3 and path.parts[:2] == ("GINO-submission", "src"):
if path.suffix == ".py":
output = directory.joinpath(*path.parts[1:])
output.parent.mkdir(parents=True, exist_ok=True)
with output.open("xb") as f:
f.write(code.read(member))
sys.path.insert(0, str(directory))
class OriginalData:
def __init__(self, c, workspace):
import numpy as np
import torch
from src.data.cfd_datamodule import CFDSDFDataModule, DictDatasetWithConstant
from neuralop.utils import UnitGaussianNormalizer
self.np = np
self.torch = torch
self.dataset_class = DictDatasetWithConstant
self.normalizer_class = UnitGaussianNormalizer
self.c = c
self.workspace = Path(workspace)
self.paper = object.__new__(CFDSDFDataModule)
self.rows = {r["case"]: r for r in json.loads(Path(c["mapping_audit"]["path"]).read_bytes())["rows"]}
self.train_ids = c["split"]["train"]
self.test_ids = c["split"]["test"]
resolution = c["protocol"]["grid"]
raw_bounds = c["split"]["raw_global_bounds"]
epsilon = c["protocol"]["bounds_expansion"]
# Identical physical grid and coordinate transforms to the attached loader.
lo = [v - epsilon for v in raw_bounds[0]]
hi = [v + epsilon for v in raw_bounds[1]]
axis = [np.linspace(a, b, n) for a, b, n in zip(lo, hi, resolution)]
self.xyz = np.stack(np.meshgrid(*axis, indexing="ij"), axis=-1).astype(np.float32)
self.bounds = torch.tensor(lo), torch.tensor(hi)
self.grid = self.paper.location_normalization(
torch.from_numpy(self.xyz), *self.bounds).permute(3, 0, 1, 2)
self.normalizer = None
def data(self, ids, role, target_log=None):
import hashlib
if role == "train":
if ids != self.train_ids or self.normalizer is not None:
raise PermissionError("Only first-pass training normalization")
elif role == "train_proxy":
if ids != self.train_ids[:111] or self.normalizer is None:
raise PermissionError("Only the fixed learning-only timing proxy")
elif role == "heldout":
if ids != self.test_ids or target_log is None or self.normalizer is None:
raise PermissionError("Held-out arrays require a completed checkpoint freeze")
else:
raise PermissionError("Unexpected data role")
torch = self.torch
folder = self.workspace / role
folder.mkdir(exist_ok=False)
sdfs, vertices, pressures = [], [], []
with ZipFile(self.c["paper_data"]["path"]) as archive:
for case in ids:
row = self.rows[case]
mesh = archive.read(f"car-pressure-data/data/mesh_{case:03d}.ply")
if hashlib.sha256(mesh).hexdigest() != row["mesh_sha256"]:
raise ValueError(f"Mesh hash mismatch: {case}")
path = folder / f"mesh_{case:03d}.ply"
path.write_bytes(mesh)
sdf, xyz, _ = self.paper.sdf_vertices_closest_from_mesh(path, self.xyz, False)
path.unlink()
if len(xyz) != row["mesh_vertices"] or sdf.shape != tuple(self.c["protocol"]["grid"]):
raise ValueError(f"Unexpected geometry dimensions: {case}")
sdfs.append(torch.as_tensor(sdf, dtype=torch.float32))
vertices.append(self.paper.location_normalization(
torch.as_tensor(xyz, dtype=torch.float32), *self.bounds))
if role == "heldout":
target_log.write(json.dumps(dict(case=case, role=role, sha256=row["pressure_sha256"])) + "\n")
target_log.flush()
pressure = archive.read(f"car-pressure-data/data/press_{case:03d}.npy")
if hashlib.sha256(pressure).hexdigest() != row["pressure_sha256"]:
raise ValueError(f"Pressure hash mismatch: {case}")
raw = self.np.load(io.BytesIO(pressure), allow_pickle=False).reshape(-1).astype(self.np.float32)
retained = self.np.concatenate((raw[:16], raw[112:]))
if len(retained) != len(xyz) or not self.np.isfinite(retained).all():
raise ValueError(f"Pressure mapping mismatch: {case}")
pressures.append(torch.from_numpy(retained))
p = torch.stack(pressures)
if role == "train":
self.normalizer = self.normalizer_class(
p, eps=self.c["protocol"]["pressure_epsilon"], reduce_dim=[0, 1], verbose=False)
p = self.normalizer.encode(p)
return self.dataset_class(
{"sdf": torch.stack(sdfs), "vertices": torch.stack(vertices), "pressure": p},
{"sdf_query_points": self.grid})
def paper_model(c):
# Use the same factory call schema as the paper. The attached factory is
# missing this symbol import, so the class is imported directly.
from src.networks.SDFFNOGNO import SDFFNOGNO
p = c["protocol"]
return SDFFNOGNO(n_modes=p["modes"], hidden_channels=p["hidden_channels"],
norm="group_norm", use_mlp=True, mlp={"expansion": 1., "dropout": 0.},
domain_padding=.125, factorization="tucker", rank=.4, out_channels=1,
r=p["radius"], resolution=p["grid"], gno_implementation="torch_scatter")
def cuda_peaks(torch):
return dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
peak_reserved_bytes=torch.cuda.max_memory_reserved())
def train_seed(c, dataset, seed, epochs, progress):
import numpy as np
import torch
from src.losses import LpLoss
torch.manual_seed(seed)
np.random.seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
model = paper_model(c).to("cuda")
optimizer = torch.optim.Adam(model.parameters(), lr=c["protocol"]["learning_rate"], weight_decay=1e-4)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=.5)
loss_fn = LpLoss(size_average=True)
loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=0)
times = []
for epoch in range(1, epochs + 1):
model.train()
began = time.monotonic()
examples = 0
for item in loader:
optimizer.zero_grad()
loss = model.loss_dict(item, loss_fn=loss_fn)["loss"].mean()
if not bool(torch.isfinite(loss)):
raise ValueError("Nonfinite training loss")
loss.backward()
optimizer.step()
examples += 1
scheduler.step()
torch.cuda.synchronize()
if examples != 500:
raise ValueError("Incomplete full training epoch")
seconds = time.monotonic() - began
times.append(seconds)
save(progress, dict(stage="EPOCH_COMPLETE", seed=seed, epoch=epoch, examples=examples,
seconds=seconds, **cuda_peaks(torch)))
return model, times
def case_scores(model, data, cases, normalizer):
import torch
loader = torch.utils.data.DataLoader(data, batch_size=1, shuffle=False, num_workers=0)
model.eval()
normalizer.to("cuda")
rows = []
with torch.no_grad():
for case, item in zip(cases, loader):
raw_input, output_vertices = model.data_dict_to_input(item)
prediction = normalizer.decode(model(raw_input, output_vertices)).reshape(-1)
truth = normalizer.decode(item["pressure"].to("cuda")).reshape(-1)
if prediction.numel() != 3586 or prediction.shape != truth.shape:
raise ValueError(f"Incomplete pressure vector: {case}")
denominator = torch.linalg.vector_norm(truth)
value = (torch.linalg.vector_norm(prediction - truth) / denominator).item()
if not math.isfinite(value) or denominator.item() <= 0:
raise ValueError(f"Undefined decoded relative L2: {case}")
rows.append(dict(case=case, relative_l2=value))
normalizer.to("cpu")
if len(rows) != len(cases):
raise ValueError("Incomplete case coverage")
return rows
def test_environment(c):
import importlib.metadata
import importlib.util
import platform
if ".".join(platform.python_version_tuple()[:2]) != c["environment"]["python_version"]:
raise RuntimeError("Pinned Python minor release required")
for name, expected in c["environment"]["package_versions"].items():
if importlib.metadata.version(name) != expected:
raise RuntimeError(f"Runtime distribution mismatch: {name}")
direct_url = importlib.metadata.distribution("neuraloperator").read_text("direct_url.json")
if not direct_url or json.loads(direct_url).get("vcs_info", {}).get("commit_id") != c["environment"]["neuralop_commit"]:
raise RuntimeError("Historical NeuralOperator commit not installed from its reviewed VCS pin")
module = importlib.util.find_spec("neuralop")
if module is None or module.origin is None:
raise RuntimeError("Historical NeuralOperator import missing")
root = Path(module.origin).parent
for relative, digest in c["environment"]["neuralop_file_sha256"].items():
checked(root / relative, digest)
def work(root, c):
began = time.monotonic()
test_environment(c)
import torch
if (os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"]
or torch.version.cuda != c["environment"]["torch_cuda"]):
raise PermissionError("Runtime CUDA binding mismatch")
torch.set_num_threads(c["limits"]["torch_threads"])
torch.set_num_interop_threads(1)
torch.cuda.set_device(0)
if "V100" not in torch.cuda.get_device_name(0):
raise ValueError("Selected device is not the fixed V100")
torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
records = root / "worker-records"
records.mkdir(exist_ok=False)
with TemporaryDirectory(prefix="original-gino-", dir=root) as scratch:
extract_paper_code(c, scratch)
data = OriginalData(c, scratch)
train = data.data(data.train_ids, "train")
preparation_seconds = time.monotonic() - began
if c["mode"] == "measurement":
model, epoch_times = train_seed(c, train, c["protocol"]["seeds"][0], 1, records / "progress.json")
began_proxy = time.monotonic()
ids = data.train_ids[:111]
proxy = data.data(ids, "train_proxy")
proxy_preparation_seconds = time.monotonic() - began_proxy
began_eval = time.monotonic()
case_scores(model, proxy, ids, data.normalizer)
torch.cuda.synchronize()
result = dict(status="COMPLETE", mode="measurement", epochs=1, training_examples=500,
heldout_labels_read=False, proxy_examples=111,
preparation_seconds=preparation_seconds, epoch_seconds=epoch_times[0],
proxy_preparation_seconds=proxy_preparation_seconds,
proxy_inference_seconds=time.monotonic() - began_eval,
reuse_measurement_weights=False, **cuda_peaks(torch))
else:
checkpoints = {}
for seed in c["protocol"]["seeds"]:
model, _ = train_seed(c, train, seed, 100, records / "progress.json")
file = root / f"seed-{seed}-epoch-100.pt"
torch.save(model.state_dict(), file)
checkpoints[str(seed)] = dict(path=str(file), sha256=sha(file))
del model
predictions = root / "checkpoints-frozen.json"
save(predictions, checkpoints)
freeze_hash = sha(predictions)
checkpoints = json.loads(checked(predictions, freeze_hash).read_bytes())
for item in checkpoints.values():
checked(item["path"], item["sha256"])
del train
with (root / "heldout-target-access.jsonl").open("x") as log:
heldout = data.data(data.test_ids, "heldout", log)
rows = {}
for seed, item in checkpoints.items():
model = paper_model(c).to("cuda")
model.load_state_dict(torch.load(checked(item["path"], item["sha256"]),
map_location="cpu"), strict=True)
scores = case_scores(model, heldout, data.test_ids, data.normalizer)
rows[seed] = dict(cases=scores, mean_percent=100 * math.fsum(x["relative_l2"] for x in scores) / len(scores))
del model
mean = math.fsum(seed["mean_percent"] for seed in rows.values()) / len(rows)
lower, upper = c["protocol"]["acceptance_percent"]
result = dict(status="COMPLETE", mode="benchmark", seeds=rows, mean_percent=mean,
within_registered_range=lower <= mean <= upper,
heldout_labels_read=True, split_sha256=c["split_receipt"]["sha256"],
checkpoints_manifest_sha256=freeze_hash, **cuda_peaks(torch))
save(records / "result.json", result)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", type=Path)
parser.add_argument("--check-environment", action="store_true")
arg = parser.parse_args()
if arg.check_environment:
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
raise PermissionError("Set CUDA_VISIBLE_DEVICES empty for the isolated import check")
from track_b_gino_shapenet_entry import contract
c = contract(arg.root / "contract.json")
test_environment(c)
with TemporaryDirectory(prefix="track-b-gino-import-") as directory:
extract_paper_code(c, directory)
from src.data.cfd_datamodule import CFDSDFDataModule
from src.networks.SDFFNOGNO import SDFFNOGNO
print(json.dumps(dict(status="PASS", imports=[CFDSDFDataModule.__name__, SDFFNOGNO.__name__],
models_created=False, arrays_loaded=False)))
else:
c = authorized_child(arg.root)
work(arg.root, c)