Files
MeltMind/tools/track_b_gino_shapenet_worker.py
T

316 lines
15 KiB
Python

"""Paper GINO-decoder worker with a measurement-only stage and held-out scoring."""
import argparse
import io
import json
import math
import os
from pathlib import Path, PurePosixPath
import sys
from tempfile import TemporaryDirectory
import time
from zipfile import ZipFile
from track_b_gino_shapenet_entry import authorized_child, checked, save, sha
def extract_paper_code(c, directory):
with ZipFile(c["paper_supplement"]["path"]) as outer:
archive = outer.read("GINO-supp/code.zip")
import hashlib
if hashlib.sha256(archive).hexdigest() != c["paper_code_zip_sha256"]:
raise ValueError("NeurIPS code archive identity mismatch")
directory = Path(directory)
with ZipFile(io.BytesIO(archive)) as code:
for member in code.infolist():
path = PurePosixPath(member.filename)
if path.is_absolute() or ".." in path.parts:
raise ValueError("Invalid archive member path")
if not member.is_dir() and len(path.parts) >= 3 and path.parts[:2] == ("GINO-submission", "src"):
if path.suffix == ".py":
output = directory.joinpath(*path.parts[1:])
output.parent.mkdir(parents=True, exist_ok=True)
with output.open("xb") as f:
f.write(code.read(member))
sys.path.insert(0, str(directory))
class OriginalData:
def __init__(self, c, workspace):
import numpy as np
import torch
from src.data.cfd_datamodule import CFDSDFDataModule, DictDatasetWithConstant
from neuralop.utils import UnitGaussianNormalizer
self.np = np
self.torch = torch
self.dataset_class = DictDatasetWithConstant
self.normalizer_class = UnitGaussianNormalizer
self.c = c
self.workspace = Path(workspace)
self.paper = object.__new__(CFDSDFDataModule)
self.rows = {r["case"]: r for r in json.loads(Path(c["mapping_audit"]["path"]).read_bytes())["rows"]}
self.train_ids = c["split"]["train"]
self.test_ids = c["split"]["test"]
resolution = c["protocol"]["grid"]
raw_bounds = c["split"]["raw_global_bounds"]
epsilon = c["protocol"]["bounds_expansion"]
# Identical physical grid and coordinate transforms to the attached loader.
lo = [v - epsilon for v in raw_bounds[0]]
hi = [v + epsilon for v in raw_bounds[1]]
axis = [np.linspace(a, b, n) for a, b, n in zip(lo, hi, resolution)]
self.xyz = np.stack(np.meshgrid(*axis, indexing="ij"), axis=-1).astype(np.float32)
self.bounds = torch.tensor(lo), torch.tensor(hi)
self.grid = self.paper.location_normalization(
torch.from_numpy(self.xyz), *self.bounds).permute(3, 0, 1, 2)
self.normalizer = None
def data(self, ids, role, target_log=None):
import hashlib
if role == "train":
if ids != self.train_ids or self.normalizer is not None:
raise PermissionError("Only first-pass training normalization")
elif role == "train_proxy":
if ids != self.train_ids[:111] or self.normalizer is None:
raise PermissionError("Only the fixed learning-only timing proxy")
elif role == "heldout":
if ids != self.test_ids or target_log is None or self.normalizer is None:
raise PermissionError("Held-out arrays require a completed checkpoint freeze")
else:
raise PermissionError("Unexpected data role")
torch = self.torch
folder = self.workspace / role
folder.mkdir(exist_ok=False)
sdfs, vertices, pressures = [], [], []
with ZipFile(self.c["paper_data"]["path"]) as archive:
for case in ids:
row = self.rows[case]
mesh = archive.read(f"car-pressure-data/data/mesh_{case:03d}.ply")
if hashlib.sha256(mesh).hexdigest() != row["mesh_sha256"]:
raise ValueError(f"Mesh hash mismatch: {case}")
path = folder / f"mesh_{case:03d}.ply"
path.write_bytes(mesh)
sdf, xyz, _ = self.paper.sdf_vertices_closest_from_mesh(path, self.xyz, False)
path.unlink()
if len(xyz) != row["mesh_vertices"] or sdf.shape != tuple(self.c["protocol"]["grid"]):
raise ValueError(f"Unexpected geometry dimensions: {case}")
sdfs.append(torch.as_tensor(sdf, dtype=torch.float32))
vertices.append(self.paper.location_normalization(
torch.as_tensor(xyz, dtype=torch.float32), *self.bounds))
if role == "heldout":
target_log.write(json.dumps(dict(case=case, role=role, sha256=row["pressure_sha256"])) + "\n")
target_log.flush()
pressure = archive.read(f"car-pressure-data/data/press_{case:03d}.npy")
if hashlib.sha256(pressure).hexdigest() != row["pressure_sha256"]:
raise ValueError(f"Pressure hash mismatch: {case}")
raw = self.np.load(io.BytesIO(pressure), allow_pickle=False).reshape(-1).astype(self.np.float32)
retained = self.np.concatenate((raw[:16], raw[112:]))
if len(retained) != len(xyz) or not self.np.isfinite(retained).all():
raise ValueError(f"Pressure mapping mismatch: {case}")
pressures.append(torch.from_numpy(retained))
p = torch.stack(pressures)
if role == "train":
self.normalizer = self.normalizer_class(
p, eps=self.c["protocol"]["pressure_epsilon"], reduce_dim=[0, 1], verbose=False)
p = self.normalizer.encode(p)
return self.dataset_class(
{"sdf": torch.stack(sdfs), "vertices": torch.stack(vertices), "pressure": p},
{"sdf_query_points": self.grid})
def paper_model(c):
# Use the same factory call schema as the paper. The attached factory is
# missing this symbol import, so the class is imported directly.
from src.networks.SDFFNOGNO import SDFFNOGNO
p = c["protocol"]
return SDFFNOGNO(n_modes=p["modes"], hidden_channels=p["hidden_channels"],
norm="group_norm", use_mlp=True, mlp={"expansion": 1., "dropout": 0.},
domain_padding=.125, factorization="tucker", rank=.4, out_channels=1,
r=p["radius"], resolution=p["grid"], gno_implementation="torch_scatter")
def cuda_peaks(torch):
return dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
peak_reserved_bytes=torch.cuda.max_memory_reserved())
def train_seed(c, dataset, seed, epochs, progress):
import numpy as np
import torch
from src.losses import LpLoss
torch.manual_seed(seed)
np.random.seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True
model = paper_model(c).to("cuda")
optimizer = torch.optim.Adam(model.parameters(), lr=c["protocol"]["learning_rate"], weight_decay=1e-4)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=.5)
loss_fn = LpLoss(size_average=True)
loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=0)
times = []
for epoch in range(1, epochs + 1):
model.train()
began = time.monotonic()
examples = 0
for item in loader:
optimizer.zero_grad()
loss = model.loss_dict(item, loss_fn=loss_fn)["loss"].mean()
if not bool(torch.isfinite(loss)):
raise ValueError("Nonfinite training loss")
loss.backward()
optimizer.step()
examples += 1
scheduler.step()
torch.cuda.synchronize()
if examples != 500:
raise ValueError("Incomplete full training epoch")
seconds = time.monotonic() - began
times.append(seconds)
save(progress, dict(stage="EPOCH_COMPLETE", seed=seed, epoch=epoch, examples=examples,
seconds=seconds, **cuda_peaks(torch)))
return model, times
def case_scores(model, data, cases, normalizer):
import torch
loader = torch.utils.data.DataLoader(data, batch_size=1, shuffle=False, num_workers=0)
model.eval()
normalizer.to("cuda")
rows = []
with torch.no_grad():
for case, item in zip(cases, loader):
raw_input, output_vertices = model.data_dict_to_input(item)
prediction = normalizer.decode(model(raw_input, output_vertices)).reshape(-1)
truth = normalizer.decode(item["pressure"].to("cuda")).reshape(-1)
if prediction.numel() != 3586 or prediction.shape != truth.shape:
raise ValueError(f"Incomplete pressure vector: {case}")
denominator = torch.linalg.vector_norm(truth)
value = (torch.linalg.vector_norm(prediction - truth) / denominator).item()
if not math.isfinite(value) or denominator.item() <= 0:
raise ValueError(f"Undefined decoded relative L2: {case}")
rows.append(dict(case=case, relative_l2=value))
normalizer.to("cpu")
if len(rows) != len(cases):
raise ValueError("Incomplete case coverage")
return rows
def test_environment(c):
import importlib.metadata
import importlib.util
import platform
if ".".join(platform.python_version_tuple()[:2]) != c["environment"]["python_version"]:
raise RuntimeError("Pinned Python minor release required")
for name, expected in c["environment"]["package_versions"].items():
if importlib.metadata.version(name) != expected:
raise RuntimeError(f"Runtime distribution mismatch: {name}")
direct_url = importlib.metadata.distribution("neuraloperator").read_text("direct_url.json")
if not direct_url or json.loads(direct_url).get("vcs_info", {}).get("commit_id") != c["environment"]["neuralop_commit"]:
raise RuntimeError("Historical NeuralOperator commit not installed from its reviewed VCS pin")
module = importlib.util.find_spec("neuralop")
if module is None or module.origin is None:
raise RuntimeError("Historical NeuralOperator import missing")
root = Path(module.origin).parent
for relative, digest in c["environment"]["neuralop_file_sha256"].items():
checked(root / relative, digest)
def work(root, c):
began = time.monotonic()
test_environment(c)
import torch
if (os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"]
or torch.version.cuda != c["environment"]["torch_cuda"]):
raise PermissionError("Runtime CUDA binding mismatch")
torch.set_num_threads(c["limits"]["torch_threads"])
torch.set_num_interop_threads(1)
torch.cuda.set_device(0)
if "V100" not in torch.cuda.get_device_name(0):
raise ValueError("Selected device is not the fixed V100")
torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
records = root / "worker-records"
records.mkdir(exist_ok=False)
with TemporaryDirectory(prefix="original-gino-", dir=root) as scratch:
extract_paper_code(c, scratch)
data = OriginalData(c, scratch)
train = data.data(data.train_ids, "train")
preparation_seconds = time.monotonic() - began
if c["mode"] == "measurement":
model, epoch_times = train_seed(c, train, c["protocol"]["seeds"][0], 1, records / "progress.json")
began_proxy = time.monotonic()
ids = data.train_ids[:111]
proxy = data.data(ids, "train_proxy")
proxy_preparation_seconds = time.monotonic() - began_proxy
began_eval = time.monotonic()
case_scores(model, proxy, ids, data.normalizer)
torch.cuda.synchronize()
result = dict(status="COMPLETE", mode="measurement", epochs=1, training_examples=500,
heldout_labels_read=False, proxy_examples=111,
preparation_seconds=preparation_seconds, epoch_seconds=epoch_times[0],
proxy_preparation_seconds=proxy_preparation_seconds,
proxy_inference_seconds=time.monotonic() - began_eval,
reuse_measurement_weights=False, **cuda_peaks(torch))
else:
checkpoints = {}
for seed in c["protocol"]["seeds"]:
model, _ = train_seed(c, train, seed, 100, records / "progress.json")
file = root / f"seed-{seed}-epoch-100.pt"
torch.save(model.state_dict(), file)
checkpoints[str(seed)] = dict(path=str(file), sha256=sha(file))
del model
predictions = root / "checkpoints-frozen.json"
save(predictions, checkpoints)
freeze_hash = sha(predictions)
checkpoints = json.loads(checked(predictions, freeze_hash).read_bytes())
for item in checkpoints.values():
checked(item["path"], item["sha256"])
del train
with (root / "heldout-target-access.jsonl").open("x") as log:
heldout = data.data(data.test_ids, "heldout", log)
rows = {}
for seed, item in checkpoints.items():
model = paper_model(c).to("cuda")
model.load_state_dict(torch.load(checked(item["path"], item["sha256"]),
map_location="cpu"), strict=True)
scores = case_scores(model, heldout, data.test_ids, data.normalizer)
rows[seed] = dict(cases=scores, mean_percent=100 * math.fsum(x["relative_l2"] for x in scores) / len(scores))
del model
mean = math.fsum(seed["mean_percent"] for seed in rows.values()) / len(rows)
lower, upper = c["protocol"]["acceptance_percent"]
result = dict(status="COMPLETE", mode="benchmark", seeds=rows, mean_percent=mean,
within_registered_range=lower <= mean <= upper,
heldout_labels_read=True, split_sha256=c["split_receipt"]["sha256"],
checkpoints_manifest_sha256=freeze_hash, **cuda_peaks(torch))
save(records / "result.json", result)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", type=Path)
parser.add_argument("--check-environment", action="store_true")
arg = parser.parse_args()
if arg.check_environment:
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
raise PermissionError("Set CUDA_VISIBLE_DEVICES empty for the isolated import check")
from track_b_gino_shapenet_entry import contract
c = contract(arg.root / "contract.json")
test_environment(c)
with TemporaryDirectory(prefix="track-b-gino-import-") as directory:
extract_paper_code(c, directory)
from src.data.cfd_datamodule import CFDSDFDataModule
from src.networks.SDFFNOGNO import SDFFNOGNO
print(json.dumps(dict(status="PASS", imports=[CFDSDFDataModule.__name__, SDFFNOGNO.__name__],
models_created=False, arrays_loaded=False)))
else:
c = authorized_child(arg.root)
work(arg.root, c)