316 lines
15 KiB
Python
316 lines
15 KiB
Python
"""Paper GINO-decoder worker with a measurement-only stage and held-out scoring."""
|
|
|
|
import argparse
|
|
import io
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import sys
|
|
from tempfile import TemporaryDirectory
|
|
import time
|
|
from zipfile import ZipFile
|
|
|
|
from track_b_gino_shapenet_entry import authorized_child, checked, save, sha
|
|
|
|
|
|
def extract_paper_code(c, directory):
|
|
with ZipFile(c["paper_supplement"]["path"]) as outer:
|
|
archive = outer.read("GINO-supp/code.zip")
|
|
import hashlib
|
|
if hashlib.sha256(archive).hexdigest() != c["paper_code_zip_sha256"]:
|
|
raise ValueError("NeurIPS code archive identity mismatch")
|
|
directory = Path(directory)
|
|
with ZipFile(io.BytesIO(archive)) as code:
|
|
for member in code.infolist():
|
|
path = PurePosixPath(member.filename)
|
|
if path.is_absolute() or ".." in path.parts:
|
|
raise ValueError("Invalid archive member path")
|
|
if not member.is_dir() and len(path.parts) >= 3 and path.parts[:2] == ("GINO-submission", "src"):
|
|
if path.suffix == ".py":
|
|
output = directory.joinpath(*path.parts[1:])
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
with output.open("xb") as f:
|
|
f.write(code.read(member))
|
|
sys.path.insert(0, str(directory))
|
|
|
|
|
|
class OriginalData:
|
|
def __init__(self, c, workspace):
|
|
import numpy as np
|
|
import torch
|
|
from src.data.cfd_datamodule import CFDSDFDataModule, DictDatasetWithConstant
|
|
from neuralop.utils import UnitGaussianNormalizer
|
|
|
|
self.np = np
|
|
self.torch = torch
|
|
self.dataset_class = DictDatasetWithConstant
|
|
self.normalizer_class = UnitGaussianNormalizer
|
|
self.c = c
|
|
self.workspace = Path(workspace)
|
|
self.paper = object.__new__(CFDSDFDataModule)
|
|
self.rows = {r["case"]: r for r in json.loads(Path(c["mapping_audit"]["path"]).read_bytes())["rows"]}
|
|
self.train_ids = c["split"]["train"]
|
|
self.test_ids = c["split"]["test"]
|
|
resolution = c["protocol"]["grid"]
|
|
raw_bounds = c["split"]["raw_global_bounds"]
|
|
epsilon = c["protocol"]["bounds_expansion"]
|
|
# Identical physical grid and coordinate transforms to the attached loader.
|
|
lo = [v - epsilon for v in raw_bounds[0]]
|
|
hi = [v + epsilon for v in raw_bounds[1]]
|
|
axis = [np.linspace(a, b, n) for a, b, n in zip(lo, hi, resolution)]
|
|
self.xyz = np.stack(np.meshgrid(*axis, indexing="ij"), axis=-1).astype(np.float32)
|
|
self.bounds = torch.tensor(lo), torch.tensor(hi)
|
|
self.grid = self.paper.location_normalization(
|
|
torch.from_numpy(self.xyz), *self.bounds).permute(3, 0, 1, 2)
|
|
self.normalizer = None
|
|
|
|
def data(self, ids, role, target_log=None):
|
|
import hashlib
|
|
|
|
if role == "train":
|
|
if ids != self.train_ids or self.normalizer is not None:
|
|
raise PermissionError("Only first-pass training normalization")
|
|
elif role == "train_proxy":
|
|
if ids != self.train_ids[:111] or self.normalizer is None:
|
|
raise PermissionError("Only the fixed learning-only timing proxy")
|
|
elif role == "heldout":
|
|
if ids != self.test_ids or target_log is None or self.normalizer is None:
|
|
raise PermissionError("Held-out arrays require a completed checkpoint freeze")
|
|
else:
|
|
raise PermissionError("Unexpected data role")
|
|
torch = self.torch
|
|
folder = self.workspace / role
|
|
folder.mkdir(exist_ok=False)
|
|
sdfs, vertices, pressures = [], [], []
|
|
with ZipFile(self.c["paper_data"]["path"]) as archive:
|
|
for case in ids:
|
|
row = self.rows[case]
|
|
mesh = archive.read(f"car-pressure-data/data/mesh_{case:03d}.ply")
|
|
if hashlib.sha256(mesh).hexdigest() != row["mesh_sha256"]:
|
|
raise ValueError(f"Mesh hash mismatch: {case}")
|
|
path = folder / f"mesh_{case:03d}.ply"
|
|
path.write_bytes(mesh)
|
|
sdf, xyz, _ = self.paper.sdf_vertices_closest_from_mesh(path, self.xyz, False)
|
|
path.unlink()
|
|
if len(xyz) != row["mesh_vertices"] or sdf.shape != tuple(self.c["protocol"]["grid"]):
|
|
raise ValueError(f"Unexpected geometry dimensions: {case}")
|
|
sdfs.append(torch.as_tensor(sdf, dtype=torch.float32))
|
|
vertices.append(self.paper.location_normalization(
|
|
torch.as_tensor(xyz, dtype=torch.float32), *self.bounds))
|
|
if role == "heldout":
|
|
target_log.write(json.dumps(dict(case=case, role=role, sha256=row["pressure_sha256"])) + "\n")
|
|
target_log.flush()
|
|
pressure = archive.read(f"car-pressure-data/data/press_{case:03d}.npy")
|
|
if hashlib.sha256(pressure).hexdigest() != row["pressure_sha256"]:
|
|
raise ValueError(f"Pressure hash mismatch: {case}")
|
|
raw = self.np.load(io.BytesIO(pressure), allow_pickle=False).reshape(-1).astype(self.np.float32)
|
|
retained = self.np.concatenate((raw[:16], raw[112:]))
|
|
if len(retained) != len(xyz) or not self.np.isfinite(retained).all():
|
|
raise ValueError(f"Pressure mapping mismatch: {case}")
|
|
pressures.append(torch.from_numpy(retained))
|
|
p = torch.stack(pressures)
|
|
if role == "train":
|
|
self.normalizer = self.normalizer_class(
|
|
p, eps=self.c["protocol"]["pressure_epsilon"], reduce_dim=[0, 1], verbose=False)
|
|
p = self.normalizer.encode(p)
|
|
return self.dataset_class(
|
|
{"sdf": torch.stack(sdfs), "vertices": torch.stack(vertices), "pressure": p},
|
|
{"sdf_query_points": self.grid})
|
|
|
|
|
|
def paper_model(c):
|
|
# Use the same factory call schema as the paper. The attached factory is
|
|
# missing this symbol import, so the class is imported directly.
|
|
from src.networks.SDFFNOGNO import SDFFNOGNO
|
|
|
|
p = c["protocol"]
|
|
return SDFFNOGNO(n_modes=p["modes"], hidden_channels=p["hidden_channels"],
|
|
norm="group_norm", use_mlp=True, mlp={"expansion": 1., "dropout": 0.},
|
|
domain_padding=.125, factorization="tucker", rank=.4, out_channels=1,
|
|
r=p["radius"], resolution=p["grid"], gno_implementation="torch_scatter")
|
|
|
|
|
|
def cuda_peaks(torch):
|
|
return dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
|
|
peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
|
|
|
|
|
def train_seed(c, dataset, seed, epochs, progress):
|
|
import numpy as np
|
|
import torch
|
|
from src.losses import LpLoss
|
|
|
|
torch.manual_seed(seed)
|
|
np.random.seed(seed)
|
|
torch.cuda.manual_seed(seed)
|
|
torch.backends.cudnn.deterministic = True
|
|
model = paper_model(c).to("cuda")
|
|
optimizer = torch.optim.Adam(model.parameters(), lr=c["protocol"]["learning_rate"], weight_decay=1e-4)
|
|
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=.5)
|
|
loss_fn = LpLoss(size_average=True)
|
|
loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=0)
|
|
times = []
|
|
for epoch in range(1, epochs + 1):
|
|
model.train()
|
|
began = time.monotonic()
|
|
examples = 0
|
|
for item in loader:
|
|
optimizer.zero_grad()
|
|
loss = model.loss_dict(item, loss_fn=loss_fn)["loss"].mean()
|
|
if not bool(torch.isfinite(loss)):
|
|
raise ValueError("Nonfinite training loss")
|
|
loss.backward()
|
|
optimizer.step()
|
|
examples += 1
|
|
scheduler.step()
|
|
torch.cuda.synchronize()
|
|
if examples != 500:
|
|
raise ValueError("Incomplete full training epoch")
|
|
seconds = time.monotonic() - began
|
|
times.append(seconds)
|
|
save(progress, dict(stage="EPOCH_COMPLETE", seed=seed, epoch=epoch, examples=examples,
|
|
seconds=seconds, **cuda_peaks(torch)))
|
|
return model, times
|
|
|
|
|
|
def case_scores(model, data, cases, normalizer):
|
|
import torch
|
|
|
|
loader = torch.utils.data.DataLoader(data, batch_size=1, shuffle=False, num_workers=0)
|
|
model.eval()
|
|
normalizer.to("cuda")
|
|
rows = []
|
|
with torch.no_grad():
|
|
for case, item in zip(cases, loader):
|
|
raw_input, output_vertices = model.data_dict_to_input(item)
|
|
prediction = normalizer.decode(model(raw_input, output_vertices)).reshape(-1)
|
|
truth = normalizer.decode(item["pressure"].to("cuda")).reshape(-1)
|
|
if prediction.numel() != 3586 or prediction.shape != truth.shape:
|
|
raise ValueError(f"Incomplete pressure vector: {case}")
|
|
denominator = torch.linalg.vector_norm(truth)
|
|
value = (torch.linalg.vector_norm(prediction - truth) / denominator).item()
|
|
if not math.isfinite(value) or denominator.item() <= 0:
|
|
raise ValueError(f"Undefined decoded relative L2: {case}")
|
|
rows.append(dict(case=case, relative_l2=value))
|
|
normalizer.to("cpu")
|
|
if len(rows) != len(cases):
|
|
raise ValueError("Incomplete case coverage")
|
|
return rows
|
|
|
|
|
|
def test_environment(c):
|
|
import importlib.metadata
|
|
import importlib.util
|
|
import platform
|
|
|
|
if ".".join(platform.python_version_tuple()[:2]) != c["environment"]["python_version"]:
|
|
raise RuntimeError("Pinned Python minor release required")
|
|
|
|
for name, expected in c["environment"]["package_versions"].items():
|
|
if importlib.metadata.version(name) != expected:
|
|
raise RuntimeError(f"Runtime distribution mismatch: {name}")
|
|
direct_url = importlib.metadata.distribution("neuraloperator").read_text("direct_url.json")
|
|
if not direct_url or json.loads(direct_url).get("vcs_info", {}).get("commit_id") != c["environment"]["neuralop_commit"]:
|
|
raise RuntimeError("Historical NeuralOperator commit not installed from its reviewed VCS pin")
|
|
module = importlib.util.find_spec("neuralop")
|
|
if module is None or module.origin is None:
|
|
raise RuntimeError("Historical NeuralOperator import missing")
|
|
root = Path(module.origin).parent
|
|
for relative, digest in c["environment"]["neuralop_file_sha256"].items():
|
|
checked(root / relative, digest)
|
|
|
|
|
|
def work(root, c):
|
|
began = time.monotonic()
|
|
test_environment(c)
|
|
import torch
|
|
|
|
if (os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"]
|
|
or torch.version.cuda != c["environment"]["torch_cuda"]):
|
|
raise PermissionError("Runtime CUDA binding mismatch")
|
|
torch.set_num_threads(c["limits"]["torch_threads"])
|
|
torch.set_num_interop_threads(1)
|
|
torch.cuda.set_device(0)
|
|
if "V100" not in torch.cuda.get_device_name(0):
|
|
raise ValueError("Selected device is not the fixed V100")
|
|
torch.cuda.reset_peak_memory_stats()
|
|
torch.backends.cuda.matmul.allow_tf32 = False
|
|
torch.backends.cudnn.allow_tf32 = False
|
|
records = root / "worker-records"
|
|
records.mkdir(exist_ok=False)
|
|
with TemporaryDirectory(prefix="original-gino-", dir=root) as scratch:
|
|
extract_paper_code(c, scratch)
|
|
data = OriginalData(c, scratch)
|
|
train = data.data(data.train_ids, "train")
|
|
preparation_seconds = time.monotonic() - began
|
|
if c["mode"] == "measurement":
|
|
model, epoch_times = train_seed(c, train, c["protocol"]["seeds"][0], 1, records / "progress.json")
|
|
began_proxy = time.monotonic()
|
|
ids = data.train_ids[:111]
|
|
proxy = data.data(ids, "train_proxy")
|
|
proxy_preparation_seconds = time.monotonic() - began_proxy
|
|
began_eval = time.monotonic()
|
|
case_scores(model, proxy, ids, data.normalizer)
|
|
torch.cuda.synchronize()
|
|
result = dict(status="COMPLETE", mode="measurement", epochs=1, training_examples=500,
|
|
heldout_labels_read=False, proxy_examples=111,
|
|
preparation_seconds=preparation_seconds, epoch_seconds=epoch_times[0],
|
|
proxy_preparation_seconds=proxy_preparation_seconds,
|
|
proxy_inference_seconds=time.monotonic() - began_eval,
|
|
reuse_measurement_weights=False, **cuda_peaks(torch))
|
|
else:
|
|
checkpoints = {}
|
|
for seed in c["protocol"]["seeds"]:
|
|
model, _ = train_seed(c, train, seed, 100, records / "progress.json")
|
|
file = root / f"seed-{seed}-epoch-100.pt"
|
|
torch.save(model.state_dict(), file)
|
|
checkpoints[str(seed)] = dict(path=str(file), sha256=sha(file))
|
|
del model
|
|
predictions = root / "checkpoints-frozen.json"
|
|
save(predictions, checkpoints)
|
|
freeze_hash = sha(predictions)
|
|
checkpoints = json.loads(checked(predictions, freeze_hash).read_bytes())
|
|
for item in checkpoints.values():
|
|
checked(item["path"], item["sha256"])
|
|
del train
|
|
with (root / "heldout-target-access.jsonl").open("x") as log:
|
|
heldout = data.data(data.test_ids, "heldout", log)
|
|
rows = {}
|
|
for seed, item in checkpoints.items():
|
|
model = paper_model(c).to("cuda")
|
|
model.load_state_dict(torch.load(checked(item["path"], item["sha256"]),
|
|
map_location="cpu"), strict=True)
|
|
scores = case_scores(model, heldout, data.test_ids, data.normalizer)
|
|
rows[seed] = dict(cases=scores, mean_percent=100 * math.fsum(x["relative_l2"] for x in scores) / len(scores))
|
|
del model
|
|
mean = math.fsum(seed["mean_percent"] for seed in rows.values()) / len(rows)
|
|
lower, upper = c["protocol"]["acceptance_percent"]
|
|
result = dict(status="COMPLETE", mode="benchmark", seeds=rows, mean_percent=mean,
|
|
within_registered_range=lower <= mean <= upper,
|
|
heldout_labels_read=True, split_sha256=c["split_receipt"]["sha256"],
|
|
checkpoints_manifest_sha256=freeze_hash, **cuda_peaks(torch))
|
|
save(records / "result.json", result)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("root", type=Path)
|
|
parser.add_argument("--check-environment", action="store_true")
|
|
arg = parser.parse_args()
|
|
if arg.check_environment:
|
|
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
|
|
raise PermissionError("Set CUDA_VISIBLE_DEVICES empty for the isolated import check")
|
|
from track_b_gino_shapenet_entry import contract
|
|
c = contract(arg.root / "contract.json")
|
|
test_environment(c)
|
|
with TemporaryDirectory(prefix="track-b-gino-import-") as directory:
|
|
extract_paper_code(c, directory)
|
|
from src.data.cfd_datamodule import CFDSDFDataModule
|
|
from src.networks.SDFFNOGNO import SDFFNOGNO
|
|
print(json.dumps(dict(status="PASS", imports=[CFDSDFDataModule.__name__, SDFFNOGNO.__name__],
|
|
models_created=False, arrays_loaded=False)))
|
|
else:
|
|
c = authorized_child(arg.root)
|
|
work(arg.root, c)
|