Files
MeltMind/tools/audit_track_b_shapenet611.py
T

226 lines
11 KiB
Python

"""Audit the GINO paper's ShapeNet files and the processed Zenodo v2 archive."""
import argparse
import hashlib
import io
import json
from pathlib import Path, PurePosixPath
import tarfile
from zipfile import ZipFile
import numpy as np
ZENODO_MD5 = "24a46fe791085201d48ee5db7b6cfc86"
LOADER_COMMIT = "a4133fc897dd3a2d01fb33b81bdfe63c3031c725"
RETAINED_PRESSURE_SLICES = [[0, 16], [112, None]]
def digest(contents):
return hashlib.sha256(contents).hexdigest()
def read_member(archive, member):
if not member.isfile():
raise ValueError(f"Not a regular file: {member.name}")
with archive.extractfile(member) as stream:
return stream.read()
def manifest_ids(contents):
# The official loader reads the first line, splits on commas and strips
# trailing whitespace without sorting or reshuffling the case IDs.
with io.StringIO(contents.decode("utf-8")) as stream:
ids = [case.rstrip() for case in stream.readline().split(",")]
if not all(case and PurePosixPath(case).name == case and case not in (".", "..") for case in ids):
raise ValueError("Invalid case ID in original manifest")
if len(set(ids)) != len(ids):
raise ValueError("Duplicate case ID in original manifest")
return ids
def vertex_count(contents):
with io.BytesIO(contents) as ply:
if ply.readline().strip() != b"ply":
raise ValueError("Invalid PLY signature")
counts = []
for line in ply:
fields = line.split()
if fields[:2] == [b"element", b"vertex"]:
counts.append(int(fields[2]))
if fields == [b"end_header"]:
if len(counts) == 1 and counts[0] > 0 and ply.read(1):
return counts[0]
break
raise ValueError("Missing or invalid PLY vertex count")
def pressure_mapping(contents, vertices):
raw = np.load(io.BytesIO(contents), allow_pickle=False)
if raw.ndim == 2 and raw.shape[0] == 1:
raw = raw[0]
if raw.ndim != 1 or len(raw) <= 112 or not np.issubdtype(raw.dtype, np.floating):
raise ValueError("Unexpected original pressure shape or dtype")
# #518 removes precisely these extra pressure entries, after normalization.
# Applying the same index selection to raw values identifies their mapping.
retained = np.concatenate([raw[:16], raw[112:]])
retained32 = np.asarray(retained, dtype="<f4")
if not 0 < len(retained) <= vertices or not np.isfinite(raw).all() or not np.isfinite(retained32).all():
raise ValueError("Corrected pressure cannot be matched to the first output vertices")
return dict(raw_pressure_count=len(raw), output_vertices=len(retained),
raw_pressure_dtype=str(raw.dtype), pressure_count_excluded=96,
retained_pressure_f32_sha256=digest(retained32.tobytes()))
def paper_split(raw_manifest):
ids = [int(token) for token in raw_manifest.decode("utf-8").split("\n")]
if len(ids) != 611 or len(set(ids)) != len(ids):
raise ValueError("Paper manifest must contain 611 distinct mesh IDs")
return ids[:500], ids[-111:]
def audit_paper(archive_path):
archive_hash = hashlib.sha256()
with archive_path.open("rb") as stream:
for block in iter(lambda: stream.read(4 * 2**20), b""):
archive_hash.update(block)
with ZipFile(archive_path) as archive:
manifests = [name for name in archive.namelist()
if PurePosixPath(name).name == "watertight_meshes.txt"]
if len(manifests) != 1 or len(set(archive.namelist())) != len(archive.namelist()):
raise ValueError("Paper archive manifest missing or duplicate member name")
manifest, = manifests
parent = PurePosixPath(manifest).parent
raw_manifest = archive.read(manifest)
bounds = archive.read(str(parent / "watertight_global_bounds.txt"))
values = np.loadtxt(io.BytesIO(bounds))
if values.shape != (2, 3) or not np.isfinite(values).all() or not np.all(values[0] < values[1]):
raise ValueError("Invalid original global bounds")
train, test = paper_split(raw_manifest)
rows = []
for role, cases in (("train", train), ("test", test)):
for pos, case in enumerate(cases):
mesh = archive.read(str(parent / "data" / f"mesh_{case:03d}.ply"))
pressure = archive.read(str(parent / "data" / f"press_{case:03d}.npy"))
vertices = vertex_count(mesh)
mapping = pressure_mapping(pressure, vertices)
if vertices != 3586 or mapping["raw_pressure_count"] != 3682 or mapping["output_vertices"] != vertices:
raise ValueError(f"Original loader requires one corrected pressure for every vertex in {case}")
rows.append(dict(case=case, split=role, manifest_position=pos, mesh_vertices=vertices,
mesh_sha256=digest(mesh), pressure_sha256=digest(pressure), **mapping))
return dict(status="PAPER_ARCHIVE_SPLIT_AND_MAPPING_AUDITED", archive_sha256=archive_hash.hexdigest(),
source_drive_id="1Vb740MGw7dMN943bRTNvy_f_qFMB9sjX",
manifest_sha256=digest(raw_manifest), global_bounds_sha256=digest(bounds),
raw_global_bounds=values.tolist(), pressure_selection=RETAINED_PRESSURE_SLICES,
selected_counts={"train": 500, "test": 111}, rows=rows)
def compare(paper, zenodo):
original = {r["case"]: r for r in paper["rows"]}
current = {int(r["case"]): r for r in zenodo["rows"] if r["case"].isascii() and r["case"].isdigit()}
if len(current) != len(zenodo["rows"]):
raise ValueError("Zenodo sample IDs do not unambiguously identify numeric paper cases")
missing = sorted(original.keys() - current.keys())
added = sorted(current.keys() - original.keys())
joined = sorted(original.keys() & current.keys())
fields = ("split", "manifest_position", "mesh_vertices", "mesh_sha256",
"retained_pressure_f32_sha256")
differences = {k: [i for i in joined if original[i][k] != current[i][k]] for k in fields}
matched = not any((missing, added, *differences.values()))
return dict(status="MATCHED_EXACTLY" if matched else "IDENTITY_DIFFERENCES_REQUIRE_REVIEW",
paper_archive_sha256=paper["archive_sha256"], v2_archive_sha256=zenodo["archive_sha256"],
missing_from_v2=missing, added_in_v2=added, differences=differences,
joined_cases=len(joined), original_paper_case_ids_verified=matched)
def audit(archive_path):
with archive_path.open("rb") as stream:
md5 = hashlib.md5()
sha256 = hashlib.sha256()
for block in iter(lambda: stream.read(4 * 2**20), b""):
md5.update(block)
sha256.update(block)
if md5.hexdigest() != ZENODO_MD5:
raise ValueError("Bytes do not match Zenodo 13936501 v2")
archive_sha = sha256.hexdigest()
with tarfile.open(archive_path, "r:gz") as archive:
members = {}
for member in archive:
name = PurePosixPath(member.name)
if name.is_absolute() or ".." in name.parts or name in members:
raise ValueError(f"Invalid or duplicate archive path: {name}")
members[name] = member
manifests = [name for name in members if name.name == "train.txt"
and name.with_name("test.txt") in members]
if len(manifests) != 1:
raise ValueError("Expected one official train/test manifest pair")
train_path, = manifests
raw_train = read_member(archive, members[train_path])
raw_test = read_member(archive, members[train_path.with_name("test.txt")])
train, test = manifest_ids(raw_train), manifest_ids(raw_test)
if len(train) < 500 or len(test) < 111 or set(train) & set(test):
raise ValueError("Official split cannot supply disjoint 500/111 cases")
data_root = train_path.parent / "data"
needed = {}
for role, cases in (("train", train[:500]), ("test", test[:111])):
for position, case in enumerate(cases):
mesh = data_root / case / "tri_mesh.ply"
pressure = data_root / case / "press.npy"
needed[mesh] = (role, position, case, "mesh")
needed[pressure] = (role, position, case, "pressure")
if set(needed) - members.keys():
raise ValueError("Official split has missing mesh or pressure payloads")
# Stream the archive again in archive order instead of decompressing the
# gzip prefix again for each of the 1,222 payloads.
rows = {}
with tarfile.open(archive_path, "r|gz") as archive:
for member in archive:
name = PurePosixPath(member.name)
if name not in needed:
continue
role, position, case, field = needed[name]
row = rows.setdefault(case, dict(case=case, split=role, manifest_position=position))
contents = read_member(archive, member)
row[field + "_sha256"] = digest(contents)
if field == "mesh":
row["mesh_vertices"] = vertex_count(contents)
else:
# The mesh may occur after the pressure in the source stream.
row.update(pressure_mapping(contents, float("inf")))
if len(rows) != 611 or any(
"mesh_vertices" not in row or "output_vertices" not in row
or row["output_vertices"] > row["mesh_vertices"] for row in rows.values()
):
raise ValueError("Mesh and corrected pressure support differ")
return dict(status="V2_OFFICIAL_LOADER_MAPPING_AUDITED",
zenodo_record="13936501", archive_md5=ZENODO_MD5, archive_sha256=archive_sha,
official_loader_commit=LOADER_COMMIT, original_paper_case_ids_verified=False,
pressure_selection=RETAINED_PRESSURE_SLICES,
manifest_sha256={"train": digest(raw_train), "test": digest(raw_test)},
manifest_counts={"train": len(train), "test": len(test)},
selected_counts={"train": 500, "test": 111},
rows=[rows[case] for case in train[:500] + test[:111]])
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("archive", type=Path)
parser.add_argument("receipt", type=Path)
parser.add_argument("--format", choices=("paper", "zenodo"), default="zenodo")
parser.add_argument("--paper-receipt", type=Path)
args = parser.parse_args()
receipt = audit_paper(args.archive) if args.format == "paper" else audit(args.archive)
if args.paper_receipt:
if args.format != "zenodo":
parser.error("--paper-receipt applies to a Zenodo audit")
original = json.loads(args.paper_receipt.read_bytes())
if original.get("status") != "PAPER_ARCHIVE_SPLIT_AND_MAPPING_AUDITED":
raise ValueError("Original paper audit required for comparison")
receipt["paper_comparison"] = compare(original, receipt)
receipt["original_paper_case_ids_verified"] = receipt["paper_comparison"]["original_paper_case_ids_verified"]
with args.receipt.open("x") as stream:
json.dump(receipt, stream, indent=2, allow_nan=False)
stream.write("\n")