226 lines
11 KiB
Python
226 lines
11 KiB
Python
"""Audit the GINO paper's ShapeNet files and the processed Zenodo v2 archive."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
import tarfile
|
|
from zipfile import ZipFile
|
|
|
|
import numpy as np
|
|
|
|
|
|
ZENODO_MD5 = "24a46fe791085201d48ee5db7b6cfc86"
|
|
LOADER_COMMIT = "a4133fc897dd3a2d01fb33b81bdfe63c3031c725"
|
|
RETAINED_PRESSURE_SLICES = [[0, 16], [112, None]]
|
|
|
|
|
|
def digest(contents):
|
|
return hashlib.sha256(contents).hexdigest()
|
|
|
|
|
|
def read_member(archive, member):
|
|
if not member.isfile():
|
|
raise ValueError(f"Not a regular file: {member.name}")
|
|
with archive.extractfile(member) as stream:
|
|
return stream.read()
|
|
|
|
|
|
def manifest_ids(contents):
|
|
# The official loader reads the first line, splits on commas and strips
|
|
# trailing whitespace without sorting or reshuffling the case IDs.
|
|
with io.StringIO(contents.decode("utf-8")) as stream:
|
|
ids = [case.rstrip() for case in stream.readline().split(",")]
|
|
if not all(case and PurePosixPath(case).name == case and case not in (".", "..") for case in ids):
|
|
raise ValueError("Invalid case ID in original manifest")
|
|
if len(set(ids)) != len(ids):
|
|
raise ValueError("Duplicate case ID in original manifest")
|
|
return ids
|
|
|
|
|
|
def vertex_count(contents):
|
|
with io.BytesIO(contents) as ply:
|
|
if ply.readline().strip() != b"ply":
|
|
raise ValueError("Invalid PLY signature")
|
|
counts = []
|
|
for line in ply:
|
|
fields = line.split()
|
|
if fields[:2] == [b"element", b"vertex"]:
|
|
counts.append(int(fields[2]))
|
|
if fields == [b"end_header"]:
|
|
if len(counts) == 1 and counts[0] > 0 and ply.read(1):
|
|
return counts[0]
|
|
break
|
|
raise ValueError("Missing or invalid PLY vertex count")
|
|
|
|
|
|
def pressure_mapping(contents, vertices):
|
|
raw = np.load(io.BytesIO(contents), allow_pickle=False)
|
|
if raw.ndim == 2 and raw.shape[0] == 1:
|
|
raw = raw[0]
|
|
if raw.ndim != 1 or len(raw) <= 112 or not np.issubdtype(raw.dtype, np.floating):
|
|
raise ValueError("Unexpected original pressure shape or dtype")
|
|
# #518 removes precisely these extra pressure entries, after normalization.
|
|
# Applying the same index selection to raw values identifies their mapping.
|
|
retained = np.concatenate([raw[:16], raw[112:]])
|
|
retained32 = np.asarray(retained, dtype="<f4")
|
|
if not 0 < len(retained) <= vertices or not np.isfinite(raw).all() or not np.isfinite(retained32).all():
|
|
raise ValueError("Corrected pressure cannot be matched to the first output vertices")
|
|
return dict(raw_pressure_count=len(raw), output_vertices=len(retained),
|
|
raw_pressure_dtype=str(raw.dtype), pressure_count_excluded=96,
|
|
retained_pressure_f32_sha256=digest(retained32.tobytes()))
|
|
|
|
|
|
def paper_split(raw_manifest):
|
|
ids = [int(token) for token in raw_manifest.decode("utf-8").split("\n")]
|
|
if len(ids) != 611 or len(set(ids)) != len(ids):
|
|
raise ValueError("Paper manifest must contain 611 distinct mesh IDs")
|
|
return ids[:500], ids[-111:]
|
|
|
|
|
|
def audit_paper(archive_path):
|
|
archive_hash = hashlib.sha256()
|
|
with archive_path.open("rb") as stream:
|
|
for block in iter(lambda: stream.read(4 * 2**20), b""):
|
|
archive_hash.update(block)
|
|
with ZipFile(archive_path) as archive:
|
|
manifests = [name for name in archive.namelist()
|
|
if PurePosixPath(name).name == "watertight_meshes.txt"]
|
|
if len(manifests) != 1 or len(set(archive.namelist())) != len(archive.namelist()):
|
|
raise ValueError("Paper archive manifest missing or duplicate member name")
|
|
manifest, = manifests
|
|
parent = PurePosixPath(manifest).parent
|
|
raw_manifest = archive.read(manifest)
|
|
bounds = archive.read(str(parent / "watertight_global_bounds.txt"))
|
|
values = np.loadtxt(io.BytesIO(bounds))
|
|
if values.shape != (2, 3) or not np.isfinite(values).all() or not np.all(values[0] < values[1]):
|
|
raise ValueError("Invalid original global bounds")
|
|
train, test = paper_split(raw_manifest)
|
|
rows = []
|
|
for role, cases in (("train", train), ("test", test)):
|
|
for pos, case in enumerate(cases):
|
|
mesh = archive.read(str(parent / "data" / f"mesh_{case:03d}.ply"))
|
|
pressure = archive.read(str(parent / "data" / f"press_{case:03d}.npy"))
|
|
vertices = vertex_count(mesh)
|
|
mapping = pressure_mapping(pressure, vertices)
|
|
if vertices != 3586 or mapping["raw_pressure_count"] != 3682 or mapping["output_vertices"] != vertices:
|
|
raise ValueError(f"Original loader requires one corrected pressure for every vertex in {case}")
|
|
rows.append(dict(case=case, split=role, manifest_position=pos, mesh_vertices=vertices,
|
|
mesh_sha256=digest(mesh), pressure_sha256=digest(pressure), **mapping))
|
|
return dict(status="PAPER_ARCHIVE_SPLIT_AND_MAPPING_AUDITED", archive_sha256=archive_hash.hexdigest(),
|
|
source_drive_id="1Vb740MGw7dMN943bRTNvy_f_qFMB9sjX",
|
|
manifest_sha256=digest(raw_manifest), global_bounds_sha256=digest(bounds),
|
|
raw_global_bounds=values.tolist(), pressure_selection=RETAINED_PRESSURE_SLICES,
|
|
selected_counts={"train": 500, "test": 111}, rows=rows)
|
|
|
|
|
|
def compare(paper, zenodo):
|
|
original = {r["case"]: r for r in paper["rows"]}
|
|
current = {int(r["case"]): r for r in zenodo["rows"] if r["case"].isascii() and r["case"].isdigit()}
|
|
if len(current) != len(zenodo["rows"]):
|
|
raise ValueError("Zenodo sample IDs do not unambiguously identify numeric paper cases")
|
|
missing = sorted(original.keys() - current.keys())
|
|
added = sorted(current.keys() - original.keys())
|
|
joined = sorted(original.keys() & current.keys())
|
|
fields = ("split", "manifest_position", "mesh_vertices", "mesh_sha256",
|
|
"retained_pressure_f32_sha256")
|
|
differences = {k: [i for i in joined if original[i][k] != current[i][k]] for k in fields}
|
|
matched = not any((missing, added, *differences.values()))
|
|
return dict(status="MATCHED_EXACTLY" if matched else "IDENTITY_DIFFERENCES_REQUIRE_REVIEW",
|
|
paper_archive_sha256=paper["archive_sha256"], v2_archive_sha256=zenodo["archive_sha256"],
|
|
missing_from_v2=missing, added_in_v2=added, differences=differences,
|
|
joined_cases=len(joined), original_paper_case_ids_verified=matched)
|
|
|
|
|
|
def audit(archive_path):
|
|
with archive_path.open("rb") as stream:
|
|
md5 = hashlib.md5()
|
|
sha256 = hashlib.sha256()
|
|
for block in iter(lambda: stream.read(4 * 2**20), b""):
|
|
md5.update(block)
|
|
sha256.update(block)
|
|
if md5.hexdigest() != ZENODO_MD5:
|
|
raise ValueError("Bytes do not match Zenodo 13936501 v2")
|
|
archive_sha = sha256.hexdigest()
|
|
with tarfile.open(archive_path, "r:gz") as archive:
|
|
members = {}
|
|
for member in archive:
|
|
name = PurePosixPath(member.name)
|
|
if name.is_absolute() or ".." in name.parts or name in members:
|
|
raise ValueError(f"Invalid or duplicate archive path: {name}")
|
|
members[name] = member
|
|
manifests = [name for name in members if name.name == "train.txt"
|
|
and name.with_name("test.txt") in members]
|
|
if len(manifests) != 1:
|
|
raise ValueError("Expected one official train/test manifest pair")
|
|
train_path, = manifests
|
|
raw_train = read_member(archive, members[train_path])
|
|
raw_test = read_member(archive, members[train_path.with_name("test.txt")])
|
|
train, test = manifest_ids(raw_train), manifest_ids(raw_test)
|
|
if len(train) < 500 or len(test) < 111 or set(train) & set(test):
|
|
raise ValueError("Official split cannot supply disjoint 500/111 cases")
|
|
data_root = train_path.parent / "data"
|
|
needed = {}
|
|
for role, cases in (("train", train[:500]), ("test", test[:111])):
|
|
for position, case in enumerate(cases):
|
|
mesh = data_root / case / "tri_mesh.ply"
|
|
pressure = data_root / case / "press.npy"
|
|
needed[mesh] = (role, position, case, "mesh")
|
|
needed[pressure] = (role, position, case, "pressure")
|
|
if set(needed) - members.keys():
|
|
raise ValueError("Official split has missing mesh or pressure payloads")
|
|
|
|
# Stream the archive again in archive order instead of decompressing the
|
|
# gzip prefix again for each of the 1,222 payloads.
|
|
rows = {}
|
|
with tarfile.open(archive_path, "r|gz") as archive:
|
|
for member in archive:
|
|
name = PurePosixPath(member.name)
|
|
if name not in needed:
|
|
continue
|
|
role, position, case, field = needed[name]
|
|
row = rows.setdefault(case, dict(case=case, split=role, manifest_position=position))
|
|
contents = read_member(archive, member)
|
|
row[field + "_sha256"] = digest(contents)
|
|
if field == "mesh":
|
|
row["mesh_vertices"] = vertex_count(contents)
|
|
else:
|
|
# The mesh may occur after the pressure in the source stream.
|
|
row.update(pressure_mapping(contents, float("inf")))
|
|
if len(rows) != 611 or any(
|
|
"mesh_vertices" not in row or "output_vertices" not in row
|
|
or row["output_vertices"] > row["mesh_vertices"] for row in rows.values()
|
|
):
|
|
raise ValueError("Mesh and corrected pressure support differ")
|
|
|
|
return dict(status="V2_OFFICIAL_LOADER_MAPPING_AUDITED",
|
|
zenodo_record="13936501", archive_md5=ZENODO_MD5, archive_sha256=archive_sha,
|
|
official_loader_commit=LOADER_COMMIT, original_paper_case_ids_verified=False,
|
|
pressure_selection=RETAINED_PRESSURE_SLICES,
|
|
manifest_sha256={"train": digest(raw_train), "test": digest(raw_test)},
|
|
manifest_counts={"train": len(train), "test": len(test)},
|
|
selected_counts={"train": 500, "test": 111},
|
|
rows=[rows[case] for case in train[:500] + test[:111]])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("archive", type=Path)
|
|
parser.add_argument("receipt", type=Path)
|
|
parser.add_argument("--format", choices=("paper", "zenodo"), default="zenodo")
|
|
parser.add_argument("--paper-receipt", type=Path)
|
|
args = parser.parse_args()
|
|
receipt = audit_paper(args.archive) if args.format == "paper" else audit(args.archive)
|
|
if args.paper_receipt:
|
|
if args.format != "zenodo":
|
|
parser.error("--paper-receipt applies to a Zenodo audit")
|
|
original = json.loads(args.paper_receipt.read_bytes())
|
|
if original.get("status") != "PAPER_ARCHIVE_SPLIT_AND_MAPPING_AUDITED":
|
|
raise ValueError("Original paper audit required for comparison")
|
|
receipt["paper_comparison"] = compare(original, receipt)
|
|
receipt["original_paper_case_ids_verified"] = receipt["paper_comparison"]["original_paper_case_ids_verified"]
|
|
with args.receipt.open("x") as stream:
|
|
json.dump(receipt, stream, indent=2, allow_nan=False)
|
|
stream.write("\n")
|