Files
MeltMind/tools/verify_archive.py
T

88 lines
4.4 KiB
Python

"""Read-only source verification and reproducible ZIP inventory (stdlib only)."""
import argparse
import collections
import datetime
import fcntl
import hashlib
import json
from pathlib import Path
import re
import resource
import time
import zipfile
EXPECTED_SIZE = 32757812726
EXPECTED_MD5 = "75ec15002234c2cdcdf4c751bdefab85"
NAME = re.compile(r"(\d{8})_([0-9a-f]{24})_step_(\d{3})(_sim)?\.(log|yml|vtu)")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("archive", type=Path)
parser.add_argument("output", type=Path)
args = parser.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
with (args.output / ".verify.lock").open("w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
start = time.monotonic()
before = args.archive.stat()
if before.st_size != EXPECTED_SIZE:
raise ValueError(f"Archive size {before.st_size} != {EXPECTED_SIZE}")
md5, sha256 = hashlib.md5(), hashlib.sha256()
read = 0
with args.archive.open("rb") as stream:
while block := stream.read(8 * 1024 * 1024):
md5.update(block)
sha256.update(block)
read += len(block)
if read % (4 * 1024**3) == 0:
print(f"Hashed {read}/{before.st_size} bytes", flush=True)
if md5.hexdigest() != EXPECTED_MD5:
raise ValueError(f"MD5 mismatch: {md5.hexdigest()}")
rows, cases, groups = [], collections.defaultdict(set), collections.defaultdict(set)
with zipfile.ZipFile(args.archive) as archive:
infos = archive.infolist()
if len({i.filename for i in infos}) != len(infos):
raise ValueError("Duplicate ZIP member name")
for info in infos:
match = NAME.fullmatch(info.filename)
if match is None or info.flag_bits & 1:
raise ValueError(f"Unexpected/encrypted member: {info.filename}")
prefix, token, step, sim, extension = match.groups()
if bool(sim) != (extension == "vtu"):
raise ValueError(info.filename)
case = f"{prefix}_{token}_step_{step}"
cases[case].add(extension)
groups[token].add(case)
rows.append(dict(name=info.filename, case=case, source_token=token,
extension=extension, bytes=info.file_size,
compressed_bytes=info.compress_size,
crc32=f"{info.CRC:08x}", offset=info.header_offset))
if len(rows) != 1872 or len(cases) != 624:
raise ValueError("Unexpected member/case counts")
if any(v != {"log", "yml", "vtu"} for v in cases.values()):
raise ValueError("Incomplete triplet")
after = args.archive.stat()
if (before.st_ino, before.st_size, before.st_mtime_ns) != (after.st_ino, after.st_size, after.st_mtime_ns):
raise ValueError("Archive changed during verification")
(args.output / "zip_inventory.json").write_text(json.dumps(rows, indent=2) + "\n")
(args.output / "source_groups.json").write_text(json.dumps({k: sorted(v) for k, v in sorted(groups.items())}, indent=2) + "\n")
result = dict(status="official_size_and_md5_pass", source=str(args.archive.resolve()),
source_url="https://zenodo.org/records/18598121", bytes=read,
md5=md5.hexdigest(), sha256=sha256.hexdigest(),
mtime_ns=after.st_mtime_ns, inode=after.st_ino,
members=len(rows), triplets=len(cases), source_tokens=len(groups),
max_group=max(map(len, groups.values())),
total_uncompressed_bytes=sum(r["bytes"] for r in rows),
finished_utc=datetime.datetime.now(datetime.timezone.utc).isoformat(),
wall_seconds=time.monotonic()-start,
peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
crc_scope="ZIP stored CRC inventory only; selected members checked on read; no full decompression",
grouping_scope="Filename source tokens, not verified independent CAD families")
(args.output / "archive_verification.json").write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps(result, indent=2), flush=True)
if __name__ == "__main__":
main()