88 lines
4.4 KiB
Python
88 lines
4.4 KiB
Python
"""Read-only source verification and reproducible ZIP inventory (stdlib only)."""
|
|
import argparse
|
|
import collections
|
|
import datetime
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import resource
|
|
import time
|
|
import zipfile
|
|
|
|
EXPECTED_SIZE = 32757812726
|
|
EXPECTED_MD5 = "75ec15002234c2cdcdf4c751bdefab85"
|
|
NAME = re.compile(r"(\d{8})_([0-9a-f]{24})_step_(\d{3})(_sim)?\.(log|yml|vtu)")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("archive", type=Path)
|
|
parser.add_argument("output", type=Path)
|
|
args = parser.parse_args()
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
with (args.output / ".verify.lock").open("w") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
start = time.monotonic()
|
|
before = args.archive.stat()
|
|
if before.st_size != EXPECTED_SIZE:
|
|
raise ValueError(f"Archive size {before.st_size} != {EXPECTED_SIZE}")
|
|
md5, sha256 = hashlib.md5(), hashlib.sha256()
|
|
read = 0
|
|
with args.archive.open("rb") as stream:
|
|
while block := stream.read(8 * 1024 * 1024):
|
|
md5.update(block)
|
|
sha256.update(block)
|
|
read += len(block)
|
|
if read % (4 * 1024**3) == 0:
|
|
print(f"Hashed {read}/{before.st_size} bytes", flush=True)
|
|
if md5.hexdigest() != EXPECTED_MD5:
|
|
raise ValueError(f"MD5 mismatch: {md5.hexdigest()}")
|
|
rows, cases, groups = [], collections.defaultdict(set), collections.defaultdict(set)
|
|
with zipfile.ZipFile(args.archive) as archive:
|
|
infos = archive.infolist()
|
|
if len({i.filename for i in infos}) != len(infos):
|
|
raise ValueError("Duplicate ZIP member name")
|
|
for info in infos:
|
|
match = NAME.fullmatch(info.filename)
|
|
if match is None or info.flag_bits & 1:
|
|
raise ValueError(f"Unexpected/encrypted member: {info.filename}")
|
|
prefix, token, step, sim, extension = match.groups()
|
|
if bool(sim) != (extension == "vtu"):
|
|
raise ValueError(info.filename)
|
|
case = f"{prefix}_{token}_step_{step}"
|
|
cases[case].add(extension)
|
|
groups[token].add(case)
|
|
rows.append(dict(name=info.filename, case=case, source_token=token,
|
|
extension=extension, bytes=info.file_size,
|
|
compressed_bytes=info.compress_size,
|
|
crc32=f"{info.CRC:08x}", offset=info.header_offset))
|
|
if len(rows) != 1872 or len(cases) != 624:
|
|
raise ValueError("Unexpected member/case counts")
|
|
if any(v != {"log", "yml", "vtu"} for v in cases.values()):
|
|
raise ValueError("Incomplete triplet")
|
|
after = args.archive.stat()
|
|
if (before.st_ino, before.st_size, before.st_mtime_ns) != (after.st_ino, after.st_size, after.st_mtime_ns):
|
|
raise ValueError("Archive changed during verification")
|
|
(args.output / "zip_inventory.json").write_text(json.dumps(rows, indent=2) + "\n")
|
|
(args.output / "source_groups.json").write_text(json.dumps({k: sorted(v) for k, v in sorted(groups.items())}, indent=2) + "\n")
|
|
result = dict(status="official_size_and_md5_pass", source=str(args.archive.resolve()),
|
|
source_url="https://zenodo.org/records/18598121", bytes=read,
|
|
md5=md5.hexdigest(), sha256=sha256.hexdigest(),
|
|
mtime_ns=after.st_mtime_ns, inode=after.st_ino,
|
|
members=len(rows), triplets=len(cases), source_tokens=len(groups),
|
|
max_group=max(map(len, groups.values())),
|
|
total_uncompressed_bytes=sum(r["bytes"] for r in rows),
|
|
finished_utc=datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
wall_seconds=time.monotonic()-start,
|
|
peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
|
|
crc_scope="ZIP stored CRC inventory only; selected members checked on read; no full decompression",
|
|
grouping_scope="Filename source tokens, not verified independent CAD families")
|
|
(args.output / "archive_verification.json").write_text(json.dumps(result, indent=2) + "\n")
|
|
print(json.dumps(result, indent=2), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|