"""Full-corpus geometry-only screening; no split freezing or duplicate deletion.""" import argparse from collections import defaultdict import hashlib import json from pathlib import Path import resource import time from xml.etree import ElementTree as ET from zipfile import ZipFile import numpy as np from audit_samples import decode_array, log_summary, number def geometry_prefix(stream): points=None for event,e in ET.iterparse(stream,events=("start","end")): if event=="start" and e.tag=="VTKFile": if e.get("byte_order")!="LittleEndian" or e.get("compressor")!="vtkZLibDataCompressor": raise ValueError("Unsupported encoding") header=e.get("header_type","UInt32") elif event=="start" and e.tag=="Piece": counts=dict(points=int(e.attrib["NumberOfPoints"]),cells=int(e.attrib["NumberOfCells"])) elif event=="end" and e.tag=="DataArray" and e.get("Name")=="Points": points=decode_array(e,header) if points.shape!=(counts["points"],3) or not np.isfinite(points).all(): raise ValueError("Invalid geometry points") return points,counts raise ValueError("Points not found") def main(): p=argparse.ArgumentParser();p.add_argument("archive",type=Path);p.add_argument("output",type=Path) a=p.parse_args();a.output.mkdir(parents=True,exist_ok=True) verified=json.loads((a.output/"archive_verification.json").read_text());st=a.archive.stat() if (st.st_size,st.st_mtime_ns)!=(verified["bytes"],verified["mtime_ns"]):raise ValueError("Stale verification") start=time.monotonic();rows=[];exact=defaultdict(list) with ZipFile(a.archive) as z: for n in sorted(n for n in z.namelist() if n.endswith(".yml")): stem=n[:-4] log=z.read(stem+".log").decode().replace("\r","") with z.open(stem+"_sim.vtu") as stream: points,counts=geometry_prefix(stream) ordered=points[np.lexsort(points.T[::-1])] # Exact sorted coordinate bytes: a conservative same-point-cloud flag. digest=hashlib.sha256(ordered.astype("1], candidate_pairs=candidates, split_status="not_frozen_pending_source_semantics_and_near_duplicate_review", near_duplicate_scope="Coarse descriptors flag candidates only; cannot certify distinct geometry families", crc_scope="all LOG members fully read/CRC checked; VTU prefix only, source archive MD5 already verified", wall_seconds=time.monotonic()-start,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) (a.output/"geometry_log_census.json").write_text(json.dumps(rows,indent=2)+"\n") (a.output/"geometry_screening.json").write_text(json.dumps(summary,indent=2)+"\n") print(json.dumps({k:v for k,v in summary.items() if k not in ("candidate_pairs","exact_same_point_cloud_sets")},indent=2),flush=True) if __name__=="__main__":main()