94 lines
5.5 KiB
Python
94 lines
5.5 KiB
Python
"""Full-corpus geometry-only screening; no split freezing or duplicate deletion."""
|
|
import argparse
|
|
from collections import defaultdict
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import resource
|
|
import time
|
|
from xml.etree import ElementTree as ET
|
|
from zipfile import ZipFile
|
|
import numpy as np
|
|
from audit_samples import decode_array, log_summary, number
|
|
|
|
|
|
def geometry_prefix(stream):
|
|
points=None
|
|
for event,e in ET.iterparse(stream,events=("start","end")):
|
|
if event=="start" and e.tag=="VTKFile":
|
|
if e.get("byte_order")!="LittleEndian" or e.get("compressor")!="vtkZLibDataCompressor":
|
|
raise ValueError("Unsupported encoding")
|
|
header=e.get("header_type","UInt32")
|
|
elif event=="start" and e.tag=="Piece":
|
|
counts=dict(points=int(e.attrib["NumberOfPoints"]),cells=int(e.attrib["NumberOfCells"]))
|
|
elif event=="end" and e.tag=="DataArray" and e.get("Name")=="Points":
|
|
points=decode_array(e,header)
|
|
if points.shape!=(counts["points"],3) or not np.isfinite(points).all():
|
|
raise ValueError("Invalid geometry points")
|
|
return points,counts
|
|
raise ValueError("Points not found")
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument("archive",type=Path);p.add_argument("output",type=Path)
|
|
a=p.parse_args();a.output.mkdir(parents=True,exist_ok=True)
|
|
verified=json.loads((a.output/"archive_verification.json").read_text());st=a.archive.stat()
|
|
if (st.st_size,st.st_mtime_ns)!=(verified["bytes"],verified["mtime_ns"]):raise ValueError("Stale verification")
|
|
start=time.monotonic();rows=[];exact=defaultdict(list)
|
|
with ZipFile(a.archive) as z:
|
|
for n in sorted(n for n in z.namelist() if n.endswith(".yml")):
|
|
stem=n[:-4]
|
|
log=z.read(stem+".log").decode().replace("\r","")
|
|
with z.open(stem+"_sim.vtu") as stream: points,counts=geometry_prefix(stream)
|
|
ordered=points[np.lexsort(points.T[::-1])]
|
|
# Exact sorted coordinate bytes: a conservative same-point-cloud flag.
|
|
digest=hashlib.sha256(ordered.astype("<f8").tobytes()).hexdigest()
|
|
exact[digest].append(stem)
|
|
centered=points-points.mean(0)
|
|
covariance=centered.T@centered/len(points)
|
|
eig=np.linalg.eigvalsh(covariance)
|
|
bb=points.max(0)-points.min(0)
|
|
logs=log_summary(log)
|
|
row=dict(case=stem,source_token=stem.split("_")[1],**counts,
|
|
bbox_size_raw=bb.tolist(),covariance_eigenvalues=eig.tolist(),
|
|
exact_sorted_points_sha256=digest,
|
|
log_nodes=number(log,"Number of tetrahedron nodes"),
|
|
log_elements=number(log,"Number of tetrahedral elements"),
|
|
log_volume=logs["Total volume"],melt=logs["Melt temperature"],mold=logs["Mold temperature"],
|
|
configured_fill_time=logs["Fill time"],vp_time=logs["Time at velocity/pressure switch-over"],
|
|
vp_pressure=logs["Injection pressure at velocity/pressure switch-over"])
|
|
rows.append(row)
|
|
if len(rows)%100==0: print(f"Geometry/log census {len(rows)}/624",flush=True)
|
|
# Coarse scale-independent covariance spectrum and extent ratios only screen
|
|
# candidate pairs. They are not invariant to mesh density or a proof of identity.
|
|
features=[]
|
|
for r in rows:
|
|
ev=np.asarray(r["covariance_eigenvalues"]);bb=np.sort(r["bbox_size_raw"])
|
|
features.append(np.r_[ev[:2]/max(ev[-1],1e-30),bb[:2]/max(bb[-1],1e-30)])
|
|
features=np.array(features)
|
|
candidates=[]
|
|
for i,r in enumerate(rows):
|
|
dist=np.linalg.norm(features-features[i],axis=1)
|
|
dist[[j for j,s in enumerate(rows) if s["source_token"]==r["source_token"]]]=np.inf
|
|
j=int(np.argmin(dist))
|
|
if i<j: candidates.append(dict(a=r["case"],b=rows[j]["case"],descriptor_distance=float(dist[j])))
|
|
candidates=sorted(candidates,key=lambda r:r["descriptor_distance"])[:30]
|
|
mismatches=[r["case"] for r in rows if (r["log_nodes"] and r["log_nodes"]["value"]!=r["points"]) or (r["log_elements"] and r["log_elements"]["value"]!=r["cells"])]
|
|
summary=dict(cases=len(rows),source_tokens=len({r["source_token"] for r in rows}),
|
|
point_quantiles=np.quantile([r["points"] for r in rows],[0,.5,.9,1]).tolist(),
|
|
cell_quantiles=np.quantile([r["cells"] for r in rows],[0,.5,.9,1]).tolist(),
|
|
missing_log_mesh=[r["case"] for r in rows if not r["log_nodes"] or not r["log_elements"]],
|
|
log_vtu_mesh_count_mismatches=mismatches,
|
|
exact_same_point_cloud_sets=[v for v in exact.values() if len(v)>1],
|
|
candidate_pairs=candidates,
|
|
split_status="not_frozen_pending_source_semantics_and_near_duplicate_review",
|
|
near_duplicate_scope="Coarse descriptors flag candidates only; cannot certify distinct geometry families",
|
|
crc_scope="all LOG members fully read/CRC checked; VTU prefix only, source archive MD5 already verified",
|
|
wall_seconds=time.monotonic()-start,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
|
(a.output/"geometry_log_census.json").write_text(json.dumps(rows,indent=2)+"\n")
|
|
(a.output/"geometry_screening.json").write_text(json.dumps(summary,indent=2)+"\n")
|
|
print(json.dumps({k:v for k,v in summary.items() if k not in ("candidate_pairs","exact_same_point_cloud_sets")},indent=2),flush=True)
|
|
|
|
|
|
if __name__=="__main__":main()
|