Files
MeltMind/tools/census_geometry.py

94 lines
5.5 KiB
Python

"""Full-corpus geometry-only screening; no split freezing or duplicate deletion."""
import argparse
from collections import defaultdict
import hashlib
import json
from pathlib import Path
import resource
import time
from xml.etree import ElementTree as ET
from zipfile import ZipFile
import numpy as np
from audit_samples import decode_array, log_summary, number
def geometry_prefix(stream):
points=None
for event,e in ET.iterparse(stream,events=("start","end")):
if event=="start" and e.tag=="VTKFile":
if e.get("byte_order")!="LittleEndian" or e.get("compressor")!="vtkZLibDataCompressor":
raise ValueError("Unsupported encoding")
header=e.get("header_type","UInt32")
elif event=="start" and e.tag=="Piece":
counts=dict(points=int(e.attrib["NumberOfPoints"]),cells=int(e.attrib["NumberOfCells"]))
elif event=="end" and e.tag=="DataArray" and e.get("Name")=="Points":
points=decode_array(e,header)
if points.shape!=(counts["points"],3) or not np.isfinite(points).all():
raise ValueError("Invalid geometry points")
return points,counts
raise ValueError("Points not found")
def main():
p=argparse.ArgumentParser();p.add_argument("archive",type=Path);p.add_argument("output",type=Path)
a=p.parse_args();a.output.mkdir(parents=True,exist_ok=True)
verified=json.loads((a.output/"archive_verification.json").read_text());st=a.archive.stat()
if (st.st_size,st.st_mtime_ns)!=(verified["bytes"],verified["mtime_ns"]):raise ValueError("Stale verification")
start=time.monotonic();rows=[];exact=defaultdict(list)
with ZipFile(a.archive) as z:
for n in sorted(n for n in z.namelist() if n.endswith(".yml")):
stem=n[:-4]
log=z.read(stem+".log").decode().replace("\r","")
with z.open(stem+"_sim.vtu") as stream: points,counts=geometry_prefix(stream)
ordered=points[np.lexsort(points.T[::-1])]
# Exact sorted coordinate bytes: a conservative same-point-cloud flag.
digest=hashlib.sha256(ordered.astype("<f8").tobytes()).hexdigest()
exact[digest].append(stem)
centered=points-points.mean(0)
covariance=centered.T@centered/len(points)
eig=np.linalg.eigvalsh(covariance)
bb=points.max(0)-points.min(0)
logs=log_summary(log)
row=dict(case=stem,source_token=stem.split("_")[1],**counts,
bbox_size_raw=bb.tolist(),covariance_eigenvalues=eig.tolist(),
exact_sorted_points_sha256=digest,
log_nodes=number(log,"Number of tetrahedron nodes"),
log_elements=number(log,"Number of tetrahedral elements"),
log_volume=logs["Total volume"],melt=logs["Melt temperature"],mold=logs["Mold temperature"],
configured_fill_time=logs["Fill time"],vp_time=logs["Time at velocity/pressure switch-over"],
vp_pressure=logs["Injection pressure at velocity/pressure switch-over"])
rows.append(row)
if len(rows)%100==0: print(f"Geometry/log census {len(rows)}/624",flush=True)
# Coarse scale-independent covariance spectrum and extent ratios only screen
# candidate pairs. They are not invariant to mesh density or a proof of identity.
features=[]
for r in rows:
ev=np.asarray(r["covariance_eigenvalues"]);bb=np.sort(r["bbox_size_raw"])
features.append(np.r_[ev[:2]/max(ev[-1],1e-30),bb[:2]/max(bb[-1],1e-30)])
features=np.array(features)
candidates=[]
for i,r in enumerate(rows):
dist=np.linalg.norm(features-features[i],axis=1)
dist[[j for j,s in enumerate(rows) if s["source_token"]==r["source_token"]]]=np.inf
j=int(np.argmin(dist))
if i<j: candidates.append(dict(a=r["case"],b=rows[j]["case"],descriptor_distance=float(dist[j])))
candidates=sorted(candidates,key=lambda r:r["descriptor_distance"])[:30]
mismatches=[r["case"] for r in rows if (r["log_nodes"] and r["log_nodes"]["value"]!=r["points"]) or (r["log_elements"] and r["log_elements"]["value"]!=r["cells"])]
summary=dict(cases=len(rows),source_tokens=len({r["source_token"] for r in rows}),
point_quantiles=np.quantile([r["points"] for r in rows],[0,.5,.9,1]).tolist(),
cell_quantiles=np.quantile([r["cells"] for r in rows],[0,.5,.9,1]).tolist(),
missing_log_mesh=[r["case"] for r in rows if not r["log_nodes"] or not r["log_elements"]],
log_vtu_mesh_count_mismatches=mismatches,
exact_same_point_cloud_sets=[v for v in exact.values() if len(v)>1],
candidate_pairs=candidates,
split_status="not_frozen_pending_source_semantics_and_near_duplicate_review",
near_duplicate_scope="Coarse descriptors flag candidates only; cannot certify distinct geometry families",
crc_scope="all LOG members fully read/CRC checked; VTU prefix only, source archive MD5 already verified",
wall_seconds=time.monotonic()-start,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
(a.output/"geometry_log_census.json").write_text(json.dumps(rows,indent=2)+"\n")
(a.output/"geometry_screening.json").write_text(json.dumps(summary,indent=2)+"\n")
print(json.dumps({k:v for k,v in summary.items() if k not in ("candidate_pairs","exact_same_point_cloud_sets")},indent=2),flush=True)
if __name__=="__main__":main()