Files
MeltMind/tools/audit_samples.py

245 lines
13 KiB
Python

"""Bounded Greif-v5 numerical intake. Reads original ZIP; never changes raw data.
Supported input is deliberately narrow: one-piece, little-endian VTU, inline
base64/zlib arrays. Unsupported encodings fail rather than silently guessing.
Format references: VTK XML specification; meshio v5.3.5 VTU reader/writer.
All cached geometry and fields retain exported units, with no implicit conversion.
"""
import argparse
import base64
from collections import Counter, defaultdict
from decimal import Decimal
import hashlib
import json
from pathlib import Path
import re
import resource
import time
from xml.etree import ElementTree as ET
from zipfile import ZipFile
import zlib
import numpy as np
import yaml
DTYPES = {f"{kind}{bits}": np.dtype(f"<{code}{bits//8}")
for kind, code, sizes in [("Float", "f", [32, 64]), ("Int", "i", [8,16,32,64]), ("UInt", "u", [8,16,32,64])]
for bits in sizes}
TIME = re.compile(r"^(.*?)_Block:(\d+)_Time:([0-9.eE+\-]+)s")
def decode_array(element, header_type="UInt32"):
if element.get("format") != "binary":
raise ValueError("Only inline binary arrays are supported")
encoded = "".join((element.text or "").split())
ht = DTYPES[header_type]
if header_type not in ("UInt32", "UInt64"):
raise ValueError("Unsupported header type")
first_chars = 4*((ht.itemsize+2)//3)
nblocks = int(np.frombuffer(base64.b64decode(encoded[:first_chars])[:ht.itemsize], dtype=ht)[0])
if not 0 < nblocks < 1_000_000:
raise ValueError("Invalid compressed block count")
hbytes = (3+nblocks)*ht.itemsize
hchars = 4*((hbytes+2)//3)
header_bytes = base64.b64decode(encoded[:hchars], validate=True)
if len(header_bytes) != hbytes:
raise ValueError("Compressed header length mismatch")
header = np.frombuffer(header_bytes, dtype=ht)
regular, last = int(header[1]), int(header[2])
if not 0 < last <= regular <= 64*1024**2:
raise ValueError("Invalid block size")
payload = base64.b64decode(encoded[hchars:], validate=True)
if sum(map(int, header[3:])) != len(payload):
raise ValueError("Compressed payload length mismatch")
chunks, offset = [], 0
for i, size in enumerate(header[3:]):
size = int(size)
expected = last if i == nblocks-1 else regular
decoder = zlib.decompressobj()
chunk = decoder.decompress(payload[offset:offset+size], expected+1)
if len(chunk) != expected or not decoder.eof or decoder.unused_data or decoder.unconsumed_tail:
raise ValueError("zlib block size/end mismatch")
chunks.append(chunk)
offset += size
raw = b"".join(chunks)
dtype = DTYPES[element.attrib["type"]]
components = int(element.get("NumberOfComponents", "1"))
if components <= 0 or len(raw) % (components*dtype.itemsize):
raise ValueError("Component/byte count mismatch")
array = np.frombuffer(raw, dtype=dtype)
return array.reshape(-1, components) if components > 1 else array
def stats(array):
flat = array.ravel()
finite = np.isfinite(flat)
good = flat[finite]
return dict(shape=list(array.shape), dtype=str(array.dtype),
nonfinite=int((~finite).sum()), nan=int(np.isnan(flat).sum()),
positive_inf=int(np.isposinf(flat).sum()), negative_inf=int(np.isneginf(flat).sum()),
negative=int((good < 0).sum()),
zero=int((good == 0).sum()),
min=float(good.min()) if good.size else None,
max=float(good.max()) if good.size else None,
quantiles=np.quantile(good, [0,.01,.5,.99,1]).tolist() if good.size else [])
def number(text, label):
match = re.search(r"^[ \t]*"+re.escape(label)+r"[ \t]*=[ \t]*([-+0-9.eE]+)[ \t]*([^\n]*)", text, re.M)
return dict(value=float(match[1]), unit=match[2].strip()) if match else None
def log_summary(text):
names = ["Total volume", "Volume of tetrahedral elements", "Fill time",
"Melt temperature", "Mold temperature", "Maximum injection pressure",
"Time at velocity/pressure switch-over", "Injection pressure at velocity/pressure switch-over",
"Volume filled at velocity/pressure switch-over"]
result = {name: number(text, name) for name in names}
result["evidence_lines"] = [line.strip() for line in text.splitlines()
if re.search(r"Number of.*(node|element)|Filling.*time|Filling phase|Total volume|Volume of tetrahedral|[Ff]ill time|[Tt]emperature.*=|pressure at velocity|Time at velocity", line)]
return result
def tetra_geometry(points, cells):
if cells["types"].ndim != 1 or not np.all(cells["types"] == 10):
raise ValueError("Numerical geometry currently requires linear tetrahedra only")
nc = len(cells["types"])
if not np.array_equal(cells["offsets"], np.arange(1,nc+1)*4):
raise ValueError("Tetrahedron offsets mismatch")
tet = cells["connectivity"].reshape(-1,4)
if tet.min() < 0 or tet.max() >= len(points):
raise ValueError("Connectivity outside node range")
q = points[tet]
signed = np.einsum("ij,ij->i", np.cross(q[:,1]-q[:,0],q[:,2]-q[:,0]),q[:,3]-q[:,0])/6
volumes = np.abs(signed)
weights = np.bincount(tet.ravel(), weights=np.repeat(volumes/4,4), minlength=len(points))
face_ids = np.array([[0,1,2],[0,1,3],[0,2,3],[1,2,3]])
faces, counts = np.unique(np.sort(tet[:,face_ids].reshape(-1,3),axis=1),axis=0,return_counts=True)
boundary = faces[counts==1]
edges = np.unique(np.sort(tet[:,np.array([[0,1],[0,2],[0,3],[1,2],[1,3],[2,3]])].reshape(-1,2),axis=1),axis=0)
from scipy.sparse import coo_matrix
from scipy.sparse.csgraph import connected_components
graph = coo_matrix((np.ones(2*len(edges)),(np.r_[edges[:,0],edges[:,1]],np.r_[edges[:,1],edges[:,0]])),shape=(len(points),len(points))).tocsr()
ncomp, labels = connected_components(graph)
report = dict(cell_type="VTK_TETRA=10", cell_count=nc, node_count=len(points),
raw_bbox_min=points.min(0).tolist(), raw_bbox_max=points.max(0).tolist(),
total_raw_volume=float(volumes.sum()), zero_volume_cells=int((volumes==0).sum()),
negative_orientation_cells=int((signed<0).sum()),
zero_weight_nodes=int((weights==0).sum()), boundary_triangles=len(boundary),
nonmanifold_faces=int((counts>2).sum()), edges=len(edges), components=ncomp,
component_node_counts=np.bincount(labels).tolist(),
node_volume_sum_error=float(abs(weights.sum()-volumes.sum())))
return report,dict(tetra=tet,node_volume_raw=weights,boundary_triangles=boundary,edges=edges)
def audit_case(archive, stem, output):
started=time.monotonic()
text=archive.read(stem+".log").decode().replace("\r","")
gate=yaml.safe_load(archive.read(stem+".yml"))
fields, kept, cells, times = [], {}, {}, defaultdict(list)
stack=[]
pieces=0
with archive.open(stem+"_sim.vtu") as stream:
for event,e in ET.iterparse(stream,events=("start","end")):
if event=="start":
stack.append(e.tag)
if e.tag=="VTKFile":
if e.get("byte_order")!="LittleEndian" or e.get("compressor")!="vtkZLibDataCompressor":
raise ValueError("Unsupported VTU encoding")
header=e.get("header_type","UInt32")
elif e.tag=="Piece":
pieces+=1
if pieces!=1: raise ValueError("Multiple pieces unsupported")
npnt,ncell=int(e.attrib["NumberOfPoints"]),int(e.attrib["NumberOfCells"])
else:
if e.tag=="DataArray":
association=stack[-2]
name=e.attrib["Name"]
values=decode_array(e,header)
expected=npnt if association in ("PointData","Points") else ncell if association=="CellData" else None
if expected is not None and len(values)!=expected:
raise ValueError(f"Association size mismatch {name}")
record=dict(name=name,association=association,**stats(values))
fields.append(record)
match=TIME.match(name)
if match:
family,block,seconds=match.groups()
frame=dict(block=int(block),time=seconds,name=name,min=record["min"],max=record["max"],zero=record["zero"],negative=record["negative"],nonfinite=record["nonfinite"])
if "fill_time_raw" in kept and association=="PointData":
finite_nodes=np.isfinite(values).all(axis=1) if values.ndim==2 else np.isfinite(values)
wet=kept["fill_time_raw"]<=float(seconds)
frame.update(finite_nodes=int(finite_nodes.sum()),
fill_time_wet_nodes=int(wet.sum()),
nonfinite_at_arrived_nodes=int((~finite_nodes & wet).sum()),
finite_at_not_yet_arrived_nodes=int((finite_nodes & ~wet).sum()))
missing_arrived = ~finite_nodes & wet
ages = float(seconds)-kept["fill_time_raw"][missing_arrived]
frame["missing_arrived_age_seconds"] = stats(ages)
gate_index=int(np.argmin(np.linalg.norm(kept["points_raw"]-np.asarray(gate["inlet_location"]),axis=1)))
frame["gate_nearest_value"]=values[gate_index].tolist() if np.isfinite(values[gate_index]).all() else None
times[family].append(frame)
if association=="Points": kept["points_raw"]=values.copy()
elif association=="Cells": cells[name]=values.copy()
elif name=="Fill_time": kept["fill_time_raw"]=values.copy()
e.clear()
stack.pop()
# iterparse consumes the complete member, triggering zipfile CRC validation.
if pieces!=1 or "fill_time_raw" not in kept: raise ValueError("Required arrays missing")
geometry,arrays=tetra_geometry(kept["points_raw"],cells)
kept.update(arrays)
gate_raw=np.asarray(gate["inlet_location"],dtype=float)
delta=kept["points_raw"]-gate_raw
distances=np.linalg.norm(delta,axis=1)
idx=int(np.argmin(distances))
geometry["gate_nearest_node"]=idx
geometry["gate_nearest_distance_raw"]=float(distances[idx])
geometry["gate_node_fill_time_raw"]=float(kept["fill_time_raw"][idx])
logs=log_summary(text)
volume=logs["Volume of tetrahedral elements"]
geometry["log_volume_to_raw_volume_ratio"]=(volume["value"]/geometry["total_raw_volume"]) if volume else None
series={}
for family,items in times.items():
by_block=sorted(items,key=lambda x:x["block"])
by_time=sorted(items,key=lambda x:Decimal(x["time"]))
ts=[Decimal(x["time"]) for x in by_time]
for i, frame in enumerate(by_time):
frame["previous_frame_interval_seconds"] = float(ts[i]-ts[i-1]) if i else None
series[family]=dict(count=len(items),duplicate_times=len(ts)-len(set(ts)),
duplicate_blocks=len(items)-len({i["block"] for i in items}),
block_order_time_decreases=sum(Decimal(b["time"])<Decimal(a["time"]) for a,b in zip(by_block,by_block[1:])),
frames_by_time=by_time)
time_sets={k:{Decimal(i["time"]) for i in v} for k,v in times.items()}
comparisons={a+" == "+b:time_sets[a]==time_sets[b] for a in time_sets for b in time_sets if a<b}
np.savez_compressed(output/(stem+".npz"),**kept,gate_location_raw=gate_raw,gate_normal=np.asarray(gate["inlet_normal"]))
result=dict(case=stem,fields=fields,series=series,time_grid_equal=comparisons,geometry=geometry,
log=logs,gate=gate,field_count=len(fields),
any_nonfinite=any(f["nonfinite"] for f in fields),
exported_units="unresolved: raw values preserved; log-volume comparison is evidence, not a universal conversion",
wall_seconds=time.monotonic()-started,
peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
(output/(stem+".audit.json")).write_text(json.dumps(result,indent=2,allow_nan=False)+"\n")
(output/(stem+".log.txt")).write_text(text)
return result
def main():
parser=argparse.ArgumentParser()
parser.add_argument("archive",type=Path)
parser.add_argument("output",type=Path)
parser.add_argument("--ids",nargs="+",default=["00000850","00000778"])
args=parser.parse_args()
args.output.mkdir(parents=True,exist_ok=True)
verification=json.loads((args.output.parent/"archive_verification.json").read_text())
st=args.archive.stat()
if verification["status"]!="official_size_and_md5_pass" or (st.st_size,st.st_mtime_ns,st.st_ino)!=(verification["bytes"],verification["mtime_ns"],verification["inode"]):
raise ValueError("Missing or stale archive verification")
with ZipFile(args.archive) as archive:
for prefix in args.ids:
names=[n for n in archive.namelist() if n.startswith(prefix+"_") and n.endswith(".yml")]
if len(names)!=1: raise ValueError(prefix)
result=audit_case(archive,names[0][:-4],args.output)
print(json.dumps({k:result[k] for k in ("case","geometry","field_count","any_nonfinite","wall_seconds","peak_rss_kib")},indent=2),flush=True)
if __name__=="__main__": main()