Files
MeltMind/tools/gre24_prepare_v2.py

147 lines
7.6 KiB
Python

"""Independent Gre24 geometry/feature preparation; full tet nodes, no S2 mask."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import time
import traceback
from xml.etree import ElementTree as ET
from zipfile import ZipFile
import numpy as np
import yaml
from audit_samples import decode_array
from gre24_features_v2 import features
from s2_recovery_supervisor import save
def sha(path):
h=hashlib.sha256()
with Path(path).open('rb') as f:
for block in iter(lambda:f.read(2**20),b''):h.update(block)
return h.hexdigest()
def checked(root):
c=json.loads((root/'contract.json').read_text())
for p,h in c['code_sha256'].items():
if sha(p)!=h:raise ValueError('Pinned source changed: '+p)
st=Path(c['archive']).stat();a=c['archive_stat']
if (st.st_size,st.st_mtime_ns,st.st_ino)!=(a['bytes'],a['mtime_ns'],a['inode']):
raise ValueError('Immutable source archive changed')
return c
def read_geometry(archive,case):
cells={};points=None;stack=[]
with ZipFile(archive) as z:
config=yaml.safe_load(z.read(case+'.yml'))
with z.open(case+'_sim.vtu') as f:
for event,e in ET.iterparse(f,events=('start','end')):
if event=='start':
stack.append(e.tag)
if e.tag=='VTKFile':
if e.get('byte_order')!='LittleEndian' or e.get('compressor')!='vtkZLibDataCompressor':
raise ValueError('Unsupported VTU')
header=e.get('header_type','UInt32')
if e.tag in ('PointData','CellData'):
raise ValueError('Field section before complete geometry')
else:
if e.tag=='DataArray':
if stack[-2]=='Points':points=decode_array(e,header).copy()
elif stack[-2]=='Cells':cells[e.get('Name')]=decode_array(e,header).copy()
e.clear()
if points is not None and all(k in cells for k in ['connectivity','offsets','types']):break
stack.pop()
offsets=np.r_[0,cells['offsets']];indices=np.flatnonzero(cells['types']==10)
if not len(indices) or np.any(np.diff(offsets)[indices]!=4):raise ValueError('No valid tetrahedra')
raw_tet=cells['connectivity'][offsets[indices,None]+np.arange(4)]
raw_ids=np.unique(raw_tet);t=np.searchsorted(raw_ids,raw_tet)
types,counts=np.unique(cells['types'],return_counts=True)
return points[raw_ids],t,raw_ids,dict(raw_nodes=len(points),tet_nodes=len(raw_ids),
outside_tet_nodes=len(points)-len(raw_ids),raw_cell_types=dict(zip(map(str,types),map(int,counts))),
inlet_config=config,gate_source='YAML inlet_location; no explicit node/face markers in this source')
def read_targets(archive,case):
stack=[];count=None
with ZipFile(archive) as z,z.open(case+'_sim.vtu') as f:
for event,e in ET.iterparse(f,events=('start','end')):
if event=='start':
stack.append(e.tag)
if e.tag=='VTKFile':header=e.get('header_type','UInt32')
elif e.tag=='Piece':
if count is not None:raise ValueError('Multiple pieces')
count=int(e.get('NumberOfPoints'))
else:
if e.tag=='DataArray':
if stack[-2]=='PointData' and e.get('Name')=='Fill_time':
v=decode_array(e,header).copy()
if v.shape!=(count,):raise ValueError('Fill_time size mismatch')
return v
e.clear()
stack.pop()
raise ValueError('Missing Fill_time')
def worker(root,index):
c=checked(root);spec=c['parts'][index];case=spec['case'];dest=root/'parts'/case/'output'
dest.mkdir(parents=True,exist_ok=False)
start=time.process_time();wall=time.monotonic()
def progress(stage,**kw):
save(dest/'progress.json',dict(stage=stage,case=case,pid=os.getpid(),unix=time.time(),
CPU_s=time.process_time()-start,wall_s=time.monotonic()-wall,**kw))
try:
progress('READ_GEOMETRY')
x,t,raw,meta=read_geometry(c['archive'],case)
save(dest/'geometry-metadata.json',meta)
if spec.get('reuse_feature_root'):
old=Path(spec['reuse_feature_root']);old_meta=json.loads((old/'result.json').read_text())
if sha(old/'result.json')!=spec['reuse_result_sha256'] or old_meta['case']!=case:
raise ValueError('Reused cost evidence changed')
if old_meta['features']['speed_min']**2<=1e-4:
raise ValueError('Old unscaled result is not eligible for exact reuse')
data={}
for name in ['features','Vf','Dw','Di','Df','mass','all_wall_tet_ids']:
if sha(old/(name+'.npy'))!=old_meta['output_sha256'][name+'.npy']:
raise ValueError('Reused feature changed')
data[name]=np.load(old/(name+'.npy'),allow_pickle=False)
if not np.array_equal(raw,np.load(old/'raw_ids.npy',allow_pickle=False)):
raise ValueError('Reused raw IDs changed')
feature_meta=dict(old_meta['features'],reused_from=str(old),Df_coordinate_and_speed_scale=1.)
else:
data,feature_meta=features(x,t,raw,meta['inlet_config']['inlet_location'],progress=progress)
progress('SAVE_FEATURES')
for name in ['features','Vf','Dw','Di','Df','mass','all_wall_tet_ids']:np.save(dest/(name+'.npy'),data[name])
np.save(dest/'raw_ids.npy',raw)
if c.get('feature_only_cost_test',False):
save(dest/'result.json',dict(status='PASS',case=case,index=index,geometry=meta,features=feature_meta,
output_sha256={p.name:sha(p) for p in dest.glob('*.npy')},
worker_cpu_seconds=time.process_time()-start,wall_seconds=time.monotonic()-wall,
decoded_labels=[],test_scores_computed=False,training=False,cost_test_only=True))
progress('PASS_COST_TEST_ONLY');return 0
progress('READ_FILL_TIME_AFTER_FEATURES_FIXED')
yall=read_targets(c['archive'],case);y=yall[raw]
if not np.isfinite(y).all() or np.any(y<0):raise ValueError('Invalid fill time on full tetrahedral domain')
outside=np.ones(len(yall),dtype=bool);outside[raw]=False;outside_y=yall[outside]
np.save(dest/'seconds.npy',y);np.save(dest/'target.npy',y/spec['t_scale_seconds'])
np.save(dest/'outside_tet_raw_ids.npy',np.flatnonzero(outside));np.save(dest/'outside_tet_fill_time.npy',outside_y)
outputs={p.name:sha(p) for p in dest.glob('*.npy')}
save(dest/'result.json',dict(status='PASS',case=case,index=index,geometry=meta,features=feature_meta,
t_scale_seconds=spec['t_scale_seconds'],time_scale_source=spec['time_scale_source'],
decoded_labels=['Fill_time'],outside_label_status=dict(nodes=len(outside_y),
finite=int(np.isfinite(outside_y).sum()),negative=int((outside_y<0).sum())),
output_sha256=outputs,worker_cpu_seconds=time.process_time()-start,wall_seconds=time.monotonic()-wall,
test_scores_computed=False,training=False))
progress('PASS');return 0
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED',case=case,type=type(exc).__name__,message=str(exc),
traceback=traceback.format_exc(),worker_cpu_seconds=time.process_time()-start,
wall_seconds=time.monotonic()-wall,automatic_retry=False))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int)
a=p.parse_args();raise SystemExit(worker(a.root,a.index))