Files
MeltMind/tools/s2_preprocess.py

215 lines
15 KiB
Python

"""Label-free full-part S2 preparation; original caches and protocols stay unchanged."""
import argparse
import json
from pathlib import Path
import resource
import time
import traceback
from zipfile import ZipFile
import numpy as np
import yaml
from boundary_distance import SurfaceDistance, triangle_distance_squared
from g1_candidate_fingerprints import read_tets
import run_e6_real_kernel as e6
import o2_rep as rep
from s2_models import features
from s2_e1_supervisor import save
from freeze_g1_components import sha
GEOMETRY_MEMBERS = {'points_raw','tetra','node_volume_raw','boundary_triangles','edges','gate_location_raw'}
O2_MEMBERS = {'tensor','kernel_mass','effective_samples','valid','neff_pass','g','B','mu','status','tau0'}
def load_selected(path, members, allowed):
if not set(members) <= allowed: raise ValueError('Array outside explicit whitelist')
with ZipFile(path) as z:
return {name:np.load(z.open(name+'.npy'),allow_pickle=False) for name in members}
def scales_and_draw(d, mass, raw_ids, prefix, existing=None):
order=np.argsort(d,kind='stable')
median=float(d[order[np.searchsorted(np.cumsum(mass[order]),mass.sum()/2)]])
if not np.isfinite(median) or median<=0:raise ValueError('Nonpositive full-node weighted median; no substitute radius')
candidates=np.flatnonzero(d>0)
if existing:
with np.load(existing,allow_pickle=False) as z:
draws=z['draw_ids'];ids=z['unique_ids'];counts=z['multiplicity'];inverse=z['draw_to_unique']
assert np.array_equal(z['draw_raw_ids'],raw_ids[draws])
else:
rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence([2026111400,prefix])))
draws=rng.choice(candidates,size=2000,replace=True,p=mass[candidates]/mass[candidates].sum(dtype=np.float64))
ids,inverse,counts=np.unique(draws,return_inverse=True,return_counts=True)
assert len(draws)==2000 and np.all(d[draws]>0) and np.array_equal(ids[inverse],draws) and counts.sum()==2000
return dict(draw_ids=draws,draw_raw_ids=raw_ids[draws],unique_ids=ids,unique_raw_ids=raw_ids[ids],
multiplicity=counts,draw_to_unique=inverse),dict(t_part=4*median,radius=10*median,
positive_mass_fraction=float(mass[d>0].sum()/mass.sum()),weighted_median_distance=median)
def edge_summary(x, edges, ids, spectral, length_scale):
lookup=np.full(len(x),-1,np.int64);lookup[ids]=np.arange(len(ids))
edge_idx,side=np.nonzero(lookup[edges]>=0);local=lookup[edges[edge_idx,side]]
n=len(local);q=np.empty((n,2));valid=np.empty(n,bool);direction=np.empty(n,bool)
for start in range(0,n,4096):
stop=min(start+4096,n);e=edges[edge_idx[start:stop]];j=local[start:stop]
result=rep.contract_direction(spectral['B'][j],spectral['mu'][j],spectral['valid'][j],
x[e[:,1]]-x[e[:,0]],length_scale,'edge')
q[start:stop]=result['q'];valid[start:stop]=result['channel_valid'];direction[start:stop]=result['direction_valid']
accepted=valid&np.isfinite(q).all(1)
degree=np.bincount(local,minlength=len(ids));good_degree=np.bincount(local[accepted],minlength=len(ids))
total=np.zeros((len(ids),2));maximum=np.full((len(ids),2),-np.inf)
np.add.at(total,local[accepted],q[accepted]);np.maximum.at(maximum,local[accepted],q[accepted])
mean=np.full_like(total,np.nan);has=good_degree>0;mean[has]=total[has]/good_degree[has,None];maximum[~has]=np.nan
summary=np.column_stack([mean[:,0],maximum[:,0],mean[:,1],maximum[:,1]])
complete=(degree==np.bincount(edges.ravel(),minlength=len(x))[ids])&has&np.isfinite(summary).all(1)
return summary,complete,dict(edge_indices=edge_idx,endpoint_side=side,anchor_local=local,
q=q,channel_valid=valid,direction_valid=direction),degree,good_degree
def s2_eligibility(distance, spectral, complete, summary):
# Gate q is intentionally absent; original gate geometry remains in outputs.
return (distance>0)&spectral['valid']&spectral['neff_pass']&complete&np.isfinite(spectral['g']).all(1)&np.isfinite(summary).all(1)
def run(root,index):
c=json.loads((root/'contract.json').read_text());spec=c['cases'][index];case=spec['case']
dest=root/'parts'/case;dest.mkdir(parents=True,exist_ok=False)
start=time.monotonic();tick=start;timings={};stage='verify_inputs'
def progress(name,**kw):
nonlocal stage
stage=name;save(dest/'progress.json',dict(stage=name,elapsed_seconds=time.monotonic()-start,**kw))
def done(name):
nonlocal tick
now=time.monotonic();timings[name]=now-tick;tick=now
try:
for p,h in c['code_sha256'].items():assert sha(p)==h,p
for p,h in spec['input_sha256'].items():assert sha(p)==h,p
progress('geometry')
with np.load(spec['g1_geometry'],allow_pickle=False) as z:
cached_x=z['points'];faces=z['boundary'];center=z['center'];rg=float(z['rg'])
if spec.get('legacy_geometry'):
data=load_selected(spec['legacy_geometry'],sorted(GEOMETRY_MEMBERS),GEOMETRY_MEMBERS)
x=data['points_raw'];tet=data['tetra'];mass=data['node_volume_raw'];edges=data['edges'];gate=data['gate_location_raw']
raw_ids=np.arange(len(x));assert np.array_equal(data['boundary_triangles'],faces)
geometry_mode='existing_cache_explicit_geometry_members_only'
else:
ver=c['archive_stat'];st=Path(c['archive']).stat()
assert (st.st_size,st.st_mtime_ns,st.st_ino)==(ver['bytes'],ver['mtime_ns'],ver['inode'])
with ZipFile(c['archive']) as z:
with z.open(case+'_sim.vtu') as stream:raw_x,raw_tet=read_tets(stream)
gate=np.asarray(yaml.safe_load(z.read(case+'.yml'))['inlet_location'],dtype=float)
raw_ids=np.unique(raw_tet);x=raw_x[raw_ids];tet=np.searchsorted(raw_ids,raw_tet)
del raw_x,raw_tet
_,_,mat,vol,mass,edges,lengths=features.mesh_geometry(x,tet)
del mat,vol,lengths
geometry_mode='missing_tet_topology_from_geometry_prefix_existing_g1_boundary_reused'
assert np.array_equal(x,cached_x);del cached_x
assert len(x)==spec['tet_nodes'] and len(tet)==spec['tet_cells']
assert np.isfinite(mass).all() and (mass>0).all() and np.isfinite(gate).all()
center_from_mass=np.sum(mass[:,None]*x,axis=0)/mass.sum()
rg_from_mass=np.sqrt(np.sum(mass*np.sum((x-center_from_mass)**2,axis=1))/mass.sum())
assert np.allclose(center,center_from_mass,rtol=1e-10,atol=1e-10) and np.isclose(rg,rg_from_mass,rtol=1e-10)
np.savez_compressed(dest/'geometry.npz',points_raw=x,tetra=tet,node_volume_raw=mass,edges=edges,
boundary_triangles=faces,raw_node_ids=raw_ids,gate_location_raw=gate)
done('geometry_load_topology_write')
progress('wall_distance')
boundary=np.zeros(len(x),bool);boundary[np.unique(faces)]=True
if spec.get('existing_distance'):
d=np.load(spec['existing_distance'],allow_pickle=False);distance_mode='existing_verified_distance';brute_error=None
else:
d=np.zeros(len(x));interior=np.flatnonzero(~boundary);surface=SurfaceDistance(x[faces])
for begin in range(0,len(interior),128):
ids=interior[begin:begin+128];d[ids]=surface.query(x[ids])
progress('wall_distance',done=min(begin+128,len(interior)),total=len(interior))
probe=np.random.default_rng(1729).choice(len(x),min(30,len(x)),replace=False)
brute=np.array([np.sqrt(triangle_distance_squared(q,x[faces]).min()) for q in x[probe]])
brute_error=float(abs(brute-d[probe]).max());assert brute_error<=1e-10*max(1.,np.ptp(x,axis=0).max())
del surface
distance_mode='new_missing_distance_exact_triangle_with_30_brute_checks'
assert np.isfinite(d).all() and (d>=0).all() and np.array_equal(d==0,boundary)
np.save(dest/'boundary_distance_raw.npy',d);done('wall_distance_load_or_compute_and_write')
progress('fixed_draw')
draw,scales=scales_and_draw(d,mass,raw_ids,spec['numeric_prefix'],spec.get('existing_primary_draw'))
ids=draw['unique_ids'];counts=draw['multiplicity'];np.savez_compressed(dest/'draw.npz',**draw);done('draw_and_scales')
progress('base_features')
geo,gate_node=features.graph_distances(x,tet,d,gate,exclude_wall_edges=False)
base=features.base_features(x,d,geo,gate,center,rg,float(mass.sum()),scales['t_part'],spec['t_set'])
np.save(dest/'base.npy',base);del base,geo;done('unrestricted_geodesic_base_write')
progress('o2')
if spec.get('existing_o2_nodes'):
old=json.loads(Path(spec['existing_o2_result']).read_text())
assert old['acceptance_status']=='PASS' and old['pose']=='original' and old['nodes']==len(x)
assert np.isclose(old['radius'],scales['radius'],rtol=0,atol=1e-13)
loaded=load_selected(spec['existing_o2_nodes'],sorted(O2_MEMBERS),O2_MEMBERS)
spectral={k:v[ids] for k,v in loaded.items()};del loaded
o2_mode='existing_full_original_O2_v1.1a_subset_no_reintegration';visits=None;raw_meta=None
else:
raw,raw_meta=e6.raw_field(x,tet,d);sx,sm,st=e6.samples(x,tet,mass,'tetra_four_point',raw);del raw
visits=0;real_tree=e6.cKDTree
class ProgressTree:
def __init__(self,*args,**kwargs):self.tree=real_tree(*args,**kwargs);self.count=0
def query_ball_point(self,*args,**kwargs):
nonlocal visits
result=self.tree.query_ball_point(*args,**kwargs);visits+=len(result);self.count+=1
if self.count%32==0:progress('o2_kernel',anchors_done=self.count,anchors_total=len(ids),neighbor_visits=visits)
return result
e6.cKDTree=ProgressTree
try:a=e6.integrate(x[ids],sx,sm,st,scales['radius'])
finally:e6.cKDTree=real_tree
del sx,sm,st
spectral=dict(**a,**rep.readout(a['tensor'],a['kernel_mass'],d[ids]>0,a['effective_samples']))
checks=rep.acceptance(spectral)
if checks['valid_count']:
assert checks['mu_sum_error']<=1e-10 and checks['mu_min']>=-1e-10 and checks['ordering_violation']<=1e-10
assert checks['B_symmetry_error']<=1e-10 and checks['B_min_eigenvalue']>=-1e-10 and checks['reconstruction_max']<=1e-10
o2_mode='fixed_full_source_four_point_kernel_at_unique_drawn_anchors'
done('o2_raw_source_kernel_readout_or_cache')
progress('edge_queries_and_s2_eligibility')
length_scale=float(np.median(np.linalg.norm(x[edges[:,1]]-x[edges[:,0]],axis=1)))
summary,complete,edge_data,degree,good_degree=edge_summary(x,edges,ids,spectral,length_scale)
eligible=s2_eligibility(d[ids],spectral,complete,summary)
rejection_masks=dict(nonpositive_distance=d[ids]<=0,spectral_invalid=~spectral['valid'],
neff_failed=~spectral['neff_pass'],edge_incomplete=~complete,
nonfinite_query=~(np.isfinite(spectral['g']).all(1)&np.isfinite(summary).all(1)))
rejections={name:dict(unique=int(mask.sum()),slots=int(counts[mask].sum())) for name,mask in rejection_masks.items()}
np.savez_compressed(dest/'queries.npz',anchor_ids=ids,raw_node_ids=raw_ids[ids],multiplicity=counts,
distance=d[ids],node_mass=mass[ids],g=spectral['g'],edge_summary=summary,eligible_s2=eligible,
spectral_valid=spectral['valid'],neff_pass=spectral['neff_pass'],effective_samples=spectral['effective_samples'],
edge_block_complete=complete,edge_count=degree,valid_edge_count=good_degree,near_wall=d[ids]/scales['t_part']<=.25)
np.savez_compressed(dest/'o2-evidence.npz',**spectral)
np.savez_compressed(dest/'edge-evidence.npz',**edge_data)
del spectral,edge_data
done('edge_queries_eligibility_write')
progress('r2_features')
r2_error=None
try:
r2=features.r2_features(x,tet,d,gate,spec['t_set'])
np.savez_compressed(dest/'r2.npz',features=r2['features'],Di=r2['Di'],Df=r2['Df'],Vf=r2['Vf'],S=r2['S'])
r2_meta=dict(columns=list(r2['columns']),radius=r2['radius'],gate_node=r2['gate_node']);del r2
except (ValueError,np.linalg.LinAlgError) as exc:
r2_error=dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc());r2_meta=None
save(dest/'r2-failure.json',r2_error)
done('r2_features_and_write')
k=int(counts[eligible].sum());h=scales['positive_mass_fraction']
result=dict(case=case,status='COMPLETE' if r2_error is None else 'PARTIAL_R2_FAILED',nodes=len(x),tets=len(tet),
geometry_mode=geometry_mode,distance_mode=distance_mode,distance_brute_max_abs_error=brute_error,
o2_mode=o2_mode,raw_field=raw_meta,neighbor_visits=visits,scales=dict(**scales,Rg=rg,volume=float(mass.sum()),L_med=length_scale,t_set=spec['t_set']),
draw_slots=2000,unique_drawn=len(ids),eligible_unique=int(eligible.sum()),eligible_slots=k,conditional_coverage=k/2000,
full_lumped_coverage=h*k/2000,boundary_mass_fraction=1-h,eligible_near_wall_slots=int(counts[eligible&(d[ids]/scales['t_part']<=.25)].sum()),
rejection_counts_overlapping=rejections,total_ineligible_unique=int((~eligible).sum()),
gate_q_used_for_eligibility=False,eligibility_version='S2-v1-no-gate-q',base_columns=list(features.BASE_COLUMNS),
base_normalization='NOT_FITTED; future fold/pool learning only',base_gate_node=gate_node,r2=r2_meta,r2_error=r2_error,
label_arrays_decoded=[],model_fit=False,gpu=False,timings=timings,wall_seconds=time.monotonic()-start,
cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),peak_rss_bytes=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024,
output_sha256={p.name:sha(p) for p in sorted(dest.iterdir()) if p.is_file() and p.name!='progress.json'})
save(dest/'result.json',result);progress('complete' if not r2_error else 'partial_r2_failed')
print(json.dumps({k:result[k] for k in ['case','status','wall_seconds','peak_rss_bytes','eligible_slots','r2_error']}),flush=True)
return 0 if r2_error is None else 2
except Exception as exc:
save(dest/'failure.json',dict(stage=stage,error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),timings=timings,wall_seconds=time.monotonic()-start))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args()
raise SystemExit(run(a.root,a.index))