233 lines
16 KiB
Python
233 lines
16 KiB
Python
"""Pilot preparation adapter v2: additionally reuse S1 primary sampled O2 caches.
|
|
|
|
Based on s2_preprocess.py v1; its ten-part execution remains immutable.
|
|
"""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import resource
|
|
import time
|
|
import traceback
|
|
from zipfile import ZipFile
|
|
import numpy as np
|
|
import yaml
|
|
from boundary_distance import SurfaceDistance, triangle_distance_squared
|
|
from g1_candidate_fingerprints import read_tets
|
|
import run_e6_real_kernel as e6
|
|
import o2_rep as rep
|
|
from s2_models import features
|
|
from s2_e1_supervisor import save
|
|
from freeze_g1_components import sha
|
|
|
|
GEOMETRY_MEMBERS = {'points_raw','tetra','node_volume_raw','boundary_triangles','edges','gate_location_raw','raw_node_ids'}
|
|
O2_MEMBERS = {'tensor','kernel_mass','effective_samples','valid','neff_pass','g','B','mu','status','tau0'}
|
|
|
|
|
|
def load_selected(path, members, allowed):
|
|
if not set(members) <= allowed: raise ValueError('Array outside explicit whitelist')
|
|
with ZipFile(path) as z:
|
|
return {name:np.load(z.open(name+'.npy'),allow_pickle=False) for name in members}
|
|
|
|
|
|
def scales_and_draw(d, mass, raw_ids, prefix, existing=None):
|
|
order=np.argsort(d,kind='stable')
|
|
median=float(d[order[np.searchsorted(np.cumsum(mass[order]),mass.sum()/2)]])
|
|
if not np.isfinite(median) or median<=0:raise ValueError('Nonpositive full-node weighted median; no substitute radius')
|
|
candidates=np.flatnonzero(d>0)
|
|
if existing:
|
|
with np.load(existing,allow_pickle=False) as z:
|
|
draws=z['draw_ids'];ids=z['unique_ids'];counts=z['multiplicity'];inverse=z['draw_to_unique']
|
|
assert np.array_equal(z['draw_raw_ids'],raw_ids[draws])
|
|
else:
|
|
rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence([2026111400,prefix])))
|
|
draws=rng.choice(candidates,size=2000,replace=True,p=mass[candidates]/mass[candidates].sum(dtype=np.float64))
|
|
ids,inverse,counts=np.unique(draws,return_inverse=True,return_counts=True)
|
|
assert len(draws)==2000 and np.all(d[draws]>0) and np.array_equal(ids[inverse],draws) and counts.sum()==2000
|
|
return dict(draw_ids=draws,draw_raw_ids=raw_ids[draws],unique_ids=ids,unique_raw_ids=raw_ids[ids],
|
|
multiplicity=counts,draw_to_unique=inverse),dict(t_part=4*median,radius=10*median,
|
|
positive_mass_fraction=float(mass[d>0].sum()/mass.sum()),weighted_median_distance=median)
|
|
|
|
|
|
def edge_summary(x, edges, ids, spectral, length_scale):
|
|
lookup=np.full(len(x),-1,np.int64);lookup[ids]=np.arange(len(ids))
|
|
edge_idx,side=np.nonzero(lookup[edges]>=0);local=lookup[edges[edge_idx,side]]
|
|
n=len(local);q=np.empty((n,2));valid=np.empty(n,bool);direction=np.empty(n,bool)
|
|
for start in range(0,n,4096):
|
|
stop=min(start+4096,n);e=edges[edge_idx[start:stop]];j=local[start:stop]
|
|
result=rep.contract_direction(spectral['B'][j],spectral['mu'][j],spectral['valid'][j],
|
|
x[e[:,1]]-x[e[:,0]],length_scale,'edge')
|
|
q[start:stop]=result['q'];valid[start:stop]=result['channel_valid'];direction[start:stop]=result['direction_valid']
|
|
accepted=valid&np.isfinite(q).all(1)
|
|
degree=np.bincount(local,minlength=len(ids));good_degree=np.bincount(local[accepted],minlength=len(ids))
|
|
total=np.zeros((len(ids),2));maximum=np.full((len(ids),2),-np.inf)
|
|
np.add.at(total,local[accepted],q[accepted]);np.maximum.at(maximum,local[accepted],q[accepted])
|
|
mean=np.full_like(total,np.nan);has=good_degree>0;mean[has]=total[has]/good_degree[has,None];maximum[~has]=np.nan
|
|
summary=np.column_stack([mean[:,0],maximum[:,0],mean[:,1],maximum[:,1]])
|
|
complete=(degree==np.bincount(edges.ravel(),minlength=len(x))[ids])&has&np.isfinite(summary).all(1)
|
|
return summary,complete,dict(edge_indices=edge_idx,endpoint_side=side,anchor_local=local,
|
|
q=q,channel_valid=valid,direction_valid=direction),degree,good_degree
|
|
|
|
|
|
def s2_eligibility(distance, spectral, complete, summary):
|
|
# Gate q is intentionally absent; original gate geometry remains in outputs.
|
|
return (distance>0)&spectral['valid']&spectral['neff_pass']&complete&np.isfinite(spectral['g']).all(1)&np.isfinite(summary).all(1)
|
|
|
|
|
|
def load_sampled_o2(spec,ids,counts,raw_ids,radius,nodes):
|
|
old=json.loads(Path(spec['existing_sampled_o2_result']).read_text())
|
|
assert old['status']=='COMPLETE' and old['pose']=='original' and old['full_nodes']==nodes
|
|
assert np.isclose(old['radius'],radius,rtol=0,atol=1e-13)
|
|
with np.load(spec['existing_sampled_o2_nodes'],allow_pickle=False) as z:
|
|
assert np.array_equal(z['anchor_ids'],ids) and np.array_equal(z['multiplicity'],counts)
|
|
assert np.array_equal(z['raw_node_ids'],raw_ids[ids])
|
|
return load_selected(spec['existing_sampled_o2_nodes'],sorted(O2_MEMBERS),O2_MEMBERS)
|
|
|
|
|
|
def run(root,index):
|
|
c=json.loads((root/'contract.json').read_text());spec=c['cases'][index];case=spec['case']
|
|
dest=root/'parts'/case;dest.mkdir(parents=True,exist_ok=False)
|
|
start=time.monotonic();tick=start;timings={};stage='verify_inputs'
|
|
def progress(name,**kw):
|
|
nonlocal stage
|
|
stage=name;save(dest/'progress.json',dict(stage=name,elapsed_seconds=time.monotonic()-start,**kw))
|
|
def done(name):
|
|
nonlocal tick
|
|
now=time.monotonic();timings[name]=now-tick;tick=now
|
|
try:
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
|
for p,h in spec['input_sha256'].items():assert sha(p)==h,p
|
|
progress('geometry')
|
|
with np.load(spec['g1_geometry'],allow_pickle=False) as z:
|
|
cached_x=z['points'];faces=z['boundary'];center=z['center'];rg=float(z['rg'])
|
|
if spec.get('legacy_geometry'):
|
|
with ZipFile(spec['legacy_geometry']) as z: has_raw_ids='raw_node_ids.npy' in z.namelist()
|
|
members=GEOMETRY_MEMBERS if has_raw_ids else GEOMETRY_MEMBERS-{'raw_node_ids'}
|
|
data=load_selected(spec['legacy_geometry'],sorted(members),GEOMETRY_MEMBERS)
|
|
x=data['points_raw'];tet=data['tetra'];mass=data['node_volume_raw'];edges=data['edges'];gate=data['gate_location_raw']
|
|
raw_ids=data.get('raw_node_ids',np.arange(len(x)));assert np.array_equal(data['boundary_triangles'],faces)
|
|
geometry_mode='existing_cache_explicit_geometry_members_only'
|
|
else:
|
|
ver=c['archive_stat'];st=Path(c['archive']).stat()
|
|
assert (st.st_size,st.st_mtime_ns,st.st_ino)==(ver['bytes'],ver['mtime_ns'],ver['inode'])
|
|
with ZipFile(c['archive']) as z:
|
|
with z.open(case+'_sim.vtu') as stream:raw_x,raw_tet=read_tets(stream)
|
|
gate=np.asarray(yaml.safe_load(z.read(case+'.yml'))['inlet_location'],dtype=float)
|
|
raw_ids=np.unique(raw_tet);x=raw_x[raw_ids];tet=np.searchsorted(raw_ids,raw_tet)
|
|
del raw_x,raw_tet
|
|
_,_,mat,vol,mass,edges,lengths=features.mesh_geometry(x,tet)
|
|
del mat,vol,lengths
|
|
geometry_mode='missing_tet_topology_from_geometry_prefix_existing_g1_boundary_reused'
|
|
assert np.array_equal(x,cached_x);del cached_x
|
|
assert len(x)==spec['tet_nodes'] and len(tet)==spec['tet_cells']
|
|
assert np.isfinite(mass).all() and (mass>0).all() and np.isfinite(gate).all()
|
|
center_from_mass=np.sum(mass[:,None]*x,axis=0)/mass.sum()
|
|
rg_from_mass=np.sqrt(np.sum(mass*np.sum((x-center_from_mass)**2,axis=1))/mass.sum())
|
|
assert np.allclose(center,center_from_mass,rtol=1e-10,atol=1e-10) and np.isclose(rg,rg_from_mass,rtol=1e-10)
|
|
np.savez_compressed(dest/'geometry.npz',points_raw=x,tetra=tet,node_volume_raw=mass,edges=edges,
|
|
boundary_triangles=faces,raw_node_ids=raw_ids,gate_location_raw=gate)
|
|
done('geometry_load_topology_write')
|
|
progress('wall_distance')
|
|
boundary=np.zeros(len(x),bool);boundary[np.unique(faces)]=True
|
|
if spec.get('existing_distance'):
|
|
d=np.load(spec['existing_distance'],allow_pickle=False);distance_mode='existing_verified_distance';brute_error=None
|
|
else:
|
|
d=np.zeros(len(x));interior=np.flatnonzero(~boundary);surface=SurfaceDistance(x[faces])
|
|
for begin in range(0,len(interior),128):
|
|
ids=interior[begin:begin+128];d[ids]=surface.query(x[ids])
|
|
progress('wall_distance',done=min(begin+128,len(interior)),total=len(interior))
|
|
probe=np.random.default_rng(1729).choice(len(x),min(30,len(x)),replace=False)
|
|
brute=np.array([np.sqrt(triangle_distance_squared(q,x[faces]).min()) for q in x[probe]])
|
|
brute_error=float(abs(brute-d[probe]).max());assert brute_error<=1e-10*max(1.,np.ptp(x,axis=0).max())
|
|
del surface
|
|
distance_mode='new_missing_distance_exact_triangle_with_30_brute_checks'
|
|
assert np.isfinite(d).all() and (d>=0).all() and np.array_equal(d==0,boundary)
|
|
np.save(dest/'boundary_distance_raw.npy',d);done('wall_distance_load_or_compute_and_write')
|
|
progress('fixed_draw')
|
|
draw,scales=scales_and_draw(d,mass,raw_ids,spec['numeric_prefix'],spec.get('existing_primary_draw'))
|
|
ids=draw['unique_ids'];counts=draw['multiplicity'];np.savez_compressed(dest/'draw.npz',**draw);done('draw_and_scales')
|
|
progress('base_features')
|
|
geo,gate_node=features.graph_distances(x,tet,d,gate,exclude_wall_edges=False)
|
|
base=features.base_features(x,d,geo,gate,center,rg,float(mass.sum()),scales['t_part'],spec['t_set'])
|
|
np.save(dest/'base.npy',base);del base,geo;done('unrestricted_geodesic_base_write')
|
|
progress('o2')
|
|
if spec.get('existing_sampled_o2_nodes'):
|
|
spectral=load_sampled_o2(spec,ids,counts,raw_ids,scales['radius'],len(x))
|
|
o2_mode='existing_S1_primary_sampled_spectrum_no_reintegration';visits=None;raw_meta=None
|
|
elif spec.get('existing_o2_nodes'):
|
|
old=json.loads(Path(spec['existing_o2_result']).read_text())
|
|
assert old['acceptance_status']=='PASS' and old['pose']=='original' and old['nodes']==len(x)
|
|
assert np.isclose(old['radius'],scales['radius'],rtol=0,atol=1e-13)
|
|
loaded=load_selected(spec['existing_o2_nodes'],sorted(O2_MEMBERS),O2_MEMBERS)
|
|
spectral={k:v[ids] for k,v in loaded.items()};del loaded
|
|
o2_mode='existing_full_original_O2_v1.1a_subset_no_reintegration';visits=None;raw_meta=None
|
|
else:
|
|
raw,raw_meta=e6.raw_field(x,tet,d);sx,sm,st=e6.samples(x,tet,mass,'tetra_four_point',raw);del raw
|
|
visits=0;real_tree=e6.cKDTree
|
|
class ProgressTree:
|
|
def __init__(self,*args,**kwargs):self.tree=real_tree(*args,**kwargs);self.count=0
|
|
def query_ball_point(self,*args,**kwargs):
|
|
nonlocal visits
|
|
result=self.tree.query_ball_point(*args,**kwargs);visits+=len(result);self.count+=1
|
|
if self.count%32==0:progress('o2_kernel',anchors_done=self.count,anchors_total=len(ids),neighbor_visits=visits)
|
|
return result
|
|
e6.cKDTree=ProgressTree
|
|
try:a=e6.integrate(x[ids],sx,sm,st,scales['radius'])
|
|
finally:e6.cKDTree=real_tree
|
|
del sx,sm,st
|
|
spectral=dict(**a,**rep.readout(a['tensor'],a['kernel_mass'],d[ids]>0,a['effective_samples']))
|
|
checks=rep.acceptance(spectral)
|
|
if checks['valid_count']:
|
|
assert checks['mu_sum_error']<=1e-10 and checks['mu_min']>=-1e-10 and checks['ordering_violation']<=1e-10
|
|
assert checks['B_symmetry_error']<=1e-10 and checks['B_min_eigenvalue']>=-1e-10 and checks['reconstruction_max']<=1e-10
|
|
o2_mode='fixed_full_source_four_point_kernel_at_unique_drawn_anchors'
|
|
done('o2_raw_source_kernel_readout_or_cache')
|
|
progress('edge_queries_and_s2_eligibility')
|
|
length_scale=float(np.median(np.linalg.norm(x[edges[:,1]]-x[edges[:,0]],axis=1)))
|
|
summary,complete,edge_data,degree,good_degree=edge_summary(x,edges,ids,spectral,length_scale)
|
|
eligible=s2_eligibility(d[ids],spectral,complete,summary)
|
|
rejection_masks=dict(nonpositive_distance=d[ids]<=0,spectral_invalid=~spectral['valid'],
|
|
neff_failed=~spectral['neff_pass'],edge_incomplete=~complete,
|
|
nonfinite_query=~(np.isfinite(spectral['g']).all(1)&np.isfinite(summary).all(1)))
|
|
rejections={name:dict(unique=int(mask.sum()),slots=int(counts[mask].sum())) for name,mask in rejection_masks.items()}
|
|
np.savez_compressed(dest/'queries.npz',anchor_ids=ids,raw_node_ids=raw_ids[ids],multiplicity=counts,
|
|
distance=d[ids],node_mass=mass[ids],g=spectral['g'],edge_summary=summary,eligible_s2=eligible,
|
|
spectral_valid=spectral['valid'],neff_pass=spectral['neff_pass'],effective_samples=spectral['effective_samples'],
|
|
edge_block_complete=complete,edge_count=degree,valid_edge_count=good_degree,near_wall=d[ids]/scales['t_part']<=.25)
|
|
np.savez_compressed(dest/'o2-evidence.npz',**spectral)
|
|
np.savez_compressed(dest/'edge-evidence.npz',**edge_data)
|
|
del spectral,edge_data
|
|
done('edge_queries_eligibility_write')
|
|
progress('r2_features')
|
|
r2_error=None
|
|
try:
|
|
r2=features.r2_features(x,tet,d,gate,spec['t_set'])
|
|
np.savez_compressed(dest/'r2.npz',features=r2['features'],Di=r2['Di'],Df=r2['Df'],Vf=r2['Vf'],S=r2['S'])
|
|
r2_meta=dict(columns=list(r2['columns']),radius=r2['radius'],gate_node=r2['gate_node']);del r2
|
|
except (ValueError,np.linalg.LinAlgError) as exc:
|
|
r2_error=dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc());r2_meta=None
|
|
save(dest/'r2-failure.json',r2_error)
|
|
done('r2_features_and_write')
|
|
k=int(counts[eligible].sum());h=scales['positive_mass_fraction']
|
|
result=dict(case=case,status='COMPLETE' if r2_error is None else 'PARTIAL_R2_FAILED',nodes=len(x),tets=len(tet),
|
|
geometry_mode=geometry_mode,distance_mode=distance_mode,distance_brute_max_abs_error=brute_error,
|
|
o2_mode=o2_mode,raw_field=raw_meta,neighbor_visits=visits,scales=dict(**scales,Rg=rg,volume=float(mass.sum()),L_med=length_scale,t_set=spec['t_set']),
|
|
draw_slots=2000,unique_drawn=len(ids),eligible_unique=int(eligible.sum()),eligible_slots=k,conditional_coverage=k/2000,
|
|
full_lumped_coverage=h*k/2000,boundary_mass_fraction=1-h,eligible_near_wall_slots=int(counts[eligible&(d[ids]/scales['t_part']<=.25)].sum()),
|
|
rejection_counts_overlapping=rejections,total_ineligible_unique=int((~eligible).sum()),
|
|
gate_q_used_for_eligibility=False,eligibility_version='S2-v1-no-gate-q',base_columns=list(features.BASE_COLUMNS),
|
|
base_normalization='NOT_FITTED; future fold/pool learning only',base_gate_node=gate_node,r2=r2_meta,r2_error=r2_error,
|
|
label_arrays_decoded=[],model_fit=False,gpu=False,timings=timings,wall_seconds=time.monotonic()-start,
|
|
cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),peak_rss_bytes=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024,
|
|
output_sha256={p.name:sha(p) for p in sorted(dest.iterdir()) if p.is_file() and p.name!='progress.json'})
|
|
save(dest/'result.json',result);progress('complete' if not r2_error else 'partial_r2_failed')
|
|
print(json.dumps({k:result[k] for k in ['case','status','wall_seconds','peak_rss_bytes','eligible_slots','r2_error']}),flush=True)
|
|
return 0 if r2_error is None else 2
|
|
except Exception as exc:
|
|
save(dest/'failure.json',dict(stage=stage,error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),timings=timings,wall_seconds=time.monotonic()-start))
|
|
raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args()
|
|
raise SystemExit(run(a.root,a.index))
|