Files
MeltMind/tools/phase2_e2_statistics.py
T

74 lines
8.2 KiB
Python

"""Frozen E2 raw deployment contrast and weighted descriptive adjustment."""
import numpy as np
from scipy.spatial import ConvexHull,QhullError
SVD_RCOND=1e-12
SUPPORT_RTOL=1e-10
METRICS=('relative_l2','last_location','kendall_tau_b')
PAIRS=(('T','G'),('T','C'),('C','G'))
def weights_for_components(component_ids,multiplicity=None):
ids=sorted(set(component_ids));index={g:i for i,g in enumerate(ids)};group=np.array([index[g] for g in component_ids]);sizes=np.bincount(group,minlength=len(ids))
mult=np.ones(len(ids)) if multiplicity is None else np.asarray(multiplicity,float)
if not len(ids) or mult.shape!=(len(ids),) or not np.isfinite(mult).all() or np.any(mult<0) or not np.isclose(mult.sum(),len(ids)):raise ValueError('Bootstrap must draw original C components')
return mult[group]/(len(ids)*sizes[group]),ids,group
def zero_support(x):
x=np.asarray(x,float)
if x.ndim!=2 or x.shape[1]!=2 or not len(x) or not np.isfinite(x).all():return dict(supported=False,reason='NO_FINITE_DUAL_SUPPORT')
scale=max(1.,float(np.abs(x).max()));tol=SUPPORT_RTOL*scale;mean=x.mean(0);u,s,vt=np.linalg.svd(x-mean,full_matrices=False);rank=int(np.sum(s>SVD_RCOND*max(1.,s[0] if len(s) else 0.)))
if rank==0:return dict(supported=bool(np.linalg.norm(mean)<=tol),affine_rank=0,zero_distance=float(np.linalg.norm(mean)),tolerance=tol)
if rank==1:
direction=vt[0];distance=float(np.linalg.norm(mean-(mean@direction)*direction));v=x@direction;ok=distance<=tol and v.min()<=tol and v.max()>=-tol
return dict(supported=bool(ok),affine_rank=1,zero_distance=distance,projected_range=[float(v.min()),float(v.max())],tolerance=tol)
try:
hull=ConvexHull(x);outside=float(np.max(hull.equations[:,-1]));return dict(supported=bool(outside<=tol),affine_rank=2,max_origin_facet_violation=outside,tolerance=tol)
except QhullError:return dict(supported=False,reason='CONVEX_HULL_NUMERICAL_FAILURE',affine_rank=2,tolerance=tol)
def adjusted_estimate(d,x,valid,w):
d=np.asarray(d,float);x=np.asarray(x,float);valid=np.asarray(valid,bool);w=np.asarray(w,float);active=w>0
if not np.isfinite(d[active]).all():return dict(value=None,reason='UNDEFINED_FULL_ROSTER_DIFFERENCE')
raw=float(np.dot(w[active],d[active]));use=active&valid&np.isfinite(x).all(1);mass=float(w[use].sum())
if not np.any(use):return dict(value=None,raw=raw,reason='NO_DUAL_VALID_SUPPORT',dual_valid_weight=mass)
X=np.column_stack([np.ones(use.sum()),x[use]]);sw=np.sqrt(w[use]);design=X*sw[:,None];target=d[use]*sw;u,s,vt=np.linalg.svd(design,full_matrices=False);rank=int(np.sum(s>SVD_RCOND*s[0]));basis=vt[:rank];intercept=np.array([1.,0.,0.]);residual=float(np.linalg.norm(intercept-basis.T@(basis@intercept)));estimable=residual<=SUPPORT_RTOL
coef=basis.T@((u[:,:rank].T@target)/s[:rank]);support=zero_support(x[use]);model_value=raw-float(np.sum(w[use]*(x[use]@coef[1:])))
reason=None if estimable and support['supported'] else 'ZERO_POINT_NOT_ESTIMABLE' if not estimable else 'ZERO_POINT_OUTSIDE_SUPPORT'
return dict(value=model_value if reason is None else None,raw=raw,model_adjusted_value=model_value,alpha=float(coef[0]) if estimable else None,beta_minimum_norm=coef[1:].tolist(),rank=rank,zero_point_estimable=estimable,estimability_residual=residual,zero_support=support,dual_valid_parts=int(use.sum()),dual_valid_weight=mass,zero_columns=np.flatnonzero(np.all(x[use]==0,axis=0)).tolist(),reason=reason)
def interval(values,point,replicates):
a=np.asarray(values,float);ok=np.isfinite(a);failed=int((~ok).sum());usable=point is not None and failed/replicates<.05
lo=hi=None
if usable:lo,hi=map(float,np.percentile(a[ok],[2.5,97.5],method='linear'))
return dict(value=point,low=lo,high=hi,failed_replicates=failed,failed_fraction=failed/replicates,replicates=replicates,usable=bool(usable),reason=None if usable else 'POINT_UNIDENTIFIED_OR_AT_LEAST_5PCT_FAILED_REPLICATES')
def raw_gate(field,last,total_components=204):
if any(m['defined_components']!=total_components or m['high'] is None for m in [field,last]):return 'PARK_UNDEFINED'
return 'EXPAND_FIXED_TYPE_CONTEXT' if field['high']<0 or (last['high']<0 and field['high']<.001) else 'PRUNE_FIXED_TYPE_CONTEXT_RECIPE'
def analyse(rows,replicates=2000,seed=2026111408,expected_parts=552,expected_components=204):
if len(rows)!=expected_parts or len({r['case'] for r in rows})!=expected_parts or len({r['component_id'] for r in rows})!=expected_components:raise ValueError('Frozen evaluation roster incomplete or duplicated')
component=[r['component_id'] for r in rows];w,ids,group=weights_for_components(component);C=len(ids);sizes=np.bincount(group);rng=np.random.default_rng(seed);draws=rng.integers(0,C,size=(replicates,C));mult=np.array([np.bincount(d,minlength=C) for d in draws]);arrays=dict(component_draws=draws.astype(np.int32));comparisons={}
for a,b in PAIRS:
name=a+'-'+b;valid=np.zeros(len(rows),bool);x=np.full((len(rows),2),np.nan);categories=[]
for i,r in enumerate(rows):
ga=r['geometry']['arms'][a];gb=r['geometry']['arms'][b];da=ga.get('descriptor');db=gb.get('descriptor')
good=ga['valid'] and gb['valid'] and da is not None and db is not None
if good:
va=np.array([da['cut_area'],da['rms_diameter']]);vb=np.array([db['cut_area'],db['rms_diameter']]);good=bool(np.isfinite(va).all() and np.isfinite(vb).all() and np.all(va>0) and np.all(vb>0))
if good:x[i]=np.log(va/vb);valid[i]=True
categories.append('both_adjustable' if good else 'both_descriptor_undefined' if ga['valid'] and gb['valid'] else 'a_only' if ga['valid'] else 'b_only' if gb['valid'] else 'neither')
pair=dict(raw={},adjusted={},covariates=dict(names=['log_cut_area_ratio','log_RMS_diameter_ratio'],centered=False,dual_valid_parts=int(valid.sum()),dual_valid_weight=float(w[valid].sum()),range_min=x[valid].min(0).tolist() if valid.any() else None,range_max=x[valid].max(0).tolist() if valid.any() else None,points=[dict(case=r['case'],component_id=r['component_id'],x=x[i].tolist() if valid[i] else None,category=categories[i],weight=float(w[i])) for i,r in enumerate(rows)]))
for metric in METRICS:
va=np.array([np.nan if r['metrics'][a][metric]['value'] is None else r['metrics'][a][metric]['value'] for r in rows]);vb=np.array([np.nan if r['metrics'][b][metric]['value'] is None else r['metrics'][b][metric]['value'] for r in rows]);d=va-vb;component_d=np.array([d[group==g].mean() for g in range(C)]);boot=component_d[draws].mean(1);defined=int(np.isfinite(component_d).sum());point=float(component_d.mean()) if defined==C else None
raw=interval(boot,point,replicates);raw.update(defined_components=defined,total_components=C,undefined_component_ids=[ids[g] for g in range(C) if not np.isfinite(component_d[g])],arm_means={a:float(np.mean([va[group==g].mean() for g in range(C)])) if np.isfinite(va).all() else None,b:float(np.mean([vb[group==g].mean() for g in range(C)])) if np.isfinite(vb).all() else None});pair['raw'][metric]=raw;arrays[name+'_'+metric+'_raw']=boot
if metric=='kendall_tau_b':continue
adj=adjusted_estimate(d,x,valid,w);values=np.full(replicates,np.nan);failures={};ranks=[]
for k in range(replicates):
wk=mult[k][group]/(C*sizes[group]);v=adjusted_estimate(d,x,valid,wk)
if v['value'] is not None:values[k]=v['value']
else:failures[v['reason']]=failures.get(v['reason'],0)+1
ranks.append(v.get('rank'))
summary=interval(values,adj['value'],replicates);summary.update(point_diagnostics=adj,failed_reasons=failures,bootstrap_regression_refitted=True,rank_counts={str(k):ranks.count(k) for k in set(ranks)},raw_contribution_by_category={category:float(np.sum(w[np.array(categories)==category]*d[np.array(categories)==category])) if np.isfinite(d[np.array(categories)==category]).all() else None for category in sorted(set(categories))});pair['adjusted'][metric]=summary;arrays[name+'_'+metric+'_adjusted']=values
comparisons[name]=pair
decision=raw_gate(comparisons['T-G']['raw']['relative_l2'],comparisons['T-G']['raw']['last_location'],C)
return dict(parts=len(rows),components=C,decision=decision,comparisons=comparisons,seed=seed,replicates=replicates,weights='1/(original component count * original component part count); multiplicity on resample',deployment_gate='raw full-roster T-G only',adjusted_scope='descriptive mechanism; conditional on fixed models/folds and one shuffle; not causal',svd_rcond=SVD_RCOND,support_rtol=SUPPORT_RTOL),arrays