52 lines
4.1 KiB
Python
52 lines
4.1 KiB
Python
"""Frozen item1 contrasts on stored part metrics; common component bootstrap draws."""
|
|
import math
|
|
import numpy as np
|
|
ARMS=('epoch25','epoch50','S1_epoch100_reuse','step50_U','step50_2U','step100_U','step100_2U','c25F','c50F','c100F')
|
|
METRICS=('relative_l2','last_location','kendall_tau_b')
|
|
|
|
def classify(s,g,delta=.001,complete=True):
|
|
if not complete or any(v is None or not math.isfinite(v) for x in [s,g] for v in [x.get('low'),x.get('high')]):return 'UNIDENTIFIED_OR_OVERFIT_SIGNS'
|
|
eq=lambda x:-delta<=x['low']<=x['high']<=delta
|
|
positive=lambda x:x['low']>delta
|
|
if positive(s) and eq(g):return 'DATA_BENEFIT_AT_TESTED_RANGE'
|
|
if eq(s) and positive(g):return 'UPDATE_BUDGET_AND_SCHEDULE_BENEFIT'
|
|
if positive(s) and positive(g):return 'BOTH_BENEFITS'
|
|
if eq(s) and eq(g):return 'LOCAL_PLATEAU_ONLY'
|
|
return 'UNIDENTIFIED_OR_OVERFIT_SIGNS'
|
|
|
|
def contrasts(a,counts):
|
|
den=math.log2(counts['100']/counts['50']);den0=math.log2(counts['50']/counts['25'])
|
|
if den<=0 or den0<=0:raise ValueError('Actual nested component counts')
|
|
sU=(a['step50_U']-a['step100_U'])/den;s2=(a['step50_2U']-a['step100_2U'])/den
|
|
out=dict(s_U=sU,s_2U=s2,g_50=a['step50_U']-a['step50_2U'],g_100=a['step100_U']-a['step100_2U'],s_2U_minus_s_U=s2-sU,epoch_s_25to50=(a['epoch25']-a['epoch50'])/den0,epoch_s_50to100=(a['epoch50']-a['S1_epoch100_reuse'])/den,clock_s_25to50=(a['c25F']-a['c50F'])/den0,clock_s_50to100=(a['c50F']-a['c100F'])/den)
|
|
for arm,pool in [('epoch25',25),('epoch50',50),('S1_epoch100_reuse',100),('step50_U',50),('step50_2U',50),('step100_U',100),('step100_2U',100)]:out[arm+'_minus_clock']=a[arm]-a['c'+str(pool)+'F']
|
|
return out
|
|
|
|
def summarize(rows,counts,replicates=2000,seed=2026111408):
|
|
out={}
|
|
for domain in ['own_learning','common_validation','common_test']:
|
|
domainrows=[r for r in rows if r['domain']==domain];result={}
|
|
for metric in METRICS:
|
|
values={};excluded={};parts={}
|
|
for arm in ARMS:
|
|
rs=[r for r in domainrows if r['arm']==arm];parts[arm]=len(rs);groups={}
|
|
for r in rs:groups.setdefault(r['component_id'],[]).append(r)
|
|
values[arm]={};excluded[arm]={}
|
|
for g,gr in sorted(groups.items()):
|
|
x=[r['metrics'][metric]['value'] for r in gr]
|
|
if any(v is None or not math.isfinite(v) for v in x):excluded[arm][g]=[dict(case=r['case'],reason=r['metrics'][metric].get('reason')) for r in gr if r['metrics'][metric]['value'] is None or not math.isfinite(r['metrics'][metric]['value'])]
|
|
else:values[arm][g]=float(np.mean(x))
|
|
result[metric]=dict(means={a:float(np.mean(list(v.values()))) if v else None for a,v in values.items()},parts=parts,components={a:len(v) for a,v in values.items()},excluded=excluded)
|
|
if domain=='own_learning':
|
|
result[metric]['contrasts']='Not computed across different learning domains';continue
|
|
ids=sorted(set.intersection(*(set(values[a]) for a in ARMS)));expected=len({r['component_id'] for r in domainrows});summary={}
|
|
if ids:
|
|
arrays={a:np.array([values[a][i] for i in ids]) for a in ARMS};rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence(seed)));draw=rng.integers(0,len(ids),size=(replicates,len(ids)));point=contrasts({a:x.mean() for a,x in arrays.items()},counts);boot=contrasts({a:x[draw].mean(1) for a,x in arrays.items()},counts)
|
|
for name,value in point.items():
|
|
sample=boot[name];undefined=int(np.sum(~np.isfinite(sample)));lo,hi=(None,None) if undefined else np.percentile(sample,[2.5,97.5],method='linear').tolist();summary[name]=dict(value=float(value),low=lo,high=hi,undefined_replicates=undefined)
|
|
result[metric].update(contrasts=summary,paired_component_ids=ids,paired_components=len(ids),expected_components=expected,complete_support=len(ids)==expected and expected>0,replicates=replicates,seed=seed,joint_draws_all_contrasts=True)
|
|
if domain!='own_learning':
|
|
field=result['relative_l2'];ci=field['contrasts'];result['field_classification']=classify(ci.get('s_2U',{}),ci.get('g_100',{}),complete=field['complete_support'])
|
|
out[domain]=result
|
|
return dict(domains=out,external_result_descriptive_reused_fold=True,uncertainty='Evaluation components only; no training seed/pool uncertainty',update_effect='Joint complete schedule and update budget change',no_training_selection=True)
|