Publish fixed regional-context item b entry and adopted item1b review with capacity hold
This commit is contained in:
1 parent
ffd5c92958
commit
31a20abd43
34 files changed
+16026
No files matched your search
@@ -0,0 +1,71 @@
|
||||
"""Reuse frozen B13 readout; learning-only normalization, joint missingness, no partition."""
|
||||
import json,time
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h1_worker_v2 import checked,save
|
||||
from phase2_h2_data import standardization as original_standardization,load_part as original_load
|
||||
|
||||
|
||||
def valid_rows(extra,status,nodes):
|
||||
x=np.asarray(extra)
|
||||
if x.shape!=(nodes,2) or np.isinf(x).any():
|
||||
raise ValueError('Context shape/infinity input inconsistency')
|
||||
finite=np.isfinite(x)
|
||||
if np.any(finite[:,0]!=finite[:,1]):
|
||||
raise ValueError('Unexpected single-column missingness')
|
||||
if status=='GEOMETRY_COMPLETE' and finite.all():return np.ones(nodes,dtype=bool)
|
||||
if status=='SCIENTIFIC_MISSING' and np.isnan(x).all():return np.zeros(nodes,dtype=bool)
|
||||
raise ValueError('Frozen whole-part context status inconsistent with array')
|
||||
|
||||
|
||||
def read_context(old,c,case):
|
||||
p=c['context'][case];r=old['parts'][case]
|
||||
if p['nodes']!=r['nodes'] or p['raw_ids_sha256']!=r['output_sha256']['raw_ids.npy'] or p['features_sha256']!=r['output_sha256']['features.npy']:
|
||||
raise ValueError('Context original node/feature identity')
|
||||
result=json.loads(Path(checked(p['result'],p['result_sha256'])).read_text())
|
||||
if result['case']!=case or result['status']!=p['status'] or result['extra_sha256']!=p['extra_sha256']:
|
||||
raise ValueError('Context result identity')
|
||||
x=np.load(checked(p['extra'],p['extra_sha256']),allow_pickle=False)
|
||||
return np.asarray(x,np.float64),valid_rows(x,p['status'],p['nodes'])
|
||||
|
||||
|
||||
def moments(chunks):
|
||||
n=0;mean=np.zeros(2,np.float64);m2=np.zeros(2,np.float64)
|
||||
for x,valid in chunks:
|
||||
values=x[valid];k=len(values)
|
||||
if not k:continue
|
||||
avg=values.mean(0);delta=avg-mean
|
||||
m2+=np.square(values-avg).sum(0)+delta*delta*n*k/(n+k)
|
||||
mean+=delta*k/(n+k);n+=k
|
||||
if n==0:raise ValueError('No valid learning nodes; no invented normalization')
|
||||
std=np.sqrt(m2/n);scale=np.where(std==0,1.,std)
|
||||
if not np.isfinite(mean).all() or not np.isfinite(scale).all():raise ValueError('Context normalization overflow')
|
||||
return dict(mean=mean.tolist(),scale=scale.tolist(),population_std=std.tolist(),valid_nodes=n)
|
||||
|
||||
|
||||
def encode(extra,valid,stats):
|
||||
out=np.zeros((len(extra),3),np.float32)
|
||||
out[valid,:2]=((extra[valid]-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
|
||||
out[valid,2]=1
|
||||
if not np.isfinite(out).all():raise ValueError('Context standardized overflow')
|
||||
return out
|
||||
|
||||
|
||||
def standardization(old,c,root):
|
||||
start=time.process_time();stats=original_standardization(old,root);seen=[]
|
||||
def chunks():
|
||||
for case in old['folds'][0]['learning']:
|
||||
x,valid=read_context(old,c,case);seen.append(dict(case=case,nodes=len(x),valid_nodes=int(valid.sum())))
|
||||
yield x,valid
|
||||
context=moments(chunks());context.update(parts=seen,weighting='unit supervised node as original H2',labels_read=False,cpu_seconds=time.process_time()-start)
|
||||
stats['context']=context;save(root/'standardization.json',stats);return stats
|
||||
|
||||
|
||||
def load_part(old,c,case,stats,role,root):
|
||||
# The original role guard runs before any extra context read.
|
||||
data=original_load(old,case,stats,role,root)
|
||||
extra,valid=read_context(old,c,case)
|
||||
data['x']=torch.cat([data['x'],torch.from_numpy(encode(extra,valid,stats['context']))],dim=1)
|
||||
data['context_valid_nodes']=int(valid.sum())
|
||||
return data
|
||||
Reference in new issue
Block a user