Files
MeltMind/tools/phase2_h2_data.py
T

88 lines
6.6 KiB
Python

"""Frozen H1 interface and label-scoped H2 inputs; no test target access."""
import json
from pathlib import Path
import time
import numpy as np
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import connected_components
import torch
from phase2_h1_worker_v2 import checked,sha,save
from phase2_h2_model import edge_attributes
from phase2_residual_math import residual_target
BASE=Path('/home/frank14f/optane/MeltMind')
def read(p):return json.loads(Path(p).read_text())
def config(root):
c=read(root/'contract.json')
if c['approval']!='1794900702.867059' or c['mode']!='PILOT_ONLY':raise PermissionError('No approved pilot contract')
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
if torch.__version__!=c['versions']['torch'] or np.__version__!=c['versions']['numpy']:raise ValueError('Library identity changed')
return c
def cached(c,case,name):
r=c['parts'][case];return np.load(checked(Path(r['output'])/(name+'.npy'),r['output_sha256'][name+'.npy']),allow_pickle=False)
def standardization(c,root):
n=0;mean=np.zeros(19,np.float64);M2=np.zeros(19,np.float64);seen=[];start=time.process_time()
for case in c['folds'][0]['learning']:
x=np.asarray(cached(c,case,'features'),np.float64)
if x.shape!=(c['parts'][case]['nodes'],19) or not np.isfinite(x).all():raise ValueError('Learning features invalid')
k=len(x);avg=x.mean(0);delta=avg-mean;batch=np.square(x-avg).sum(0)
M2+=batch+delta*delta*(n*k/(n+k));mean+=delta*(k/(n+k));n+=k;seen.append(case)
save(root/'progress.json',dict(stage='LEARNING_ONLY_STANDARDIZATION',parts=len(seen),nodes=n,cpu_seconds=time.process_time()-start,labels_read=False))
std=np.sqrt(M2/n);scale=np.where(std==0,1.,std)
if not np.isfinite(scale).all() or n!=c['folds'][0]['learning_nodes']:raise ValueError('Standardization coverage')
out=dict(mean=mean.tolist(),scale=scale.tolist(),population_std=std.tolist(),nodes=n,parts=seen,labels_read=False,cpu_seconds=time.process_time()-start)
save(root/'standardization.json',out);return out
def load_part(c,case,stats,role,root):
allowed=c['folds'][0][role] if role in ('learning','validation') else []
if case not in allowed:raise PermissionError('Pilot role violation')
r=c['parts'][case];g=c['geometry'][case]
with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z:
points=z['points'].copy();raw=z['raw_ids'].copy();mass=z['mass'].copy()
indptr=z['indptr'].copy();sender=z['indices'].astype(np.int64);old_rg=float(z['rg'])
n=len(raw);receiver=np.repeat(np.arange(n,dtype=np.int64),np.diff(indptr))
if not np.array_equal(raw,cached(c,case,'raw_ids')) or not np.array_equal(mass,cached(c,case,'mass')):raise ValueError('Frozen geometry/node mass identity')
if n!=r['nodes'] or len(sender)!=g['directed_edges']:raise ValueError('Frozen graph counts')
a,rg=edge_attributes(points,mass,receiver,sender)
if not np.isclose(rg,old_rg,rtol=1e-10,atol=1e-12):raise ValueError('Original mass Rg differs')
graph=csr_matrix((np.ones(len(sender)),sender,indptr),shape=(n,n));count,comp=connected_components(graph,directed=False)
x=cached(c,case,'features');F=cached(c,case,'Vf')
if F.dtype!=np.float64 or F.shape!=(n,) or not np.isfinite(F).all():raise ValueError('Original F must be FP64')
x=((np.asarray(x,np.float64)-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
if not np.isfinite(x).all():raise ValueError('Standardized feature overflow')
result=dict(x=torch.from_numpy(x),mass=torch.from_numpy(mass.astype(np.float32)),receiver=torch.from_numpy(receiver),sender=torch.from_numpy(sender),attributes=torch.from_numpy(a),component_ids=torch.from_numpy(comp.astype(np.int64)),
F=F,raw_ids=raw,physical_components=int(count),nodes=n,directed_edges=len(sender),Rg=rg,t_set=r['t_set'])
if role=='learning':
# Append the intended role/hash before opening any cached target.
with (root/'learning-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,role=role,target_sha256=r['output_sha256']['target.npy'],clock_sha256=c['folds'][0]['clock_sha256']))+'\n')
y=cached(c,case,'target');result['target']=torch.from_numpy(residual_target(y,F,c['folds'][0]['c'],fit_dtype='float32'))
return result
def to_device(data,device):
return {k:(v.to(device) if isinstance(v,torch.Tensor) else v) for k,v in data.items()}
def arguments(data):return tuple(data[k] for k in ['x','mass','receiver','sender','attributes','component_ids'])
def projection(c,epoch_seconds,validation_seconds,pilot_seconds):
f0=c['folds'][0];per_fold=[]
for f in c['folds']:
learning_ratio=max(f['learning_nodes']/f0['learning_nodes'],f['learning_edges']/f0['learning_edges'],len(f['learning'])/len(f0['learning']))
validation_ratio=max(f['validation_nodes']/f0['validation_nodes'],f['validation_edges']/f0['validation_edges'],len(f['validation'])/len(f0['validation']))
prediction_ratio=max(f['test_nodes']/f0['validation_nodes'],f['test_edges']/f0['validation_edges'],len(f['test'])/len(f0['validation']))
per_fold.append(dict(fold=f['fold'],train_200_seconds=200*epoch_seconds*learning_ratio,validation_20_seconds=20*validation_seconds*validation_ratio,test_prediction_seconds=validation_seconds*prediction_ratio,
learning_ratio=learning_ratio,validation_ratio=validation_ratio,test_ratio=prediction_ratio))
# Every measured forward/backward time includes load, transfers and optimizer.
training=1.25*sum(f['train_200_seconds']+f['validation_20_seconds'] for f in per_fold)
prediction=1.25*sum(f['test_prediction_seconds'] for f in per_fold)
total=pilot_seconds+training+prediction+c['future_setup_reserve_seconds']
passed=training<=12*3600 and prediction<=2*3600 and total<=16*3600
return dict(per_fold=per_fold,measured_full_epoch_seconds=epoch_seconds,measured_validation_inference_seconds=validation_seconds,
safety_factor=1.25,training_with_validation_seconds=training,prediction_seconds=prediction,pilot_seconds=pilot_seconds,future_setup_seconds=c['future_setup_reserve_seconds'],total_gpu_seconds=total,
fits_12h_training_2h_prediction_16h_total=passed,decision='COST_FEASIBLE_FULL_SCOPE_AUTHORIZED' if passed else 'PARKED_RESOURCE',
full_run_authorized_by='1794900702.867059',full_run_started=False,epoch_target=200,folds=5,new_fits=5,validation_targets_read=False,
limitation='Validation forward timing only; no target metrics in pilot. CPU final field/topology scoring is unmeasured and cannot be claimed affordable from this GPU projection.')