88 lines
6.6 KiB
Python
88 lines
6.6 KiB
Python
"""Frozen H1 interface and label-scoped H2 inputs; no test target access."""
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import numpy as np
|
|
from scipy.sparse import csr_matrix
|
|
from scipy.sparse.csgraph import connected_components
|
|
import torch
|
|
from phase2_h1_worker_v2 import checked,sha,save
|
|
from phase2_h2_model import edge_attributes
|
|
from phase2_residual_math import residual_target
|
|
|
|
BASE=Path('/home/frank14f/optane/MeltMind')
|
|
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
|
|
def config(root):
|
|
c=read(root/'contract.json')
|
|
if c['approval']!='1794900702.867059' or c['mode']!='PILOT_ONLY':raise PermissionError('No approved pilot contract')
|
|
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
|
|
if torch.__version__!=c['versions']['torch'] or np.__version__!=c['versions']['numpy']:raise ValueError('Library identity changed')
|
|
return c
|
|
|
|
def cached(c,case,name):
|
|
r=c['parts'][case];return np.load(checked(Path(r['output'])/(name+'.npy'),r['output_sha256'][name+'.npy']),allow_pickle=False)
|
|
|
|
def standardization(c,root):
|
|
n=0;mean=np.zeros(19,np.float64);M2=np.zeros(19,np.float64);seen=[];start=time.process_time()
|
|
for case in c['folds'][0]['learning']:
|
|
x=np.asarray(cached(c,case,'features'),np.float64)
|
|
if x.shape!=(c['parts'][case]['nodes'],19) or not np.isfinite(x).all():raise ValueError('Learning features invalid')
|
|
k=len(x);avg=x.mean(0);delta=avg-mean;batch=np.square(x-avg).sum(0)
|
|
M2+=batch+delta*delta*(n*k/(n+k));mean+=delta*(k/(n+k));n+=k;seen.append(case)
|
|
save(root/'progress.json',dict(stage='LEARNING_ONLY_STANDARDIZATION',parts=len(seen),nodes=n,cpu_seconds=time.process_time()-start,labels_read=False))
|
|
std=np.sqrt(M2/n);scale=np.where(std==0,1.,std)
|
|
if not np.isfinite(scale).all() or n!=c['folds'][0]['learning_nodes']:raise ValueError('Standardization coverage')
|
|
out=dict(mean=mean.tolist(),scale=scale.tolist(),population_std=std.tolist(),nodes=n,parts=seen,labels_read=False,cpu_seconds=time.process_time()-start)
|
|
save(root/'standardization.json',out);return out
|
|
|
|
def load_part(c,case,stats,role,root):
|
|
allowed=c['folds'][0][role] if role in ('learning','validation') else []
|
|
if case not in allowed:raise PermissionError('Pilot role violation')
|
|
r=c['parts'][case];g=c['geometry'][case]
|
|
with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z:
|
|
points=z['points'].copy();raw=z['raw_ids'].copy();mass=z['mass'].copy()
|
|
indptr=z['indptr'].copy();sender=z['indices'].astype(np.int64);old_rg=float(z['rg'])
|
|
n=len(raw);receiver=np.repeat(np.arange(n,dtype=np.int64),np.diff(indptr))
|
|
if not np.array_equal(raw,cached(c,case,'raw_ids')) or not np.array_equal(mass,cached(c,case,'mass')):raise ValueError('Frozen geometry/node mass identity')
|
|
if n!=r['nodes'] or len(sender)!=g['directed_edges']:raise ValueError('Frozen graph counts')
|
|
a,rg=edge_attributes(points,mass,receiver,sender)
|
|
if not np.isclose(rg,old_rg,rtol=1e-10,atol=1e-12):raise ValueError('Original mass Rg differs')
|
|
graph=csr_matrix((np.ones(len(sender)),sender,indptr),shape=(n,n));count,comp=connected_components(graph,directed=False)
|
|
x=cached(c,case,'features');F=cached(c,case,'Vf')
|
|
if F.dtype!=np.float64 or F.shape!=(n,) or not np.isfinite(F).all():raise ValueError('Original F must be FP64')
|
|
x=((np.asarray(x,np.float64)-np.asarray(stats['mean']))/np.asarray(stats['scale'])).astype(np.float32)
|
|
if not np.isfinite(x).all():raise ValueError('Standardized feature overflow')
|
|
result=dict(x=torch.from_numpy(x),mass=torch.from_numpy(mass.astype(np.float32)),receiver=torch.from_numpy(receiver),sender=torch.from_numpy(sender),attributes=torch.from_numpy(a),component_ids=torch.from_numpy(comp.astype(np.int64)),
|
|
F=F,raw_ids=raw,physical_components=int(count),nodes=n,directed_edges=len(sender),Rg=rg,t_set=r['t_set'])
|
|
if role=='learning':
|
|
# Append the intended role/hash before opening any cached target.
|
|
with (root/'learning-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,role=role,target_sha256=r['output_sha256']['target.npy'],clock_sha256=c['folds'][0]['clock_sha256']))+'\n')
|
|
y=cached(c,case,'target');result['target']=torch.from_numpy(residual_target(y,F,c['folds'][0]['c'],fit_dtype='float32'))
|
|
return result
|
|
|
|
def to_device(data,device):
|
|
return {k:(v.to(device) if isinstance(v,torch.Tensor) else v) for k,v in data.items()}
|
|
|
|
def arguments(data):return tuple(data[k] for k in ['x','mass','receiver','sender','attributes','component_ids'])
|
|
|
|
def projection(c,epoch_seconds,validation_seconds,pilot_seconds):
|
|
f0=c['folds'][0];per_fold=[]
|
|
for f in c['folds']:
|
|
learning_ratio=max(f['learning_nodes']/f0['learning_nodes'],f['learning_edges']/f0['learning_edges'],len(f['learning'])/len(f0['learning']))
|
|
validation_ratio=max(f['validation_nodes']/f0['validation_nodes'],f['validation_edges']/f0['validation_edges'],len(f['validation'])/len(f0['validation']))
|
|
prediction_ratio=max(f['test_nodes']/f0['validation_nodes'],f['test_edges']/f0['validation_edges'],len(f['test'])/len(f0['validation']))
|
|
per_fold.append(dict(fold=f['fold'],train_200_seconds=200*epoch_seconds*learning_ratio,validation_20_seconds=20*validation_seconds*validation_ratio,test_prediction_seconds=validation_seconds*prediction_ratio,
|
|
learning_ratio=learning_ratio,validation_ratio=validation_ratio,test_ratio=prediction_ratio))
|
|
# Every measured forward/backward time includes load, transfers and optimizer.
|
|
training=1.25*sum(f['train_200_seconds']+f['validation_20_seconds'] for f in per_fold)
|
|
prediction=1.25*sum(f['test_prediction_seconds'] for f in per_fold)
|
|
total=pilot_seconds+training+prediction+c['future_setup_reserve_seconds']
|
|
passed=training<=12*3600 and prediction<=2*3600 and total<=16*3600
|
|
return dict(per_fold=per_fold,measured_full_epoch_seconds=epoch_seconds,measured_validation_inference_seconds=validation_seconds,
|
|
safety_factor=1.25,training_with_validation_seconds=training,prediction_seconds=prediction,pilot_seconds=pilot_seconds,future_setup_seconds=c['future_setup_reserve_seconds'],total_gpu_seconds=total,
|
|
fits_12h_training_2h_prediction_16h_total=passed,decision='COST_FEASIBLE_FULL_SCOPE_AUTHORIZED' if passed else 'PARKED_RESOURCE',
|
|
full_run_authorized_by='1794900702.867059',full_run_started=False,epoch_target=200,folds=5,new_fits=5,validation_targets_read=False,
|
|
limitation='Validation forward timing only; no target metrics in pilot. CPU final field/topology scoring is unmeasured and cannot be claimed affordable from this GPU projection.')
|