Freeze H2 graph-token implementation and nonzero gradient CPU acceptance
This commit is contained in:
1 parent
c1e2077e8a
commit
7054acdbe0
9 files changed
+1516
No files matched your search
@@ -0,0 +1,108 @@
|
||||
"""H2-GEO reviewed fixed FP32 graph/token recipe; no data or launches."""
|
||||
import math
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.utils.checkpoint import checkpoint
|
||||
|
||||
EDGE_CHUNK=65536
|
||||
NODE_CHUNK=8192
|
||||
ATOL=3e-5
|
||||
RTOL=3e-4
|
||||
|
||||
def lr_epoch(epoch):
|
||||
if not 1<=epoch<=200:raise ValueError('Fixed 1-based epochs 1..200')
|
||||
if epoch<=10:return 1e-4+(epoch-1)*9e-4/9
|
||||
return 1e-5+(1e-3-1e-5)*(1+math.cos(math.pi*(epoch-10)/190))/2
|
||||
|
||||
def namespace_components(part_ids,local_ids):
|
||||
keys=list(zip(part_ids,local_ids));lookup={k:i for i,k in enumerate(sorted(set(keys)))}
|
||||
return np.array([lookup[k] for k in keys],np.int64)
|
||||
|
||||
def edge_attributes(points,mass,receiver,sender):
|
||||
x=np.asarray(points,np.float64);m=np.asarray(mass,np.float64)
|
||||
if not np.isfinite(x).all() or not np.isfinite(m).all() or np.any(m<=0):raise ValueError('Invalid geometry/mass')
|
||||
center=np.sum(x*m[:,None],axis=0)/m.sum()
|
||||
rg=float(np.sqrt(np.sum(m*np.sum((x-center)**2,axis=1))/m.sum()))
|
||||
if not math.isfinite(rg) or rg<=0:raise ValueError('Invalid mass Rg')
|
||||
xi=(x-center)/rg;delta=xi[sender]-xi[receiver];length=np.linalg.norm(delta,axis=1)
|
||||
if np.any(length<=0):raise ValueError('Nonpositive true edge')
|
||||
return np.column_stack([delta,length]).astype(np.float32),rg
|
||||
|
||||
class GeoBlock(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.local_norm=nn.LayerNorm(64);self.global_norm=nn.LayerNorm(64);self.ffn_norm=nn.LayerNorm(64)
|
||||
self.edge=nn.Sequential(nn.Linear(4,16),nn.GELU(),nn.Linear(16,1),nn.Softplus())
|
||||
self.local_value=nn.Linear(64,64,bias=False)
|
||||
self.assignment=nn.Linear(64,64);self.global_value=nn.Linear(64,64,bias=False)
|
||||
self.attention=nn.MultiheadAttention(64,4,dropout=0,batch_first=True)
|
||||
self.ffn=nn.Sequential(nn.Linear(64,128),nn.GELU(),nn.Linear(128,64))
|
||||
|
||||
def local(self,h,m,receiver,sender,attributes,edge_chunk):
|
||||
values=self.local_value(h);num=h.new_zeros(h.shape);den=h.new_zeros((len(h),1))
|
||||
for lo in range(0,len(sender),edge_chunk):
|
||||
hi=lo+edge_chunk;r=receiver[lo:hi];s=sender[lo:hi]
|
||||
weight=self.edge(attributes[lo:hi])*m[s,None]
|
||||
num.index_add_(0,r,weight*values[s]);den.index_add_(0,r,weight)
|
||||
occupied=torch.bincount(receiver,minlength=len(h))>0
|
||||
if not bool(torch.isfinite(den).all()) or bool((den[occupied]<=0).any()):raise ValueError('Invalid local denominator')
|
||||
safe=torch.where(occupied[:,None],den,torch.ones_like(den))
|
||||
return num/safe
|
||||
|
||||
def global_message(self,h,m,components,node_chunk,detach_global=False):
|
||||
out=h.new_zeros(h.shape)
|
||||
for ids in components:
|
||||
numerator=h.new_zeros((64,64));denominator=h.new_zeros((64,1));chunks=[]
|
||||
for lo in range(0,len(ids),node_chunk):
|
||||
ix=ids[lo:lo+node_chunk];hh=h[ix]
|
||||
assignment=torch.softmax(self.assignment(hh),dim=-1)
|
||||
weighted=assignment*m[ix,None]
|
||||
numerator=numerator+weighted.T@self.global_value(hh)
|
||||
denominator=denominator+weighted.sum(0)[:,None]
|
||||
chunks.append((ix,assignment))
|
||||
if not bool(torch.isfinite(denominator).all()) or bool((denominator<=0).any()):raise ValueError('Invalid token denominator')
|
||||
z=numerator/denominator
|
||||
if detach_global:z=z.detach() # Only the synthetic negative control enables this.
|
||||
z=self.attention(z[None],z[None],z[None],need_weights=False)[0][0]
|
||||
for ix,assignment in chunks:out.index_copy_(0,ix,assignment@z)
|
||||
return out
|
||||
|
||||
def forward(self,h,m,receiver,sender,attributes,components,edge_chunk,node_chunk,detach_global=False):
|
||||
h=h+self.local(self.local_norm(h),m,receiver,sender,attributes,edge_chunk)
|
||||
h=h+self.global_message(self.global_norm(h),m,components,node_chunk,detach_global)
|
||||
return h+self.ffn(self.ffn_norm(h))
|
||||
|
||||
class H2Geo(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__();self.encoder=nn.Sequential(nn.Linear(19,64),nn.GELU(),nn.Linear(64,64))
|
||||
self.blocks=nn.ModuleList([GeoBlock() for _ in range(4)])
|
||||
self.head=nn.Sequential(nn.Linear(83,64),nn.GELU(),nn.Linear(64,1))
|
||||
nn.init.zeros_(self.head[-1].weight);nn.init.zeros_(self.head[-1].bias)
|
||||
|
||||
def forward(self,x,m,receiver,sender,attributes,component_ids,*,edge_chunk=EDGE_CHUNK,node_chunk=NODE_CHUNK,checkpoint_blocks=True,detach_global=False):
|
||||
if x.dtype!=torch.float32 or m.dtype!=torch.float32 or attributes.dtype!=torch.float32:raise ValueError('Only FP32 model/aggregations')
|
||||
if not bool(torch.isfinite(m).all()) or bool((m<=0).any()):raise ValueError('Nonpositive mass; no epsilon')
|
||||
if bool((component_ids[receiver]!=component_ids[sender]).any()):raise ValueError('Cross-component edge')
|
||||
components=[torch.nonzero(component_ids==c,as_tuple=True)[0] for c in torch.unique(component_ids)]
|
||||
h=self.encoder(x)
|
||||
for block in self.blocks:
|
||||
def run(v,b=block):return b(v,m,receiver,sender,attributes,components,edge_chunk,node_chunk,detach_global)
|
||||
h=checkpoint(run,h,use_reentrant=False,preserve_rng_state=False) if checkpoint_blocks and self.training and torch.is_grad_enabled() else run(h)
|
||||
outputs=[]
|
||||
for lo in range(0,len(x),node_chunk):outputs.append(self.head(torch.cat([h[lo:lo+node_chunk],x[lo:lo+node_chunk]],dim=-1))[:,0])
|
||||
return torch.cat(outputs)
|
||||
|
||||
def optimizer(model):
|
||||
decay=[];no_decay=[]
|
||||
for name,p in model.named_parameters():
|
||||
(no_decay if name.endswith('bias') or p.ndim==1 else decay).append(p)
|
||||
return torch.optim.AdamW([dict(params=decay,weight_decay=1e-4),dict(params=no_decay,weight_decay=0.)],lr=lr_epoch(1),betas=(.9,.999),eps=1e-8)
|
||||
|
||||
def weighted_loss(prediction,target,parts,part_nodes,total_nodes):
|
||||
if len(prediction)!=part_nodes or prediction.shape!=target.shape:raise ValueError('Full part loss shape')
|
||||
return torch.mean((prediction-target)**2)*(parts*part_nodes/total_nodes)
|
||||
|
||||
def comparison(a,b):
|
||||
a=a.detach();b=b.detach();difference=float((a-b).abs().max());scale=float(a.abs().max())
|
||||
return dict(max_absolute=difference,reference_scale=scale,relative_to_scale=difference/max(scale,1e-30),passed=difference<=ATOL+RTOL*scale)
|
||||
Reference in new issue
Block a user