Files
MeltMind/tools/probe_torch.py
T

62 lines
3.2 KiB
Python

"""Bounded real-mesh tensor-operator probe, NOT a model benchmark or training run."""
import argparse
import json
import os
from pathlib import Path
import resource
import sys
import time
import numpy as np
import torch
def main():
p=argparse.ArgumentParser();p.add_argument("sample",type=Path);p.add_argument("output",type=Path);a=p.parse_args()
torch.set_num_threads(2);torch.manual_seed(1729)
if not torch.cuda.is_available():raise RuntimeError("CUDA unavailable")
if torch.cuda.get_device_capability()!=(7,0):raise RuntimeError("Expected V100 sm_70")
data=np.load(a.sample,allow_pickle=False);n=len(data["points_raw"])
edges=torch.as_tensor(data["edges"],device="cuda",dtype=torch.long)
src=torch.cat((edges[:,0],edges[:,1]));dst=torch.cat((edges[:,1],edges[:,0]))
x=torch.randn(n,96,device="cuda")
w=(torch.randn(96,96,device="cuda")/96**.5).requires_grad_()
degree=torch.bincount(dst,minlength=n).float().clamp_min(1)[:,None]
volumes=torch.as_tensor(data["node_volume_raw"],device="cuda",dtype=torch.float32)
volumes=volumes/volumes.sum()
def operation(amp):
with torch.autocast("cuda",dtype=torch.float16,enabled=amp):
h=x@w
summed=torch.zeros(n,96,device="cuda",dtype=torch.float32)
summed.index_add_(0,dst,h.float()[src])
local=summed/degree
global_mean=(local*volumes[:,None]).sum(0)
return local,global_mean
torch.cuda.reset_peak_memory_stats()
reference,ref_global=operation(False)
candidate,amp_global=operation(True)
error=(torch.linalg.vector_norm(candidate-reference)/torch.linalg.vector_norm(reference)).item()
if not torch.isfinite(candidate).all() or error>0.01:raise ValueError(f"AMP comparison failed {error}")
del reference,candidate,ref_global,amp_global
timings=[]
for i in range(25):
torch.cuda.synchronize();start=time.perf_counter()
local,pooled=operation(True)
loss=local.square().mean()+pooled.square().mean();loss.backward()
if not torch.isfinite(w.grad).all():raise ValueError("Nonfinite gradient")
w.grad=None
del local,pooled,loss
torch.cuda.synchronize()
if i>=5:timings.append(time.perf_counter()-start)
result=dict(status="pass",purpose="operator compatibility/AMP/autograd only; no optimizer, labels, model or training",
python=sys.version,executable=sys.executable,torch=torch.__version__,cuda=torch.version.cuda,
gpu=torch.cuda.get_device_name(),capability=list(torch.cuda.get_device_capability()),
physical_gpu=os.environ.get("CUDA_VISIBLE_DEVICES"),nodes=n,undirected_edges=len(edges),width=96,
warmup_iterations=5,measured_iterations=20,fp16_vs_fp32_relative_l2=error,
forward_backward_seconds_p50=float(np.median(timings)),forward_backward_seconds_p90=float(np.quantile(timings,.9)),
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
a.output.write_text(json.dumps(result,indent=2)+"\n");print(json.dumps(result,indent=2))
if __name__=="__main__":main()