62 lines
3.2 KiB
Python
62 lines
3.2 KiB
Python
"""Bounded real-mesh tensor-operator probe, NOT a model benchmark or training run."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import sys
|
|
import time
|
|
import numpy as np
|
|
import torch
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument("sample",type=Path);p.add_argument("output",type=Path);a=p.parse_args()
|
|
torch.set_num_threads(2);torch.manual_seed(1729)
|
|
if not torch.cuda.is_available():raise RuntimeError("CUDA unavailable")
|
|
if torch.cuda.get_device_capability()!=(7,0):raise RuntimeError("Expected V100 sm_70")
|
|
data=np.load(a.sample,allow_pickle=False);n=len(data["points_raw"])
|
|
edges=torch.as_tensor(data["edges"],device="cuda",dtype=torch.long)
|
|
src=torch.cat((edges[:,0],edges[:,1]));dst=torch.cat((edges[:,1],edges[:,0]))
|
|
x=torch.randn(n,96,device="cuda")
|
|
w=(torch.randn(96,96,device="cuda")/96**.5).requires_grad_()
|
|
degree=torch.bincount(dst,minlength=n).float().clamp_min(1)[:,None]
|
|
volumes=torch.as_tensor(data["node_volume_raw"],device="cuda",dtype=torch.float32)
|
|
volumes=volumes/volumes.sum()
|
|
def operation(amp):
|
|
with torch.autocast("cuda",dtype=torch.float16,enabled=amp):
|
|
h=x@w
|
|
summed=torch.zeros(n,96,device="cuda",dtype=torch.float32)
|
|
summed.index_add_(0,dst,h.float()[src])
|
|
local=summed/degree
|
|
global_mean=(local*volumes[:,None]).sum(0)
|
|
return local,global_mean
|
|
torch.cuda.reset_peak_memory_stats()
|
|
reference,ref_global=operation(False)
|
|
candidate,amp_global=operation(True)
|
|
error=(torch.linalg.vector_norm(candidate-reference)/torch.linalg.vector_norm(reference)).item()
|
|
if not torch.isfinite(candidate).all() or error>0.01:raise ValueError(f"AMP comparison failed {error}")
|
|
del reference,candidate,ref_global,amp_global
|
|
timings=[]
|
|
for i in range(25):
|
|
torch.cuda.synchronize();start=time.perf_counter()
|
|
local,pooled=operation(True)
|
|
loss=local.square().mean()+pooled.square().mean();loss.backward()
|
|
if not torch.isfinite(w.grad).all():raise ValueError("Nonfinite gradient")
|
|
w.grad=None
|
|
del local,pooled,loss
|
|
torch.cuda.synchronize()
|
|
if i>=5:timings.append(time.perf_counter()-start)
|
|
result=dict(status="pass",purpose="operator compatibility/AMP/autograd only; no optimizer, labels, model or training",
|
|
python=sys.version,executable=sys.executable,torch=torch.__version__,cuda=torch.version.cuda,
|
|
gpu=torch.cuda.get_device_name(),capability=list(torch.cuda.get_device_capability()),
|
|
physical_gpu=os.environ.get("CUDA_VISIBLE_DEVICES"),nodes=n,undirected_edges=len(edges),width=96,
|
|
warmup_iterations=5,measured_iterations=20,fp16_vs_fp32_relative_l2=error,
|
|
forward_backward_seconds_p50=float(np.median(timings)),forward_backward_seconds_p90=float(np.quantile(timings,.9)),
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
|
|
peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
|
a.output.write_text(json.dumps(result,indent=2)+"\n");print(json.dumps(result,indent=2))
|
|
|
|
|
|
if __name__=="__main__":main()
|