51 lines
2.8 KiB
Python
51 lines
2.8 KiB
Python
"""Synthetic precision regression checks, without corpus or GPU use."""
|
|
import unittest
|
|
import torch
|
|
from s2_models.network import QueryModel as OriginalModel
|
|
from s2_models_v2.network import QueryModel as PrecisionModel
|
|
from s2_models.normalization import BaseStandardizer
|
|
from s2_models.attention import Physics_Attention_Irregular_Mesh as OriginalAttention
|
|
from s2_models_v2.attention import Physics_Attention_Irregular_Mesh as PrecisionAttention
|
|
|
|
|
|
class PrecisionTests(unittest.TestCase):
|
|
def setUp(self):
|
|
torch.set_num_threads(1);torch.manual_seed(9)
|
|
|
|
def test_fp32_model_outputs_and_gradients_unchanged_for_both_arms(self):
|
|
stats=BaseStandardizer('0','100',('synthetic',),32,(0.,)*10,(1.,)*10)
|
|
for arm in ['N','E']:
|
|
for width in [128,256]:
|
|
with self.subTest(arm=arm,width=width):
|
|
original=OriginalModel(arm,width,standardizer=stats)
|
|
fixed=PrecisionModel(arm,width,standardizer=stats)
|
|
fixed.load_state_dict(original.state_dict())
|
|
base=torch.randn(32,10);ids=torch.arange(8);gaps=torch.randn(8,2);edge=torch.randn(8,4) if arm=='E' else None
|
|
a=original(base,ids,gaps,edge);b=fixed(base,ids,gaps,edge)
|
|
self.assertTrue(torch.equal(a,b));a.square().sum().backward();b.square().sum().backward()
|
|
self.assertEqual(original.parameter_counts(),fixed.parameter_counts())
|
|
for p,q in zip(original.parameters(),fixed.parameters()):self.assertTrue(torch.equal(p.grad,q.grad))
|
|
|
|
def test_large_slice_fp16_overflow_removed_and_backward_finite(self):
|
|
original=OriginalAttention(8,heads=1,dim_head=8,slice_num=1)
|
|
fixed=PrecisionAttention(8,heads=1,dim_head=8,slice_num=1)
|
|
with torch.no_grad():
|
|
original.in_project_fx.weight.zero_();original.in_project_fx.bias.fill_(8.)
|
|
fixed.load_state_dict(original.state_dict())
|
|
x=torch.ones(1,100000,8,requires_grad=True)
|
|
with torch.no_grad():reference=original(x)
|
|
numerator=torch.einsum('bhnc,bhng->bhgc',torch.full((1,1,100000,8),8.,dtype=torch.float16),torch.ones(1,1,100000,1,dtype=torch.float16))
|
|
self.assertTrue(bool(torch.isinf(numerator).all()))
|
|
with torch.autocast('cpu',dtype=torch.float16):
|
|
a=original(x);b=fixed(x)
|
|
self.assertFalse(bool(torch.isfinite(a).all()))
|
|
self.assertTrue(bool(torch.isfinite(b).all()))
|
|
torch.testing.assert_close(b.float(),reference,rtol=5e-3,atol=5e-3)
|
|
b.float().square().mean().backward()
|
|
self.assertTrue(bool(torch.isfinite(x.grad).all()))
|
|
for p in fixed.parameters():
|
|
if p.grad is not None:self.assertTrue(bool(torch.isfinite(p.grad).all()))
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|