Files
MeltMind/tools/test_s2_precision_pilot.py

51 lines
2.8 KiB
Python

"""Synthetic precision regression checks, without corpus or GPU use."""
import unittest
import torch
from s2_models.network import QueryModel as OriginalModel
from s2_models_v2.network import QueryModel as PrecisionModel
from s2_models.normalization import BaseStandardizer
from s2_models.attention import Physics_Attention_Irregular_Mesh as OriginalAttention
from s2_models_v2.attention import Physics_Attention_Irregular_Mesh as PrecisionAttention
class PrecisionTests(unittest.TestCase):
def setUp(self):
torch.set_num_threads(1);torch.manual_seed(9)
def test_fp32_model_outputs_and_gradients_unchanged_for_both_arms(self):
stats=BaseStandardizer('0','100',('synthetic',),32,(0.,)*10,(1.,)*10)
for arm in ['N','E']:
for width in [128,256]:
with self.subTest(arm=arm,width=width):
original=OriginalModel(arm,width,standardizer=stats)
fixed=PrecisionModel(arm,width,standardizer=stats)
fixed.load_state_dict(original.state_dict())
base=torch.randn(32,10);ids=torch.arange(8);gaps=torch.randn(8,2);edge=torch.randn(8,4) if arm=='E' else None
a=original(base,ids,gaps,edge);b=fixed(base,ids,gaps,edge)
self.assertTrue(torch.equal(a,b));a.square().sum().backward();b.square().sum().backward()
self.assertEqual(original.parameter_counts(),fixed.parameter_counts())
for p,q in zip(original.parameters(),fixed.parameters()):self.assertTrue(torch.equal(p.grad,q.grad))
def test_large_slice_fp16_overflow_removed_and_backward_finite(self):
original=OriginalAttention(8,heads=1,dim_head=8,slice_num=1)
fixed=PrecisionAttention(8,heads=1,dim_head=8,slice_num=1)
with torch.no_grad():
original.in_project_fx.weight.zero_();original.in_project_fx.bias.fill_(8.)
fixed.load_state_dict(original.state_dict())
x=torch.ones(1,100000,8,requires_grad=True)
with torch.no_grad():reference=original(x)
numerator=torch.einsum('bhnc,bhng->bhgc',torch.full((1,1,100000,8),8.,dtype=torch.float16),torch.ones(1,1,100000,1,dtype=torch.float16))
self.assertTrue(bool(torch.isinf(numerator).all()))
with torch.autocast('cpu',dtype=torch.float16):
a=original(x);b=fixed(x)
self.assertFalse(bool(torch.isfinite(a).all()))
self.assertTrue(bool(torch.isfinite(b).all()))
torch.testing.assert_close(b.float(),reference,rtol=5e-3,atol=5e-3)
b.float().square().mean().backward()
self.assertTrue(bool(torch.isfinite(x.grad).all()))
for p in fixed.parameters():
if p.grad is not None:self.assertTrue(bool(torch.isfinite(p.grad).all()))
if __name__=='__main__':unittest.main()