"""Synthetic precision regression checks, without corpus or GPU use.""" import unittest import torch from s2_models.network import QueryModel as OriginalModel from s2_models_v2.network import QueryModel as PrecisionModel from s2_models.normalization import BaseStandardizer from s2_models.attention import Physics_Attention_Irregular_Mesh as OriginalAttention from s2_models_v2.attention import Physics_Attention_Irregular_Mesh as PrecisionAttention class PrecisionTests(unittest.TestCase): def setUp(self): torch.set_num_threads(1);torch.manual_seed(9) def test_fp32_model_outputs_and_gradients_unchanged_for_both_arms(self): stats=BaseStandardizer('0','100',('synthetic',),32,(0.,)*10,(1.,)*10) for arm in ['N','E']: for width in [128,256]: with self.subTest(arm=arm,width=width): original=OriginalModel(arm,width,standardizer=stats) fixed=PrecisionModel(arm,width,standardizer=stats) fixed.load_state_dict(original.state_dict()) base=torch.randn(32,10);ids=torch.arange(8);gaps=torch.randn(8,2);edge=torch.randn(8,4) if arm=='E' else None a=original(base,ids,gaps,edge);b=fixed(base,ids,gaps,edge) self.assertTrue(torch.equal(a,b));a.square().sum().backward();b.square().sum().backward() self.assertEqual(original.parameter_counts(),fixed.parameter_counts()) for p,q in zip(original.parameters(),fixed.parameters()):self.assertTrue(torch.equal(p.grad,q.grad)) def test_large_slice_fp16_overflow_removed_and_backward_finite(self): original=OriginalAttention(8,heads=1,dim_head=8,slice_num=1) fixed=PrecisionAttention(8,heads=1,dim_head=8,slice_num=1) with torch.no_grad(): original.in_project_fx.weight.zero_();original.in_project_fx.bias.fill_(8.) fixed.load_state_dict(original.state_dict()) x=torch.ones(1,100000,8,requires_grad=True) with torch.no_grad():reference=original(x) numerator=torch.einsum('bhnc,bhng->bhgc',torch.full((1,1,100000,8),8.,dtype=torch.float16),torch.ones(1,1,100000,1,dtype=torch.float16)) self.assertTrue(bool(torch.isinf(numerator).all())) with torch.autocast('cpu',dtype=torch.float16): a=original(x);b=fixed(x) self.assertFalse(bool(torch.isfinite(a).all())) self.assertTrue(bool(torch.isfinite(b).all())) torch.testing.assert_close(b.float(),reference,rtol=5e-3,atol=5e-3) b.float().square().mean().backward() self.assertTrue(bool(torch.isfinite(x.grad).all())) for p in fixed.parameters(): if p.grad is not None:self.assertTrue(bool(torch.isfinite(p.grad).all())) if __name__=='__main__':unittest.main()