refinement: free-memory preflight for every level before any device work
Restores the PeriodicBlockStepper safeguard dropped in 8e1feff, for all
FixedPatchStepper levels: both G buffers, halo pairs and boundary columns
of every level are compared with cuda.mem_get_info() before the stream,
any module or any buffer exists, raising MemoryError('level buffers exceed
free device memory'). Transfer tables are not counted (lower bound).
Tests: the single-level MemoryError case is back beside the param-layout
check (no stream, no allocation); host test_single_level covers depth 1
and depth 2 (no stream, build or allocation) and the wait failure after a
good step (count stays 1). Driver-mocking fixtures report ample free memory.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8c776309c9
commit
19db7d362f
@@ -28,6 +28,7 @@ def host_module():
|
||||
from CelerisLab.body.jets import surface_link_velocity
|
||||
module.__dict__.update(cuda=Mock(),compiler=Mock(),
|
||||
surface_link_velocity=surface_link_velocity)
|
||||
module.cuda.mem_get_info.return_value=(1<<40,1<<40) # free-memory preflight passes
|
||||
exec(compile(tree,str(path),'exec'),module.__dict__)
|
||||
return module
|
||||
|
||||
|
||||
@@ -100,6 +100,29 @@ class SingleLevel(unittest.TestCase):
|
||||
with self.assertRaisesRegex(RuntimeError, "failed"):
|
||||
run.download()
|
||||
|
||||
def test_free_memory_preflight_before_stream_and_buffers(self):
|
||||
for cfg, options in ((LBMConfig(nx=32, ny=24, viscosity=.1),
|
||||
dict(refinements=[], level_blocks=[(12, 16)])),
|
||||
(LBMConfig(nx=48, ny=48, viscosity=.02), # every level counts
|
||||
dict(refinements=[[(6, 6, 26, 26)]], level_blocks=[(4, 4)] * 2))):
|
||||
with self.subTest(depth=len(options["refinements"]) + 1):
|
||||
host = host_module()
|
||||
host.cuda.mem_get_info.return_value = (1024, 1 << 30)
|
||||
with self.assertRaisesRegex(MemoryError, "exceed free device memory"):
|
||||
host.FixedPatchStepper(cfg, context=Mock(activate=nullcontext, sm_arch="sm_70"),
|
||||
**options)
|
||||
host.cuda.Stream.assert_not_called()
|
||||
host.cuda.mem_alloc.assert_not_called()
|
||||
host.compiler.build_module.assert_not_called()
|
||||
|
||||
def test_wait_failure_after_a_good_step_keeps_its_count(self):
|
||||
run, _, _ = backend()
|
||||
run.step(1)
|
||||
run._stream.synchronize.side_effect = RuntimeError("device fault")
|
||||
with self.assertRaises(RuntimeError):
|
||||
run.step(2)
|
||||
self.assertEqual((run.step_count, run._status), (1, "failed"))
|
||||
|
||||
def test_final_wait_failure_fails_instance_and_keeps_step_count(self):
|
||||
run, _, _ = backend()
|
||||
run._stream.synchronize.side_effect = RuntimeError("device error")
|
||||
|
||||
Reference in New Issue
Block a user