Files
MeltMind/tools/test_s2_main_memory_guard.py
T

62 lines
5.1 KiB
Python

import json
import os
import sys
from pathlib import Path
import tempfile
import threading
import unittest
from unittest.mock import patch
import s2_main_memory_guard as guard
class RecordedPeakTests(unittest.TestCase):
def test_only_progress_reserved_breach_stops_live_worker(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'run').mkdir();killed=threading.Event()
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+1)))
def supervise(*args,**kwargs):
self.assertTrue(killed.wait(2));return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=-9)
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[123456789]),patch.object(guard,'stop_group',side_effect=lambda pid:killed.set()):
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
self.assertIsNone(result);self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT')
self.assertEqual(receipt['recorded_memory']['maxima']['peak_reserved_bytes'],guard.VRAM_BYTES+1)
def test_failed_worker_final_scan_catches_breach_without_result(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'run').mkdir()
def supervise(*args,**kwargs):
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+3)))
return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=1)
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[]):
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT');self.assertEqual(receipt['returncode'],1)
self.assertIsNone(result)
def test_maximum_survives_lower_replacement_and_deletion(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'progress.json';tracker=guard.PeakRecords()
p.write_text(json.dumps(dict(peak_reserved_bytes=guard.VRAM_BYTES+1)));self.assertTrue(tracker.scan(tmp))
p.write_text(json.dumps(dict(peak_reserved_bytes=7)));self.assertTrue(tracker.scan(tmp))
p.unlink();self.assertTrue(tracker.scan(tmp));self.assertEqual(tracker.maxima['peak_reserved_bytes'],guard.VRAM_BYTES+1)
def test_nested_history_and_result_both_contribute(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'history.json').write_text(json.dumps([dict(peak_reserved_bytes=20)]))
(root/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=30,peak_reserved_bytes=10)))
tracker=guard.PeakRecords();self.assertFalse(tracker.scan(root));self.assertEqual(tracker.maxima,dict(peak_allocated_bytes=30,peak_reserved_bytes=20))
def test_invalid_recorded_peak_rejected(self):
for value in [float('nan'),-1,True,'100']:
with self.subTest(value=value),self.assertRaises(ValueError):guard.peaks_in(dict(peak_reserved_bytes=value))
def test_allocator_is_required_before_device_or_worker_call(self):
with patch.dict(os.environ,{},clear=True),patch.object(guard,'device_usage') as device:
with self.assertRaises(ValueError):guard.guarded_worker(Path('/unused'),'run','synthetic',20,[],cumulative_worker_cpu_limit_seconds=20)
device.assert_not_called()
def test_allocator_reaches_supervised_cpu_child_without_cuda_work(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp)
code="import os,json,pathlib; assert os.environ['PYTORCH_CUDA_ALLOC_CONF']=='expandable_segments:True'; assert os.environ['CUDA_VISIBLE_DEVICES']=='synthetic'; p=pathlib.Path('"+str(root/ 'run')+"'); p.mkdir(); (p/'result.json').write_text(json.dumps(dict(status='COMPLETE',peak_allocated_bytes=0,peak_reserved_bytes=0)))"
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)):
receipt,result=guard.guarded_worker(root,'run','synthetic',20,[sys.executable,'-c',code],cumulative_worker_cpu_limit_seconds=20)
self.assertEqual(receipt['status'],'PASS');self.assertEqual(result['status'],'COMPLETE')
self.assertEqual(receipt['allocator_conf'],guard.ALLOCATOR_CONF)
if __name__=='__main__':unittest.main(verbosity=2)