import json import os import sys from pathlib import Path import tempfile import threading import unittest from unittest.mock import patch import s2_main_memory_guard as guard class RecordedPeakTests(unittest.TestCase): def test_only_progress_reserved_breach_stops_live_worker(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);(root/'run').mkdir();killed=threading.Event() (root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+1))) def supervise(*args,**kwargs): self.assertTrue(killed.wait(2));return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=-9) with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[123456789]),patch.object(guard,'stop_group',side_effect=lambda pid:killed.set()): receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20) self.assertIsNone(result);self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT') self.assertEqual(receipt['recorded_memory']['maxima']['peak_reserved_bytes'],guard.VRAM_BYTES+1) def test_failed_worker_final_scan_catches_breach_without_result(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);(root/'run').mkdir() def supervise(*args,**kwargs): (root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+3))) return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=1) with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[]): receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20) self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT');self.assertEqual(receipt['returncode'],1) self.assertIsNone(result) def test_maximum_survives_lower_replacement_and_deletion(self): with tempfile.TemporaryDirectory() as tmp: p=Path(tmp)/'progress.json';tracker=guard.PeakRecords() p.write_text(json.dumps(dict(peak_reserved_bytes=guard.VRAM_BYTES+1)));self.assertTrue(tracker.scan(tmp)) p.write_text(json.dumps(dict(peak_reserved_bytes=7)));self.assertTrue(tracker.scan(tmp)) p.unlink();self.assertTrue(tracker.scan(tmp));self.assertEqual(tracker.maxima['peak_reserved_bytes'],guard.VRAM_BYTES+1) def test_nested_history_and_result_both_contribute(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp);(root/'history.json').write_text(json.dumps([dict(peak_reserved_bytes=20)])) (root/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=30,peak_reserved_bytes=10))) tracker=guard.PeakRecords();self.assertFalse(tracker.scan(root));self.assertEqual(tracker.maxima,dict(peak_allocated_bytes=30,peak_reserved_bytes=20)) def test_invalid_recorded_peak_rejected(self): for value in [float('nan'),-1,True,'100']: with self.subTest(value=value),self.assertRaises(ValueError):guard.peaks_in(dict(peak_reserved_bytes=value)) def test_allocator_is_required_before_device_or_worker_call(self): with patch.dict(os.environ,{},clear=True),patch.object(guard,'device_usage') as device: with self.assertRaises(ValueError):guard.guarded_worker(Path('/unused'),'run','synthetic',20,[],cumulative_worker_cpu_limit_seconds=20) device.assert_not_called() def test_allocator_reaches_supervised_cpu_child_without_cuda_work(self): with tempfile.TemporaryDirectory() as tmp: root=Path(tmp) code="import os,json,pathlib; assert os.environ['PYTORCH_CUDA_ALLOC_CONF']=='expandable_segments:True'; assert os.environ['CUDA_VISIBLE_DEVICES']=='synthetic'; p=pathlib.Path('"+str(root/ 'run')+"'); p.mkdir(); (p/'result.json').write_text(json.dumps(dict(status='COMPLETE',peak_allocated_bytes=0,peak_reserved_bytes=0)))" with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)): receipt,result=guard.guarded_worker(root,'run','synthetic',20,[sys.executable,'-c',code],cumulative_worker_cpu_limit_seconds=20) self.assertEqual(receipt['status'],'PASS');self.assertEqual(result['status'],'COMPLETE') self.assertEqual(receipt['allocator_conf'],guard.ALLOCATOR_CONF) if __name__=='__main__':unittest.main(verbosity=2)