import unittest import tempfile import sys import json import threading from pathlib import Path from unittest.mock import patch import s2_pilot_gpu_guard as guard from s2_gpu_process_supervisor import supervise from s2_pilot_gpu_guard import parse_device_usage class GPUUsageTests(unittest.TestCase): def test_exact_device_memory_units(self): self.assertEqual(parse_device_usage('15360, 100\n'),(15*2**30,100)) self.assertEqual(parse_device_usage('5, 0'),(5*2**20,0)) def test_missing_or_ambiguous_counters_fail_closed(self): for text in ['', 'N/A, 0','5, 0\n5, 0','-1, 0','2, 101','2']: with self.assertRaises(ValueError):parse_device_usage(text) def test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use(self): with tempfile.TemporaryDirectory() as root: # wait4 includes transient inherited torch parent RSS before exec. limits=dict(total_cpu_seconds=5,job_cpu_seconds=2,job_wall_seconds=2,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1) r=supervise([sys.executable,'-c','import os; assert os.environ["CUDA_VISIBLE_DEVICES"]=="synthetic-device"'],root,'synthetic',limits,gpu_uuid='synthetic-device') self.assertEqual(r['status'],'PASS');self.assertTrue(r['gpu_visible']);self.assertEqual(r['gpu_uuid'],'synthetic-device') def test_vram_monitor_error_and_whole_host_rss_stop_without_second_run(self): for trigger in ['vram','monitor','host']: with self.subTest(trigger=trigger), tempfile.TemporaryDirectory() as tmp: root=Path(tmp);killed=threading.Event() c=dict(gpu_uuid='synthetic-device',code_sha256={},runs=[dict(name=f'{arm}{width}',arm=arm,width=width,seed=0) for width in [128,256] for arm in ['N','E']]) (root/'training-contract.json').write_text(json.dumps(c)) calls=0 def usage(uuid): nonlocal calls calls+=1 if calls==1:return 5*2**20,0 if trigger=='monitor':raise RuntimeError('synthetic monitor failure') return (16*2**30 if trigger=='vram' else 5*2**20),0 def fake_supervise(*a,**kw): self.assertTrue(killed.wait(2));return dict(status='PASS',stop_reason=None) with patch.object(guard,'device_usage',usage),patch.object(guard,'tree_usage',return_value=(0,17*2**30 if trigger=='host' else 0,1)),patch.object(guard,'active_training_pids',return_value=[123456789]),patch.object(guard,'stop_group',side_effect=lambda pid:killed.set()),patch.object(guard,'supervise',side_effect=fake_supervise) as worker: guard.run(root) self.assertEqual(worker.call_count,1) result=json.loads((root/'pilot-complete.json').read_text()) self.assertEqual(result['status'],'PILOT_INCOMPLETE');self.assertIsNone(result['selected_global_width']) self.assertEqual(result['runs'][0]['stop_reason'],{'vram':'DEVICE_VRAM_LIMIT','monitor':'GPU_MONITOR_FAILED','host':'WHOLE_RUN_HOST_RSS_LIMIT'}[trigger]) if __name__=='__main__':unittest.main(verbosity=2)