50 lines
3.1 KiB
Python
50 lines
3.1 KiB
Python
import unittest
|
|
import tempfile
|
|
import sys
|
|
import json
|
|
import threading
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
import s2_pilot_gpu_guard as guard
|
|
from s2_gpu_process_supervisor import supervise
|
|
from s2_pilot_gpu_guard import parse_device_usage
|
|
|
|
|
|
class GPUUsageTests(unittest.TestCase):
|
|
def test_exact_device_memory_units(self):
|
|
self.assertEqual(parse_device_usage('15360, 100\n'),(15*2**30,100))
|
|
self.assertEqual(parse_device_usage('5, 0'),(5*2**20,0))
|
|
def test_missing_or_ambiguous_counters_fail_closed(self):
|
|
for text in ['', 'N/A, 0','5, 0\n5, 0','-1, 0','2, 101','2']:
|
|
with self.assertRaises(ValueError):parse_device_usage(text)
|
|
def test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
# wait4 includes transient inherited torch parent RSS before exec.
|
|
limits=dict(total_cpu_seconds=5,job_cpu_seconds=2,job_wall_seconds=2,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1)
|
|
r=supervise([sys.executable,'-c','import os; assert os.environ["CUDA_VISIBLE_DEVICES"]=="synthetic-device"'],root,'synthetic',limits,gpu_uuid='synthetic-device')
|
|
self.assertEqual(r['status'],'PASS');self.assertTrue(r['gpu_visible']);self.assertEqual(r['gpu_uuid'],'synthetic-device')
|
|
def test_vram_monitor_error_and_whole_host_rss_stop_without_second_run(self):
|
|
for trigger in ['vram','monitor','host']:
|
|
with self.subTest(trigger=trigger), tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);killed=threading.Event()
|
|
c=dict(gpu_uuid='synthetic-device',code_sha256={},runs=[dict(name=f'{arm}{width}',arm=arm,width=width,seed=0) for width in [128,256] for arm in ['N','E']])
|
|
(root/'training-contract.json').write_text(json.dumps(c))
|
|
calls=0
|
|
def usage(uuid):
|
|
nonlocal calls
|
|
calls+=1
|
|
if calls==1:return 5*2**20,0
|
|
if trigger=='monitor':raise RuntimeError('synthetic monitor failure')
|
|
return (16*2**30 if trigger=='vram' else 5*2**20),0
|
|
def fake_supervise(*a,**kw):
|
|
self.assertTrue(killed.wait(2));return dict(status='PASS',stop_reason=None)
|
|
with patch.object(guard,'device_usage',usage),patch.object(guard,'tree_usage',return_value=(0,17*2**30 if trigger=='host' else 0,1)),patch.object(guard,'active_training_pids',return_value=[123456789]),patch.object(guard,'stop_group',side_effect=lambda pid:killed.set()),patch.object(guard,'supervise',side_effect=fake_supervise) as worker:
|
|
guard.run(root)
|
|
self.assertEqual(worker.call_count,1)
|
|
result=json.loads((root/'pilot-complete.json').read_text())
|
|
self.assertEqual(result['status'],'PILOT_INCOMPLETE');self.assertIsNone(result['selected_global_width'])
|
|
self.assertEqual(result['runs'][0]['stop_reason'],{'vram':'DEVICE_VRAM_LIMIT','monitor':'GPU_MONITOR_FAILED','host':'WHOLE_RUN_HOST_RSS_LIMIT'}[trigger])
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|