Add independent recorded-peak guard and freeze fresh main-matrix engineering plan
This commit is contained in:
1 parent
59fd40adba
commit
0af3dab8d1
16 files changed
+3342
No files matched your search
@@ -0,0 +1,133 @@
|
||||
"""Independent future-run guard; no matrix launcher or training entry point.
|
||||
|
||||
Preserves the old pilot guard and its receipts. Checks recorded cumulative CUDA
|
||||
peaks while the worker is alive and again after any exit, including failure.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise, save, stop_group, disk_usage, tree_usage
|
||||
|
||||
ALLOCATOR_CONF='expandable_segments:True'
|
||||
VRAM_BYTES=15*2**30
|
||||
HOST_BYTES=16*2**30
|
||||
DISK_BYTES=5*2**30
|
||||
PEAK_KEYS=('peak_allocated_bytes','peak_reserved_bytes')
|
||||
|
||||
|
||||
def peaks_in(value):
|
||||
found={}
|
||||
if isinstance(value,dict):
|
||||
for key,child in value.items():
|
||||
if key in PEAK_KEYS:
|
||||
if isinstance(child,bool) or not isinstance(child,(int,float)) or not math.isfinite(child) or child<0:
|
||||
raise ValueError('Invalid recorded CUDA peak: '+key)
|
||||
found[key]=child
|
||||
nested=peaks_in(child)
|
||||
for k,v in nested.items():found[k]=max(found.get(k,0),v)
|
||||
elif isinstance(value,list):
|
||||
for child in value:
|
||||
for k,v in peaks_in(child).items():found[k]=max(found.get(k,0),v)
|
||||
return found
|
||||
|
||||
|
||||
class PeakRecords:
|
||||
def __init__(self):
|
||||
self.maxima={k:0 for k in PEAK_KEYS};self.witnesses={};self.cache={};self.observations=[]
|
||||
def scan(self,root):
|
||||
# Includes progress, result and failure records, plus epoch/history JSON.
|
||||
# Running maxima survive file replacement, regression and deletion.
|
||||
for path in sorted(Path(root).rglob('*.json')):
|
||||
try:
|
||||
stat=path.stat();version=(stat.st_mtime_ns,stat.st_size,stat.st_ino)
|
||||
if self.cache.get(str(path))==version:continue
|
||||
raw=path.read_bytes()
|
||||
except FileNotFoundError:continue
|
||||
peaks=peaks_in(json.loads(raw));self.cache[str(path)]=version
|
||||
if not peaks:continue
|
||||
record=dict(path=str(path),sha256=hashlib.sha256(raw).hexdigest(),observed_unix=time.time(),peaks=peaks)
|
||||
self.observations.append(record)
|
||||
for key,value in peaks.items():
|
||||
if key not in self.witnesses or value>self.maxima[key]:
|
||||
self.maxima[key]=value;self.witnesses[key]=record
|
||||
return max(self.maxima.values())>VRAM_BYTES
|
||||
def as_dict(self):
|
||||
return dict(maxima=self.maxima,witnesses=self.witnesses,observations=self.observations,
|
||||
limit_bytes=VRAM_BYTES,breach=max(self.maxima.values())>VRAM_BYTES)
|
||||
|
||||
|
||||
def active_command_pids(command):
|
||||
result=[]
|
||||
try:children=psutil.Process(os.getpid()).children(recursive=True)
|
||||
except psutil.NoSuchProcess:return result
|
||||
for p in children:
|
||||
try:
|
||||
if p.cmdline()==command:result.append(p.pid)
|
||||
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
|
||||
return result
|
||||
|
||||
|
||||
def guarded_worker(root,name,uuid,wall_limit,command,*,cumulative_worker_cpu_limit_seconds):
|
||||
root=Path(root)
|
||||
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!=ALLOCATOR_CONF:
|
||||
raise ValueError('Explicit PYTORCH_CUDA_ALLOC_CONF='+ALLOCATOR_CONF+' required before spawn')
|
||||
assert 0<wall_limit<=1800 and cumulative_worker_cpu_limit_seconds>0
|
||||
begin=time.monotonic();initial,util=device_usage(uuid)
|
||||
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
|
||||
tracker=PeakRecords();done=threading.Event()
|
||||
monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,reasons=[],errors=[],samples=0)
|
||||
def reason(value):
|
||||
if value not in monitor['reasons']:monitor['reasons'].append(value)
|
||||
def scan_records():
|
||||
try:
|
||||
if tracker.scan(root/name):reason('RECORDED_VRAM_LIMIT')
|
||||
except Exception as exc:
|
||||
reason('RECORDED_MEMORY_READ_FAILED');monitor['errors'].append(str(exc))
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
scan_records()
|
||||
try:
|
||||
used,_=device_usage(uuid);monitor['samples']+=1
|
||||
monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
|
||||
if used>VRAM_BYTES:reason('DEVICE_VRAM_LIMIT')
|
||||
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
|
||||
if rss>HOST_BYTES:reason('WHOLE_RUN_HOST_RSS_LIMIT')
|
||||
if disk_usage(root)>=DISK_BYTES:reason('RUN_DISK_LIMIT')
|
||||
except Exception as exc:
|
||||
reason('GPU_MONITOR_FAILED');monitor['errors'].append(str(exc))
|
||||
if monitor['reasons']:
|
||||
for pid in active_command_pids(command):stop_group(pid)
|
||||
done.wait(.5)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
remaining=wall_limit-(time.monotonic()-begin)-2.5
|
||||
if remaining<=0:raise RuntimeError('No wall budget left before spawn')
|
||||
receipt=supervise(command,root/'guards',name,dict(total_cpu_seconds=cumulative_worker_cpu_limit_seconds,
|
||||
job_cpu_seconds=wall_limit,job_wall_seconds=remaining,tree_rss_bytes=HOST_BYTES,
|
||||
disk_bytes=DISK_BYTES,max_processes=2),poll=.1,gpu_uuid=uuid)
|
||||
finally:
|
||||
done.set();thread.join(timeout=3)
|
||||
if thread.is_alive():reason('GPU_MONITOR_DID_NOT_STOP')
|
||||
else:scan_records() # Mandatory even if the worker failed or wrote no result.
|
||||
result_path=root/name/'result.json';result=None
|
||||
if result_path.exists():
|
||||
try:result=json.loads(result_path.read_text())
|
||||
except Exception as exc:reason('RESULT_READ_FAILED');monitor['errors'].append(str(exc))
|
||||
if receipt['status']=='PASS' and (not result or result.get('status')!='COMPLETE'):reason('MISSING_COMPLETE_RESULT')
|
||||
if receipt['status']=='PASS' and not all(k in tracker.witnesses for k in PEAK_KEYS):reason('MISSING_RECORDED_MEMORY_PEAKS')
|
||||
elapsed=time.monotonic()-begin
|
||||
if elapsed>wall_limit:reason('TOTAL_RUN_WALL_LIMIT')
|
||||
receipt['underlying_worker_status']=receipt['status'];receipt['underlying_stop_reason']=receipt.get('stop_reason')
|
||||
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
|
||||
receipt.update(device_monitor=monitor,recorded_memory=tracker.as_dict(),total_run_wall_seconds=elapsed,
|
||||
allocator_conf=os.environ['PYTORCH_CUDA_ALLOC_CONF'],gpu_uuid=uuid,
|
||||
result_sha256=hashlib.sha256(result_path.read_bytes()).hexdigest() if result else None)
|
||||
save(root/(name+'-guard-receipt.json'),receipt)
|
||||
return receipt,result
|
||||
@@ -0,0 +1,61 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import s2_main_memory_guard as guard
|
||||
|
||||
|
||||
class RecordedPeakTests(unittest.TestCase):
|
||||
def test_only_progress_reserved_breach_stops_live_worker(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);(root/'run').mkdir();killed=threading.Event()
|
||||
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+1)))
|
||||
def supervise(*args,**kwargs):
|
||||
self.assertTrue(killed.wait(2));return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=-9)
|
||||
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[123456789]),patch.object(guard,'stop_group',side_effect=lambda pid:killed.set()):
|
||||
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
|
||||
self.assertIsNone(result);self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT')
|
||||
self.assertEqual(receipt['recorded_memory']['maxima']['peak_reserved_bytes'],guard.VRAM_BYTES+1)
|
||||
def test_failed_worker_final_scan_catches_breach_without_result(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);(root/'run').mkdir()
|
||||
def supervise(*args,**kwargs):
|
||||
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+3)))
|
||||
return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=1)
|
||||
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[]):
|
||||
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
|
||||
self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT');self.assertEqual(receipt['returncode'],1)
|
||||
self.assertIsNone(result)
|
||||
def test_maximum_survives_lower_replacement_and_deletion(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp)/'progress.json';tracker=guard.PeakRecords()
|
||||
p.write_text(json.dumps(dict(peak_reserved_bytes=guard.VRAM_BYTES+1)));self.assertTrue(tracker.scan(tmp))
|
||||
p.write_text(json.dumps(dict(peak_reserved_bytes=7)));self.assertTrue(tracker.scan(tmp))
|
||||
p.unlink();self.assertTrue(tracker.scan(tmp));self.assertEqual(tracker.maxima['peak_reserved_bytes'],guard.VRAM_BYTES+1)
|
||||
def test_nested_history_and_result_both_contribute(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);(root/'history.json').write_text(json.dumps([dict(peak_reserved_bytes=20)]))
|
||||
(root/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=30,peak_reserved_bytes=10)))
|
||||
tracker=guard.PeakRecords();self.assertFalse(tracker.scan(root));self.assertEqual(tracker.maxima,dict(peak_allocated_bytes=30,peak_reserved_bytes=20))
|
||||
def test_invalid_recorded_peak_rejected(self):
|
||||
for value in [float('nan'),-1,True,'100']:
|
||||
with self.subTest(value=value),self.assertRaises(ValueError):guard.peaks_in(dict(peak_reserved_bytes=value))
|
||||
def test_allocator_is_required_before_device_or_worker_call(self):
|
||||
with patch.dict(os.environ,{},clear=True),patch.object(guard,'device_usage') as device:
|
||||
with self.assertRaises(ValueError):guard.guarded_worker(Path('/unused'),'run','synthetic',20,[],cumulative_worker_cpu_limit_seconds=20)
|
||||
device.assert_not_called()
|
||||
def test_allocator_reaches_supervised_cpu_child_without_cuda_work(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp)
|
||||
code="import os,json,pathlib; assert os.environ['PYTORCH_CUDA_ALLOC_CONF']=='expandable_segments:True'; assert os.environ['CUDA_VISIBLE_DEVICES']=='synthetic'; p=pathlib.Path('"+str(root/ 'run')+"'); p.mkdir(); (p/'result.json').write_text(json.dumps(dict(status='COMPLETE',peak_allocated_bytes=0,peak_reserved_bytes=0)))"
|
||||
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)):
|
||||
receipt,result=guard.guarded_worker(root,'run','synthetic',20,[sys.executable,'-c',code],cumulative_worker_cpu_limit_seconds=20)
|
||||
self.assertEqual(receipt['status'],'PASS');self.assertEqual(result['status'],'COMPLETE')
|
||||
self.assertEqual(receipt['allocator_conf'],guard.ALLOCATOR_CONF)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user