Files
MeltMind/tools/s2_main_memory_guard.py

134 lines
6.5 KiB
Python

"""Independent future-run guard; no matrix launcher or training entry point.
Preserves the old pilot guard and its receipts. Checks recorded cumulative CUDA
peaks while the worker is alive and again after any exit, including failure.
"""
import hashlib
import json
import math
import os
from pathlib import Path
import threading
import time
import psutil
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise, save, stop_group, disk_usage, tree_usage
ALLOCATOR_CONF='expandable_segments:True'
VRAM_BYTES=15*2**30
HOST_BYTES=16*2**30
DISK_BYTES=5*2**30
PEAK_KEYS=('peak_allocated_bytes','peak_reserved_bytes')
def peaks_in(value):
found={}
if isinstance(value,dict):
for key,child in value.items():
if key in PEAK_KEYS:
if isinstance(child,bool) or not isinstance(child,(int,float)) or not math.isfinite(child) or child<0:
raise ValueError('Invalid recorded CUDA peak: '+key)
found[key]=child
nested=peaks_in(child)
for k,v in nested.items():found[k]=max(found.get(k,0),v)
elif isinstance(value,list):
for child in value:
for k,v in peaks_in(child).items():found[k]=max(found.get(k,0),v)
return found
class PeakRecords:
def __init__(self):
self.maxima={k:0 for k in PEAK_KEYS};self.witnesses={};self.cache={};self.observations=[]
def scan(self,root):
# Includes progress, result and failure records, plus epoch/history JSON.
# Running maxima survive file replacement, regression and deletion.
for path in sorted(Path(root).rglob('*.json')):
try:
stat=path.stat();version=(stat.st_mtime_ns,stat.st_size,stat.st_ino)
if self.cache.get(str(path))==version:continue
raw=path.read_bytes()
except FileNotFoundError:continue
peaks=peaks_in(json.loads(raw));self.cache[str(path)]=version
if not peaks:continue
record=dict(path=str(path),sha256=hashlib.sha256(raw).hexdigest(),observed_unix=time.time(),peaks=peaks)
self.observations.append(record)
for key,value in peaks.items():
if key not in self.witnesses or value>self.maxima[key]:
self.maxima[key]=value;self.witnesses[key]=record
return max(self.maxima.values())>VRAM_BYTES
def as_dict(self):
return dict(maxima=self.maxima,witnesses=self.witnesses,observations=self.observations,
limit_bytes=VRAM_BYTES,breach=max(self.maxima.values())>VRAM_BYTES)
def active_command_pids(command):
result=[]
try:children=psutil.Process(os.getpid()).children(recursive=True)
except psutil.NoSuchProcess:return result
for p in children:
try:
if p.cmdline()==command:result.append(p.pid)
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
return result
def guarded_worker(root,name,uuid,wall_limit,command,*,cumulative_worker_cpu_limit_seconds):
root=Path(root)
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!=ALLOCATOR_CONF:
raise ValueError('Explicit PYTORCH_CUDA_ALLOC_CONF='+ALLOCATOR_CONF+' required before spawn')
assert 0<wall_limit<=1800 and cumulative_worker_cpu_limit_seconds>0
begin=time.monotonic();initial,util=device_usage(uuid)
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
tracker=PeakRecords();done=threading.Event()
monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,reasons=[],errors=[],samples=0)
def reason(value):
if value not in monitor['reasons']:monitor['reasons'].append(value)
def scan_records():
try:
if tracker.scan(root/name):reason('RECORDED_VRAM_LIMIT')
except Exception as exc:
reason('RECORDED_MEMORY_READ_FAILED');monitor['errors'].append(str(exc))
def watch():
while not done.is_set():
scan_records()
try:
used,_=device_usage(uuid);monitor['samples']+=1
monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
if used>VRAM_BYTES:reason('DEVICE_VRAM_LIMIT')
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
if rss>HOST_BYTES:reason('WHOLE_RUN_HOST_RSS_LIMIT')
if disk_usage(root)>=DISK_BYTES:reason('RUN_DISK_LIMIT')
except Exception as exc:
reason('GPU_MONITOR_FAILED');monitor['errors'].append(str(exc))
if monitor['reasons']:
for pid in active_command_pids(command):stop_group(pid)
done.wait(.5)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
remaining=wall_limit-(time.monotonic()-begin)-2.5
if remaining<=0:raise RuntimeError('No wall budget left before spawn')
receipt=supervise(command,root/'guards',name,dict(total_cpu_seconds=cumulative_worker_cpu_limit_seconds,
job_cpu_seconds=wall_limit,job_wall_seconds=remaining,tree_rss_bytes=HOST_BYTES,
disk_bytes=DISK_BYTES,max_processes=2),poll=.1,gpu_uuid=uuid)
finally:
done.set();thread.join(timeout=3)
if thread.is_alive():reason('GPU_MONITOR_DID_NOT_STOP')
else:scan_records() # Mandatory even if the worker failed or wrote no result.
result_path=root/name/'result.json';result=None
if result_path.exists():
try:result=json.loads(result_path.read_text())
except Exception as exc:reason('RESULT_READ_FAILED');monitor['errors'].append(str(exc))
if receipt['status']=='PASS' and (not result or result.get('status')!='COMPLETE'):reason('MISSING_COMPLETE_RESULT')
if receipt['status']=='PASS' and not all(k in tracker.witnesses for k in PEAK_KEYS):reason('MISSING_RECORDED_MEMORY_PEAKS')
elapsed=time.monotonic()-begin
if elapsed>wall_limit:reason('TOTAL_RUN_WALL_LIMIT')
receipt['underlying_worker_status']=receipt['status'];receipt['underlying_stop_reason']=receipt.get('stop_reason')
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
receipt.update(device_monitor=monitor,recorded_memory=tracker.as_dict(),total_run_wall_seconds=elapsed,
allocator_conf=os.environ['PYTORCH_CUDA_ALLOC_CONF'],gpu_uuid=uuid,
result_sha256=hashlib.sha256(result_path.read_bytes()).hexdigest() if result else None)
save(root/(name+'-guard-receipt.json'),receipt)
return receipt,result