Files
MeltMind/tools/gre24_gpu_guard.py
T

83 lines
4.0 KiB
Python

"""Single-GPU wall accounting and recorded/device VRAM checks including failure."""
import hashlib
import json
import math
from pathlib import Path
import subprocess
import threading
import time
import psutil
from gre24_job_guard import supervise,stop_group
def gpu_state(uuid=None):
args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits']
if uuid:args+=['-i',uuid]
out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout
rows=[]
for line in out.splitlines():
u,mem,util=[v.strip() for v in line.split(',')]
rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util)))
if not rows:raise RuntimeError('No GPU state returned')
return rows
def recorded_peaks(root):
peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
def walk(value,path,digest):
if isinstance(value,dict):
for k,v in value.items():
if k in peaks:
if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0:
raise ValueError('Invalid CUDA peak record')
if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest}
else:walk(v,path,digest)
elif isinstance(value,list):
for v in value:walk(v,path,digest)
for path in sorted(Path(root).glob('*.json')):
raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest())
return peaks,witnesses
def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap):
start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0}
maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
def scan():
current,seen=recorded_peaks(fit_root)
for k,v in current.items():
if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k]
if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']:
monitor['reasons'].append('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();state=gpu_state(uuid)[0];monitor['samples']+=1
monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes'])
if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']:
monitor['reasons'].append('DEVICE_VRAM_LIMIT')
except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc))
if monitor['reasons']:
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
done.wait(.5)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit,
job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30,
disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2)
finally:
done.set();thread.join(timeout=12)
if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP')
try:scan()
except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc))
if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS')
elapsed=time.monotonic()-start
if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT')
receipt['underlying_worker_status']=receipt['status']
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses,
total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap)
return receipt