83 lines
4.0 KiB
Python
83 lines
4.0 KiB
Python
"""Single-GPU wall accounting and recorded/device VRAM checks including failure."""
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import psutil
|
|
from gre24_job_guard import supervise,stop_group
|
|
|
|
|
|
def gpu_state(uuid=None):
|
|
args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits']
|
|
if uuid:args+=['-i',uuid]
|
|
out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout
|
|
rows=[]
|
|
for line in out.splitlines():
|
|
u,mem,util=[v.strip() for v in line.split(',')]
|
|
rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util)))
|
|
if not rows:raise RuntimeError('No GPU state returned')
|
|
return rows
|
|
|
|
|
|
def recorded_peaks(root):
|
|
peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
|
def walk(value,path,digest):
|
|
if isinstance(value,dict):
|
|
for k,v in value.items():
|
|
if k in peaks:
|
|
if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0:
|
|
raise ValueError('Invalid CUDA peak record')
|
|
if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest}
|
|
else:walk(v,path,digest)
|
|
elif isinstance(value,list):
|
|
for v in value:walk(v,path,digest)
|
|
for path in sorted(Path(root).glob('*.json')):
|
|
raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest())
|
|
return peaks,witnesses
|
|
|
|
|
|
def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap):
|
|
start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0}
|
|
maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
|
def scan():
|
|
current,seen=recorded_peaks(fit_root)
|
|
for k,v in current.items():
|
|
if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k]
|
|
if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']:
|
|
monitor['reasons'].append('RECORDED_VRAM_LIMIT')
|
|
def watch():
|
|
while not done.is_set():
|
|
try:
|
|
scan();state=gpu_state(uuid)[0];monitor['samples']+=1
|
|
monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes'])
|
|
if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']:
|
|
monitor['reasons'].append('DEVICE_VRAM_LIMIT')
|
|
except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc))
|
|
if monitor['reasons']:
|
|
for p in psutil.Process().children(recursive=True):
|
|
try:
|
|
if p.cmdline()==command:stop_group(p.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
done.wait(.5)
|
|
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
|
try:
|
|
receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit,
|
|
job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30,
|
|
disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2)
|
|
finally:
|
|
done.set();thread.join(timeout=12)
|
|
if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP')
|
|
try:scan()
|
|
except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc))
|
|
if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS')
|
|
elapsed=time.monotonic()-start
|
|
if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT')
|
|
receipt['underlying_worker_status']=receipt['status']
|
|
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
|
|
receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses,
|
|
total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap)
|
|
return receipt
|