"""Single-GPU wall accounting and recorded/device VRAM checks including failure.""" import hashlib import json import math from pathlib import Path import subprocess import threading import time import psutil from gre24_job_guard import supervise,stop_group def gpu_state(uuid=None): args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits'] if uuid:args+=['-i',uuid] out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout rows=[] for line in out.splitlines(): u,mem,util=[v.strip() for v in line.split(',')] rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util))) if not rows:raise RuntimeError('No GPU state returned') return rows def recorded_peaks(root): peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={} def walk(value,path,digest): if isinstance(value,dict): for k,v in value.items(): if k in peaks: if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0: raise ValueError('Invalid CUDA peak record') if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest} else:walk(v,path,digest) elif isinstance(value,list): for v in value:walk(v,path,digest) for path in sorted(Path(root).glob('*.json')): raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest()) return peaks,witnesses def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap): start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0} maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={} def scan(): current,seen=recorded_peaks(fit_root) for k,v in current.items(): if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k] if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']: monitor['reasons'].append('RECORDED_VRAM_LIMIT') def watch(): while not done.is_set(): try: scan();state=gpu_state(uuid)[0];monitor['samples']+=1 monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes']) if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']: monitor['reasons'].append('DEVICE_VRAM_LIMIT') except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc)) if monitor['reasons']: for p in psutil.Process().children(recursive=True): try: if p.cmdline()==command:stop_group(p.pid) except psutil.NoSuchProcess:pass done.wait(.5) thread=threading.Thread(target=watch,daemon=True);thread.start() try: receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit, job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30, disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2) finally: done.set();thread.join(timeout=12) if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP') try:scan() except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc)) if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS') elapsed=time.monotonic()-start if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT') receipt['underlying_worker_status']=receipt['status'] if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0]) receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses, total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap) return receipt