Freeze Gre24 model chain and retain synthetic acceptance evidence
This commit is contained in:
1 parent
8409910b1e
commit
eb9ed0c762
124 files changed
+5993
No files matched your search
@@ -0,0 +1,82 @@
|
||||
"""Single-GPU wall accounting and recorded/device VRAM checks including failure."""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from gre24_job_guard import supervise,stop_group
|
||||
|
||||
|
||||
def gpu_state(uuid=None):
|
||||
args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits']
|
||||
if uuid:args+=['-i',uuid]
|
||||
out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout
|
||||
rows=[]
|
||||
for line in out.splitlines():
|
||||
u,mem,util=[v.strip() for v in line.split(',')]
|
||||
rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util)))
|
||||
if not rows:raise RuntimeError('No GPU state returned')
|
||||
return rows
|
||||
|
||||
|
||||
def recorded_peaks(root):
|
||||
peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
||||
def walk(value,path,digest):
|
||||
if isinstance(value,dict):
|
||||
for k,v in value.items():
|
||||
if k in peaks:
|
||||
if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0:
|
||||
raise ValueError('Invalid CUDA peak record')
|
||||
if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest}
|
||||
else:walk(v,path,digest)
|
||||
elif isinstance(value,list):
|
||||
for v in value:walk(v,path,digest)
|
||||
for path in sorted(Path(root).glob('*.json')):
|
||||
raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest())
|
||||
return peaks,witnesses
|
||||
|
||||
|
||||
def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap):
|
||||
start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0}
|
||||
maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
||||
def scan():
|
||||
current,seen=recorded_peaks(fit_root)
|
||||
for k,v in current.items():
|
||||
if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k]
|
||||
if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']:
|
||||
monitor['reasons'].append('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();state=gpu_state(uuid)[0];monitor['samples']+=1
|
||||
monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes'])
|
||||
if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']:
|
||||
monitor['reasons'].append('DEVICE_VRAM_LIMIT')
|
||||
except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc))
|
||||
if monitor['reasons']:
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
done.wait(.5)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit,
|
||||
job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30,
|
||||
disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2)
|
||||
finally:
|
||||
done.set();thread.join(timeout=12)
|
||||
if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP')
|
||||
try:scan()
|
||||
except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc))
|
||||
if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS')
|
||||
elapsed=time.monotonic()-start
|
||||
if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT')
|
||||
receipt['underlying_worker_status']=receipt['status']
|
||||
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
|
||||
receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses,
|
||||
total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap)
|
||||
return receipt
|
||||
Reference in new issue
Block a user