Files
MeltMind/tools/s2_gpu_idle_gate.py

54 lines
2.7 KiB
Python

"""Bounded, recorded idle observations; never signal foreign processes."""
import csv
import json
import subprocess
import time
from pathlib import Path
import psutil
def inventory():
r=subprocess.run(['nvidia-smi','--query-gpu=uuid,name','--format=csv,noheader'],check=True,capture_output=True,text=True,timeout=3)
return {row[0].strip():row[1].strip() for row in csv.reader(r.stdout.splitlines())}
def snapshot(uuid):
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
rows=list(csv.reader(r.stdout.splitlines()));assert len(rows)==1
memory,util=[int(v.strip()) for v in rows[0]];assert memory>=0 and 0<=util<=100
r=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
occupants=[]
for row in csv.reader(r.stdout.splitlines()):
if not row or row[0].strip()!=uuid:continue
pid=int(row[1]);user=None
try:user=psutil.Process(pid).username()
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
occupants.append(dict(pid=pid,user=user,process_name=row[2].strip(),memory_MiB=row[3].strip()))
return dict(unix=time.time(),uuid=uuid,memory_MiB=memory,utilization_percent=util,occupants=occupants,
idle=memory<=128 and util==0 and not occupants)
def wait_idle(uuid,limit_seconds,record,*,sample=snapshot,clock=time.monotonic,sleep=time.sleep):
begin=clock();consecutive=0;count=0;last=None
while True:
try:last=sample(uuid)
except Exception as exc:
last=dict(uuid=uuid,unix=time.time(),idle=False,error=str(exc),occupants=[])
count+=1;consecutive=consecutive+1 if last['idle'] else 0
record(dict(**last,poll=count,consecutive_idle=consecutive,elapsed_seconds=clock()-begin))
if clock()-begin>limit_seconds:break
if consecutive>=3:return dict(status='IDLE_CONFIRMED',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
remaining=limit_seconds-(clock()-begin)
if remaining<=0:break
sleep(min(10,remaining))
return dict(status='IDLE_TIMEOUT',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
def recorded_wait(root,name,uuid,limit_seconds):
path=Path(root)/'idle-gates';path.mkdir(exist_ok=True)
def record(row):
with (path/(name+'.jsonl')).open('a') as f:f.write(json.dumps(row)+'\n')
result=wait_idle(uuid,limit_seconds,record)
with (path/(name+'.json')).open('x') as f:json.dump(result,f,indent=2);f.write('\n')
return result