54 lines
2.7 KiB
Python
54 lines
2.7 KiB
Python
"""Bounded, recorded idle observations; never signal foreign processes."""
|
|
import csv
|
|
import json
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
import psutil
|
|
|
|
|
|
def inventory():
|
|
r=subprocess.run(['nvidia-smi','--query-gpu=uuid,name','--format=csv,noheader'],check=True,capture_output=True,text=True,timeout=3)
|
|
return {row[0].strip():row[1].strip() for row in csv.reader(r.stdout.splitlines())}
|
|
|
|
|
|
def snapshot(uuid):
|
|
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
|
|
rows=list(csv.reader(r.stdout.splitlines()));assert len(rows)==1
|
|
memory,util=[int(v.strip()) for v in rows[0]];assert memory>=0 and 0<=util<=100
|
|
r=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
|
|
occupants=[]
|
|
for row in csv.reader(r.stdout.splitlines()):
|
|
if not row or row[0].strip()!=uuid:continue
|
|
pid=int(row[1]);user=None
|
|
try:user=psutil.Process(pid).username()
|
|
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
|
|
occupants.append(dict(pid=pid,user=user,process_name=row[2].strip(),memory_MiB=row[3].strip()))
|
|
return dict(unix=time.time(),uuid=uuid,memory_MiB=memory,utilization_percent=util,occupants=occupants,
|
|
idle=memory<=128 and util==0 and not occupants)
|
|
|
|
|
|
def wait_idle(uuid,limit_seconds,record,*,sample=snapshot,clock=time.monotonic,sleep=time.sleep):
|
|
begin=clock();consecutive=0;count=0;last=None
|
|
while True:
|
|
try:last=sample(uuid)
|
|
except Exception as exc:
|
|
last=dict(uuid=uuid,unix=time.time(),idle=False,error=str(exc),occupants=[])
|
|
count+=1;consecutive=consecutive+1 if last['idle'] else 0
|
|
record(dict(**last,poll=count,consecutive_idle=consecutive,elapsed_seconds=clock()-begin))
|
|
if clock()-begin>limit_seconds:break
|
|
if consecutive>=3:return dict(status='IDLE_CONFIRMED',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
|
|
remaining=limit_seconds-(clock()-begin)
|
|
if remaining<=0:break
|
|
sleep(min(10,remaining))
|
|
return dict(status='IDLE_TIMEOUT',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
|
|
|
|
|
|
def recorded_wait(root,name,uuid,limit_seconds):
|
|
path=Path(root)/'idle-gates';path.mkdir(exist_ok=True)
|
|
def record(row):
|
|
with (path/(name+'.jsonl')).open('a') as f:f.write(json.dumps(row)+'\n')
|
|
result=wait_idle(uuid,limit_seconds,record)
|
|
with (path/(name+'.json')).open('x') as f:json.dump(result,f,indent=2);f.write('\n')
|
|
return result
|