"""Bounded, recorded idle observations; never signal foreign processes.""" import csv import json import subprocess import time from pathlib import Path import psutil def inventory(): r=subprocess.run(['nvidia-smi','--query-gpu=uuid,name','--format=csv,noheader'],check=True,capture_output=True,text=True,timeout=3) return {row[0].strip():row[1].strip() for row in csv.reader(r.stdout.splitlines())} def snapshot(uuid): r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3) rows=list(csv.reader(r.stdout.splitlines()));assert len(rows)==1 memory,util=[int(v.strip()) for v in rows[0]];assert memory>=0 and 0<=util<=100 r=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3) occupants=[] for row in csv.reader(r.stdout.splitlines()): if not row or row[0].strip()!=uuid:continue pid=int(row[1]);user=None try:user=psutil.Process(pid).username() except (psutil.NoSuchProcess,psutil.AccessDenied):pass occupants.append(dict(pid=pid,user=user,process_name=row[2].strip(),memory_MiB=row[3].strip())) return dict(unix=time.time(),uuid=uuid,memory_MiB=memory,utilization_percent=util,occupants=occupants, idle=memory<=128 and util==0 and not occupants) def wait_idle(uuid,limit_seconds,record,*,sample=snapshot,clock=time.monotonic,sleep=time.sleep): begin=clock();consecutive=0;count=0;last=None while True: try:last=sample(uuid) except Exception as exc: last=dict(uuid=uuid,unix=time.time(),idle=False,error=str(exc),occupants=[]) count+=1;consecutive=consecutive+1 if last['idle'] else 0 record(dict(**last,poll=count,consecutive_idle=consecutive,elapsed_seconds=clock()-begin)) if clock()-begin>limit_seconds:break if consecutive>=3:return dict(status='IDLE_CONFIRMED',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last) remaining=limit_seconds-(clock()-begin) if remaining<=0:break sleep(min(10,remaining)) return dict(status='IDLE_TIMEOUT',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last) def recorded_wait(root,name,uuid,limit_seconds): path=Path(root)/'idle-gates';path.mkdir(exist_ok=True) def record(row): with (path/(name+'.jsonl')).open('a') as f:f.write(json.dumps(row)+'\n') result=wait_idle(uuid,limit_seconds,record) with (path/(name+'.json')).open('x') as f:json.dump(result,f,indent=2);f.write('\n') return result