Freeze hash-bound zero-run v2 continuation with recorded idle settling
This commit is contained in:
1 parent
a15f2882ee
commit
ea5ac05191
8 files changed
+366
No files matched your search
@@ -0,0 +1,53 @@
|
||||
"""Bounded, recorded idle observations; never signal foreign processes."""
|
||||
import csv
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
|
||||
|
||||
def inventory():
|
||||
r=subprocess.run(['nvidia-smi','--query-gpu=uuid,name','--format=csv,noheader'],check=True,capture_output=True,text=True,timeout=3)
|
||||
return {row[0].strip():row[1].strip() for row in csv.reader(r.stdout.splitlines())}
|
||||
|
||||
|
||||
def snapshot(uuid):
|
||||
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
|
||||
rows=list(csv.reader(r.stdout.splitlines()));assert len(rows)==1
|
||||
memory,util=[int(v.strip()) for v in rows[0]];assert memory>=0 and 0<=util<=100
|
||||
r=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
|
||||
occupants=[]
|
||||
for row in csv.reader(r.stdout.splitlines()):
|
||||
if not row or row[0].strip()!=uuid:continue
|
||||
pid=int(row[1]);user=None
|
||||
try:user=psutil.Process(pid).username()
|
||||
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
|
||||
occupants.append(dict(pid=pid,user=user,process_name=row[2].strip(),memory_MiB=row[3].strip()))
|
||||
return dict(unix=time.time(),uuid=uuid,memory_MiB=memory,utilization_percent=util,occupants=occupants,
|
||||
idle=memory<=128 and util==0 and not occupants)
|
||||
|
||||
|
||||
def wait_idle(uuid,limit_seconds,record,*,sample=snapshot,clock=time.monotonic,sleep=time.sleep):
|
||||
begin=clock();consecutive=0;count=0;last=None
|
||||
while True:
|
||||
try:last=sample(uuid)
|
||||
except Exception as exc:
|
||||
last=dict(uuid=uuid,unix=time.time(),idle=False,error=str(exc),occupants=[])
|
||||
count+=1;consecutive=consecutive+1 if last['idle'] else 0
|
||||
record(dict(**last,poll=count,consecutive_idle=consecutive,elapsed_seconds=clock()-begin))
|
||||
if clock()-begin>limit_seconds:break
|
||||
if consecutive>=3:return dict(status='IDLE_CONFIRMED',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
|
||||
remaining=limit_seconds-(clock()-begin)
|
||||
if remaining<=0:break
|
||||
sleep(min(10,remaining))
|
||||
return dict(status='IDLE_TIMEOUT',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
|
||||
|
||||
|
||||
def recorded_wait(root,name,uuid,limit_seconds):
|
||||
path=Path(root)/'idle-gates';path.mkdir(exist_ok=True)
|
||||
def record(row):
|
||||
with (path/(name+'.jsonl')).open('a') as f:f.write(json.dumps(row)+'\n')
|
||||
result=wait_idle(uuid,limit_seconds,record)
|
||||
with (path/(name+'.json')).open('x') as f:json.dump(result,f,indent=2);f.write('\n')
|
||||
return result
|
||||
Reference in new issue
Block a user