49 lines
2.3 KiB
Python
49 lines
2.3 KiB
Python
"""Recovery-only consecutive failed-sample tolerance; missing is never a breach."""
|
|
import json,subprocess,threading,time
|
|
from pathlib import Path
|
|
from s2_precision_continuation_guard import parse_device_usage
|
|
|
|
|
|
class MonitorFailure(RuntimeError):pass
|
|
|
|
|
|
class Monitor:
|
|
def __init__(self,path):
|
|
self.path=Path(path);self.errors=0;self.last_error=None;self.streaks={};self.max_streaks={};self.failed=False;self.records=[];self.lock=threading.Lock()
|
|
|
|
def sample(self,operation,fn,previous=None):
|
|
try:value=fn()
|
|
except Exception as exc:
|
|
with self.lock:
|
|
self.errors+=1;streak=self.streaks.get(operation,0)+1;self.streaks[operation]=streak
|
|
self.max_streaks[operation]=max(streak,self.max_streaks.get(operation,0));self.failed|=streak>=3
|
|
row=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc),consecutive_failures=streak,failed=streak>=3)
|
|
self.last_error=row;self.records.append(row)
|
|
# A logging failure is not silently discarded: stderr retains it and
|
|
# the caller stops because failure evidence cannot be persisted.
|
|
try:
|
|
with self.path.open('a') as f:f.write(json.dumps(row)+'\n')
|
|
except Exception as log_exc:
|
|
import sys
|
|
print(json.dumps(dict(monitor_error=row,logging_error=str(log_exc))),file=sys.stderr,flush=True)
|
|
self.failed=True
|
|
return previous,False
|
|
with self.lock:self.streaks[operation]=0
|
|
return value,True
|
|
|
|
def retry(self,operation,fn,poll):
|
|
while True:
|
|
value,ok=self.sample(operation,fn)
|
|
if ok:return value
|
|
if self.failed:raise MonitorFailure('Three consecutive failed samples: '+operation)
|
|
time.sleep(poll)
|
|
|
|
def policy(self):
|
|
return dict(rule='three_consecutive_failed_samples_per_operation',failure_limit=3,failed=self.failed,
|
|
errors=self.errors,streaks=dict(self.streaks),max_streaks=dict(self.max_streaks))
|
|
|
|
|
|
def device_usage(uuid):
|
|
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=10)
|
|
return parse_device_usage(r.stdout)
|