"""Recovery-only consecutive failed-sample tolerance; missing is never a breach.""" import json,subprocess,threading,time from pathlib import Path from s2_precision_continuation_guard import parse_device_usage class MonitorFailure(RuntimeError):pass class Monitor: def __init__(self,path): self.path=Path(path);self.errors=0;self.last_error=None;self.streaks={};self.max_streaks={};self.failed=False;self.records=[];self.lock=threading.Lock() def sample(self,operation,fn,previous=None): try:value=fn() except Exception as exc: with self.lock: self.errors+=1;streak=self.streaks.get(operation,0)+1;self.streaks[operation]=streak self.max_streaks[operation]=max(streak,self.max_streaks.get(operation,0));self.failed|=streak>=3 row=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc),consecutive_failures=streak,failed=streak>=3) self.last_error=row;self.records.append(row) # A logging failure is not silently discarded: stderr retains it and # the caller stops because failure evidence cannot be persisted. try: with self.path.open('a') as f:f.write(json.dumps(row)+'\n') except Exception as log_exc: import sys print(json.dumps(dict(monitor_error=row,logging_error=str(log_exc))),file=sys.stderr,flush=True) self.failed=True return previous,False with self.lock:self.streaks[operation]=0 return value,True def retry(self,operation,fn,poll): while True: value,ok=self.sample(operation,fn) if ok:return value if self.failed:raise MonitorFailure('Three consecutive failed samples: '+operation) time.sleep(poll) def policy(self): return dict(rule='three_consecutive_failed_samples_per_operation',failure_limit=3,failed=self.failed, errors=self.errors,streaks=dict(self.streaks),max_streaks=dict(self.max_streaks)) def device_usage(uuid): r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=10) return parse_device_usage(r.stdout)