Files
MeltMind/tools/s2_recovery_monitoring.py

41 lines
1.6 KiB
Python

"""Race-tolerant sampled infrastructure monitoring, without worker changes."""
import json
from pathlib import Path
import stat
import sys
import time
def disk_usage(root):
logical=allocated=0
for p in Path(root).rglob('*'):
try:s=p.stat(follow_symlinks=False)
except FileNotFoundError:continue # atomic progress replacement / vanished entry
if stat.S_ISREG(s.st_mode):
logical+=s.st_size;allocated+=s.st_blocks*512
return max(logical,allocated)
class Monitor:
def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None
def record(self,operation,exc):
self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc))
try:
with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n')
except Exception as log_exc:
print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True)
def sample(self,operation,fn,previous):
try:return fn(),True
except Exception as exc:self.record(operation,exc);return previous,False
def retry(self,operation,fn,poll):
while True:
value,ok=self.sample(operation,fn,None)
if ok:return value
time.sleep(poll)
def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30):
if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT'
if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT'
return None