41 lines
1.6 KiB
Python
41 lines
1.6 KiB
Python
"""Race-tolerant sampled infrastructure monitoring, without worker changes."""
|
|
import json
|
|
from pathlib import Path
|
|
import stat
|
|
import sys
|
|
import time
|
|
|
|
|
|
def disk_usage(root):
|
|
logical=allocated=0
|
|
for p in Path(root).rglob('*'):
|
|
try:s=p.stat(follow_symlinks=False)
|
|
except FileNotFoundError:continue # atomic progress replacement / vanished entry
|
|
if stat.S_ISREG(s.st_mode):
|
|
logical+=s.st_size;allocated+=s.st_blocks*512
|
|
return max(logical,allocated)
|
|
|
|
|
|
class Monitor:
|
|
def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None
|
|
def record(self,operation,exc):
|
|
self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc))
|
|
try:
|
|
with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n')
|
|
except Exception as log_exc:
|
|
print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True)
|
|
def sample(self,operation,fn,previous):
|
|
try:return fn(),True
|
|
except Exception as exc:self.record(operation,exc);return previous,False
|
|
def retry(self,operation,fn,poll):
|
|
while True:
|
|
value,ok=self.sample(operation,fn,None)
|
|
if ok:return value
|
|
time.sleep(poll)
|
|
|
|
|
|
def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30):
|
|
if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT'
|
|
if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT'
|
|
return None
|