"""Race-tolerant sampled infrastructure monitoring, without worker changes.""" import json from pathlib import Path import stat import sys import time def disk_usage(root): logical=allocated=0 for p in Path(root).rglob('*'): try:s=p.stat(follow_symlinks=False) except FileNotFoundError:continue # atomic progress replacement / vanished entry if stat.S_ISREG(s.st_mode): logical+=s.st_size;allocated+=s.st_blocks*512 return max(logical,allocated) class Monitor: def __init__(self,path):self.path=Path(path);self.errors=0;self.last_error=None def record(self,operation,exc): self.errors+=1;self.last_error=dict(unix=time.time(),operation=operation,type=type(exc).__name__,message=str(exc)) try: with self.path.open('a') as f:f.write(json.dumps(self.last_error)+'\n') except Exception as log_exc: print(json.dumps(dict(monitor_error=self.last_error,logging_error=str(log_exc))),file=sys.stderr,flush=True) def sample(self,operation,fn,previous): try:return fn(),True except Exception as exc:self.record(operation,exc);return previous,False def retry(self,operation,fn,poll): while True: value,ok=self.sample(operation,fn,None) if ok:return value time.sleep(poll) def budget_reason(prior_cpu,settled_cpu,live_cpu,disk_bytes,*,cpu_limit=60*3600,disk_limit=100*2**30): if prior_cpu+settled_cpu+live_cpu>=cpu_limit:return 'GLOBAL_CPU_LIMIT' if disk_bytes>=disk_limit:return 'GLOBAL_DISK_LIMIT' return None