"""Diagnostic-only copy of r4bc guard, adding a running coordinator CPU allowance. Shared guards and all scientific definitions stay unchanged. """ import argparse import ctypes import fcntl import json import math import os from pathlib import Path import signal import subprocess import sys import time from s2_recovery_supervisor import save, tree_usage, stop_group from s2_recovery_monitoring import disk_usage def supervise(command, dest, limits, *, coordinator_cpu_start, coordinator_cpu_limit=90., global_stop=None, poll=.2): if not math.isfinite(coordinator_cpu_start) or not math.isfinite(coordinator_cpu_limit) or coordinator_cpu_limit <= 0: raise ValueError('Invalid coordinator CPU allowance') dest = Path(dest); dest.mkdir(parents=True, exist_ok=True) if any(not math.isfinite(v) or v <= 0 for v in limits.values()): raise ValueError('Invalid guard limit') with (dest/'execution.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists(): raise FileExistsError('Single attempt; no restart') save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time())) requested = [None] def stop(signum, frame): requested[0] = 'GUARD_SIGNAL_'+str(signum) prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)} libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int() if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0): raise OSError('Cannot enable descendant accounting') start = time.monotonic(); last = start; reason = None; errors = [] cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='') try: with (dest/'worker.log').open('x') as log: child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid())) try: while True: now = time.monotonic(); gap = max(gap, now-last); last = now try: cpu, rss, count = tree_usage(child.pid) disk = disk_usage(dest) cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count) except Exception as exc: errors.append(dict(type=type(exc).__name__, message=str(exc))) reason = 'MONITOR_ERROR' if time.process_time()-coordinator_cpu_start >= coordinator_cpu_limit: reason = reason or 'coordinator_cpu_limit' if requested[0]: reason = reason or requested[0] if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST' if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit' if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit' if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit' if proc_peak > 1: reason = reason or 'process_count_limit' if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit' if reason: stop_group(child.pid) pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG) if pid: usage = current_usage; exit_status = status; break time.sleep(poll) finally: stop_group(child.pid) if exit_status is None: _, exit_status, usage = os.wait4(child.pid, 0) child.returncode = os.waitstatus_to_exitcode(exit_status) orphan_cpu = 0.; orphan_rss = 0 while True: try: _, _, u = os.wait4(-child.pid, 0) except ChildProcessError: break orphan_cpu += u.ru_utime + u.ru_stime orphan_rss = max(orphan_rss, u.ru_maxrss*1024) wall = time.monotonic()-start charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss) # Inspect final peaks even when a numerical exception returned nonzero. if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit' if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit' if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit' coordinator_cpu = time.process_time()-coordinator_cpu_start if coordinator_cpu >= coordinator_cpu_limit: reason = reason or 'coordinator_cpu_limit_after_exit' result = dict(coordinator_cpu_seconds=coordinator_cpu, coordinator_cpu_limit=coordinator_cpu_limit, status='PASS' if child.returncode == 0 and reason is None else 'FAILED', returncode=child.returncode, stop_reason=reason, cpu_seconds=charged, wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak, monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu, limits=limits, retry=False, gpu_visible=False) save(dest/'worker.receipt.json', result) return result finally: libc.prctl(36, previous.value, 0, 0, 0) for signum, handler in prior.items(): signal.signal(signum, handler)