Files
MeltMind/tools/s2_tail_diagnostic_guard.py
T

103 lines
6.1 KiB
Python

"""Diagnostic-only copy of r4bc guard, adding a running coordinator CPU allowance.
Shared guards and all scientific definitions stay unchanged.
"""
import argparse
import ctypes
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
from s2_recovery_supervisor import save, tree_usage, stop_group
from s2_recovery_monitoring import disk_usage
def supervise(command, dest, limits, *, coordinator_cpu_start, coordinator_cpu_limit=90., global_stop=None, poll=.2):
if not math.isfinite(coordinator_cpu_start) or not math.isfinite(coordinator_cpu_limit) or coordinator_cpu_limit <= 0:
raise ValueError('Invalid coordinator CPU allowance')
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Invalid guard limit')
with (dest/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
raise FileExistsError('Single attempt; no restart')
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
requested = [None]
def stop(signum, frame):
requested[0] = 'GUARD_SIGNAL_'+str(signum)
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
raise OSError('Cannot enable descendant accounting')
start = time.monotonic(); last = start; reason = None; errors = []
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
try:
with (dest/'worker.log').open('x') as log:
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
start_new_session=True)
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
try:
while True:
now = time.monotonic(); gap = max(gap, now-last); last = now
try:
cpu, rss, count = tree_usage(child.pid)
disk = disk_usage(dest)
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
except Exception as exc:
errors.append(dict(type=type(exc).__name__, message=str(exc)))
reason = 'MONITOR_ERROR'
if time.process_time()-coordinator_cpu_start >= coordinator_cpu_limit:
reason = reason or 'coordinator_cpu_limit'
if requested[0]: reason = reason or requested[0]
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
if proc_peak > 1: reason = reason or 'process_count_limit'
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
if reason: stop_group(child.pid)
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
if pid:
usage = current_usage; exit_status = status; break
time.sleep(poll)
finally:
stop_group(child.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(child.pid, 0)
child.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.; orphan_rss = 0
while True:
try: _, _, u = os.wait4(-child.pid, 0)
except ChildProcessError: break
orphan_cpu += u.ru_utime + u.ru_stime
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
wall = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
# Inspect final peaks even when a numerical exception returned nonzero.
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
coordinator_cpu = time.process_time()-coordinator_cpu_start
if coordinator_cpu >= coordinator_cpu_limit: reason = reason or 'coordinator_cpu_limit_after_exit'
result = dict(coordinator_cpu_seconds=coordinator_cpu, coordinator_cpu_limit=coordinator_cpu_limit,
status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
limits=limits, retry=False, gpu_visible=False)
save(dest/'worker.receipt.json', result)
return result
finally:
libc.prctl(36, previous.value, 0, 0, 0)
for signum, handler in prior.items(): signal.signal(signum, handler)