103 lines
6.1 KiB
Python
103 lines
6.1 KiB
Python
"""Diagnostic-only copy of r4bc guard, adding a running coordinator CPU allowance.
|
|
Shared guards and all scientific definitions stay unchanged.
|
|
"""
|
|
import argparse
|
|
import ctypes
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from s2_recovery_supervisor import save, tree_usage, stop_group
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
|
|
def supervise(command, dest, limits, *, coordinator_cpu_start, coordinator_cpu_limit=90., global_stop=None, poll=.2):
|
|
if not math.isfinite(coordinator_cpu_start) or not math.isfinite(coordinator_cpu_limit) or coordinator_cpu_limit <= 0:
|
|
raise ValueError('Invalid coordinator CPU allowance')
|
|
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
|
|
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
|
raise ValueError('Invalid guard limit')
|
|
with (dest/'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
|
|
raise FileExistsError('Single attempt; no restart')
|
|
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
|
|
requested = [None]
|
|
def stop(signum, frame):
|
|
requested[0] = 'GUARD_SIGNAL_'+str(signum)
|
|
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
|
|
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
|
|
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
|
|
raise OSError('Cannot enable descendant accounting')
|
|
start = time.monotonic(); last = start; reason = None; errors = []
|
|
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
|
|
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
|
|
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
|
try:
|
|
with (dest/'worker.log').open('x') as log:
|
|
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
|
|
start_new_session=True)
|
|
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
|
|
try:
|
|
while True:
|
|
now = time.monotonic(); gap = max(gap, now-last); last = now
|
|
try:
|
|
cpu, rss, count = tree_usage(child.pid)
|
|
disk = disk_usage(dest)
|
|
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
|
|
except Exception as exc:
|
|
errors.append(dict(type=type(exc).__name__, message=str(exc)))
|
|
reason = 'MONITOR_ERROR'
|
|
if time.process_time()-coordinator_cpu_start >= coordinator_cpu_limit:
|
|
reason = reason or 'coordinator_cpu_limit'
|
|
if requested[0]: reason = reason or requested[0]
|
|
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
|
|
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
|
|
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
|
|
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
|
|
if proc_peak > 1: reason = reason or 'process_count_limit'
|
|
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
|
|
if reason: stop_group(child.pid)
|
|
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
|
|
if pid:
|
|
usage = current_usage; exit_status = status; break
|
|
time.sleep(poll)
|
|
finally:
|
|
stop_group(child.pid)
|
|
if exit_status is None:
|
|
_, exit_status, usage = os.wait4(child.pid, 0)
|
|
child.returncode = os.waitstatus_to_exitcode(exit_status)
|
|
orphan_cpu = 0.; orphan_rss = 0
|
|
while True:
|
|
try: _, _, u = os.wait4(-child.pid, 0)
|
|
except ChildProcessError: break
|
|
orphan_cpu += u.ru_utime + u.ru_stime
|
|
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
|
|
wall = time.monotonic()-start
|
|
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
|
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
|
|
# Inspect final peaks even when a numerical exception returned nonzero.
|
|
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
|
|
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
|
|
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
|
|
coordinator_cpu = time.process_time()-coordinator_cpu_start
|
|
if coordinator_cpu >= coordinator_cpu_limit: reason = reason or 'coordinator_cpu_limit_after_exit'
|
|
result = dict(coordinator_cpu_seconds=coordinator_cpu, coordinator_cpu_limit=coordinator_cpu_limit,
|
|
status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
|
|
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
|
|
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
|
|
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
|
|
limits=limits, retry=False, gpu_visible=False)
|
|
save(dest/'worker.receipt.json', result)
|
|
return result
|
|
finally:
|
|
libc.prctl(36, previous.value, 0, 0, 0)
|
|
for signum, handler in prior.items(): signal.signal(signum, handler)
|
|
|