Files
MeltMind/tools/r4bc_guard.py
T

119 lines
6.9 KiB
Python

"""Single attempt CPU guard; global stop requests retain wait4 accounting."""
import argparse
import ctypes
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
from s2_recovery_supervisor import save, tree_usage, stop_group
from s2_recovery_monitoring import disk_usage
def supervise(command, dest, limits, *, global_stop=None, poll=.2):
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Invalid guard limit')
with (dest/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
raise FileExistsError('Single attempt; no restart')
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
requested = [None]
def stop(signum, frame):
requested[0] = 'GUARD_SIGNAL_'+str(signum)
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
raise OSError('Cannot enable descendant accounting')
start = time.monotonic(); last = start; reason = None; errors = []
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
try:
with (dest/'worker.log').open('x') as log:
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
start_new_session=True)
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
try:
while True:
now = time.monotonic(); gap = max(gap, now-last); last = now
try:
cpu, rss, count = tree_usage(child.pid)
disk = disk_usage(dest)
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
except Exception as exc:
errors.append(dict(type=type(exc).__name__, message=str(exc)))
reason = 'MONITOR_ERROR'
if requested[0]: reason = reason or requested[0]
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
if proc_peak > 1: reason = reason or 'process_count_limit'
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
if reason: stop_group(child.pid)
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
if pid:
usage = current_usage; exit_status = status; break
time.sleep(poll)
finally:
stop_group(child.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(child.pid, 0)
child.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.; orphan_rss = 0
while True:
try: _, _, u = os.wait4(-child.pid, 0)
except ChildProcessError: break
orphan_cpu += u.ru_utime + u.ru_stime
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
wall = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
# Inspect final peaks even when a numerical exception returned nonzero.
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
result = dict(status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
limits=limits, retry=False, gpu_visible=False)
save(dest/'worker.receipt.json', result)
return result
finally:
libc.prctl(36, previous.value, 0, 0, 0)
for signum, handler in prior.items(): signal.signal(signum, handler)
def run(root, index, slot):
from r4bc_common import contract, job_dir, read, sha
c = contract(root); os.sched_setaffinity(0, [c['worker_cpus'][slot]])
dest = job_dir(root, index)
receipt = supervise([sys.executable, 'tools/r4bc_fit.py', str(root), str(index)], dest,
c['fit_limits'], global_stop=Path(root)/'global-stop.json')
result_path = dest/'output/result.json'; failure_path = dest/'output/failure.json'
if receipt['status'] == 'PASS' and not result_path.exists():
receipt.update(status='FAILED', stop_reason='MISSING_RESULT')
failure = read(failure_path) if failure_path.exists() else None
global_failure = bool(receipt['monitor_errors']) or bool(failure and failure.get('global_stop'))
if global_failure and not (Path(root)/'global-stop.json').exists():
save(Path(root)/'global-stop.json', dict(reason='WORKER_INPUT_OR_MONITOR_FAILURE', job=index, failure=failure))
record = dict(index=index, job=c['jobs'][index], worker=receipt, global_failure=global_failure,
guard_cpu_seconds=time.process_time(), result_path=str(result_path) if result_path.exists() else None,
result_sha256=sha(result_path) if result_path.exists() else None)
save(dest/'guard-result.json', record)
return 0 if receipt['status'] == 'PASS' else 1
if __name__ == '__main__':
parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path)
parser.add_argument('index', type=int); parser.add_argument('slot', type=int)
args = parser.parse_args(); raise SystemExit(run(args.root, args.index, args.slot))