119 lines
6.9 KiB
Python
119 lines
6.9 KiB
Python
"""Single attempt CPU guard; global stop requests retain wait4 accounting."""
|
|
import argparse
|
|
import ctypes
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from s2_recovery_supervisor import save, tree_usage, stop_group
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
|
|
def supervise(command, dest, limits, *, global_stop=None, poll=.2):
|
|
dest = Path(dest); dest.mkdir(parents=True, exist_ok=True)
|
|
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
|
raise ValueError('Invalid guard limit')
|
|
with (dest/'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists():
|
|
raise FileExistsError('Single attempt; no restart')
|
|
save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time()))
|
|
requested = [None]
|
|
def stop(signum, frame):
|
|
requested[0] = 'GUARD_SIGNAL_'+str(signum)
|
|
prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)}
|
|
libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int()
|
|
if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0):
|
|
raise OSError('Cannot enable descendant accounting')
|
|
start = time.monotonic(); last = start; reason = None; errors = []
|
|
cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None
|
|
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1',
|
|
NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
|
try:
|
|
with (dest/'worker.log').open('x') as log:
|
|
child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT,
|
|
start_new_session=True)
|
|
save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid()))
|
|
try:
|
|
while True:
|
|
now = time.monotonic(); gap = max(gap, now-last); last = now
|
|
try:
|
|
cpu, rss, count = tree_usage(child.pid)
|
|
disk = disk_usage(dest)
|
|
cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count)
|
|
except Exception as exc:
|
|
errors.append(dict(type=type(exc).__name__, message=str(exc)))
|
|
reason = 'MONITOR_ERROR'
|
|
if requested[0]: reason = reason or requested[0]
|
|
if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST'
|
|
if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit'
|
|
if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit'
|
|
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit'
|
|
if proc_peak > 1: reason = reason or 'process_count_limit'
|
|
if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit'
|
|
if reason: stop_group(child.pid)
|
|
pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG)
|
|
if pid:
|
|
usage = current_usage; exit_status = status; break
|
|
time.sleep(poll)
|
|
finally:
|
|
stop_group(child.pid)
|
|
if exit_status is None:
|
|
_, exit_status, usage = os.wait4(child.pid, 0)
|
|
child.returncode = os.waitstatus_to_exitcode(exit_status)
|
|
orphan_cpu = 0.; orphan_rss = 0
|
|
while True:
|
|
try: _, _, u = os.wait4(-child.pid, 0)
|
|
except ChildProcessError: break
|
|
orphan_cpu += u.ru_utime + u.ru_stime
|
|
orphan_rss = max(orphan_rss, u.ru_maxrss*1024)
|
|
wall = time.monotonic()-start
|
|
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
|
rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss)
|
|
# Inspect final peaks even when a numerical exception returned nonzero.
|
|
if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit'
|
|
if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit'
|
|
if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit'
|
|
result = dict(status='PASS' if child.returncode == 0 and reason is None else 'FAILED',
|
|
returncode=child.returncode, stop_reason=reason, cpu_seconds=charged,
|
|
wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak,
|
|
monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu,
|
|
limits=limits, retry=False, gpu_visible=False)
|
|
save(dest/'worker.receipt.json', result)
|
|
return result
|
|
finally:
|
|
libc.prctl(36, previous.value, 0, 0, 0)
|
|
for signum, handler in prior.items(): signal.signal(signum, handler)
|
|
|
|
|
|
def run(root, index, slot):
|
|
from r4bc_common import contract, job_dir, read, sha
|
|
c = contract(root); os.sched_setaffinity(0, [c['worker_cpus'][slot]])
|
|
dest = job_dir(root, index)
|
|
receipt = supervise([sys.executable, 'tools/r4bc_fit.py', str(root), str(index)], dest,
|
|
c['fit_limits'], global_stop=Path(root)/'global-stop.json')
|
|
result_path = dest/'output/result.json'; failure_path = dest/'output/failure.json'
|
|
if receipt['status'] == 'PASS' and not result_path.exists():
|
|
receipt.update(status='FAILED', stop_reason='MISSING_RESULT')
|
|
failure = read(failure_path) if failure_path.exists() else None
|
|
global_failure = bool(receipt['monitor_errors']) or bool(failure and failure.get('global_stop'))
|
|
if global_failure and not (Path(root)/'global-stop.json').exists():
|
|
save(Path(root)/'global-stop.json', dict(reason='WORKER_INPUT_OR_MONITOR_FAILURE', job=index, failure=failure))
|
|
record = dict(index=index, job=c['jobs'][index], worker=receipt, global_failure=global_failure,
|
|
guard_cpu_seconds=time.process_time(), result_path=str(result_path) if result_path.exists() else None,
|
|
result_sha256=sha(result_path) if result_path.exists() else None)
|
|
save(dest/'guard-result.json', record)
|
|
return 0 if receipt['status'] == 'PASS' else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path)
|
|
parser.add_argument('index', type=int); parser.add_argument('slot', type=int)
|
|
args = parser.parse_args(); raise SystemExit(run(args.root, args.index, args.slot))
|