"""Single attempt CPU guard; global stop requests retain wait4 accounting.""" import argparse import ctypes import fcntl import json import math import os from pathlib import Path import signal import subprocess import sys import time from s2_recovery_supervisor import save, tree_usage, stop_group from s2_recovery_monitoring import disk_usage def supervise(command, dest, limits, *, global_stop=None, poll=.2): dest = Path(dest); dest.mkdir(parents=True, exist_ok=True) if any(not math.isfinite(v) or v <= 0 for v in limits.values()): raise ValueError('Invalid guard limit') with (dest/'execution.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if (dest/'worker.started.json').exists() or (dest/'worker.receipt.json').exists(): raise FileExistsError('Single attempt; no restart') save(dest/'worker.started.json', dict(command=command, limits=limits, unix=time.time())) requested = [None] def stop(signum, frame): requested[0] = 'GUARD_SIGNAL_'+str(signum) prior = {s: signal.signal(s, stop) for s in (signal.SIGTERM, signal.SIGINT)} libc = ctypes.CDLL(None, use_errno=True); previous = ctypes.c_int() if libc.prctl(37, ctypes.byref(previous), 0, 0, 0) or libc.prctl(36, 1, 0, 0, 0): raise OSError('Cannot enable descendant accounting') start = time.monotonic(); last = start; reason = None; errors = [] cpu_peak = rss_peak = proc_peak = gap = 0; usage = None; exit_status = None env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='') try: with (dest/'worker.log').open('x') as log: child = subprocess.Popen(command, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) save(dest/'worker-pid.json', dict(pid=child.pid, unix=time.time(), guard_pid=os.getpid())) try: while True: now = time.monotonic(); gap = max(gap, now-last); last = now try: cpu, rss, count = tree_usage(child.pid) disk = disk_usage(dest) cpu_peak = max(cpu_peak, cpu); rss_peak = max(rss_peak, rss); proc_peak = max(proc_peak, count) except Exception as exc: errors.append(dict(type=type(exc).__name__, message=str(exc))) reason = 'MONITOR_ERROR' if requested[0]: reason = reason or requested[0] if global_stop and Path(global_stop).exists(): reason = reason or 'GLOBAL_STOP_REQUEST' if now-start >= limits['wall_seconds']: reason = reason or 'wall_limit' if cpu_peak >= limits['cpu_seconds']: reason = reason or 'cpu_limit' if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit' if proc_peak > 1: reason = reason or 'process_count_limit' if not errors and disk >= limits['disk_bytes']: reason = reason or 'disk_limit' if reason: stop_group(child.pid) pid, status, current_usage = os.wait4(child.pid, 0 if reason else os.WNOHANG) if pid: usage = current_usage; exit_status = status; break time.sleep(poll) finally: stop_group(child.pid) if exit_status is None: _, exit_status, usage = os.wait4(child.pid, 0) child.returncode = os.waitstatus_to_exitcode(exit_status) orphan_cpu = 0.; orphan_rss = 0 while True: try: _, _, u = os.wait4(-child.pid, 0) except ChildProcessError: break orphan_cpu += u.ru_utime + u.ru_stime orphan_rss = max(orphan_rss, u.ru_maxrss*1024) wall = time.monotonic()-start charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) rss_peak = max(rss_peak, usage.ru_maxrss*1024, orphan_rss) # Inspect final peaks even when a numerical exception returned nonzero. if charged > limits['cpu_seconds']: reason = reason or 'cpu_limit_after_exit' if wall > limits['wall_seconds']: reason = reason or 'wall_limit_after_exit' if rss_peak > limits['rss_bytes']: reason = reason or 'rss_limit_after_exit' result = dict(status='PASS' if child.returncode == 0 and reason is None else 'FAILED', returncode=child.returncode, stop_reason=reason, cpu_seconds=charged, wall_seconds=wall, peak_tree_rss_bytes=rss_peak, peak_process_count=proc_peak, monitor_errors=errors, max_poll_gap_seconds=gap, reaped_orphan_cpu_seconds=orphan_cpu, limits=limits, retry=False, gpu_visible=False) save(dest/'worker.receipt.json', result) return result finally: libc.prctl(36, previous.value, 0, 0, 0) for signum, handler in prior.items(): signal.signal(signum, handler) def run(root, index, slot): from r4bc_common import contract, job_dir, read, sha c = contract(root); os.sched_setaffinity(0, [c['worker_cpus'][slot]]) dest = job_dir(root, index) receipt = supervise([sys.executable, 'tools/r4bc_fit.py', str(root), str(index)], dest, c['fit_limits'], global_stop=Path(root)/'global-stop.json') result_path = dest/'output/result.json'; failure_path = dest/'output/failure.json' if receipt['status'] == 'PASS' and not result_path.exists(): receipt.update(status='FAILED', stop_reason='MISSING_RESULT') failure = read(failure_path) if failure_path.exists() else None global_failure = bool(receipt['monitor_errors']) or bool(failure and failure.get('global_stop')) if global_failure and not (Path(root)/'global-stop.json').exists(): save(Path(root)/'global-stop.json', dict(reason='WORKER_INPUT_OR_MONITOR_FAILURE', job=index, failure=failure)) record = dict(index=index, job=c['jobs'][index], worker=receipt, global_failure=global_failure, guard_cpu_seconds=time.process_time(), result_path=str(result_path) if result_path.exists() else None, result_sha256=sha(result_path) if result_path.exists() else None) save(dest/'guard-result.json', record) return 0 if receipt['status'] == 'PASS' else 1 if __name__ == '__main__': parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path) parser.add_argument('index', type=int); parser.add_argument('slot', type=int) args = parser.parse_args(); raise SystemExit(run(args.root, args.index, args.slot))