"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility. Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot is measured and fails the job. Run only reviewed label-free worker commands. """ import fcntl import ctypes import json import math import os from pathlib import Path import signal import subprocess import time import psutil def save(path, data): path = Path(path) tmp = path.with_suffix(path.suffix + '.tmp') tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') tmp.replace(path) def disk_usage(root): logical = allocated = 0 for p in Path(root).rglob('*'): if p.is_file() and not p.is_symlink(): s = p.stat() logical += s.st_size allocated += s.st_blocks * 512 return max(logical, allocated) def tree_usage(pid): try: root = psutil.Process(pid) processes = [root] + root.children(recursive=True) except psutil.NoSuchProcess: return 0., 0, 0 cpu = 0. rss = count = 0 for p in processes: try: t = p.cpu_times() # Linux child times include reaped descendants, not live children. cpu += t.user + t.system + t.children_user + t.children_system rss += p.memory_info().rss count += 1 except psutil.NoSuchProcess: pass return cpu, rss, count def stop_group(pid): try: os.killpg(pid, signal.SIGKILL) except ProcessLookupError: pass def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None): """One locked attempt; no automatic retry/replacement; account failed CPU.""" root = Path(root) root.mkdir(parents=True, exist_ok=True) with (root/'execution.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) started_path = root/(name+'.started.json') receipt_path = root/(name+'.receipt.json') if started_path.exists() or receipt_path.exists(): raise FileExistsError('Attempt already exists; no silent retry') if any(not math.isfinite(v) or v <= 0 for v in limits.values()): raise ValueError('Every limit must be finite and positive') ledger_path = root/'ledger.json' ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]} if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']: raise RuntimeError('Total CPU exhausted before spawn') if disk_usage(root) >= limits['disk_bytes']: raise RuntimeError('Disk exhausted before spawn') save(started_path, {'command':command,'limits':limits,'started_unix':time.time()}) start = last = time.monotonic() cpu_peak = rss_peak = max_processes = max_gap = 0 reason = None exit_status = usage = None env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or '')) with (root/(name+'.log')).open('x') as log: libc = ctypes.CDLL(None, use_errno=True) previous_subreaper = ctypes.c_int() if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0: raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping') proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True) try: while True: now = time.monotonic() max_gap = max(max_gap, now-last) last = now cpu, rss, count = tree_usage(proc.pid) cpu_peak = max(cpu_peak,cpu) rss_peak = max(rss_peak,rss) max_processes = max(max_processes,count) used_disk = disk_usage(root) if now-start >= limits['job_wall_seconds']: reason = 'wall_limit' elif rss > limits['tree_rss_bytes']: reason = 'tree_rss_limit' elif count > limits['max_processes']: reason = 'process_count_limit' elif used_disk >= limits['disk_bytes']: reason = 'disk_limit' elif cpu_peak >= limits['job_cpu_seconds']: reason = 'job_cpu_limit' elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']: reason = 'total_cpu_limit' if reason: stop_group(proc.pid) pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG) if pid: exit_status, usage = status, child_usage break time.sleep(poll) finally: # No detached work may outlive a supervised attempt. stop_group(proc.pid) if exit_status is None: _, exit_status, usage = os.wait4(proc.pid, 0) proc.returncode = os.waitstatus_to_exitcode(exit_status) orphan_cpu = 0. orphan_peak = 0 # Killed descendants are adopted here rather than losing their # final CPU usage when the root worker cannot reap them itself. while True: try: _, _, orphan_usage = os.wait4(-proc.pid, 0) except ChildProcessError: break orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024) if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state') elapsed = time.monotonic()-start charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak) used_disk = disk_usage(root) if reason is None: if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit' elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit' elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit' elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit' elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit' receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED', command=command,returncode=proc.returncode,stop_reason=reason, cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak, peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk, max_poll_gap_seconds=max_gap,poll_target_seconds=poll, reaped_orphan_cpu_seconds=orphan_cpu, supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure', gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False) ledger['cpu_seconds_charged'] += charged ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged}) save(receipt_path,receipt) save(ledger_path,ledger) return receipt