"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility. Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged. """ import fcntl import ctypes import json import math import os from pathlib import Path import signal import subprocess import time import psutil from s2_recovery_monitoring import disk_usage,Monitor def save(path, data): path = Path(path) tmp = path.with_suffix(path.suffix + '.tmp') tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') tmp.replace(path) def tree_usage(pid): try: root = psutil.Process(pid) processes = [root] + root.children(recursive=True) except psutil.NoSuchProcess: return 0., 0, 0 cpu = 0. rss = count = 0 for p in processes: try: t = p.cpu_times() # Linux child times include reaped descendants, not live children. cpu += t.user + t.system + t.children_user + t.children_system rss += p.memory_info().rss count += 1 except psutil.NoSuchProcess: pass return cpu, rss, count def stop_group(pid): try: os.killpg(pid, signal.SIGKILL) except ProcessLookupError: pass def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None): """One locked attempt; no automatic retry/replacement; account failed CPU.""" root = Path(root) root.mkdir(parents=True, exist_ok=True) monitor=Monitor(root/(name+'.monitor-errors.jsonl')) with (root/'execution.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) started_path = root/(name+'.started.json') receipt_path = root/(name+'.receipt.json') if started_path.exists() or receipt_path.exists(): raise FileExistsError('Attempt already exists; no silent retry') if any(not math.isfinite(v) or v <= 0 for v in limits.values()): raise ValueError('Every limit must be finite and positive') ledger_path = root/'ledger.json' ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]} if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']: raise RuntimeError('Total CPU exhausted before spawn') if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']: raise RuntimeError('Disk exhausted before spawn') save(started_path, {'command':command,'limits':limits,'started_unix':time.time()}) start = last = time.monotonic() cpu_peak = rss_peak = max_processes = max_gap = 0 cpu=rss=count=used_disk=0 reason = None exit_status = usage = None env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or '')) with (root/(name+'.log')).open('x') as log: libc = ctypes.CDLL(None, use_errno=True) previous_subreaper = ctypes.c_int() if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0: raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping') proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True) try: while True: now = time.monotonic() max_gap = max(max_gap, now-last) last = now (cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count)) cpu_peak = max(cpu_peak,cpu) rss_peak = max(rss_peak,rss) max_processes = max(max_processes,count) used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk) if now-start >= limits['job_wall_seconds']: reason = 'wall_limit' elif rss > limits['tree_rss_bytes']: reason = 'tree_rss_limit' elif count > limits['max_processes']: reason = 'process_count_limit' elif used_disk >= limits['disk_bytes']: reason = 'disk_limit' elif cpu_peak >= limits['job_cpu_seconds']: reason = 'job_cpu_limit' elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']: reason = 'total_cpu_limit' if reason: stop_group(proc.pid) pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG) if pid: exit_status, usage = status, child_usage break time.sleep(poll) finally: # No detached work may outlive a supervised attempt. stop_group(proc.pid) if exit_status is None: _, exit_status, usage = os.wait4(proc.pid, 0) proc.returncode = os.waitstatus_to_exitcode(exit_status) orphan_cpu = 0. orphan_peak = 0 # Killed descendants are adopted here rather than losing their # final CPU usage when the root worker cannot reap them itself. while True: try: _, _, orphan_usage = os.wait4(-proc.pid, 0) except ChildProcessError: break orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024) if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state') elapsed = time.monotonic()-start charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak) used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll) if reason is None: if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit' elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit' elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit' elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit' elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit' receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED', command=command,returncode=proc.returncode,stop_reason=reason, cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak, peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk, max_poll_gap_seconds=max_gap,poll_target_seconds=poll, reaped_orphan_cpu_seconds=orphan_cpu, supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure', gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error) ledger['cpu_seconds_charged'] += charged ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged}) monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll) monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll) return receipt