Files
MeltMind/tools/s2_recovery_gpu_supervisor.py
T

159 lines
7.8 KiB
Python

"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility.
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
"""
import fcntl
import ctypes
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import time
import psutil
from s2_recovery_monitoring import disk_usage,Monitor
def save(path, data):
path = Path(path)
tmp = path.with_suffix(path.suffix + '.tmp')
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
tmp.replace(path)
def tree_usage(pid):
try:
root = psutil.Process(pid)
processes = [root] + root.children(recursive=True)
except psutil.NoSuchProcess:
return 0., 0, 0
cpu = 0.
rss = count = 0
for p in processes:
try:
t = p.cpu_times()
# Linux child times include reaped descendants, not live children.
cpu += t.user + t.system + t.children_user + t.children_system
rss += p.memory_info().rss
count += 1
except psutil.NoSuchProcess:
pass
return cpu, rss, count
def stop_group(pid):
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None):
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
root = Path(root)
root.mkdir(parents=True, exist_ok=True)
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
with (root/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
started_path = root/(name+'.started.json')
receipt_path = root/(name+'.receipt.json')
if started_path.exists() or receipt_path.exists():
raise FileExistsError('Attempt already exists; no silent retry')
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Every limit must be finite and positive')
ledger_path = root/'ledger.json'
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
raise RuntimeError('Total CPU exhausted before spawn')
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
raise RuntimeError('Disk exhausted before spawn')
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
start = last = time.monotonic()
cpu_peak = rss_peak = max_processes = max_gap = 0
cpu=rss=count=used_disk=0
reason = None
exit_status = usage = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or ''))
with (root/(name+'.log')).open('x') as log:
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
env=env, start_new_session=True)
try:
while True:
now = time.monotonic()
max_gap = max(max_gap, now-last)
last = now
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
cpu_peak = max(cpu_peak,cpu)
rss_peak = max(rss_peak,rss)
max_processes = max(max_processes,count)
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
if now-start >= limits['job_wall_seconds']:
reason = 'wall_limit'
elif rss > limits['tree_rss_bytes']:
reason = 'tree_rss_limit'
elif count > limits['max_processes']:
reason = 'process_count_limit'
elif used_disk >= limits['disk_bytes']:
reason = 'disk_limit'
elif cpu_peak >= limits['job_cpu_seconds']:
reason = 'job_cpu_limit'
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
reason = 'total_cpu_limit'
if reason:
stop_group(proc.pid)
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
if pid:
exit_status, usage = status, child_usage
break
time.sleep(poll)
finally:
# No detached work may outlive a supervised attempt.
stop_group(proc.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(proc.pid, 0)
proc.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.
orphan_peak = 0
# Killed descendants are adopted here rather than losing their
# final CPU usage when the root worker cannot reap them itself.
while True:
try:
_, _, orphan_usage = os.wait4(-proc.pid, 0)
except ChildProcessError:
break
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
elapsed = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
if reason is None:
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
command=command,returncode=proc.returncode,stop_reason=reason,
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
reaped_orphan_cpu_seconds=orphan_cpu,
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
ledger['cpu_seconds_charged'] += charged
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
return receipt