165 lines
8.2 KiB
Python
165 lines
8.2 KiB
Python
"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility.
|
|
|
|
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
|
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
|
|
"""
|
|
import fcntl
|
|
import ctypes
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import time
|
|
|
|
import psutil
|
|
from s2_recovery_monitoring import disk_usage
|
|
from track_a_recovery_monitor import Monitor
|
|
|
|
|
|
def save(path, data):
|
|
path = Path(path)
|
|
tmp = path.with_suffix(path.suffix + '.tmp')
|
|
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
|
tmp.replace(path)
|
|
|
|
|
|
def tree_usage(pid):
|
|
try:
|
|
root = psutil.Process(pid)
|
|
processes = [root] + root.children(recursive=True)
|
|
except psutil.NoSuchProcess:
|
|
return 0., 0, 0
|
|
cpu = 0.
|
|
rss = count = 0
|
|
for p in processes:
|
|
try:
|
|
t = p.cpu_times()
|
|
# Linux child times include reaped descendants, not live children.
|
|
cpu += t.user + t.system + t.children_user + t.children_system
|
|
rss += p.memory_info().rss
|
|
count += 1
|
|
except psutil.NoSuchProcess:
|
|
pass
|
|
return cpu, rss, count
|
|
|
|
|
|
def stop_group(pid):
|
|
try:
|
|
os.killpg(pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
|
|
|
|
def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None):
|
|
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
|
root = Path(root)
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
|
|
with (root/'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
started_path = root/(name+'.started.json')
|
|
receipt_path = root/(name+'.receipt.json')
|
|
if started_path.exists() or receipt_path.exists():
|
|
raise FileExistsError('Attempt already exists; no silent retry')
|
|
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
|
raise ValueError('Every limit must be finite and positive')
|
|
ledger_path = root/'ledger.json'
|
|
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
|
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
|
raise RuntimeError('Total CPU exhausted before spawn')
|
|
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
|
|
raise RuntimeError('Disk exhausted before spawn')
|
|
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
|
start = last = time.monotonic()
|
|
cpu_peak = rss_peak = max_processes = max_gap = 0
|
|
cpu=rss=count=used_disk=0
|
|
reason = None
|
|
exit_status = usage = None
|
|
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
|
|
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or ''))
|
|
with (root/(name+'.log')).open('x') as log:
|
|
libc = ctypes.CDLL(None, use_errno=True)
|
|
previous_subreaper = ctypes.c_int()
|
|
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
|
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
|
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
|
env=env, start_new_session=True)
|
|
try:
|
|
while True:
|
|
now = time.monotonic()
|
|
max_gap = max(max_gap, now-last)
|
|
last = now
|
|
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
|
|
cpu_peak = max(cpu_peak,cpu)
|
|
rss_peak = max(rss_peak,rss)
|
|
max_processes = max(max_processes,count)
|
|
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
|
|
if monitor.failed:
|
|
reason = 'three_consecutive_monitor_failures'
|
|
elif now-start >= limits['job_wall_seconds']:
|
|
reason = 'wall_limit'
|
|
elif rss > limits['tree_rss_bytes']:
|
|
reason = 'tree_rss_limit'
|
|
elif count > limits['max_processes']:
|
|
reason = 'process_count_limit'
|
|
elif used_disk >= limits['disk_bytes']:
|
|
reason = 'disk_limit'
|
|
elif cpu_peak >= limits['job_cpu_seconds']:
|
|
reason = 'job_cpu_limit'
|
|
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
|
reason = 'total_cpu_limit'
|
|
if reason:
|
|
stop_group(proc.pid)
|
|
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
|
if pid:
|
|
exit_status, usage = status, child_usage
|
|
break
|
|
time.sleep(poll)
|
|
finally:
|
|
# No detached work may outlive a supervised attempt.
|
|
stop_group(proc.pid)
|
|
if exit_status is None:
|
|
_, exit_status, usage = os.wait4(proc.pid, 0)
|
|
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
|
orphan_cpu = 0.
|
|
orphan_peak = 0
|
|
# Killed descendants are adopted here rather than losing their
|
|
# final CPU usage when the root worker cannot reap them itself.
|
|
while True:
|
|
try:
|
|
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
|
except ChildProcessError:
|
|
break
|
|
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
|
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
|
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
|
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
|
elapsed = time.monotonic()-start
|
|
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
|
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
|
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
|
|
if reason is None:
|
|
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
|
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
|
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
|
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
|
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
|
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
|
command=command,returncode=proc.returncode,stop_reason=reason,
|
|
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
|
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
|
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
|
reaped_orphan_cpu_seconds=orphan_cpu,
|
|
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
|
gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,monitor_policy=monitor.policy())
|
|
ledger['cpu_seconds_charged'] += charged
|
|
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
|
def write_receipt():
|
|
receipt.update(monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,monitor_policy=monitor.policy())
|
|
save(receipt_path,receipt)
|
|
monitor.retry('receipt_write',write_receipt,poll)
|
|
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
|
|
return receipt
|