Implement bounded S2 preparation and sealed pilot execution
This commit is contained in:
1 parent
fb827178ec
commit
f8a6ed3f81
35 files changed
+2626
No files matched your search
@@ -0,0 +1,165 @@
|
||||
"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility.
|
||||
|
||||
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
||||
is measured and fails the job. Run only reviewed label-free worker commands.
|
||||
"""
|
||||
import fcntl
|
||||
import ctypes
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import psutil
|
||||
|
||||
|
||||
def save(path, data):
|
||||
path = Path(path)
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def disk_usage(root):
|
||||
logical = allocated = 0
|
||||
for p in Path(root).rglob('*'):
|
||||
if p.is_file() and not p.is_symlink():
|
||||
s = p.stat()
|
||||
logical += s.st_size
|
||||
allocated += s.st_blocks * 512
|
||||
return max(logical, allocated)
|
||||
|
||||
|
||||
def tree_usage(pid):
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
processes = [root] + root.children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return 0., 0, 0
|
||||
cpu = 0.
|
||||
rss = count = 0
|
||||
for p in processes:
|
||||
try:
|
||||
t = p.cpu_times()
|
||||
# Linux child times include reaped descendants, not live children.
|
||||
cpu += t.user + t.system + t.children_user + t.children_system
|
||||
rss += p.memory_info().rss
|
||||
count += 1
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
return cpu, rss, count
|
||||
|
||||
|
||||
def stop_group(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None):
|
||||
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
||||
root = Path(root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
with (root/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
started_path = root/(name+'.started.json')
|
||||
receipt_path = root/(name+'.receipt.json')
|
||||
if started_path.exists() or receipt_path.exists():
|
||||
raise FileExistsError('Attempt already exists; no silent retry')
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Every limit must be finite and positive')
|
||||
ledger_path = root/'ledger.json'
|
||||
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
||||
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
||||
raise RuntimeError('Total CPU exhausted before spawn')
|
||||
if disk_usage(root) >= limits['disk_bytes']:
|
||||
raise RuntimeError('Disk exhausted before spawn')
|
||||
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
||||
start = last = time.monotonic()
|
||||
cpu_peak = rss_peak = max_processes = max_gap = 0
|
||||
reason = None
|
||||
exit_status = usage = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
|
||||
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or ''))
|
||||
with (root/(name+'.log')).open('x') as log:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
||||
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env, start_new_session=True)
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
max_gap = max(max_gap, now-last)
|
||||
last = now
|
||||
cpu, rss, count = tree_usage(proc.pid)
|
||||
cpu_peak = max(cpu_peak,cpu)
|
||||
rss_peak = max(rss_peak,rss)
|
||||
max_processes = max(max_processes,count)
|
||||
used_disk = disk_usage(root)
|
||||
if now-start >= limits['job_wall_seconds']:
|
||||
reason = 'wall_limit'
|
||||
elif rss > limits['tree_rss_bytes']:
|
||||
reason = 'tree_rss_limit'
|
||||
elif count > limits['max_processes']:
|
||||
reason = 'process_count_limit'
|
||||
elif used_disk >= limits['disk_bytes']:
|
||||
reason = 'disk_limit'
|
||||
elif cpu_peak >= limits['job_cpu_seconds']:
|
||||
reason = 'job_cpu_limit'
|
||||
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
||||
reason = 'total_cpu_limit'
|
||||
if reason:
|
||||
stop_group(proc.pid)
|
||||
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
exit_status, usage = status, child_usage
|
||||
break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
# No detached work may outlive a supervised attempt.
|
||||
stop_group(proc.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.
|
||||
orphan_peak = 0
|
||||
# Killed descendants are adopted here rather than losing their
|
||||
# final CPU usage when the root worker cannot reap them itself.
|
||||
while True:
|
||||
try:
|
||||
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
||||
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
||||
elapsed = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
||||
used_disk = disk_usage(root)
|
||||
if reason is None:
|
||||
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
||||
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
||||
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
||||
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
||||
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
||||
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
||||
command=command,returncode=proc.returncode,stop_reason=reason,
|
||||
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
||||
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
||||
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
||||
reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
||||
gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False)
|
||||
ledger['cpu_seconds_charged'] += charged
|
||||
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
||||
save(receipt_path,receipt)
|
||||
save(ledger_path,ledger)
|
||||
return receipt
|
||||
Reference in new issue
Block a user