Implement bounded S2 preparation and sealed pilot execution

This commit is contained in:
Frank14f committed 2026-11-15 02:37:29 +08:00
1 parent fb827178ec
commit f8a6ed3f81
35 files changed
+2626

No files matched your search

+165
View File
@@ -0,0 +1,165 @@
"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility.
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
is measured and fails the job. Run only reviewed label-free worker commands.
"""
import fcntl
import ctypes
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import time
import psutil
def save(path, data):
path = Path(path)
tmp = path.with_suffix(path.suffix + '.tmp')
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
tmp.replace(path)
def disk_usage(root):
logical = allocated = 0
for p in Path(root).rglob('*'):
if p.is_file() and not p.is_symlink():
s = p.stat()
logical += s.st_size
allocated += s.st_blocks * 512
return max(logical, allocated)
def tree_usage(pid):
try:
root = psutil.Process(pid)
processes = [root] + root.children(recursive=True)
except psutil.NoSuchProcess:
return 0., 0, 0
cpu = 0.
rss = count = 0
for p in processes:
try:
t = p.cpu_times()
# Linux child times include reaped descendants, not live children.
cpu += t.user + t.system + t.children_user + t.children_system
rss += p.memory_info().rss
count += 1
except psutil.NoSuchProcess:
pass
return cpu, rss, count
def stop_group(pid):
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None):
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
root = Path(root)
root.mkdir(parents=True, exist_ok=True)
with (root/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
started_path = root/(name+'.started.json')
receipt_path = root/(name+'.receipt.json')
if started_path.exists() or receipt_path.exists():
raise FileExistsError('Attempt already exists; no silent retry')
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Every limit must be finite and positive')
ledger_path = root/'ledger.json'
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
raise RuntimeError('Total CPU exhausted before spawn')
if disk_usage(root) >= limits['disk_bytes']:
raise RuntimeError('Disk exhausted before spawn')
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
start = last = time.monotonic()
cpu_peak = rss_peak = max_processes = max_gap = 0
reason = None
exit_status = usage = None
env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1',
MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or ''))
with (root/(name+'.log')).open('x') as log:
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
env=env, start_new_session=True)
try:
while True:
now = time.monotonic()
max_gap = max(max_gap, now-last)
last = now
cpu, rss, count = tree_usage(proc.pid)
cpu_peak = max(cpu_peak,cpu)
rss_peak = max(rss_peak,rss)
max_processes = max(max_processes,count)
used_disk = disk_usage(root)
if now-start >= limits['job_wall_seconds']:
reason = 'wall_limit'
elif rss > limits['tree_rss_bytes']:
reason = 'tree_rss_limit'
elif count > limits['max_processes']:
reason = 'process_count_limit'
elif used_disk >= limits['disk_bytes']:
reason = 'disk_limit'
elif cpu_peak >= limits['job_cpu_seconds']:
reason = 'job_cpu_limit'
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
reason = 'total_cpu_limit'
if reason:
stop_group(proc.pid)
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
if pid:
exit_status, usage = status, child_usage
break
time.sleep(poll)
finally:
# No detached work may outlive a supervised attempt.
stop_group(proc.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(proc.pid, 0)
proc.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.
orphan_peak = 0
# Killed descendants are adopted here rather than losing their
# final CPU usage when the root worker cannot reap them itself.
while True:
try:
_, _, orphan_usage = os.wait4(-proc.pid, 0)
except ChildProcessError:
break
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
elapsed = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
used_disk = disk_usage(root)
if reason is None:
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
command=command,returncode=proc.returncode,stop_reason=reason,
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
reaped_orphan_cpu_seconds=orphan_cpu,
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False)
ledger['cpu_seconds_charged'] += charged
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
save(receipt_path,receipt)
save(ledger_path,ledger)
return receipt