Bind GINO cleanup to unreaped worker group and replay actual signals
This commit is contained in:
1 parent
8af022e434
commit
ee8e01937e
9 files changed
+1185
-21
No files matched your search
@@ -18,7 +18,7 @@ from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT
|
||||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||||
|
||||
|
||||
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-v2-2026-11-19')
|
||||
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-v3-2026-11-19')
|
||||
SOURCES = Path(__file__).resolve().parent
|
||||
OLD_FILES = {
|
||||
'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a',
|
||||
@@ -67,6 +67,7 @@ def expected_contract():
|
||||
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES})
|
||||
c['formal'] = dict(
|
||||
handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES,
|
||||
lifecycle_review_slack_ts='1795075951.091959',
|
||||
device_pci_ordinals=ORDINALS,
|
||||
reservation=dict(handoff='h-a429935e', timezone=TZ.key,
|
||||
start_inclusive=RESERVATION_START.isoformat(),
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
@@ -11,7 +12,7 @@ import psutil
|
||||
|
||||
from s2_gpu_idle_gate import snapshot, wait_idle
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_recovery_gpu_supervisor import stop_group, supervise
|
||||
from s2_recovery_gpu_supervisor import supervise
|
||||
from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity
|
||||
from track_b_gino_shapenet_entry import process_identity, save
|
||||
|
||||
@@ -71,7 +72,70 @@ class OccupancyMonitor:
|
||||
raise RuntimeError('DEVICE_VRAM_LIMIT')
|
||||
|
||||
|
||||
def watch(root, uuid, worker, limits, done, tracker, state):
|
||||
class OwnedGroup:
|
||||
"""Tie group signals to an unreaped session leader we directly spawned."""
|
||||
|
||||
def __init__(self):
|
||||
self.pid = None
|
||||
self.identity = None
|
||||
self.reaped = False
|
||||
self.signalled = False
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
|
||||
def observe_exit(self, pid, nonblocking):
|
||||
flags = os.WEXITED | os.WNOWAIT | (os.WNOHANG if nonblocking else 0)
|
||||
return os.waitid(os.P_PID, pid, flags)
|
||||
|
||||
def stop_locked(self, pid, reason):
|
||||
if pid != self.pid or self.reaped or self.signalled:
|
||||
return
|
||||
# WNOWAIT verifies this PID is still our child without permitting PID
|
||||
# reuse. The same lock prevents our wait4 from reaping before killpg.
|
||||
try:
|
||||
self.observe_exit(pid, nonblocking=True)
|
||||
if os.getpgid(pid) != pid:
|
||||
raise RuntimeError('WORKER_IS_NOT_ITS_OWN_SESSION_LEADER')
|
||||
if self.identity is not None and current_identity(pid) != self.identity:
|
||||
raise RuntimeError('WORKER_BIRTH_IDENTITY_DIFFERS')
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except (ChildProcessError, ProcessLookupError):
|
||||
return
|
||||
self.signalled = True
|
||||
self.events.append(dict(event='group_signal', pid=pid, signal=int(signal.SIGKILL), reason=reason))
|
||||
|
||||
def stop(self, pid):
|
||||
with self.lock:
|
||||
self.stop_locked(pid, 'GUARD_STOP')
|
||||
|
||||
def wait4(self, pid, options):
|
||||
if pid != self.pid:
|
||||
# The shared supervisor separately collects adopted descendants
|
||||
# with a negative group selector. That path never sends signals.
|
||||
return os.wait4(pid, options)
|
||||
# Never hold the lock on a blocking wait. The watcher may still need
|
||||
# it to stop the worker.
|
||||
if self.observe_exit(pid, nonblocking=bool(options & os.WNOHANG)) is None:
|
||||
return 0, 0, None
|
||||
with self.lock:
|
||||
# With subreaping enabled, children the leader left in its group
|
||||
# are now ours. Clean them up while the zombie still pins PGID.
|
||||
for child in psutil.Process().children():
|
||||
try:
|
||||
in_group = child.pid != pid and os.getpgid(child.pid) == pid
|
||||
except ProcessLookupError:
|
||||
continue
|
||||
if in_group:
|
||||
self.stop_locked(pid, 'REMAINING_GROUP_MEMBER')
|
||||
break
|
||||
result = os.wait4(pid, options)
|
||||
if result[0] == pid:
|
||||
self.reaped = True
|
||||
self.events.append(dict(event='leader_reaped', pid=pid))
|
||||
return result
|
||||
|
||||
|
||||
def watch(root, uuid, worker, group, limits, done, tracker, state):
|
||||
occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes'])
|
||||
while not done.is_set():
|
||||
try:
|
||||
@@ -83,9 +147,7 @@ def watch(root, uuid, worker, limits, done, tracker, state):
|
||||
monitor_identity(root, uuid)
|
||||
except Exception as exc:
|
||||
state['trigger'] = str(exc) or type(exc).__name__
|
||||
# Signal only the recorded worker group while its leader has the same identity.
|
||||
if current_identity(worker['pid']) == worker:
|
||||
stop_group(worker['pid'])
|
||||
group.stop(worker['pid'])
|
||||
return
|
||||
done.wait(.5)
|
||||
|
||||
@@ -95,6 +157,7 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds'])
|
||||
done = threading.Event()
|
||||
tracker = PeakRecords()
|
||||
group = OwnedGroup()
|
||||
state = dict(trigger=None, peak_device_bytes=0)
|
||||
watcher = None
|
||||
worker_receipt = None
|
||||
@@ -114,16 +177,19 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
if time.time() >= ticket['dispatch_before_unix']:
|
||||
raise PermissionError('CARD_RESERVATION_EXPIRED_BEFORE_SPAWN')
|
||||
proc = subprocess.Popen(args, **dict(kwargs, env=env))
|
||||
group.pid = proc.pid
|
||||
try:
|
||||
worker = process_identity(proc.pid)
|
||||
group.identity = worker
|
||||
save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid))
|
||||
watcher = threading.Thread(target=watch, daemon=True,
|
||||
args=(root, uuid, worker, limits, done, tracker, state))
|
||||
args=(root, uuid, worker, group, limits, done, tracker, state))
|
||||
watcher.start()
|
||||
except (Exception, KeyboardInterrupt):
|
||||
# The shared supervisor has not yet received the process to reap.
|
||||
stop_group(proc.pid)
|
||||
proc.wait()
|
||||
group.stop(proc.pid)
|
||||
_, status, _ = group.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(status)
|
||||
raise
|
||||
return proc
|
||||
|
||||
@@ -139,7 +205,8 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
remaining = deadline - time.monotonic() - limits['cleanup_seconds']
|
||||
if remaining <= 0:
|
||||
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
|
||||
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
||||
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT),
|
||||
os=SimpleNamespace(**dict(vars(os), wait4=group.wait4)), stop_group=group.stop)
|
||||
run_worker = FunctionType(supervise.__code__, ns, supervise.__name__)
|
||||
try:
|
||||
worker_receipt = run_worker(command, root, 'gino', dict(
|
||||
@@ -172,6 +239,7 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt,
|
||||
physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state,
|
||||
charged_gpu_seconds=end - ticket['guard_started_monotonic'],
|
||||
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True)
|
||||
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima,
|
||||
lifecycle=group.events, no_retry=True)
|
||||
save(root / 'complete.json', terminal)
|
||||
return terminal
|
||||
Reference in new issue
Block a user