"""Supervise a formal GINO fit or score on one physical V100.""" import os import shutil import signal import subprocess import threading import time from types import FunctionType, SimpleNamespace import psutil from s2_gpu_idle_gate import snapshot, wait_idle from s2_main_memory_guard import PeakRecords from s2_recovery_gpu_supervisor import supervise from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity from track_b_gino_shapenet_entry import process_identity, save def current_identity(pid): try: return process_identity(pid) except psutil.NoSuchProcess: return None def process_tree(worker): if current_identity(worker['pid']) != worker: return [] try: children = psutil.Process(worker['pid']).children(recursive=True) except psutil.NoSuchProcess: return [] result = [] for child in children: try: result.append(dict(pid=child.pid, created=child.create_time())) except psutil.NoSuchProcess: continue return result class OccupancyMonitor: def __init__(self, root, uuid, worker, device_vram_bytes): self.root = root self.uuid = uuid self.worker = worker self.known = {worker['pid']: worker} self.device_vram_bytes = device_vram_bytes self.peak_device_bytes = 0 def check(self): before = process_tree(self.worker) state = snapshot(self.uuid) after = process_tree(self.worker) for identity in before + after: if identity['pid'] != self.worker['pid']: self.known[identity['pid']] = identity foreign = [] for row in state['occupants']: pid = row['pid'] current = current_identity(pid) if pid not in self.known or (current is not None and current != self.known[pid]): foreign.append(dict(row, observed_process_identity=current)) if foreign: save(self.root / 'foreign-occupant.json', dict( trigger='FOREIGN_GPU_OCCUPANT', unix=state['unix'], physical_uuid=self.uuid, occupants=foreign, own_spawn=self.worker, tree_before=before, tree_after=after)) raise RuntimeError('FOREIGN_GPU_OCCUPANT') self.peak_device_bytes = max(self.peak_device_bytes, state['memory_MiB'] * 2**20) if self.peak_device_bytes > self.device_vram_bytes: raise RuntimeError('DEVICE_VRAM_LIMIT') class OwnedGroup: """Tie group signals to an unreaped session leader we directly spawned.""" def __init__(self): self.pid = None self.identity = None self.reaped = False self.signalled = False self.lock = threading.Lock() self.events = [] def observe_exit(self, pid, nonblocking): flags = os.WEXITED | os.WNOWAIT | (os.WNOHANG if nonblocking else 0) return os.waitid(os.P_PID, pid, flags) def stop_locked(self, pid, reason): if pid != self.pid or self.reaped or self.signalled: return # WNOWAIT verifies this PID is still our child without permitting PID # reuse. The same lock prevents our wait4 from reaping before killpg. try: self.observe_exit(pid, nonblocking=True) if os.getpgid(pid) != pid: raise RuntimeError('WORKER_IS_NOT_ITS_OWN_SESSION_LEADER') if self.identity is not None and current_identity(pid) != self.identity: raise RuntimeError('WORKER_BIRTH_IDENTITY_DIFFERS') os.killpg(pid, signal.SIGKILL) except (ChildProcessError, ProcessLookupError): return self.signalled = True self.events.append(dict(event='group_signal', pid=pid, signal=int(signal.SIGKILL), reason=reason)) def stop(self, pid): with self.lock: self.stop_locked(pid, 'GUARD_STOP') def wait4(self, pid, options): if pid != self.pid: # The shared supervisor separately collects adopted descendants # with a negative group selector. That path never sends signals. return os.wait4(pid, options) # Never hold the lock on a blocking wait. The watcher may still need # it to stop the worker. if self.observe_exit(pid, nonblocking=bool(options & os.WNOHANG)) is None: return 0, 0, None with self.lock: # With subreaping enabled, children the leader left in its group # are now ours. Clean them up while the zombie still pins PGID. for child in psutil.Process().children(): try: in_group = child.pid != pid and os.getpgid(child.pid) == pid except ProcessLookupError: continue if in_group: self.stop_locked(pid, 'REMAINING_GROUP_MEMBER') break result = os.wait4(pid, options) if result[0] == pid: self.reaped = True self.events.append(dict(event='leader_reaped', pid=pid)) return result def watch(root, uuid, worker, group, limits, done, tracker, state): occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes']) while not done.is_set(): try: occupancy.check() state['peak_device_bytes'] = occupancy.peak_device_bytes tracker.scan(root / 'worker-records') if max(tracker.maxima.values()) > limits['recorded_vram_bytes']: raise RuntimeError('RECORDED_VRAM_LIMIT') monitor_identity(root, uuid) except Exception as exc: state['trigger'] = str(exc) or type(exc).__name__ group.stop(worker['pid']) return done.wait(.5) def guard(root, command, *, uuid, limits, deadline, ticket, validate_result): began = time.monotonic() admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds']) done = threading.Event() tracker = PeakRecords() group = OwnedGroup() state = dict(trigger=None, peak_device_bytes=0) watcher = None worker_receipt = None status, trigger, result_hash = 'STOPPED', None, None def spawn(args, **kwargs): nonlocal watcher env, inventory = dispatch_environment(uuid, kwargs['env']) ordinal = env['CUDA_VISIBLE_DEVICES'] if ordinal != ticket['ordinal']: raise RuntimeError('PINNED_PCI_ORDINAL_DIFFERS') if not snapshot(uuid)['idle']: raise RuntimeError('SELECTED_GPU_BECAME_BUSY') save(root / 'ordinal-dispatch.json', dict(ordinal=ordinal, physical_uuid=uuid, inventory=inventory)) if time.monotonic() >= admission: raise RuntimeError('ADMISSION_DEADLINE') if time.time() >= ticket['dispatch_before_unix']: raise PermissionError('CARD_RESERVATION_EXPIRED_BEFORE_SPAWN') proc = subprocess.Popen(args, **dict(kwargs, env=env)) group.pid = proc.pid try: worker = process_identity(proc.pid) group.identity = worker save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid)) watcher = threading.Thread(target=watch, daemon=True, args=(root, uuid, worker, group, limits, done, tracker, state)) watcher.start() except (Exception, KeyboardInterrupt): # The shared supervisor has not yet received the process to reap. group.stop(proc.pid) _, status, _ = group.wait4(proc.pid, 0) proc.returncode = os.waitstatus_to_exitcode(status) raise return proc try: if shutil.disk_usage(root).free < limits['disk_bytes']: raise RuntimeError('DISK_HEADROOM') idle = wait_idle(uuid, max(0., admission - time.monotonic()), lambda _: None, sample=snapshot) save(root / 'idle-gate.json', idle) if idle['status'] != 'IDLE_CONFIRMED' or time.monotonic() >= admission: raise RuntimeError('IDLE_DEADLINE') save(root / 'dispatch.json', dict(ticket, coordinator=process_identity(os.getpid()), start_by_monotonic=admission)) remaining = deadline - time.monotonic() - limits['cleanup_seconds'] if remaining <= 0: raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET') ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT), os=SimpleNamespace(**dict(vars(os), wait4=group.wait4)), stop_group=group.stop) run_worker = FunctionType(supervise.__code__, ns, supervise.__name__) try: worker_receipt = run_worker(command, root, 'gino', dict( total_cpu_seconds=limits['cpu_seconds'], job_cpu_seconds=limits['cpu_seconds'], job_wall_seconds=remaining, tree_rss_bytes=limits['rss_bytes'], disk_bytes=limits['disk_bytes'], max_processes=limits['max_processes']), poll=.2, gpu_uuid=uuid) finally: done.set() if watcher is not None: watcher.join(timeout=8) if watcher is not None and watcher.is_alive(): raise RuntimeError('MONITOR_DID_NOT_EXIT') tracker.scan(root / 'worker-records') if state['trigger']: raise RuntimeError(state['trigger']) if max(tracker.maxima.values()) > limits['recorded_vram_bytes']: raise RuntimeError('RECORDED_VRAM_LIMIT') if worker_receipt['status'] != 'PASS' or worker_receipt['monitor_errors']: raise RuntimeError(worker_receipt.get('stop_reason') or 'WORKER_FAILED') if set(tracker.witnesses) != set(tracker.maxima): raise RuntimeError('MISSING_RECORDED_MEMORY_PEAKS') result_hash = validate_result() status = 'COMPLETE' except (Exception, KeyboardInterrupt) as exc: trigger = str(exc) or type(exc).__name__ end = time.monotonic() if end > deadline and status == 'COMPLETE': status, trigger, result_hash = 'STOPPED', 'BOOKED_GPU_WALL_CAP', None terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt, physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state, charged_gpu_seconds=end - ticket['guard_started_monotonic'], booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, lifecycle=group.events, no_retry=True) save(root / 'complete.json', terminal) return terminal