Files
MeltMind/tools/track_b_gino_formal_guard.py
T

178 lines
7.7 KiB
Python

"""Supervise a formal GINO fit or score on one physical V100."""
import os
import shutil
import subprocess
import threading
import time
from types import FunctionType, SimpleNamespace
import psutil
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords
from s2_recovery_gpu_supervisor import stop_group, supervise
from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity
from track_b_gino_shapenet_entry import process_identity, save
def current_identity(pid):
try:
return process_identity(pid)
except psutil.NoSuchProcess:
return None
def process_tree(worker):
if current_identity(worker['pid']) != worker:
return []
try:
children = psutil.Process(worker['pid']).children(recursive=True)
except psutil.NoSuchProcess:
return []
result = []
for child in children:
try:
result.append(dict(pid=child.pid, created=child.create_time()))
except psutil.NoSuchProcess:
continue
return result
class OccupancyMonitor:
def __init__(self, root, uuid, worker, device_vram_bytes):
self.root = root
self.uuid = uuid
self.worker = worker
self.known = {worker['pid']: worker}
self.device_vram_bytes = device_vram_bytes
self.peak_device_bytes = 0
def check(self):
before = process_tree(self.worker)
state = snapshot(self.uuid)
after = process_tree(self.worker)
for identity in before + after:
if identity['pid'] != self.worker['pid']:
self.known[identity['pid']] = identity
foreign = []
for row in state['occupants']:
pid = row['pid']
current = current_identity(pid)
if pid not in self.known or (current is not None and current != self.known[pid]):
foreign.append(dict(row, observed_process_identity=current))
if foreign:
save(self.root / 'foreign-occupant.json', dict(
trigger='FOREIGN_GPU_OCCUPANT', unix=state['unix'], physical_uuid=self.uuid,
occupants=foreign, own_spawn=self.worker, tree_before=before, tree_after=after))
raise RuntimeError('FOREIGN_GPU_OCCUPANT')
self.peak_device_bytes = max(self.peak_device_bytes, state['memory_MiB'] * 2**20)
if self.peak_device_bytes > self.device_vram_bytes:
raise RuntimeError('DEVICE_VRAM_LIMIT')
def watch(root, uuid, worker, limits, done, tracker, state):
occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes'])
while not done.is_set():
try:
occupancy.check()
state['peak_device_bytes'] = occupancy.peak_device_bytes
tracker.scan(root / 'worker-records')
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
raise RuntimeError('RECORDED_VRAM_LIMIT')
monitor_identity(root, uuid)
except Exception as exc:
state['trigger'] = str(exc) or type(exc).__name__
# Signal only the recorded worker group while its leader has the same identity.
if current_identity(worker['pid']) == worker:
stop_group(worker['pid'])
return
done.wait(.5)
def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
began = time.monotonic()
admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds'])
done = threading.Event()
tracker = PeakRecords()
state = dict(trigger=None, peak_device_bytes=0)
watcher = None
worker_receipt = None
status, trigger, result_hash = 'STOPPED', None, None
def spawn(args, **kwargs):
nonlocal watcher
env, inventory = dispatch_environment(uuid, kwargs['env'])
ordinal = env['CUDA_VISIBLE_DEVICES']
if ordinal != ticket['ordinal']:
raise RuntimeError('PINNED_PCI_ORDINAL_DIFFERS')
if not snapshot(uuid)['idle']:
raise RuntimeError('SELECTED_GPU_BECAME_BUSY')
save(root / 'ordinal-dispatch.json', dict(ordinal=ordinal, physical_uuid=uuid, inventory=inventory))
if time.monotonic() >= admission:
raise RuntimeError('ADMISSION_DEADLINE')
if time.time() >= ticket['dispatch_before_unix']:
raise PermissionError('CARD_RESERVATION_EXPIRED_BEFORE_SPAWN')
proc = subprocess.Popen(args, **dict(kwargs, env=env))
try:
worker = process_identity(proc.pid)
save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid))
watcher = threading.Thread(target=watch, daemon=True,
args=(root, uuid, worker, limits, done, tracker, state))
watcher.start()
except (Exception, KeyboardInterrupt):
# The shared supervisor has not yet received the process to reap.
stop_group(proc.pid)
proc.wait()
raise
return proc
try:
if shutil.disk_usage(root).free < limits['disk_bytes']:
raise RuntimeError('DISK_HEADROOM')
idle = wait_idle(uuid, max(0., admission - time.monotonic()), lambda _: None, sample=snapshot)
save(root / 'idle-gate.json', idle)
if idle['status'] != 'IDLE_CONFIRMED' or time.monotonic() >= admission:
raise RuntimeError('IDLE_DEADLINE')
save(root / 'dispatch.json', dict(ticket, coordinator=process_identity(os.getpid()),
start_by_monotonic=admission))
remaining = deadline - time.monotonic() - limits['cleanup_seconds']
if remaining <= 0:
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
run_worker = FunctionType(supervise.__code__, ns, supervise.__name__)
try:
worker_receipt = run_worker(command, root, 'gino', dict(
total_cpu_seconds=limits['cpu_seconds'], job_cpu_seconds=limits['cpu_seconds'],
job_wall_seconds=remaining, tree_rss_bytes=limits['rss_bytes'],
disk_bytes=limits['disk_bytes'], max_processes=limits['max_processes']),
poll=.2, gpu_uuid=uuid)
finally:
done.set()
if watcher is not None:
watcher.join(timeout=8)
if watcher is not None and watcher.is_alive():
raise RuntimeError('MONITOR_DID_NOT_EXIT')
tracker.scan(root / 'worker-records')
if state['trigger']:
raise RuntimeError(state['trigger'])
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
raise RuntimeError('RECORDED_VRAM_LIMIT')
if worker_receipt['status'] != 'PASS' or worker_receipt['monitor_errors']:
raise RuntimeError(worker_receipt.get('stop_reason') or 'WORKER_FAILED')
if set(tracker.witnesses) != set(tracker.maxima):
raise RuntimeError('MISSING_RECORDED_MEMORY_PEAKS')
result_hash = validate_result()
status = 'COMPLETE'
except (Exception, KeyboardInterrupt) as exc:
trigger = str(exc) or type(exc).__name__
end = time.monotonic()
if end > deadline and status == 'COMPLETE':
status, trigger, result_hash = 'STOPPED', 'BOOKED_GPU_WALL_CAP', None
terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt,
physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state,
charged_gpu_seconds=end - ticket['guard_started_monotonic'],
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True)
save(root / 'complete.json', terminal)
return terminal