176 lines
7.6 KiB
Python
176 lines
7.6 KiB
Python
"""Supervise a formal GINO fit or score on one physical V100."""
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from types import FunctionType, SimpleNamespace
|
|
|
|
import psutil
|
|
|
|
from s2_gpu_idle_gate import snapshot, wait_idle
|
|
from s2_main_memory_guard import PeakRecords
|
|
from s2_recovery_gpu_supervisor import stop_group, supervise
|
|
from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity
|
|
from track_b_gino_shapenet_entry import process_identity, save
|
|
|
|
|
|
def current_identity(pid):
|
|
try:
|
|
return process_identity(pid)
|
|
except psutil.NoSuchProcess:
|
|
return None
|
|
|
|
|
|
def process_tree(worker):
|
|
if current_identity(worker['pid']) != worker:
|
|
return []
|
|
try:
|
|
children = psutil.Process(worker['pid']).children(recursive=True)
|
|
except psutil.NoSuchProcess:
|
|
return []
|
|
result = []
|
|
for child in children:
|
|
try:
|
|
result.append(dict(pid=child.pid, created=child.create_time()))
|
|
except psutil.NoSuchProcess:
|
|
continue
|
|
return result
|
|
|
|
|
|
class OccupancyMonitor:
|
|
def __init__(self, root, uuid, worker, device_vram_bytes):
|
|
self.root = root
|
|
self.uuid = uuid
|
|
self.worker = worker
|
|
self.known = {worker['pid']: worker}
|
|
self.device_vram_bytes = device_vram_bytes
|
|
self.peak_device_bytes = 0
|
|
|
|
def check(self):
|
|
before = process_tree(self.worker)
|
|
state = snapshot(self.uuid)
|
|
after = process_tree(self.worker)
|
|
for identity in before + after:
|
|
if identity['pid'] != self.worker['pid']:
|
|
self.known[identity['pid']] = identity
|
|
foreign = []
|
|
for row in state['occupants']:
|
|
pid = row['pid']
|
|
current = current_identity(pid)
|
|
if pid not in self.known or (current is not None and current != self.known[pid]):
|
|
foreign.append(dict(row, observed_process_identity=current))
|
|
if foreign:
|
|
save(self.root / 'foreign-occupant.json', dict(
|
|
trigger='FOREIGN_GPU_OCCUPANT', unix=state['unix'], physical_uuid=self.uuid,
|
|
occupants=foreign, own_spawn=self.worker, tree_before=before, tree_after=after))
|
|
raise RuntimeError('FOREIGN_GPU_OCCUPANT')
|
|
self.peak_device_bytes = max(self.peak_device_bytes, state['memory_MiB'] * 2**20)
|
|
if self.peak_device_bytes > self.device_vram_bytes:
|
|
raise RuntimeError('DEVICE_VRAM_LIMIT')
|
|
|
|
|
|
def watch(root, uuid, worker, limits, done, tracker, state):
|
|
occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes'])
|
|
while not done.is_set():
|
|
try:
|
|
occupancy.check()
|
|
state['peak_device_bytes'] = occupancy.peak_device_bytes
|
|
tracker.scan(root / 'worker-records')
|
|
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
|
|
raise RuntimeError('RECORDED_VRAM_LIMIT')
|
|
monitor_identity(root, uuid)
|
|
except Exception as exc:
|
|
state['trigger'] = str(exc) or type(exc).__name__
|
|
# Signal only the recorded worker group while its leader has the same identity.
|
|
if current_identity(worker['pid']) == worker:
|
|
stop_group(worker['pid'])
|
|
return
|
|
done.wait(.5)
|
|
|
|
|
|
def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
|
began = time.monotonic()
|
|
admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds'])
|
|
done = threading.Event()
|
|
tracker = PeakRecords()
|
|
state = dict(trigger=None, peak_device_bytes=0)
|
|
watcher = None
|
|
worker_receipt = None
|
|
status, trigger, result_hash = 'STOPPED', None, None
|
|
|
|
def spawn(args, **kwargs):
|
|
nonlocal watcher
|
|
env, inventory = dispatch_environment(uuid, kwargs['env'])
|
|
ordinal = env['CUDA_VISIBLE_DEVICES']
|
|
if ordinal != ticket['ordinal']:
|
|
raise RuntimeError('PINNED_PCI_ORDINAL_DIFFERS')
|
|
if not snapshot(uuid)['idle']:
|
|
raise RuntimeError('SELECTED_GPU_BECAME_BUSY')
|
|
save(root / 'ordinal-dispatch.json', dict(ordinal=ordinal, physical_uuid=uuid, inventory=inventory))
|
|
if time.monotonic() >= admission:
|
|
raise RuntimeError('ADMISSION_DEADLINE')
|
|
proc = subprocess.Popen(args, **dict(kwargs, env=env))
|
|
try:
|
|
worker = process_identity(proc.pid)
|
|
save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid))
|
|
watcher = threading.Thread(target=watch, daemon=True,
|
|
args=(root, uuid, worker, limits, done, tracker, state))
|
|
watcher.start()
|
|
except (Exception, KeyboardInterrupt):
|
|
# The shared supervisor has not yet received the process to reap.
|
|
stop_group(proc.pid)
|
|
proc.wait()
|
|
raise
|
|
return proc
|
|
|
|
try:
|
|
if shutil.disk_usage(root).free < limits['disk_bytes']:
|
|
raise RuntimeError('DISK_HEADROOM')
|
|
idle = wait_idle(uuid, max(0., admission - time.monotonic()), lambda _: None, sample=snapshot)
|
|
save(root / 'idle-gate.json', idle)
|
|
if idle['status'] != 'IDLE_CONFIRMED' or time.monotonic() >= admission:
|
|
raise RuntimeError('IDLE_DEADLINE')
|
|
save(root / 'dispatch.json', dict(ticket, coordinator=process_identity(os.getpid()),
|
|
start_by_monotonic=admission))
|
|
remaining = deadline - time.monotonic() - limits['cleanup_seconds']
|
|
if remaining <= 0:
|
|
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
|
|
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
|
run_worker = FunctionType(supervise.__code__, ns, supervise.__name__)
|
|
try:
|
|
worker_receipt = run_worker(command, root, 'gino', dict(
|
|
total_cpu_seconds=limits['cpu_seconds'], job_cpu_seconds=limits['cpu_seconds'],
|
|
job_wall_seconds=remaining, tree_rss_bytes=limits['rss_bytes'],
|
|
disk_bytes=limits['disk_bytes'], max_processes=limits['max_processes']),
|
|
poll=.2, gpu_uuid=uuid)
|
|
finally:
|
|
done.set()
|
|
if watcher is not None:
|
|
watcher.join(timeout=8)
|
|
if watcher is not None and watcher.is_alive():
|
|
raise RuntimeError('MONITOR_DID_NOT_EXIT')
|
|
tracker.scan(root / 'worker-records')
|
|
if state['trigger']:
|
|
raise RuntimeError(state['trigger'])
|
|
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
|
|
raise RuntimeError('RECORDED_VRAM_LIMIT')
|
|
if worker_receipt['status'] != 'PASS' or worker_receipt['monitor_errors']:
|
|
raise RuntimeError(worker_receipt.get('stop_reason') or 'WORKER_FAILED')
|
|
if set(tracker.witnesses) != set(tracker.maxima):
|
|
raise RuntimeError('MISSING_RECORDED_MEMORY_PEAKS')
|
|
result_hash = validate_result()
|
|
status = 'COMPLETE'
|
|
except (Exception, KeyboardInterrupt) as exc:
|
|
trigger = str(exc) or type(exc).__name__
|
|
end = time.monotonic()
|
|
if end > deadline and status == 'COMPLETE':
|
|
status, trigger, result_hash = 'STOPPED', 'BOOKED_GPU_WALL_CAP', None
|
|
terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt,
|
|
physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state,
|
|
charged_gpu_seconds=end - ticket['guard_started_monotonic'],
|
|
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True)
|
|
save(root / 'complete.json', terminal)
|
|
return terminal
|