174 lines
7.6 KiB
Python
174 lines
7.6 KiB
Python
"""One labelled recovery for the historical Torch UUID-counting failure."""
|
|
|
|
import argparse
|
|
import csv
|
|
from copy import deepcopy
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
from types import FunctionType, SimpleNamespace
|
|
|
|
import psutil
|
|
|
|
import track_b_gino_import_checked_entry as accepted
|
|
import track_b_gino_measurement_gpu2 as previous
|
|
import track_b_gino_shapenet_entry as original
|
|
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
|
|
|
|
|
|
ROOT = previous.ROOT.with_name("track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19")
|
|
ENTRY = Path(__file__).resolve()
|
|
WORKER = ENTRY.with_name("track_b_gino_ordinal_recovery_worker.py")
|
|
UUID = previous.UUID
|
|
OLD_CONTRACT = "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f"
|
|
OLD_TERMINAL = "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e"
|
|
|
|
|
|
def expected_contract():
|
|
c = read(checked(previous.ROOT / "contract.json", OLD_CONTRACT))
|
|
stopped = read(checked(previous.ROOT / "complete.json", OLD_TERMINAL))
|
|
if stopped["status"] != "STOPPED" or stopped["trigger"] != "WORKER_FAILED":
|
|
raise PermissionError("The original stopped GPU2 attempt is required")
|
|
c["ordinal_recovery"] = dict(handoff="h-91e61871", previous_terminal_sha256=OLD_TERMINAL,
|
|
previous_gpu_wall_seconds=stopped["cumulative_gpu_wall_seconds"])
|
|
c["code_sha256"].update({str(p): sha(p) for p in (ENTRY, WORKER)})
|
|
return c
|
|
|
|
|
|
def contract(path):
|
|
c = previous.checked_entry.contract(path)
|
|
if c != expected_contract():
|
|
raise PermissionError("Only the fixed ordinal recovery of this measurement is authorized")
|
|
return c
|
|
|
|
|
|
def query_nvidia(options):
|
|
r = subprocess.run(["nvidia-smi", options, "--format=csv,noheader"],
|
|
check=True, capture_output=True, text=True, timeout=3)
|
|
return [[v.strip() for v in row] for row in csv.reader(r.stdout.splitlines()) if row]
|
|
|
|
|
|
def device_ordinal(rows, uuid):
|
|
ordered = sorted(rows, key=lambda row: row[2].lower())
|
|
if len({row[2].lower() for row in ordered}) != len(ordered):
|
|
raise ValueError("Nonunique GPU PCI identities")
|
|
if [row[0] for row in ordered] != [str(i) for i in range(len(ordered))]:
|
|
raise ValueError("NVIDIA indices do not match CUDA PCI bus order")
|
|
match = [row for row in ordered if row[1] == uuid and "V100" in row[3]]
|
|
if len(match) != 1:
|
|
raise ValueError("Expected exactly one reviewed physical V100")
|
|
return match[0][0]
|
|
|
|
|
|
def dispatch_environment(uuid, env):
|
|
if env["CUDA_VISIBLE_DEVICES"] != uuid:
|
|
raise PermissionError("Supervisor did not select the exact physical UUID")
|
|
rows = query_nvidia("--query-gpu=index,uuid,pci.bus_id,name")
|
|
ordinal = device_ordinal(rows, uuid)
|
|
return dict(env, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=ordinal), rows
|
|
|
|
|
|
def monitor_identity(root, uuid):
|
|
pending = root / "worker-device-witness.json"
|
|
verified = root / "worker-device-verified.json"
|
|
if not pending.is_file() or verified.exists():
|
|
return
|
|
witness = read(pending)
|
|
spawn = read(root / "actual-worker-spawn.json")
|
|
if (witness["process"] != spawn["process"]
|
|
or witness["physical_uuid_expected"] != uuid
|
|
or witness["ordinal"] != spawn["ordinal"]
|
|
or witness["device_count"] != 1 or "V100" not in witness["device_name"]
|
|
or witness["first_allocation_bytes"] != 1):
|
|
raise RuntimeError("WORKER_DEVICE_WITNESS_IDENTITY_DIFFERS")
|
|
pid = witness["process"]["pid"]
|
|
observed = sorted({card for card, owner in query_nvidia("--query-compute-apps=gpu_uuid,pid")
|
|
if int(owner) == pid})
|
|
if not observed:
|
|
return
|
|
if observed != [uuid]:
|
|
out = root / "worker-device-mismatch.json"
|
|
if not out.exists():
|
|
save(out, dict(process=witness["process"], expected_uuid=uuid, observed_uuids=observed,
|
|
scientific_worker_released=False))
|
|
raise RuntimeError("WORKER_GPU_UUID_MISMATCH")
|
|
save(verified, dict(process=witness["process"], physical_uuid=uuid,
|
|
ordinal=witness["ordinal"], observed_uuids=observed))
|
|
|
|
|
|
def dispatch_guard(root, path):
|
|
root = Path(root)
|
|
c = contract(path)
|
|
old_charge = c["ordinal_recovery"]["previous_gpu_wall_seconds"]
|
|
total = c["limits"]["gpu_wall_seconds"]
|
|
spawned = None
|
|
started_dispatch = None
|
|
|
|
def runtime_contract(checked_path):
|
|
runtime = deepcopy(contract(checked_path))
|
|
runtime["limits"]["gpu_wall_seconds"] = total - old_charge
|
|
return runtime
|
|
|
|
def snapshot(uuid):
|
|
state = original.snapshot(uuid)
|
|
if spawned is not None:
|
|
receipt = root / "actual-worker-spawn.json"
|
|
if not receipt.exists():
|
|
try:
|
|
identity = process_identity(spawned.pid)
|
|
except psutil.NoSuchProcess:
|
|
return state
|
|
save(receipt, dict(process=identity, ordinal=started_dispatch["ordinal"], physical_uuid=uuid))
|
|
monitor_identity(root, uuid)
|
|
return state
|
|
|
|
def route_supervise(command, directory, mode, limits, *, poll, gpu_uuid):
|
|
if gpu_uuid != UUID or Path(command[1]).name != "track_b_gino_shapenet_worker.py":
|
|
raise PermissionError("Unexpected original guard worker route")
|
|
command[1] = str(WORKER)
|
|
|
|
def spawn(args, **kwargs):
|
|
nonlocal spawned, started_dispatch
|
|
environment, rows = dispatch_environment(gpu_uuid, kwargs["env"])
|
|
ordinal = environment["CUDA_VISIBLE_DEVICES"]
|
|
started_dispatch = dict(ordinal=ordinal, order="PCI_BUS_ID",
|
|
physical_uuid=gpu_uuid, gpu_inventory=rows)
|
|
save(root / "ordinal-dispatch.json", started_dispatch)
|
|
if time.monotonic() >= read(root / "dispatch.json")["start_by_monotonic"]:
|
|
raise RuntimeError("ORDINAL_DISPATCH_DEADLINE")
|
|
kwargs["env"] = environment
|
|
spawned = subprocess.Popen(args, **kwargs)
|
|
return spawned
|
|
|
|
ns = dict(original.supervise.__globals__,
|
|
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
|
supervise = FunctionType(original.supervise.__code__, ns, "supervise")
|
|
return supervise(command, directory, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
|
|
|
|
namespace = dict(original.dispatch_guard.__globals__, contract=runtime_contract,
|
|
snapshot=snapshot, supervise=route_supervise)
|
|
code = FunctionType(original.dispatch_guard.__code__, namespace, "dispatch_guard")(root, path)
|
|
record = read(root / "complete.json")
|
|
new_charge = record["cumulative_gpu_wall_seconds"]
|
|
save(root / "recovery-settlement.json", dict(status=record["status"], trigger=record["trigger"],
|
|
previous_gpu_wall_seconds=old_charge, recovery_gpu_wall_seconds=new_charge,
|
|
cumulative_measurement_gpu_wall_seconds=old_charge + new_charge, total_cap_gpu_wall_seconds=total,
|
|
independent_attempts=2, no_retry=True, within_total_cap=old_charge + new_charge <= total))
|
|
return code
|
|
|
|
|
|
def authorized_child(root):
|
|
namespace = dict(accepted.authorized_child.__globals__, ROOT=ROOT, contract=contract)
|
|
return FunctionType(accepted.authorized_child.__code__, namespace, "authorized_child")(root)
|
|
|
|
|
|
def run(root):
|
|
namespace = dict(accepted.run.__globals__, ROOT=ROOT, contract=contract, dispatch_guard=dispatch_guard)
|
|
return FunctionType(accepted.run.__code__, namespace, "run")(root)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("root", type=Path)
|
|
raise SystemExit(run(parser.parse_args().root))
|