Files
MeltMind/tools/track_b_gino_ordinal_recovery.py
T

174 lines
7.6 KiB
Python

"""One labelled recovery for the historical Torch UUID-counting failure."""
import argparse
import csv
from copy import deepcopy
from pathlib import Path
import subprocess
import time
from types import FunctionType, SimpleNamespace
import psutil
import track_b_gino_import_checked_entry as accepted
import track_b_gino_measurement_gpu2 as previous
import track_b_gino_shapenet_entry as original
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
ROOT = previous.ROOT.with_name("track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19")
ENTRY = Path(__file__).resolve()
WORKER = ENTRY.with_name("track_b_gino_ordinal_recovery_worker.py")
UUID = previous.UUID
OLD_CONTRACT = "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f"
OLD_TERMINAL = "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e"
def expected_contract():
c = read(checked(previous.ROOT / "contract.json", OLD_CONTRACT))
stopped = read(checked(previous.ROOT / "complete.json", OLD_TERMINAL))
if stopped["status"] != "STOPPED" or stopped["trigger"] != "WORKER_FAILED":
raise PermissionError("The original stopped GPU2 attempt is required")
c["ordinal_recovery"] = dict(handoff="h-91e61871", previous_terminal_sha256=OLD_TERMINAL,
previous_gpu_wall_seconds=stopped["cumulative_gpu_wall_seconds"])
c["code_sha256"].update({str(p): sha(p) for p in (ENTRY, WORKER)})
return c
def contract(path):
c = previous.checked_entry.contract(path)
if c != expected_contract():
raise PermissionError("Only the fixed ordinal recovery of this measurement is authorized")
return c
def query_nvidia(options):
r = subprocess.run(["nvidia-smi", options, "--format=csv,noheader"],
check=True, capture_output=True, text=True, timeout=3)
return [[v.strip() for v in row] for row in csv.reader(r.stdout.splitlines()) if row]
def device_ordinal(rows, uuid):
ordered = sorted(rows, key=lambda row: row[2].lower())
if len({row[2].lower() for row in ordered}) != len(ordered):
raise ValueError("Nonunique GPU PCI identities")
if [row[0] for row in ordered] != [str(i) for i in range(len(ordered))]:
raise ValueError("NVIDIA indices do not match CUDA PCI bus order")
match = [row for row in ordered if row[1] == uuid and "V100" in row[3]]
if len(match) != 1:
raise ValueError("Expected exactly one reviewed physical V100")
return match[0][0]
def dispatch_environment(uuid, env):
if env["CUDA_VISIBLE_DEVICES"] != uuid:
raise PermissionError("Supervisor did not select the exact physical UUID")
rows = query_nvidia("--query-gpu=index,uuid,pci.bus_id,name")
ordinal = device_ordinal(rows, uuid)
return dict(env, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=ordinal), rows
def monitor_identity(root, uuid):
pending = root / "worker-device-witness.json"
verified = root / "worker-device-verified.json"
if not pending.is_file() or verified.exists():
return
witness = read(pending)
spawn = read(root / "actual-worker-spawn.json")
if (witness["process"] != spawn["process"]
or witness["physical_uuid_expected"] != uuid
or witness["ordinal"] != spawn["ordinal"]
or witness["device_count"] != 1 or "V100" not in witness["device_name"]
or witness["first_allocation_bytes"] != 1):
raise RuntimeError("WORKER_DEVICE_WITNESS_IDENTITY_DIFFERS")
pid = witness["process"]["pid"]
observed = sorted({card for card, owner in query_nvidia("--query-compute-apps=gpu_uuid,pid")
if int(owner) == pid})
if not observed:
return
if observed != [uuid]:
out = root / "worker-device-mismatch.json"
if not out.exists():
save(out, dict(process=witness["process"], expected_uuid=uuid, observed_uuids=observed,
scientific_worker_released=False))
raise RuntimeError("WORKER_GPU_UUID_MISMATCH")
save(verified, dict(process=witness["process"], physical_uuid=uuid,
ordinal=witness["ordinal"], observed_uuids=observed))
def dispatch_guard(root, path):
root = Path(root)
c = contract(path)
old_charge = c["ordinal_recovery"]["previous_gpu_wall_seconds"]
total = c["limits"]["gpu_wall_seconds"]
spawned = None
started_dispatch = None
def runtime_contract(checked_path):
runtime = deepcopy(contract(checked_path))
runtime["limits"]["gpu_wall_seconds"] = total - old_charge
return runtime
def snapshot(uuid):
state = original.snapshot(uuid)
if spawned is not None:
receipt = root / "actual-worker-spawn.json"
if not receipt.exists():
try:
identity = process_identity(spawned.pid)
except psutil.NoSuchProcess:
return state
save(receipt, dict(process=identity, ordinal=started_dispatch["ordinal"], physical_uuid=uuid))
monitor_identity(root, uuid)
return state
def route_supervise(command, directory, mode, limits, *, poll, gpu_uuid):
if gpu_uuid != UUID or Path(command[1]).name != "track_b_gino_shapenet_worker.py":
raise PermissionError("Unexpected original guard worker route")
command[1] = str(WORKER)
def spawn(args, **kwargs):
nonlocal spawned, started_dispatch
environment, rows = dispatch_environment(gpu_uuid, kwargs["env"])
ordinal = environment["CUDA_VISIBLE_DEVICES"]
started_dispatch = dict(ordinal=ordinal, order="PCI_BUS_ID",
physical_uuid=gpu_uuid, gpu_inventory=rows)
save(root / "ordinal-dispatch.json", started_dispatch)
if time.monotonic() >= read(root / "dispatch.json")["start_by_monotonic"]:
raise RuntimeError("ORDINAL_DISPATCH_DEADLINE")
kwargs["env"] = environment
spawned = subprocess.Popen(args, **kwargs)
return spawned
ns = dict(original.supervise.__globals__,
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
supervise = FunctionType(original.supervise.__code__, ns, "supervise")
return supervise(command, directory, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
namespace = dict(original.dispatch_guard.__globals__, contract=runtime_contract,
snapshot=snapshot, supervise=route_supervise)
code = FunctionType(original.dispatch_guard.__code__, namespace, "dispatch_guard")(root, path)
record = read(root / "complete.json")
new_charge = record["cumulative_gpu_wall_seconds"]
save(root / "recovery-settlement.json", dict(status=record["status"], trigger=record["trigger"],
previous_gpu_wall_seconds=old_charge, recovery_gpu_wall_seconds=new_charge,
cumulative_measurement_gpu_wall_seconds=old_charge + new_charge, total_cap_gpu_wall_seconds=total,
independent_attempts=2, no_retry=True, within_total_cap=old_charge + new_charge <= total))
return code
def authorized_child(root):
namespace = dict(accepted.authorized_child.__globals__, ROOT=ROOT, contract=contract)
return FunctionType(accepted.authorized_child.__code__, namespace, "authorized_child")(root)
def run(root):
namespace = dict(accepted.run.__globals__, ROOT=ROOT, contract=contract, dispatch_guard=dispatch_guard)
return FunctionType(accepted.run.__code__, namespace, "run")(root)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", type=Path)
raise SystemExit(run(parser.parse_args().root))