Freeze the one GINO ordinal recovery, physical witness and cumulative budget
This commit is contained in:
1 parent
17a40a6c1b
commit
057843ae2a
7 files changed
+1303
No files matched your search
@@ -0,0 +1,132 @@
|
||||
"""Bind the one approved recovery and replay ordinal dispatch and mismatch."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from tempfile import TemporaryDirectory
|
||||
import threading
|
||||
import time
|
||||
from types import FunctionType, SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import track_b_gino_import_checked_entry as accepted
|
||||
import track_b_gino_ordinal_recovery as recovery
|
||||
import track_b_gino_shapenet_entry as original
|
||||
|
||||
|
||||
halt = threading.Event()
|
||||
|
||||
|
||||
def fake_supervisor(command, root, mode, limits, *, poll, gpu_uuid):
|
||||
"""Executed in a fixture namespace with the real recovery spawn adapter."""
|
||||
process = subprocess.Popen(command, env={"CUDA_VISIBLE_DEVICES": gpu_uuid}, start_new_session=True)
|
||||
deadline = time.monotonic() + 4
|
||||
matched = Path(root) / "worker-device-verified.json"
|
||||
while not matched.exists() and not halt.is_set() and time.monotonic() < deadline:
|
||||
time.sleep(.01)
|
||||
if not (matched.exists() or halt.is_set()):
|
||||
raise AssertionError("Fixture monitor never verified or stopped the worker")
|
||||
return dict(status="STOPPED_OR_FAILED", returncode=-9, monitor_errors=0,
|
||||
stop_reason="MOCK_WORK_REPLACED", pid=process.pid)
|
||||
|
||||
|
||||
def replay(inventory, slot):
|
||||
halt = threading.Event()
|
||||
pid = 12345
|
||||
identity = {"pid": pid, "created": 123.}
|
||||
captured = {}
|
||||
|
||||
def query(option):
|
||||
if option == "--query-gpu=index,uuid,pci.bus_id,name":
|
||||
return inventory
|
||||
if option != "--query-compute-apps=gpu_uuid,pid":
|
||||
raise AssertionError("Unexpected inventory query")
|
||||
return [[slot, str(pid)]]
|
||||
|
||||
def spawn(command, *, env, start_new_session):
|
||||
captured.update(command=list(command), device_order=env["CUDA_DEVICE_ORDER"],
|
||||
visible_devices=env["CUDA_VISIBLE_DEVICES"], start_new_session=start_new_session)
|
||||
original.save(root / "worker-device-witness.json", dict(
|
||||
process=identity, ordinal="2", device_count=1, device_name="Tesla V100-SXM2-16GB",
|
||||
physical_uuid_expected=recovery.UUID, first_allocation_bytes=1))
|
||||
return SimpleNamespace(pid=pid)
|
||||
|
||||
def stop(pid_to_stop):
|
||||
if pid_to_stop != pid:
|
||||
raise AssertionError("Attempted to stop the wrong process")
|
||||
halt.set()
|
||||
|
||||
with TemporaryDirectory(prefix="track-b-cuda-identity-replay-") as temp:
|
||||
root = Path(temp)
|
||||
path = root / "contract.json"
|
||||
original.save(path, recovery.expected_contract())
|
||||
isolated = dict(fake_supervisor.__globals__, halt=halt)
|
||||
fake = FunctionType(fake_supervisor.__code__, isolated, fake_supervisor.__name__)
|
||||
with (patch.object(recovery, "query_nvidia", side_effect=query),
|
||||
patch.object(recovery, "process_identity", return_value=identity),
|
||||
patch.object(recovery, "subprocess", SimpleNamespace(Popen=spawn, STDOUT=-2)),
|
||||
patch.object(original, "supervise", fake),
|
||||
patch.object(original, "snapshot", return_value=dict(idle=True, memory_MiB=5, occupants=[])),
|
||||
patch.object(original, "wait_idle", return_value=dict(status="IDLE_CONFIRMED")),
|
||||
patch.object(original, "active_command_pids", return_value=[pid]),
|
||||
patch.object(original, "stop_group", side_effect=stop)):
|
||||
code = recovery.dispatch_guard(root, path)
|
||||
witness, mismatch = root / "worker-device-verified.json", root / "worker-device-mismatch.json"
|
||||
terminal = original.read(root / "complete.json")
|
||||
budget = original.read(root / "recovery-settlement.json")
|
||||
assert code == 1 and bool(captured)
|
||||
assert captured["device_order"] == "PCI_BUS_ID" and captured["visible_devices"] == "2"
|
||||
assert captured["command"][1] == str(recovery.WORKER)
|
||||
assert budget["previous_gpu_wall_seconds"] == 23.69431395700667
|
||||
assert budget["cumulative_measurement_gpu_wall_seconds"] == (
|
||||
budget["previous_gpu_wall_seconds"] + budget["recovery_gpu_wall_seconds"])
|
||||
assert terminal["gpu_wall_cap_seconds"] == 3600 - budget["previous_gpu_wall_seconds"]
|
||||
if slot == recovery.UUID:
|
||||
assert witness.exists() and not mismatch.exists() and not halt.is_set()
|
||||
else:
|
||||
assert mismatch.exists() and not witness.exists() and halt.is_set()
|
||||
assert "WORKER_GPU_UUID_MISMATCH" in terminal["trigger"]
|
||||
return dict(captured=captured, verified=witness.exists(), mismatch=mismatch.exists(),
|
||||
stop_called=halt.is_set(), terminal_trigger=terminal["trigger"], budget=budget)
|
||||
|
||||
|
||||
def prepare(inventory_path):
|
||||
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
|
||||
raise PermissionError("The replay requires an empty device mask")
|
||||
source = json.loads(inventory_path.read_text())
|
||||
rows = source["inventory"]
|
||||
assert recovery.device_ordinal(rows, recovery.UUID) == "2"
|
||||
assert not any(card == recovery.UUID for card, *_ in source["occupants"])
|
||||
mismatched = [row.copy() for row in rows]
|
||||
mismatched[0][0], mismatched[2][0] = mismatched[2][0], mismatched[0][0]
|
||||
try:
|
||||
recovery.device_ordinal(mismatched, recovery.UUID)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("A conflicting PCI order was accepted")
|
||||
c = recovery.expected_contract()
|
||||
old = original.read(recovery.previous.ROOT / "contract.json")
|
||||
assert c["gpu_uuid"] == old["gpu_uuid"] and c["limits"] == old["limits"]
|
||||
assert set(k for k in c if c[k] != old.get(k)) == {"ordinal_recovery", "code_sha256"}
|
||||
assert accepted.accepted_environment(c)["heldout_target_reads"] == 0
|
||||
checks = {"matching": replay(rows, recovery.UUID),
|
||||
"mismatch": replay(rows, rows[0][1])}
|
||||
assert not (recovery.ROOT / "started.json").exists()
|
||||
recovery.ROOT.mkdir(exist_ok=False)
|
||||
original.save(recovery.ROOT / "contract.json", c)
|
||||
original.save(recovery.ROOT / "ordinal-cpu-replay.json", dict(
|
||||
status="PASS", ruling="h-91e61871", old_terminal_sha256=recovery.OLD_TERMINAL,
|
||||
contract_sha256=original.sha(recovery.ROOT / "contract.json"),
|
||||
same_scientific_protocol_and_caps=True, hardware_inventory=source,
|
||||
inconsistent_pci_order_rejected=True, advance_checks=checks, real_spawns=0, real_cuda_calls=0,
|
||||
heldout_targets_read=0, independent_review="PENDING"))
|
||||
print("PASS", original.sha(recovery.ROOT / "contract.json"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("inventory", type=Path)
|
||||
prepare(parser.parse_args().inventory)
|
||||
@@ -0,0 +1,173 @@
|
||||
"""One labelled recovery for the historical Torch UUID-counting failure."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
from types import FunctionType, SimpleNamespace
|
||||
|
||||
import psutil
|
||||
|
||||
import track_b_gino_import_checked_entry as accepted
|
||||
import track_b_gino_measurement_gpu2 as previous
|
||||
import track_b_gino_shapenet_entry as original
|
||||
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
|
||||
|
||||
|
||||
ROOT = previous.ROOT.with_name("track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19")
|
||||
ENTRY = Path(__file__).resolve()
|
||||
WORKER = ENTRY.with_name("track_b_gino_ordinal_recovery_worker.py")
|
||||
UUID = previous.UUID
|
||||
OLD_CONTRACT = "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f"
|
||||
OLD_TERMINAL = "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e"
|
||||
|
||||
|
||||
def expected_contract():
|
||||
c = read(checked(previous.ROOT / "contract.json", OLD_CONTRACT))
|
||||
stopped = read(checked(previous.ROOT / "complete.json", OLD_TERMINAL))
|
||||
if stopped["status"] != "STOPPED" or stopped["trigger"] != "WORKER_FAILED":
|
||||
raise PermissionError("The original stopped GPU2 attempt is required")
|
||||
c["ordinal_recovery"] = dict(handoff="h-91e61871", previous_terminal_sha256=OLD_TERMINAL,
|
||||
previous_gpu_wall_seconds=stopped["cumulative_gpu_wall_seconds"])
|
||||
c["code_sha256"].update({str(p): sha(p) for p in (ENTRY, WORKER)})
|
||||
return c
|
||||
|
||||
|
||||
def contract(path):
|
||||
c = previous.checked_entry.contract(path)
|
||||
if c != expected_contract():
|
||||
raise PermissionError("Only the fixed ordinal recovery of this measurement is authorized")
|
||||
return c
|
||||
|
||||
|
||||
def query_nvidia(options):
|
||||
r = subprocess.run(["nvidia-smi", options, "--format=csv,noheader"],
|
||||
check=True, capture_output=True, text=True, timeout=3)
|
||||
return [[v.strip() for v in row] for row in csv.reader(r.stdout.splitlines()) if row]
|
||||
|
||||
|
||||
def device_ordinal(rows, uuid):
|
||||
ordered = sorted(rows, key=lambda row: row[2].lower())
|
||||
if len({row[2].lower() for row in ordered}) != len(ordered):
|
||||
raise ValueError("Nonunique GPU PCI identities")
|
||||
if [row[0] for row in ordered] != [str(i) for i in range(len(ordered))]:
|
||||
raise ValueError("NVIDIA indices do not match CUDA PCI bus order")
|
||||
match = [row for row in ordered if row[1] == uuid and "V100" in row[3]]
|
||||
if len(match) != 1:
|
||||
raise ValueError("Expected exactly one reviewed physical V100")
|
||||
return match[0][0]
|
||||
|
||||
|
||||
def dispatch_environment(uuid, env):
|
||||
if env["CUDA_VISIBLE_DEVICES"] != uuid:
|
||||
raise PermissionError("Supervisor did not select the exact physical UUID")
|
||||
rows = query_nvidia("--query-gpu=index,uuid,pci.bus_id,name")
|
||||
ordinal = device_ordinal(rows, uuid)
|
||||
return dict(env, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=ordinal), rows
|
||||
|
||||
|
||||
def monitor_identity(root, uuid):
|
||||
pending = root / "worker-device-witness.json"
|
||||
verified = root / "worker-device-verified.json"
|
||||
if not pending.is_file() or verified.exists():
|
||||
return
|
||||
witness = read(pending)
|
||||
spawn = read(root / "actual-worker-spawn.json")
|
||||
if (witness["process"] != spawn["process"]
|
||||
or witness["physical_uuid_expected"] != uuid
|
||||
or witness["ordinal"] != spawn["ordinal"]
|
||||
or witness["device_count"] != 1 or "V100" not in witness["device_name"]
|
||||
or witness["first_allocation_bytes"] != 1):
|
||||
raise RuntimeError("WORKER_DEVICE_WITNESS_IDENTITY_DIFFERS")
|
||||
pid = witness["process"]["pid"]
|
||||
observed = sorted({card for card, owner in query_nvidia("--query-compute-apps=gpu_uuid,pid")
|
||||
if int(owner) == pid})
|
||||
if not observed:
|
||||
return
|
||||
if observed != [uuid]:
|
||||
out = root / "worker-device-mismatch.json"
|
||||
if not out.exists():
|
||||
save(out, dict(process=witness["process"], expected_uuid=uuid, observed_uuids=observed,
|
||||
scientific_worker_released=False))
|
||||
raise RuntimeError("WORKER_GPU_UUID_MISMATCH")
|
||||
save(verified, dict(process=witness["process"], physical_uuid=uuid,
|
||||
ordinal=witness["ordinal"], observed_uuids=observed))
|
||||
|
||||
|
||||
def dispatch_guard(root, path):
|
||||
root = Path(root)
|
||||
c = contract(path)
|
||||
old_charge = c["ordinal_recovery"]["previous_gpu_wall_seconds"]
|
||||
total = c["limits"]["gpu_wall_seconds"]
|
||||
spawned = None
|
||||
started_dispatch = None
|
||||
|
||||
def runtime_contract(checked_path):
|
||||
runtime = deepcopy(contract(checked_path))
|
||||
runtime["limits"]["gpu_wall_seconds"] = total - old_charge
|
||||
return runtime
|
||||
|
||||
def snapshot(uuid):
|
||||
state = original.snapshot(uuid)
|
||||
if spawned is not None:
|
||||
receipt = root / "actual-worker-spawn.json"
|
||||
if not receipt.exists():
|
||||
try:
|
||||
identity = process_identity(spawned.pid)
|
||||
except psutil.NoSuchProcess:
|
||||
return state
|
||||
save(receipt, dict(process=identity, ordinal=started_dispatch["ordinal"], physical_uuid=uuid))
|
||||
monitor_identity(root, uuid)
|
||||
return state
|
||||
|
||||
def route_supervise(command, directory, mode, limits, *, poll, gpu_uuid):
|
||||
if gpu_uuid != UUID or Path(command[1]).name != "track_b_gino_shapenet_worker.py":
|
||||
raise PermissionError("Unexpected original guard worker route")
|
||||
command[1] = str(WORKER)
|
||||
|
||||
def spawn(args, **kwargs):
|
||||
nonlocal spawned, started_dispatch
|
||||
environment, rows = dispatch_environment(gpu_uuid, kwargs["env"])
|
||||
ordinal = environment["CUDA_VISIBLE_DEVICES"]
|
||||
started_dispatch = dict(ordinal=ordinal, order="PCI_BUS_ID",
|
||||
physical_uuid=gpu_uuid, gpu_inventory=rows)
|
||||
save(root / "ordinal-dispatch.json", started_dispatch)
|
||||
if time.monotonic() >= read(root / "dispatch.json")["start_by_monotonic"]:
|
||||
raise RuntimeError("ORDINAL_DISPATCH_DEADLINE")
|
||||
kwargs["env"] = environment
|
||||
spawned = subprocess.Popen(args, **kwargs)
|
||||
return spawned
|
||||
|
||||
ns = dict(original.supervise.__globals__,
|
||||
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
||||
supervise = FunctionType(original.supervise.__code__, ns, "supervise")
|
||||
return supervise(command, directory, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
|
||||
|
||||
namespace = dict(original.dispatch_guard.__globals__, contract=runtime_contract,
|
||||
snapshot=snapshot, supervise=route_supervise)
|
||||
code = FunctionType(original.dispatch_guard.__code__, namespace, "dispatch_guard")(root, path)
|
||||
record = read(root / "complete.json")
|
||||
new_charge = record["cumulative_gpu_wall_seconds"]
|
||||
save(root / "recovery-settlement.json", dict(status=record["status"], trigger=record["trigger"],
|
||||
previous_gpu_wall_seconds=old_charge, recovery_gpu_wall_seconds=new_charge,
|
||||
cumulative_measurement_gpu_wall_seconds=old_charge + new_charge, total_cap_gpu_wall_seconds=total,
|
||||
independent_attempts=2, no_retry=True, within_total_cap=old_charge + new_charge <= total))
|
||||
return code
|
||||
|
||||
|
||||
def authorized_child(root):
|
||||
namespace = dict(accepted.authorized_child.__globals__, ROOT=ROOT, contract=contract)
|
||||
return FunctionType(accepted.authorized_child.__code__, namespace, "authorized_child")(root)
|
||||
|
||||
|
||||
def run(root):
|
||||
namespace = dict(accepted.run.__globals__, ROOT=ROOT, contract=contract, dispatch_guard=dispatch_guard)
|
||||
return FunctionType(accepted.run.__code__, namespace, "run")(root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("root", type=Path)
|
||||
raise SystemExit(run(parser.parse_args().root))
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Prove the worker's first allocation is on the reviewed physical V100."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
from types import FunctionType
|
||||
|
||||
from track_b_gino_ordinal_recovery import UUID, authorized_child
|
||||
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
|
||||
from track_b_gino_shapenet_entry import process_identity, read, save
|
||||
from track_b_gino_shapenet_worker import work
|
||||
|
||||
|
||||
def wait_for_verified_device(root, c):
|
||||
import torch
|
||||
|
||||
dispatch = read(root / "ordinal-dispatch.json")
|
||||
ordinal = dispatch["ordinal"]
|
||||
if (not ordinal.isdecimal() or os.environ.get("CUDA_VISIBLE_DEVICES") != ordinal
|
||||
or os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID" or dispatch["physical_uuid"] != UUID
|
||||
or torch.cuda.device_count() != 1 or torch.version.cuda != c["environment"]["torch_cuda"]):
|
||||
raise PermissionError("Exactly one device must be visible through the reviewed PCI ordinal")
|
||||
torch.cuda.set_device(0)
|
||||
name = torch.cuda.get_device_name(0)
|
||||
if "V100" not in name:
|
||||
raise PermissionError("Expected one V100 before creating the witness allocation")
|
||||
probe = torch.empty(1, dtype=torch.uint8, device="cuda:0")
|
||||
current = process_identity(os.getpid())
|
||||
save(root / "worker-device-witness.json", dict(process=current, ordinal=ordinal, device_count=1,
|
||||
device_name=name, physical_uuid_expected=UUID, first_allocation_bytes=probe.numel()))
|
||||
deadline = read(root / "dispatch.json")["start_by_monotonic"]
|
||||
verified = root / "worker-device-verified.json"
|
||||
while time.monotonic() < deadline:
|
||||
if verified.exists():
|
||||
record = read(verified)
|
||||
if (record["process"] != current or record["ordinal"] != ordinal or record["physical_uuid"] != UUID
|
||||
or record["observed_uuids"] != [UUID]):
|
||||
raise PermissionError("Verified physical GPU witness differs")
|
||||
del probe
|
||||
return ordinal
|
||||
time.sleep(.1)
|
||||
raise RuntimeError("Physical GPU identity was not verified in the bounded admission window")
|
||||
|
||||
|
||||
def run(root):
|
||||
c = authorized_child(root)
|
||||
if str(Path(sys.executable).absolute()) != c["environment"]["python"]:
|
||||
raise PermissionError("The worker did not start with the accepted interpreter")
|
||||
ordinal = wait_for_verified_device(root, c)
|
||||
# In the unchanged worker this field is read only to check the visible-device
|
||||
# environment string. The coordinator and admission receipts keep the UUID.
|
||||
c = dict(c, gpu_uuid=ordinal)
|
||||
namespace = dict(work.__globals__, extract_paper_code=extract_patched_paper_code)
|
||||
FunctionType(work.__code__, namespace, work.__name__)(root, c)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("root", type=Path)
|
||||
run(parser.parse_args().root.resolve())
|
||||
Reference in new issue
Block a user