Freeze the one GINO ordinal recovery, physical witness and cumulative budget

This commit is contained in:
NewMeltAssist committed 2026-11-19 15:18:33 +08:00
1 parent 17a40a6c1b
commit 057843ae2a
7 files changed
+1303

No files matched your search

@@ -0,0 +1,132 @@
"""Bind the one approved recovery and replay ordinal dispatch and mismatch."""
import argparse
import json
import os
from pathlib import Path
import subprocess
from tempfile import TemporaryDirectory
import threading
import time
from types import FunctionType, SimpleNamespace
from unittest.mock import patch
import track_b_gino_import_checked_entry as accepted
import track_b_gino_ordinal_recovery as recovery
import track_b_gino_shapenet_entry as original
halt = threading.Event()
def fake_supervisor(command, root, mode, limits, *, poll, gpu_uuid):
"""Executed in a fixture namespace with the real recovery spawn adapter."""
process = subprocess.Popen(command, env={"CUDA_VISIBLE_DEVICES": gpu_uuid}, start_new_session=True)
deadline = time.monotonic() + 4
matched = Path(root) / "worker-device-verified.json"
while not matched.exists() and not halt.is_set() and time.monotonic() < deadline:
time.sleep(.01)
if not (matched.exists() or halt.is_set()):
raise AssertionError("Fixture monitor never verified or stopped the worker")
return dict(status="STOPPED_OR_FAILED", returncode=-9, monitor_errors=0,
stop_reason="MOCK_WORK_REPLACED", pid=process.pid)
def replay(inventory, slot):
halt = threading.Event()
pid = 12345
identity = {"pid": pid, "created": 123.}
captured = {}
def query(option):
if option == "--query-gpu=index,uuid,pci.bus_id,name":
return inventory
if option != "--query-compute-apps=gpu_uuid,pid":
raise AssertionError("Unexpected inventory query")
return [[slot, str(pid)]]
def spawn(command, *, env, start_new_session):
captured.update(command=list(command), device_order=env["CUDA_DEVICE_ORDER"],
visible_devices=env["CUDA_VISIBLE_DEVICES"], start_new_session=start_new_session)
original.save(root / "worker-device-witness.json", dict(
process=identity, ordinal="2", device_count=1, device_name="Tesla V100-SXM2-16GB",
physical_uuid_expected=recovery.UUID, first_allocation_bytes=1))
return SimpleNamespace(pid=pid)
def stop(pid_to_stop):
if pid_to_stop != pid:
raise AssertionError("Attempted to stop the wrong process")
halt.set()
with TemporaryDirectory(prefix="track-b-cuda-identity-replay-") as temp:
root = Path(temp)
path = root / "contract.json"
original.save(path, recovery.expected_contract())
isolated = dict(fake_supervisor.__globals__, halt=halt)
fake = FunctionType(fake_supervisor.__code__, isolated, fake_supervisor.__name__)
with (patch.object(recovery, "query_nvidia", side_effect=query),
patch.object(recovery, "process_identity", return_value=identity),
patch.object(recovery, "subprocess", SimpleNamespace(Popen=spawn, STDOUT=-2)),
patch.object(original, "supervise", fake),
patch.object(original, "snapshot", return_value=dict(idle=True, memory_MiB=5, occupants=[])),
patch.object(original, "wait_idle", return_value=dict(status="IDLE_CONFIRMED")),
patch.object(original, "active_command_pids", return_value=[pid]),
patch.object(original, "stop_group", side_effect=stop)):
code = recovery.dispatch_guard(root, path)
witness, mismatch = root / "worker-device-verified.json", root / "worker-device-mismatch.json"
terminal = original.read(root / "complete.json")
budget = original.read(root / "recovery-settlement.json")
assert code == 1 and bool(captured)
assert captured["device_order"] == "PCI_BUS_ID" and captured["visible_devices"] == "2"
assert captured["command"][1] == str(recovery.WORKER)
assert budget["previous_gpu_wall_seconds"] == 23.69431395700667
assert budget["cumulative_measurement_gpu_wall_seconds"] == (
budget["previous_gpu_wall_seconds"] + budget["recovery_gpu_wall_seconds"])
assert terminal["gpu_wall_cap_seconds"] == 3600 - budget["previous_gpu_wall_seconds"]
if slot == recovery.UUID:
assert witness.exists() and not mismatch.exists() and not halt.is_set()
else:
assert mismatch.exists() and not witness.exists() and halt.is_set()
assert "WORKER_GPU_UUID_MISMATCH" in terminal["trigger"]
return dict(captured=captured, verified=witness.exists(), mismatch=mismatch.exists(),
stop_called=halt.is_set(), terminal_trigger=terminal["trigger"], budget=budget)
def prepare(inventory_path):
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
raise PermissionError("The replay requires an empty device mask")
source = json.loads(inventory_path.read_text())
rows = source["inventory"]
assert recovery.device_ordinal(rows, recovery.UUID) == "2"
assert not any(card == recovery.UUID for card, *_ in source["occupants"])
mismatched = [row.copy() for row in rows]
mismatched[0][0], mismatched[2][0] = mismatched[2][0], mismatched[0][0]
try:
recovery.device_ordinal(mismatched, recovery.UUID)
except ValueError:
pass
else:
raise AssertionError("A conflicting PCI order was accepted")
c = recovery.expected_contract()
old = original.read(recovery.previous.ROOT / "contract.json")
assert c["gpu_uuid"] == old["gpu_uuid"] and c["limits"] == old["limits"]
assert set(k for k in c if c[k] != old.get(k)) == {"ordinal_recovery", "code_sha256"}
assert accepted.accepted_environment(c)["heldout_target_reads"] == 0
checks = {"matching": replay(rows, recovery.UUID),
"mismatch": replay(rows, rows[0][1])}
assert not (recovery.ROOT / "started.json").exists()
recovery.ROOT.mkdir(exist_ok=False)
original.save(recovery.ROOT / "contract.json", c)
original.save(recovery.ROOT / "ordinal-cpu-replay.json", dict(
status="PASS", ruling="h-91e61871", old_terminal_sha256=recovery.OLD_TERMINAL,
contract_sha256=original.sha(recovery.ROOT / "contract.json"),
same_scientific_protocol_and_caps=True, hardware_inventory=source,
inconsistent_pci_order_rejected=True, advance_checks=checks, real_spawns=0, real_cuda_calls=0,
heldout_targets_read=0, independent_review="PENDING"))
print("PASS", original.sha(recovery.ROOT / "contract.json"))
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("inventory", type=Path)
prepare(parser.parse_args().inventory)
+173
View File
@@ -0,0 +1,173 @@
"""One labelled recovery for the historical Torch UUID-counting failure."""
import argparse
import csv
from copy import deepcopy
from pathlib import Path
import subprocess
import time
from types import FunctionType, SimpleNamespace
import psutil
import track_b_gino_import_checked_entry as accepted
import track_b_gino_measurement_gpu2 as previous
import track_b_gino_shapenet_entry as original
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
ROOT = previous.ROOT.with_name("track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19")
ENTRY = Path(__file__).resolve()
WORKER = ENTRY.with_name("track_b_gino_ordinal_recovery_worker.py")
UUID = previous.UUID
OLD_CONTRACT = "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f"
OLD_TERMINAL = "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e"
def expected_contract():
c = read(checked(previous.ROOT / "contract.json", OLD_CONTRACT))
stopped = read(checked(previous.ROOT / "complete.json", OLD_TERMINAL))
if stopped["status"] != "STOPPED" or stopped["trigger"] != "WORKER_FAILED":
raise PermissionError("The original stopped GPU2 attempt is required")
c["ordinal_recovery"] = dict(handoff="h-91e61871", previous_terminal_sha256=OLD_TERMINAL,
previous_gpu_wall_seconds=stopped["cumulative_gpu_wall_seconds"])
c["code_sha256"].update({str(p): sha(p) for p in (ENTRY, WORKER)})
return c
def contract(path):
c = previous.checked_entry.contract(path)
if c != expected_contract():
raise PermissionError("Only the fixed ordinal recovery of this measurement is authorized")
return c
def query_nvidia(options):
r = subprocess.run(["nvidia-smi", options, "--format=csv,noheader"],
check=True, capture_output=True, text=True, timeout=3)
return [[v.strip() for v in row] for row in csv.reader(r.stdout.splitlines()) if row]
def device_ordinal(rows, uuid):
ordered = sorted(rows, key=lambda row: row[2].lower())
if len({row[2].lower() for row in ordered}) != len(ordered):
raise ValueError("Nonunique GPU PCI identities")
if [row[0] for row in ordered] != [str(i) for i in range(len(ordered))]:
raise ValueError("NVIDIA indices do not match CUDA PCI bus order")
match = [row for row in ordered if row[1] == uuid and "V100" in row[3]]
if len(match) != 1:
raise ValueError("Expected exactly one reviewed physical V100")
return match[0][0]
def dispatch_environment(uuid, env):
if env["CUDA_VISIBLE_DEVICES"] != uuid:
raise PermissionError("Supervisor did not select the exact physical UUID")
rows = query_nvidia("--query-gpu=index,uuid,pci.bus_id,name")
ordinal = device_ordinal(rows, uuid)
return dict(env, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=ordinal), rows
def monitor_identity(root, uuid):
pending = root / "worker-device-witness.json"
verified = root / "worker-device-verified.json"
if not pending.is_file() or verified.exists():
return
witness = read(pending)
spawn = read(root / "actual-worker-spawn.json")
if (witness["process"] != spawn["process"]
or witness["physical_uuid_expected"] != uuid
or witness["ordinal"] != spawn["ordinal"]
or witness["device_count"] != 1 or "V100" not in witness["device_name"]
or witness["first_allocation_bytes"] != 1):
raise RuntimeError("WORKER_DEVICE_WITNESS_IDENTITY_DIFFERS")
pid = witness["process"]["pid"]
observed = sorted({card for card, owner in query_nvidia("--query-compute-apps=gpu_uuid,pid")
if int(owner) == pid})
if not observed:
return
if observed != [uuid]:
out = root / "worker-device-mismatch.json"
if not out.exists():
save(out, dict(process=witness["process"], expected_uuid=uuid, observed_uuids=observed,
scientific_worker_released=False))
raise RuntimeError("WORKER_GPU_UUID_MISMATCH")
save(verified, dict(process=witness["process"], physical_uuid=uuid,
ordinal=witness["ordinal"], observed_uuids=observed))
def dispatch_guard(root, path):
root = Path(root)
c = contract(path)
old_charge = c["ordinal_recovery"]["previous_gpu_wall_seconds"]
total = c["limits"]["gpu_wall_seconds"]
spawned = None
started_dispatch = None
def runtime_contract(checked_path):
runtime = deepcopy(contract(checked_path))
runtime["limits"]["gpu_wall_seconds"] = total - old_charge
return runtime
def snapshot(uuid):
state = original.snapshot(uuid)
if spawned is not None:
receipt = root / "actual-worker-spawn.json"
if not receipt.exists():
try:
identity = process_identity(spawned.pid)
except psutil.NoSuchProcess:
return state
save(receipt, dict(process=identity, ordinal=started_dispatch["ordinal"], physical_uuid=uuid))
monitor_identity(root, uuid)
return state
def route_supervise(command, directory, mode, limits, *, poll, gpu_uuid):
if gpu_uuid != UUID or Path(command[1]).name != "track_b_gino_shapenet_worker.py":
raise PermissionError("Unexpected original guard worker route")
command[1] = str(WORKER)
def spawn(args, **kwargs):
nonlocal spawned, started_dispatch
environment, rows = dispatch_environment(gpu_uuid, kwargs["env"])
ordinal = environment["CUDA_VISIBLE_DEVICES"]
started_dispatch = dict(ordinal=ordinal, order="PCI_BUS_ID",
physical_uuid=gpu_uuid, gpu_inventory=rows)
save(root / "ordinal-dispatch.json", started_dispatch)
if time.monotonic() >= read(root / "dispatch.json")["start_by_monotonic"]:
raise RuntimeError("ORDINAL_DISPATCH_DEADLINE")
kwargs["env"] = environment
spawned = subprocess.Popen(args, **kwargs)
return spawned
ns = dict(original.supervise.__globals__,
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
supervise = FunctionType(original.supervise.__code__, ns, "supervise")
return supervise(command, directory, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
namespace = dict(original.dispatch_guard.__globals__, contract=runtime_contract,
snapshot=snapshot, supervise=route_supervise)
code = FunctionType(original.dispatch_guard.__code__, namespace, "dispatch_guard")(root, path)
record = read(root / "complete.json")
new_charge = record["cumulative_gpu_wall_seconds"]
save(root / "recovery-settlement.json", dict(status=record["status"], trigger=record["trigger"],
previous_gpu_wall_seconds=old_charge, recovery_gpu_wall_seconds=new_charge,
cumulative_measurement_gpu_wall_seconds=old_charge + new_charge, total_cap_gpu_wall_seconds=total,
independent_attempts=2, no_retry=True, within_total_cap=old_charge + new_charge <= total))
return code
def authorized_child(root):
namespace = dict(accepted.authorized_child.__globals__, ROOT=ROOT, contract=contract)
return FunctionType(accepted.authorized_child.__code__, namespace, "authorized_child")(root)
def run(root):
namespace = dict(accepted.run.__globals__, ROOT=ROOT, contract=contract, dispatch_guard=dispatch_guard)
return FunctionType(accepted.run.__code__, namespace, "run")(root)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", type=Path)
raise SystemExit(run(parser.parse_args().root))
@@ -0,0 +1,62 @@
"""Prove the worker's first allocation is on the reviewed physical V100."""
import argparse
import os
from pathlib import Path
import sys
import time
from types import FunctionType
from track_b_gino_ordinal_recovery import UUID, authorized_child
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
from track_b_gino_shapenet_entry import process_identity, read, save
from track_b_gino_shapenet_worker import work
def wait_for_verified_device(root, c):
import torch
dispatch = read(root / "ordinal-dispatch.json")
ordinal = dispatch["ordinal"]
if (not ordinal.isdecimal() or os.environ.get("CUDA_VISIBLE_DEVICES") != ordinal
or os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID" or dispatch["physical_uuid"] != UUID
or torch.cuda.device_count() != 1 or torch.version.cuda != c["environment"]["torch_cuda"]):
raise PermissionError("Exactly one device must be visible through the reviewed PCI ordinal")
torch.cuda.set_device(0)
name = torch.cuda.get_device_name(0)
if "V100" not in name:
raise PermissionError("Expected one V100 before creating the witness allocation")
probe = torch.empty(1, dtype=torch.uint8, device="cuda:0")
current = process_identity(os.getpid())
save(root / "worker-device-witness.json", dict(process=current, ordinal=ordinal, device_count=1,
device_name=name, physical_uuid_expected=UUID, first_allocation_bytes=probe.numel()))
deadline = read(root / "dispatch.json")["start_by_monotonic"]
verified = root / "worker-device-verified.json"
while time.monotonic() < deadline:
if verified.exists():
record = read(verified)
if (record["process"] != current or record["ordinal"] != ordinal or record["physical_uuid"] != UUID
or record["observed_uuids"] != [UUID]):
raise PermissionError("Verified physical GPU witness differs")
del probe
return ordinal
time.sleep(.1)
raise RuntimeError("Physical GPU identity was not verified in the bounded admission window")
def run(root):
c = authorized_child(root)
if str(Path(sys.executable).absolute()) != c["environment"]["python"]:
raise PermissionError("The worker did not start with the accepted interpreter")
ordinal = wait_for_verified_device(root, c)
# In the unchanged worker this field is read only to check the visible-device
# environment string. The coordinator and admission receipts keep the UUID.
c = dict(c, gpu_uuid=ordinal)
namespace = dict(work.__globals__, extract_paper_code=extract_patched_paper_code)
FunctionType(work.__code__, namespace, work.__name__)(root, c)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", type=Path)
run(parser.parse_args().root.resolve())