263 lines
13 KiB
Python
263 lines
13 KiB
Python
"""Review-gated dispatch for the original ShapeNet GINO reproduction."""
|
|
|
|
import argparse
|
|
import ctypes
|
|
import fcntl
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import signal
|
|
import threading
|
|
import time
|
|
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h1_worker_v2 import checked, save, sha
|
|
from phase3_item1b_common import read
|
|
from phase3_shared_capacity import process_identity
|
|
from s2_gpu_idle_gate import snapshot, wait_idle
|
|
from s2_main_memory_guard import PeakRecords, active_command_pids
|
|
from s2_recovery_gpu_supervisor import stop_group, supervise
|
|
|
|
|
|
BASE = Path(__file__).resolve().parents[1]
|
|
MEASUREMENT_ROOT = BASE / "artifacts/track-b-gino-shapenet611-measurement-2026-11-19"
|
|
BENCHMARK_ROOT = BASE / "artifacts/track-b-gino-shapenet611-benchmark-2026-11-19"
|
|
RULING = "h-f84547d7"
|
|
|
|
|
|
def projected_budget(measurement, limits):
|
|
prep = measurement["preparation_seconds"]
|
|
train = measurement["epoch_seconds"]
|
|
test_prep = measurement["proxy_preparation_seconds"]
|
|
inference = measurement["proxy_inference_seconds"]
|
|
if any(not math.isfinite(x) or x <= 0 for x in (prep, train, test_prep, inference)):
|
|
raise ValueError("Incomplete empirical timing components")
|
|
if (measurement["epochs"], measurement["training_examples"], measurement["proxy_examples"],
|
|
measurement["heldout_labels_read"], measurement["reuse_measurement_weights"]) != (1, 500, 111, False, False):
|
|
raise ValueError("Timing does not represent the reviewed learning-only measurement")
|
|
# One serial worker, one preprocessing pass per domain, and three fresh
|
|
# serial 100-epoch fits. The fixed wait and cleanup are charged separately.
|
|
return math.ceil(1.25 * (prep + 3 * 100 * train + test_prep + 3 * inference)) + (
|
|
limits["idle_wait_seconds"] + limits["cleanup_seconds"])
|
|
|
|
|
|
def contract(path):
|
|
c = read(path)
|
|
if c["ruling"] != RULING or c["mode"] not in ("measurement", "benchmark"):
|
|
raise PermissionError("Only the paper GINO execution scope is supported")
|
|
if c["retry"] is not False or c["evaluate_other_models"] is not False:
|
|
raise PermissionError("Single reviewed GINO attempt required")
|
|
for path, digest in c["code_sha256"].items():
|
|
checked(path, digest)
|
|
sources = ("paper_data", "paper_supplement", "split_receipt", "mapping_audit")
|
|
for name in sources:
|
|
pin = c[name]
|
|
checked(pin["path"], pin["sha256"])
|
|
comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"]))
|
|
if c["protocol"] != comparison["protocol"]:
|
|
raise ValueError("Model/score protocol differs from the comparison contract")
|
|
split = c["split"]
|
|
original = read(c["split_receipt"]["path"])
|
|
if any(split[key] != original[key] for key in ("train", "test", "raw_global_bounds")):
|
|
raise ValueError("Original ShapeNet split or bounds changed")
|
|
if len(set(split["train"])) != 500 or len(set(split["test"])) != 111 or set(split["train"]) & set(split["test"]):
|
|
raise ValueError("Expected disjoint original 500/111 split")
|
|
p = c["protocol"]
|
|
if p["grid"] != [64, 64, 64] or p["epochs_per_seed"] != 100 or p["seeds"] != [0, 1, 2]:
|
|
raise ValueError("Fixed decoder resolution, epochs or seeds changed")
|
|
if p["acceptance_percent"] != [6.62, 7.62]:
|
|
raise ValueError("Original pre-registered team interval changed")
|
|
limit = c["limits"]
|
|
caps = ("gpu_wall_seconds", "cleanup_seconds", "idle_wait_seconds", "cpu_seconds", "rss_bytes",
|
|
"recorded_vram_bytes", "device_vram_bytes", "disk_bytes", "torch_threads", "max_processes")
|
|
if any(type(limit[k]) is not int or limit[k] <= 0 for k in caps):
|
|
raise ValueError("All runtime limits must be positive integers")
|
|
if limit["idle_wait_seconds"] + limit["cleanup_seconds"] >= limit["gpu_wall_seconds"]:
|
|
raise ValueError("No runtime after booking admission and cleanup")
|
|
if c["mode"] == "measurement":
|
|
if c["measurement_allowed_heldout_labels"] is not False:
|
|
raise PermissionError("Measurement may only use the learning split")
|
|
else:
|
|
timed = c["measured_timing"]
|
|
expected_paths = {
|
|
"path": MEASUREMENT_ROOT / "worker-records/result.json",
|
|
"guard_path": MEASUREMENT_ROOT / "complete.json",
|
|
"contract_path": MEASUREMENT_ROOT / "contract.json",
|
|
}
|
|
if any(Path(timed[key]).resolve() != expected for key, expected in expected_paths.items()):
|
|
raise ValueError("Timing must come from the sole reviewed measurement root")
|
|
timing = read(checked(timed["path"], timed["sha256"]))
|
|
witness = read(checked(timed["guard_path"], timed["guard_sha256"]))
|
|
if (witness["status"] != "COMPLETE" or witness["contract_sha256"] != timed["contract_sha256"]
|
|
or witness["result_sha256"] != timed["sha256"] or timing["mode"] != "measurement"
|
|
or timing["status"] != "COMPLETE"):
|
|
raise ValueError("Benchmark requires the authentic measured timing and guard result")
|
|
measured = contract(Path(checked(timed["contract_path"], timed["contract_sha256"])))
|
|
for key in ("gpu_uuid", "paper_data", "paper_code_zip_sha256", "comparison_contract_sha256", "environment"):
|
|
if measured[key] != c[key]:
|
|
raise ValueError(f"Timing and benchmark execution regime differs: {key}")
|
|
if measured["mode"] != "measurement" or measured["limits"]["torch_threads"] != limit["torch_threads"]:
|
|
raise ValueError("Timing is not from the same worker/CPU configuration")
|
|
validate_review(MEASUREMENT_ROOT)
|
|
if limit["gpu_wall_seconds"] != projected_budget(timing, limit):
|
|
raise ValueError("Benchmark cap must come from the measured +25% projection")
|
|
return c
|
|
|
|
|
|
def dispatch_guard(root, contract_path):
|
|
started = time.monotonic()
|
|
c = contract(contract_path)
|
|
cap = c["limits"]
|
|
deadline = started + cap["gpu_wall_seconds"]
|
|
wait_deadline = started + cap["idle_wait_seconds"]
|
|
gpu = c["gpu_uuid"]
|
|
status, reason = "STOPPED", None
|
|
receipt = None
|
|
waiting = 0.
|
|
output = None
|
|
tracker = PeakRecords()
|
|
command = [c["environment"]["python"], str(Path(__file__).with_name("track_b_gino_shapenet_worker.py")), str(root)]
|
|
try:
|
|
if shutil.disk_usage(root).free < cap["disk_bytes"]:
|
|
raise RuntimeError("DISK_HEADROOM")
|
|
# Use the shared poller, then look at wall time again after the last
|
|
# independent check and the dispatch receipt write.
|
|
begin_wait = time.monotonic()
|
|
remaining_window = max(0., wait_deadline - begin_wait)
|
|
idle = wait_idle(gpu, remaining_window, lambda _: None, sample=snapshot)
|
|
waiting = time.monotonic() - begin_wait
|
|
save(root / "idle-gate.json", idle)
|
|
if idle["status"] != "IDLE_CONFIRMED" or time.monotonic() >= wait_deadline:
|
|
raise RuntimeError("IDLE_DEADLINE")
|
|
if not snapshot(gpu)["idle"]:
|
|
raise RuntimeError("SELECTED_GPU_BECAME_BUSY")
|
|
save(root / "dispatch.json", dict(contract_sha256=sha(contract_path),
|
|
coordinator=process_identity(os.getpid()),
|
|
start_by_monotonic=wait_deadline))
|
|
now = time.monotonic()
|
|
waiting = now - begin_wait
|
|
if now >= wait_deadline:
|
|
raise RuntimeError("IDLE_DEADLINE_AFTER_FINAL_CHECK")
|
|
remaining = deadline - now - cap["cleanup_seconds"]
|
|
if remaining <= 0:
|
|
raise RuntimeError("INSUFFICIENT_REMAINING_BUDGET")
|
|
reasons = []
|
|
done = threading.Event()
|
|
|
|
def monitor():
|
|
while not done.is_set():
|
|
try:
|
|
tracker.scan(root / "worker-records")
|
|
state = snapshot(gpu)
|
|
allowed = set(active_command_pids(command))
|
|
if max(tracker.maxima.values()) > cap["recorded_vram_bytes"]:
|
|
reasons.append("RECORDED_VRAM_LIMIT")
|
|
if state["memory_MiB"] * 2**20 > cap["device_vram_bytes"]:
|
|
reasons.append("DEVICE_VRAM_LIMIT")
|
|
if any(p["pid"] not in allowed for p in state["occupants"]):
|
|
reasons.append("FOREIGN_GPU_OCCUPANT")
|
|
except Exception as exc:
|
|
reasons.append("MONITOR_FAILURE: " + str(exc))
|
|
if reasons:
|
|
for pid in active_command_pids(command):
|
|
stop_group(pid)
|
|
done.wait(.5)
|
|
|
|
watcher = threading.Thread(target=monitor, daemon=True)
|
|
watcher.start()
|
|
try:
|
|
receipt = supervise(command, root, "gino", dict(
|
|
total_cpu_seconds=cap["cpu_seconds"], job_cpu_seconds=cap["cpu_seconds"],
|
|
job_wall_seconds=remaining, tree_rss_bytes=cap["rss_bytes"],
|
|
disk_bytes=cap["disk_bytes"], max_processes=cap["max_processes"]), poll=.2, gpu_uuid=gpu)
|
|
finally:
|
|
done.set()
|
|
watcher.join(timeout=8)
|
|
if watcher.is_alive():
|
|
raise RuntimeError("MONITOR_DID_NOT_EXIT")
|
|
tracker.scan(root / "worker-records")
|
|
if max(tracker.maxima.values()) > cap["recorded_vram_bytes"]:
|
|
reasons.append("RECORDED_VRAM_LIMIT")
|
|
if reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0 or receipt["monitor_errors"] != 0:
|
|
raise RuntimeError(reasons[0] if reasons else receipt.get("stop_reason") or "WORKER_FAILED")
|
|
output = root / "worker-records/result.json"
|
|
result = read(output)
|
|
if (result["status"] != "COMPLETE" or result["mode"] != c["mode"]
|
|
or set(tracker.witnesses) != set(tracker.maxima)):
|
|
raise RuntimeError("INCOMPLETE_WORKER_RESULT")
|
|
if c["mode"] == "measurement" and (result["heldout_labels_read"] or result["epochs"] != 1):
|
|
raise RuntimeError("MEASUREMENT_SCOPE_MISMATCH")
|
|
if c["mode"] == "benchmark" and (not result["heldout_labels_read"]
|
|
or set(result["seeds"]) != {str(x) for x in c["protocol"]["seeds"]}
|
|
or any([item["case"] for item in seed["cases"]] != c["split"]["test"] for seed in result["seeds"].values())):
|
|
raise RuntimeError("BENCHMARK_SUPPORT_MISMATCH")
|
|
status = "COMPLETE"
|
|
except (Exception, KeyboardInterrupt) as exc:
|
|
reason = str(exc) or type(exc).__name__
|
|
result_hash = sha(output) if output is not None and output.is_file() else None
|
|
contract_hash = sha(contract_path)
|
|
measured = time.monotonic() - started
|
|
if measured > cap["gpu_wall_seconds"]:
|
|
status, reason = "STOPPED", "TOTAL_GPU_WALL_CAP"
|
|
save(root / "complete.json", dict(
|
|
status=status, trigger=reason, contract_sha256=contract_hash, result_sha256=result_hash,
|
|
worker=receipt, booked_idle_seconds=waiting, cumulative_gpu_wall_seconds=measured,
|
|
gpu_wall_cap_seconds=cap["gpu_wall_seconds"], recorded_peaks=tracker.maxima, no_retry=True))
|
|
return 0 if status == "COMPLETE" else 1
|
|
|
|
|
|
def authorized_child(root):
|
|
root = Path(root).resolve()
|
|
if root not in (MEASUREMENT_ROOT, BENCHMARK_ROOT):
|
|
raise PermissionError("Unrecognized Track B run root")
|
|
ticket = read(root / "dispatch.json")
|
|
parent = os.getppid()
|
|
if (ticket["coordinator"] != process_identity(parent)
|
|
or ticket["contract_sha256"] != sha(root / "contract.json")
|
|
or time.monotonic() >= ticket["start_by_monotonic"]):
|
|
raise PermissionError("Child is not attached to this reviewed guard")
|
|
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0 or os.getppid() != parent:
|
|
raise RuntimeError("Could not attach child lifetime to its coordinator")
|
|
validate_review(root)
|
|
return contract(root / "contract.json")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("root", type=Path)
|
|
args = parser.parse_args()
|
|
root = args.root.resolve()
|
|
if root not in (MEASUREMENT_ROOT, BENCHMARK_ROOT):
|
|
raise PermissionError("Unrecognized Track B run root")
|
|
validate_review(root)
|
|
path = root / "contract.json"
|
|
c = contract(path)
|
|
if (root == MEASUREMENT_ROOT) != (c["mode"] == "measurement"):
|
|
raise PermissionError("Execution root is tied to its reviewed stage")
|
|
# Separate authority for the costed measurement or benchmark.
|
|
authorization = read(root / "stage-authorization.json")
|
|
if (authorization["contract_sha256"] != sha(path)
|
|
or authorization["decision"] != "APPROVED"
|
|
or authorization["mode"] != c["mode"]):
|
|
raise PermissionError("Exact stage authorization is required")
|
|
if not os.access(c["environment"]["python"], os.X_OK):
|
|
raise FileNotFoundError("The isolated pinned GINO runtime is not available")
|
|
with (root / "entry.lock").open("a") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (root / "started.json").exists():
|
|
raise FileExistsError("One attempt already recorded")
|
|
save(root / "started.json", dict(coordinator=process_identity(os.getpid()), contract_sha256=sha(path)))
|
|
def request_stop(signum, _frame):
|
|
raise KeyboardInterrupt(signal.Signals(signum).name)
|
|
|
|
handler = signal.signal(signal.SIGTERM, request_stop)
|
|
try:
|
|
return dispatch_guard(root, path)
|
|
finally:
|
|
signal.signal(signal.SIGTERM, handler)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|