"""Review-gated dispatch for the original ShapeNet GINO reproduction.""" import argparse import ctypes import fcntl import math import os from pathlib import Path import shutil import signal import threading import time from phase2_gate_outcome_review import validate_review from phase2_h1_worker_v2 import checked, save, sha from phase3_item1b_common import read from phase3_shared_capacity import process_identity from s2_gpu_idle_gate import snapshot, wait_idle from s2_main_memory_guard import PeakRecords, active_command_pids from s2_recovery_gpu_supervisor import stop_group, supervise BASE = Path(__file__).resolve().parents[1] MEASUREMENT_ROOT = BASE / "artifacts/track-b-gino-shapenet611-measurement-2026-11-19" BENCHMARK_ROOT = BASE / "artifacts/track-b-gino-shapenet611-benchmark-2026-11-19" RULING = "h-f84547d7" def projected_budget(measurement, limits): prep = measurement["preparation_seconds"] train = measurement["epoch_seconds"] test_prep = measurement["proxy_preparation_seconds"] inference = measurement["proxy_inference_seconds"] if any(not math.isfinite(x) or x <= 0 for x in (prep, train, test_prep, inference)): raise ValueError("Incomplete empirical timing components") if (measurement["epochs"], measurement["training_examples"], measurement["proxy_examples"], measurement["heldout_labels_read"], measurement["reuse_measurement_weights"]) != (1, 500, 111, False, False): raise ValueError("Timing does not represent the reviewed learning-only measurement") # One serial worker, one preprocessing pass per domain, and three fresh # serial 100-epoch fits. The fixed wait and cleanup are charged separately. return math.ceil(1.25 * (prep + 3 * 100 * train + test_prep + 3 * inference)) + ( limits["idle_wait_seconds"] + limits["cleanup_seconds"]) def contract(path): c = read(path) if c["ruling"] != RULING or c["mode"] not in ("measurement", "benchmark"): raise PermissionError("Only the paper GINO execution scope is supported") if c["retry"] is not False or c["evaluate_other_models"] is not False: raise PermissionError("Single reviewed GINO attempt required") for path, digest in c["code_sha256"].items(): checked(path, digest) sources = ("paper_data", "paper_supplement", "split_receipt", "mapping_audit") for name in sources: pin = c[name] checked(pin["path"], pin["sha256"]) comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"])) if c["protocol"] != comparison["protocol"]: raise ValueError("Model/score protocol differs from the comparison contract") split = c["split"] original = read(c["split_receipt"]["path"]) if any(split[key] != original[key] for key in ("train", "test", "raw_global_bounds")): raise ValueError("Original ShapeNet split or bounds changed") if len(set(split["train"])) != 500 or len(set(split["test"])) != 111 or set(split["train"]) & set(split["test"]): raise ValueError("Expected disjoint original 500/111 split") p = c["protocol"] if p["grid"] != [64, 64, 64] or p["epochs_per_seed"] != 100 or p["seeds"] != [0, 1, 2]: raise ValueError("Fixed decoder resolution, epochs or seeds changed") if p["acceptance_percent"] != [6.62, 7.62]: raise ValueError("Original pre-registered team interval changed") limit = c["limits"] caps = ("gpu_wall_seconds", "cleanup_seconds", "idle_wait_seconds", "cpu_seconds", "rss_bytes", "recorded_vram_bytes", "device_vram_bytes", "disk_bytes", "torch_threads", "max_processes") if any(type(limit[k]) is not int or limit[k] <= 0 for k in caps): raise ValueError("All runtime limits must be positive integers") if limit["idle_wait_seconds"] + limit["cleanup_seconds"] >= limit["gpu_wall_seconds"]: raise ValueError("No runtime after booking admission and cleanup") if c["mode"] == "measurement": if c["measurement_allowed_heldout_labels"] is not False: raise PermissionError("Measurement may only use the learning split") else: timed = c["measured_timing"] expected_paths = { "path": MEASUREMENT_ROOT / "worker-records/result.json", "guard_path": MEASUREMENT_ROOT / "complete.json", "contract_path": MEASUREMENT_ROOT / "contract.json", } if any(Path(timed[key]).resolve() != expected for key, expected in expected_paths.items()): raise ValueError("Timing must come from the sole reviewed measurement root") timing = read(checked(timed["path"], timed["sha256"])) witness = read(checked(timed["guard_path"], timed["guard_sha256"])) if (witness["status"] != "COMPLETE" or witness["contract_sha256"] != timed["contract_sha256"] or witness["result_sha256"] != timed["sha256"] or timing["mode"] != "measurement" or timing["status"] != "COMPLETE"): raise ValueError("Benchmark requires the authentic measured timing and guard result") measured = contract(Path(checked(timed["contract_path"], timed["contract_sha256"]))) for key in ("gpu_uuid", "paper_data", "paper_code_zip_sha256", "comparison_contract_sha256", "environment"): if measured[key] != c[key]: raise ValueError(f"Timing and benchmark execution regime differs: {key}") if measured["mode"] != "measurement" or measured["limits"]["torch_threads"] != limit["torch_threads"]: raise ValueError("Timing is not from the same worker/CPU configuration") validate_review(MEASUREMENT_ROOT) if limit["gpu_wall_seconds"] != projected_budget(timing, limit): raise ValueError("Benchmark cap must come from the measured +25% projection") return c def dispatch_guard(root, contract_path): started = time.monotonic() c = contract(contract_path) cap = c["limits"] deadline = started + cap["gpu_wall_seconds"] wait_deadline = started + cap["idle_wait_seconds"] gpu = c["gpu_uuid"] status, reason = "STOPPED", None receipt = None waiting = 0. output = None tracker = PeakRecords() command = [c["environment"]["python"], str(Path(__file__).with_name("track_b_gino_shapenet_worker.py")), str(root)] try: if shutil.disk_usage(root).free < cap["disk_bytes"]: raise RuntimeError("DISK_HEADROOM") # Use the shared poller, then look at wall time again after the last # independent check and the dispatch receipt write. begin_wait = time.monotonic() remaining_window = max(0., wait_deadline - begin_wait) idle = wait_idle(gpu, remaining_window, lambda _: None, sample=snapshot) waiting = time.monotonic() - begin_wait save(root / "idle-gate.json", idle) if idle["status"] != "IDLE_CONFIRMED" or time.monotonic() >= wait_deadline: raise RuntimeError("IDLE_DEADLINE") if not snapshot(gpu)["idle"]: raise RuntimeError("SELECTED_GPU_BECAME_BUSY") save(root / "dispatch.json", dict(contract_sha256=sha(contract_path), coordinator=process_identity(os.getpid()), start_by_monotonic=wait_deadline)) now = time.monotonic() waiting = now - begin_wait if now >= wait_deadline: raise RuntimeError("IDLE_DEADLINE_AFTER_FINAL_CHECK") remaining = deadline - now - cap["cleanup_seconds"] if remaining <= 0: raise RuntimeError("INSUFFICIENT_REMAINING_BUDGET") reasons = [] done = threading.Event() def monitor(): while not done.is_set(): try: tracker.scan(root / "worker-records") state = snapshot(gpu) allowed = set(active_command_pids(command)) if max(tracker.maxima.values()) > cap["recorded_vram_bytes"]: reasons.append("RECORDED_VRAM_LIMIT") if state["memory_MiB"] * 2**20 > cap["device_vram_bytes"]: reasons.append("DEVICE_VRAM_LIMIT") if any(p["pid"] not in allowed for p in state["occupants"]): reasons.append("FOREIGN_GPU_OCCUPANT") except Exception as exc: reasons.append("MONITOR_FAILURE: " + str(exc)) if reasons: for pid in active_command_pids(command): stop_group(pid) done.wait(.5) watcher = threading.Thread(target=monitor, daemon=True) watcher.start() try: receipt = supervise(command, root, "gino", dict( total_cpu_seconds=cap["cpu_seconds"], job_cpu_seconds=cap["cpu_seconds"], job_wall_seconds=remaining, tree_rss_bytes=cap["rss_bytes"], disk_bytes=cap["disk_bytes"], max_processes=cap["max_processes"]), poll=.2, gpu_uuid=gpu) finally: done.set() watcher.join(timeout=8) if watcher.is_alive(): raise RuntimeError("MONITOR_DID_NOT_EXIT") tracker.scan(root / "worker-records") if max(tracker.maxima.values()) > cap["recorded_vram_bytes"]: reasons.append("RECORDED_VRAM_LIMIT") if reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0 or receipt["monitor_errors"] != 0: raise RuntimeError(reasons[0] if reasons else receipt.get("stop_reason") or "WORKER_FAILED") output = root / "worker-records/result.json" result = read(output) if (result["status"] != "COMPLETE" or result["mode"] != c["mode"] or set(tracker.witnesses) != set(tracker.maxima)): raise RuntimeError("INCOMPLETE_WORKER_RESULT") if c["mode"] == "measurement" and (result["heldout_labels_read"] or result["epochs"] != 1): raise RuntimeError("MEASUREMENT_SCOPE_MISMATCH") if c["mode"] == "benchmark" and (not result["heldout_labels_read"] or set(result["seeds"]) != {str(x) for x in c["protocol"]["seeds"]} or any([item["case"] for item in seed["cases"]] != c["split"]["test"] for seed in result["seeds"].values())): raise RuntimeError("BENCHMARK_SUPPORT_MISMATCH") status = "COMPLETE" except (Exception, KeyboardInterrupt) as exc: reason = str(exc) or type(exc).__name__ result_hash = sha(output) if output is not None and output.is_file() else None contract_hash = sha(contract_path) measured = time.monotonic() - started if measured > cap["gpu_wall_seconds"]: status, reason = "STOPPED", "TOTAL_GPU_WALL_CAP" save(root / "complete.json", dict( status=status, trigger=reason, contract_sha256=contract_hash, result_sha256=result_hash, worker=receipt, booked_idle_seconds=waiting, cumulative_gpu_wall_seconds=measured, gpu_wall_cap_seconds=cap["gpu_wall_seconds"], recorded_peaks=tracker.maxima, no_retry=True)) return 0 if status == "COMPLETE" else 1 def authorized_child(root): root = Path(root).resolve() if root not in (MEASUREMENT_ROOT, BENCHMARK_ROOT): raise PermissionError("Unrecognized Track B run root") ticket = read(root / "dispatch.json") parent = os.getppid() if (ticket["coordinator"] != process_identity(parent) or ticket["contract_sha256"] != sha(root / "contract.json") or time.monotonic() >= ticket["start_by_monotonic"]): raise PermissionError("Child is not attached to this reviewed guard") if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0 or os.getppid() != parent: raise RuntimeError("Could not attach child lifetime to its coordinator") validate_review(root) return contract(root / "contract.json") def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("root", type=Path) args = parser.parse_args() root = args.root.resolve() if root not in (MEASUREMENT_ROOT, BENCHMARK_ROOT): raise PermissionError("Unrecognized Track B run root") validate_review(root) path = root / "contract.json" c = contract(path) if (root == MEASUREMENT_ROOT) != (c["mode"] == "measurement"): raise PermissionError("Execution root is tied to its reviewed stage") # Separate authority for the costed measurement or benchmark. authorization = read(root / "stage-authorization.json") if (authorization["contract_sha256"] != sha(path) or authorization["decision"] != "APPROVED" or authorization["mode"] != c["mode"]): raise PermissionError("Exact stage authorization is required") if not os.access(c["environment"]["python"], os.X_OK): raise FileNotFoundError("The isolated pinned GINO runtime is not available") with (root / "entry.lock").open("a") as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if (root / "started.json").exists(): raise FileExistsError("One attempt already recorded") save(root / "started.json", dict(coordinator=process_identity(os.getpid()), contract_sha256=sha(path))) def request_stop(signum, _frame): raise KeyboardInterrupt(signal.Signals(signum).name) handler = signal.signal(signal.SIGTERM, request_stop) try: return dispatch_guard(root, path) finally: signal.signal(signal.SIGTERM, handler) if __name__ == "__main__": raise SystemExit(main())