"""One reviewed fold-0 attempt and its validation-only screen.""" import argparse from collections import defaultdict import ctypes import fcntl import json import math import os from pathlib import Path import shutil import signal import sys import threading import time from types import FunctionType from phase2_gate_outcome_review import validate_review from phase2_h1_worker_v2 import checked, save, sha from phase3_item1b_common import fold_view, read from phase3_shared_capacity import process_identity from s2_gpu_idle_gate import snapshot, wait_idle from s2_main_memory_guard import PeakRecords, active_command_pids from s2_recovery_gpu_supervisor import stop_group, supervise ROOT = Path(__file__).resolve().parents[1] / "artifacts/phase3-item1b-fold0-screen-2026-11-19" RULING = "3a48b123da54b0bcff0e7fa672fc74e8920235a0" ALLOCATOR = "expandable_segments:True" GIB = 2**30 ARMS = ("short_seed0", "long_seed0", "long_seed1") def contract(path): c = read(path) if (c["ruling"] != RULING or c["fold"] != 0 or c["screen_fold"] != 0 or c["epochs"] != 100 or c["seed"] != 0 or c["max_fits"] != 1 or c["retry"] is not False or c["test_targets_allowed"] is not False): raise PermissionError("Only the new fold-0 validation scope is authorized") for p, h in (c["code_sha256"] | c["source_sha256"]).items(): checked(p, h) comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"])) if comparison["validation_cases"] != c["job"]["validation"]: raise ValueError("Fold-0 validation identity differs from comparison") budget = c["budget"] reserves = sum(budget[k] for k in ("train_gpu_seconds", "screen_gpu_seconds", "idle_wait_seconds", "finalization_gpu_seconds")) if (budget["projected_new_gpu_seconds"] != reserves or budget["prior_item_gpu_seconds"] + reserves > budget["item_gpu_limit_seconds"] or budget["prior_phase3_gpu_seconds"] + reserves > budget["phase3_gpu_limit_seconds"]): raise PermissionError("Phase 3 or 1b remaining budget exceeded") return c def folded_inputs(c): pilot = read(c["pilot_contract"]) view = fold_view(pilot, 0) original = view["folds"][0] job = c["job"] for k in ("learning", "validation", "test", "c", "clock_sha256", "learning_nodes"): if job[k] != original[k]: raise ValueError(f"Original fold-0 identity changed: {k}") if job["id"] != 0 or job["target_steps"] != 100 * len(original["learning"]): raise ValueError("Full fold-0 learning schedule required") if set(original["validation"]) & set(original["test"]): raise ValueError("Validation and test overlap") return pilot, view def train(root, c): import numpy as np import torch from phase3_item1b_worker import run as frozen_worker pilot, _ = folded_inputs(c) if torch.__version__ != pilot["versions"]["torch"] or np.__version__ != pilot["versions"]["numpy"]: raise ValueError("Original array and Torch versions required") job = dict(c["job"], gpu_uuid=c["gpu_uuid"], worker_cpus=c["worker_cpus"]) digest = sha(root / "contract.json") def first_epoch_release(queue, fold): if queue != root or fold != 0: raise PermissionError("Wrong worker route") row = read(root / "jobs/0/worker/history/001.json") budget = c["budget"] epoch_seconds = max(row["train_epoch_seconds"], budget["train_epoch_seconds_basis"]) projection = (row["wall_seconds"] + 99 * epoch_seconds + 10 * budget["validation_seconds_basis"] + budget["training_setup_tail_seconds"]) passed = math.isfinite(projection) and projection <= budget["train_gpu_seconds"] save(root / "first-epoch-projection.json", dict( status="PASS" if passed else "STOP_BUDGET", contract_sha256=digest, measured_epoch_seconds=row["train_epoch_seconds"], projected_train_gpu_seconds=projection)) if not passed: raise RuntimeError("First-epoch projection exceeds fold-0 training cap") return True # Reuse the immutable training body. The private namespace changes only # its root/assignment adapters and the one-fold budget gate. namespace = dict(frozen_worker.__globals__, config=lambda _: (c, pilot), assigned_job=lambda *_: job, first_epoch_release=first_epoch_release) runner = FunctionType(frozen_worker.__code__, namespace, frozen_worker.__name__) runner(root, 0, digest) def screen(root, c): import numpy as np import torch from phase2_h2_data import arguments, cached, to_device from phase2_h2_model import H2Geo from phase2_h2_pilot_worker import peaks from phase2_residual_math import decode_seconds from phase3_item1_postfit_data import load_evaluation_part from s2_custody_common import prediction_inputs from s2_models.objectives import relative_l2 pilot, view = folded_inputs(c) comparison = read(c["comparison_contract"]) fold = view["folds"][0] cases = comparison["validation_cases"] if (cases != fold["validation"] or torch.__version__ != pilot["versions"]["torch"] or np.__version__ != pilot["versions"]["numpy"] or os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"] or os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR): raise PermissionError("Screen scope or environment differs") train_path = root / "jobs/0/worker/result.json" adopted = read(root / "training-adopted.json") short = read(checked(train_path, adopted["result_sha256"])) models = dict(c["references"]) models["short_seed0"] = dict( checkpoint=short["checkpoint"], checkpoint_sha256=short["checkpoint_sha256"], standardization=str(train_path.with_name("standardization.json")), standardization_sha256=short["standardization_sha256"], seed=0, epochs=100, training_contract_sha256=sha(root / "contract.json")) roster = {r["case"]: r for r in read(c["query_roster_contract"])["rows"] if r["case"] in cases} if set(roster) != set(cases) or any( roster[case]["component_id"] != pilot["parts"][case]["clock_group"] or roster[case]["v5_cache"]["output_sha256"] != pilot["parts"][case]["output_sha256"] for case in cases ): raise ValueError("Frozen validation query-to-cache identity differs") os.sched_setaffinity(0, c["worker_cpus"]) torch.set_num_threads(1) torch.set_num_interop_threads(1) torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False torch.cuda.set_device(0) torch.cuda.reset_peak_memory_stats() out = root / "screen" out.mkdir(exist_ok=False) predictions = {} with torch.no_grad(): for arm in ARMS: ref = models[arm] stats = read(checked(ref["standardization"], ref["standardization_sha256"])) blob = torch.load(checked(ref["checkpoint"], ref["checkpoint_sha256"]), map_location="cpu", weights_only=True) required = dict(completed_epochs=ref["epochs"], fold=0, seed=ref["seed"], role="last_epoch_only", contract_sha256=ref["training_contract_sha256"], optimizer_steps=ref["epochs"] * len(fold["learning"])) if any(blob.get(k) != v for k, v in required.items()): raise ValueError(f"Final checkpoint identity: {arm}") if stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]: raise ValueError("Screen learning-only normalization mismatch") model = H2Geo().cuda() model.load_state_dict(blob["model"], strict=True) del blob model.eval() folder = out / arm folder.mkdir() for case in cases: data = to_device(load_evaluation_part(view, case, stats, "validation"), "cuda") values = model(*arguments(data), checkpoint_blocks=False).cpu().numpy() seconds = decode_seconds(values, data["F"], fold["c"], data["t_set"]) if seconds.shape != (data["nodes"],) or not np.isfinite(seconds).all(): raise ValueError("Full validation prediction is missing or nonfinite") path = folder / (case + ".npy") np.save(path, seconds, allow_pickle=False) predictions.setdefault(case, {})[arm] = {"file": str(path), "sha256": sha(path)} save(out / "progress.json", dict(stage="VALIDATION_PREDICT", arm=arm, case=case, **peaks())) del data del model manifest = out / "predictions-frozen.json" save(manifest, predictions) manifest_hash = sha(manifest) predictions = read(checked(manifest, manifest_hash)) rows = [] grouped = defaultdict(list) for case in cases: q = prediction_inputs(roster[case]) ids = cached(view, case, "raw_ids") ix = np.searchsorted(ids, q["raw_ids"]) if np.any(ix >= len(ids)) or not np.array_equal(ids[ix], q["raw_ids"]): raise ValueError("Validation query does not map to original raw IDs") with (out / "validation-target-access.jsonl").open("a") as log: log.write(json.dumps(dict(case=case, role="validation", predictions_sha256=manifest_hash, target_sha256=pilot["parts"][case]["output_sha256"]["seconds.npy"])) + "\n") truth = cached(view, case, "seconds")[ix] scores = {} for arm in ARMS: row = predictions[case][arm] pred = np.load(checked(row["file"], row["sha256"]), allow_pickle=False)[ix] value = relative_l2(pred, truth, q["counts"], np.ones(len(ix), dtype=bool))["value"] if value is None or not math.isfinite(value): raise ValueError("Undefined validation part metric") scores[arm] = value component = roster[case]["component_id"] grouped[component].append([scores[arm] for arm in ARMS]) rows.append(dict(case=case, component_id=component, relative_l2=scores)) means = dict(zip(ARMS, np.mean([np.mean(a, axis=0) for a in grouped.values()], axis=0).tolist())) delta = means["short_seed0"] - means["long_seed0"] seed_delta = means["long_seed1"] - means["long_seed0"] passed = delta <= comparison["delta_short_at_most"] and abs(delta) > abs(seed_delta) save(out / "result.json", dict( status="COMPLETE", decision="SCREEN_PASS" if passed else "SCREEN_FAIL", parts=len(rows), components=len(grouped), component_equal_relative_l2=means, delta_short=delta, delta_seed=seed_delta, rows=rows, predictions_sha256=manifest_hash, test_targets_read=False, automatic_continuation=False, **peaks())) def dispatch_guard(root, contract_path): """The same entry consumes the generated contract in production and preflight.""" began_entry = time.monotonic() c = contract(contract_path) folded_inputs(c) budget = c["budget"] gpu = c["gpu_uuid"] charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0., screen_gpu_seconds=0., finalization_gpu_seconds=0.) receipts = [] status = "STOPPED" trigger = "STOP_BEFORE_DISPATCH" try: if shutil.disk_usage(root).free < c["disk_bytes"]: raise RuntimeError("Insufficient free disk for the fixed attempt") began_wait = time.monotonic() try: idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot) except BaseException: charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait raise charges["queue_wait_gpu_seconds"] = idle["wall_seconds"] save(root / "idle-gate.json", idle) trigger = idle["status"] if idle["status"] != "IDLE_CONFIRMED" or not snapshot(gpu)["idle"]: if idle["status"] == "IDLE_CONFIRMED": trigger = "SELECTED_GPU_BUSY_AFTER_IDLE_GATE" return 1 (root / "jobs/0").mkdir(parents=True, exist_ok=False) for mode, cap_key, charge_key, output in ( ("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"), ("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"), ): began = time.monotonic() if not snapshot(gpu)["idle"]: trigger = "SELECTED_GPU_BUSY_BEFORE_" + mode.upper() charges[charge_key] = time.monotonic() - began return 1 command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)] tracker = PeakRecords() done = threading.Event() reasons = [] def watch(): while not done.is_set(): try: tracker.scan(output) state = snapshot(gpu) pids = set(active_command_pids(command)) if max(tracker.maxima.values()) > c["recorded_vram_bytes"]: reasons.append("RECORDED_VRAM_LIMIT") if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]: reasons.append("DEVICE_VRAM_LIMIT") if any(p["pid"] not in pids for p in state["occupants"]): reasons.append("FOREIGN_GPU_OCCUPANT") except Exception as exc: reasons.append(f"MONITOR_FAILURE: {exc}") if reasons: for pid in active_command_pids(command): stop_group(pid) # Also catch a child spawned just after the failing # sample while the supervisor was taking its lock. done.wait(.1) continue done.wait(.5) monitor = threading.Thread(target=watch, daemon=True) monitor.start() try: if reasons: raise RuntimeError(reasons[0]) cap = budget[cap_key] - (time.monotonic() - began) - 10 receipt = supervise(command, root, mode, dict( total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key], job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"], disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]), poll=.2, gpu_uuid=gpu) finally: done.set() monitor.join(timeout=8) charges[charge_key] = time.monotonic() - began if monitor.is_alive(): reasons.append("MONITOR_DID_NOT_EXIT") else: tracker.scan(output) if max(tracker.maxima.values()) > c["recorded_vram_bytes"]: reasons.append("RECORDED_VRAM_LIMIT") if set(tracker.witnesses) != set(tracker.maxima): reasons.append("MISSING_RECORDED_VRAM_PEAKS") receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima)) if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0 or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]): worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT" trigger = reasons[0] if reasons else worker_reason return 1 result_path = output / "result.json" result = read(result_path) if result["status"] != "COMPLETE" or result["test_targets_read"] is not False: raise RuntimeError("Unexpected worker result") if mode == "train": if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"] or result["optimizer_steps"] != c["job"]["target_steps"]): raise RuntimeError("Fold-0 final checkpoint identity differs") checked(result["checkpoint"], result["checkpoint_sha256"]) save(root / "training-adopted.json", dict(result_sha256=sha(result_path))) elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]: raise RuntimeError("Validation screen scope differs") overhead = time.monotonic() - began_entry - sum(charges.values()) if overhead > budget["finalization_gpu_seconds"]: raise RuntimeError("Setup and finalization cost exceeded the fixed allowance") status, trigger = "COMPLETE", result["decision"] return 0 except Exception as exc: trigger = f"{type(exc).__name__}: {exc}" return 1 except KeyboardInterrupt: trigger = "COORDINATOR_INTERRUPTED" return 1 finally: charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values())) new_gpu = sum(charges.values()) save(root / "complete.json", dict( status=status, trigger=trigger, receipts=receipts, **charges, cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu, cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu, no_retry=True, test_targets_read=False, automatic_continuation=False)) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("mode", choices=("run", "train", "screen")) parser.add_argument("root", type=Path) args = parser.parse_args() root = args.root.resolve() if root != ROOT: raise PermissionError("Only the new fixed fold-0 root is accepted") expected_python = str(ROOT.parents[1] / "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python") if sys.executable != expected_python: raise PermissionError("Use the fixed original H2 environment") validate_review(root) contract_path = root / "contract.json" if args.mode != "run": coordinator = read(root / "started.json") if process_identity(os.getppid()) != coordinator["coordinator"]: raise PermissionError("Worker requires the live guard as its parent") if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0: raise OSError(ctypes.get_errno(), "Cannot bind worker lifetime to guard") if os.getppid() != coordinator["coordinator"]["pid"]: raise RuntimeError("Guard disappeared before worker setup") return globals()[args.mode](root, contract(contract_path)) if os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR: raise PermissionError("Explicit original CUDA allocator setup required") with (root / "entry.lock").open("a") as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if (root / "started.json").exists(): raise FileExistsError("Fold-0 attempt already started") save(root / "started.json", dict(coordinator=process_identity(os.getpid()), contract_sha256=sha(contract_path), unix=time.time())) def request_stop(signum, _frame): raise KeyboardInterrupt(signal.Signals(signum).name) old_handler = signal.signal(signal.SIGTERM, request_stop) try: return dispatch_guard(root, contract_path) finally: signal.signal(signal.SIGTERM, old_handler) if __name__ == "__main__": raise SystemExit(main())