391 lines
19 KiB
Python
391 lines
19 KiB
Python
"""One reviewed fold-0 attempt and its validation-only screen."""
|
|
|
|
import argparse
|
|
from collections import defaultdict
|
|
import ctypes
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import signal
|
|
import sys
|
|
import threading
|
|
import time
|
|
from types import FunctionType
|
|
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h1_worker_v2 import checked, save, sha
|
|
from phase3_item1b_common import fold_view, read
|
|
from phase3_shared_capacity import process_identity
|
|
from s2_gpu_idle_gate import snapshot, wait_idle
|
|
from s2_main_memory_guard import PeakRecords, active_command_pids
|
|
from s2_recovery_gpu_supervisor import stop_group, supervise
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1] / "artifacts/phase3-item1b-fold0-screen-2026-11-19"
|
|
RULING = "3a48b123da54b0bcff0e7fa672fc74e8920235a0"
|
|
ALLOCATOR = "expandable_segments:True"
|
|
GIB = 2**30
|
|
ARMS = ("short_seed0", "long_seed0", "long_seed1")
|
|
|
|
|
|
def contract(path):
|
|
c = read(path)
|
|
if (c["ruling"] != RULING or c["fold"] != 0 or c["screen_fold"] != 0
|
|
or c["epochs"] != 100 or c["seed"] != 0 or c["max_fits"] != 1
|
|
or c["retry"] is not False or c["test_targets_allowed"] is not False):
|
|
raise PermissionError("Only the new fold-0 validation scope is authorized")
|
|
for p, h in (c["code_sha256"] | c["source_sha256"]).items():
|
|
checked(p, h)
|
|
comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"]))
|
|
if comparison["validation_cases"] != c["job"]["validation"]:
|
|
raise ValueError("Fold-0 validation identity differs from comparison")
|
|
budget = c["budget"]
|
|
reserves = sum(budget[k] for k in ("train_gpu_seconds", "screen_gpu_seconds",
|
|
"idle_wait_seconds", "finalization_gpu_seconds"))
|
|
if (budget["projected_new_gpu_seconds"] != reserves
|
|
or budget["prior_item_gpu_seconds"] + reserves > budget["item_gpu_limit_seconds"]
|
|
or budget["prior_phase3_gpu_seconds"] + reserves > budget["phase3_gpu_limit_seconds"]):
|
|
raise PermissionError("Phase 3 or 1b remaining budget exceeded")
|
|
return c
|
|
|
|
|
|
def folded_inputs(c):
|
|
pilot = read(c["pilot_contract"])
|
|
view = fold_view(pilot, 0)
|
|
original = view["folds"][0]
|
|
job = c["job"]
|
|
for k in ("learning", "validation", "test", "c", "clock_sha256", "learning_nodes"):
|
|
if job[k] != original[k]:
|
|
raise ValueError(f"Original fold-0 identity changed: {k}")
|
|
if job["id"] != 0 or job["target_steps"] != 100 * len(original["learning"]):
|
|
raise ValueError("Full fold-0 learning schedule required")
|
|
if set(original["validation"]) & set(original["test"]):
|
|
raise ValueError("Validation and test overlap")
|
|
return pilot, view
|
|
|
|
|
|
def train(root, c):
|
|
import numpy as np
|
|
import torch
|
|
from phase3_item1b_worker import run as frozen_worker
|
|
|
|
pilot, _ = folded_inputs(c)
|
|
if torch.__version__ != pilot["versions"]["torch"] or np.__version__ != pilot["versions"]["numpy"]:
|
|
raise ValueError("Original array and Torch versions required")
|
|
job = dict(c["job"], gpu_uuid=c["gpu_uuid"], worker_cpus=c["worker_cpus"])
|
|
digest = sha(root / "contract.json")
|
|
|
|
def first_epoch_release(queue, fold):
|
|
if queue != root or fold != 0:
|
|
raise PermissionError("Wrong worker route")
|
|
row = read(root / "jobs/0/worker/history/001.json")
|
|
budget = c["budget"]
|
|
epoch_seconds = max(row["train_epoch_seconds"], budget["train_epoch_seconds_basis"])
|
|
projection = (row["wall_seconds"] + 99 * epoch_seconds
|
|
+ 10 * budget["validation_seconds_basis"] + budget["training_setup_tail_seconds"])
|
|
passed = math.isfinite(projection) and projection <= budget["train_gpu_seconds"]
|
|
save(root / "first-epoch-projection.json", dict(
|
|
status="PASS" if passed else "STOP_BUDGET", contract_sha256=digest,
|
|
measured_epoch_seconds=row["train_epoch_seconds"], projected_train_gpu_seconds=projection))
|
|
if not passed:
|
|
raise RuntimeError("First-epoch projection exceeds fold-0 training cap")
|
|
return True
|
|
|
|
# Reuse the immutable training body. The private namespace changes only
|
|
# its root/assignment adapters and the one-fold budget gate.
|
|
namespace = dict(frozen_worker.__globals__, config=lambda _: (c, pilot),
|
|
assigned_job=lambda *_: job, first_epoch_release=first_epoch_release)
|
|
runner = FunctionType(frozen_worker.__code__, namespace, frozen_worker.__name__)
|
|
runner(root, 0, digest)
|
|
|
|
|
|
def screen(root, c):
|
|
import numpy as np
|
|
import torch
|
|
from phase2_h2_data import arguments, cached, to_device
|
|
from phase2_h2_model import H2Geo
|
|
from phase2_h2_pilot_worker import peaks
|
|
from phase2_residual_math import decode_seconds
|
|
from phase3_item1_postfit_data import load_evaluation_part
|
|
from s2_custody_common import prediction_inputs
|
|
from s2_models.objectives import relative_l2
|
|
|
|
pilot, view = folded_inputs(c)
|
|
comparison = read(c["comparison_contract"])
|
|
fold = view["folds"][0]
|
|
cases = comparison["validation_cases"]
|
|
if (cases != fold["validation"] or torch.__version__ != pilot["versions"]["torch"]
|
|
or np.__version__ != pilot["versions"]["numpy"]
|
|
or os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"]
|
|
or os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR):
|
|
raise PermissionError("Screen scope or environment differs")
|
|
train_path = root / "jobs/0/worker/result.json"
|
|
adopted = read(root / "training-adopted.json")
|
|
short = read(checked(train_path, adopted["result_sha256"]))
|
|
models = dict(c["references"])
|
|
models["short_seed0"] = dict(
|
|
checkpoint=short["checkpoint"], checkpoint_sha256=short["checkpoint_sha256"],
|
|
standardization=str(train_path.with_name("standardization.json")),
|
|
standardization_sha256=short["standardization_sha256"], seed=0, epochs=100,
|
|
training_contract_sha256=sha(root / "contract.json"))
|
|
roster = {r["case"]: r for r in read(c["query_roster_contract"])["rows"] if r["case"] in cases}
|
|
if set(roster) != set(cases) or any(
|
|
roster[case]["component_id"] != pilot["parts"][case]["clock_group"]
|
|
or roster[case]["v5_cache"]["output_sha256"] != pilot["parts"][case]["output_sha256"]
|
|
for case in cases
|
|
):
|
|
raise ValueError("Frozen validation query-to-cache identity differs")
|
|
|
|
os.sched_setaffinity(0, c["worker_cpus"])
|
|
torch.set_num_threads(1)
|
|
torch.set_num_interop_threads(1)
|
|
torch.backends.cuda.matmul.allow_tf32 = False
|
|
torch.backends.cudnn.allow_tf32 = False
|
|
torch.cuda.set_device(0)
|
|
torch.cuda.reset_peak_memory_stats()
|
|
out = root / "screen"
|
|
out.mkdir(exist_ok=False)
|
|
predictions = {}
|
|
with torch.no_grad():
|
|
for arm in ARMS:
|
|
ref = models[arm]
|
|
stats = read(checked(ref["standardization"], ref["standardization_sha256"]))
|
|
blob = torch.load(checked(ref["checkpoint"], ref["checkpoint_sha256"]),
|
|
map_location="cpu", weights_only=True)
|
|
required = dict(completed_epochs=ref["epochs"], fold=0, seed=ref["seed"],
|
|
role="last_epoch_only", contract_sha256=ref["training_contract_sha256"],
|
|
optimizer_steps=ref["epochs"] * len(fold["learning"]))
|
|
if any(blob.get(k) != v for k, v in required.items()):
|
|
raise ValueError(f"Final checkpoint identity: {arm}")
|
|
if stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]:
|
|
raise ValueError("Screen learning-only normalization mismatch")
|
|
model = H2Geo().cuda()
|
|
model.load_state_dict(blob["model"], strict=True)
|
|
del blob
|
|
model.eval()
|
|
folder = out / arm
|
|
folder.mkdir()
|
|
for case in cases:
|
|
data = to_device(load_evaluation_part(view, case, stats, "validation"), "cuda")
|
|
values = model(*arguments(data), checkpoint_blocks=False).cpu().numpy()
|
|
seconds = decode_seconds(values, data["F"], fold["c"], data["t_set"])
|
|
if seconds.shape != (data["nodes"],) or not np.isfinite(seconds).all():
|
|
raise ValueError("Full validation prediction is missing or nonfinite")
|
|
path = folder / (case + ".npy")
|
|
np.save(path, seconds, allow_pickle=False)
|
|
predictions.setdefault(case, {})[arm] = {"file": str(path), "sha256": sha(path)}
|
|
save(out / "progress.json", dict(stage="VALIDATION_PREDICT", arm=arm, case=case, **peaks()))
|
|
del data
|
|
del model
|
|
manifest = out / "predictions-frozen.json"
|
|
save(manifest, predictions)
|
|
manifest_hash = sha(manifest)
|
|
predictions = read(checked(manifest, manifest_hash))
|
|
rows = []
|
|
grouped = defaultdict(list)
|
|
for case in cases:
|
|
q = prediction_inputs(roster[case])
|
|
ids = cached(view, case, "raw_ids")
|
|
ix = np.searchsorted(ids, q["raw_ids"])
|
|
if np.any(ix >= len(ids)) or not np.array_equal(ids[ix], q["raw_ids"]):
|
|
raise ValueError("Validation query does not map to original raw IDs")
|
|
with (out / "validation-target-access.jsonl").open("a") as log:
|
|
log.write(json.dumps(dict(case=case, role="validation", predictions_sha256=manifest_hash,
|
|
target_sha256=pilot["parts"][case]["output_sha256"]["seconds.npy"])) + "\n")
|
|
truth = cached(view, case, "seconds")[ix]
|
|
scores = {}
|
|
for arm in ARMS:
|
|
row = predictions[case][arm]
|
|
pred = np.load(checked(row["file"], row["sha256"]), allow_pickle=False)[ix]
|
|
value = relative_l2(pred, truth, q["counts"], np.ones(len(ix), dtype=bool))["value"]
|
|
if value is None or not math.isfinite(value):
|
|
raise ValueError("Undefined validation part metric")
|
|
scores[arm] = value
|
|
component = roster[case]["component_id"]
|
|
grouped[component].append([scores[arm] for arm in ARMS])
|
|
rows.append(dict(case=case, component_id=component, relative_l2=scores))
|
|
means = dict(zip(ARMS, np.mean([np.mean(a, axis=0) for a in grouped.values()], axis=0).tolist()))
|
|
delta = means["short_seed0"] - means["long_seed0"]
|
|
seed_delta = means["long_seed1"] - means["long_seed0"]
|
|
passed = delta <= comparison["delta_short_at_most"] and abs(delta) > abs(seed_delta)
|
|
save(out / "result.json", dict(
|
|
status="COMPLETE", decision="SCREEN_PASS" if passed else "SCREEN_FAIL",
|
|
parts=len(rows), components=len(grouped), component_equal_relative_l2=means,
|
|
delta_short=delta, delta_seed=seed_delta, rows=rows, predictions_sha256=manifest_hash,
|
|
test_targets_read=False, automatic_continuation=False, **peaks()))
|
|
|
|
|
|
def dispatch_guard(root, contract_path):
|
|
"""The same entry consumes the generated contract in production and preflight."""
|
|
began_entry = time.monotonic()
|
|
c = contract(contract_path)
|
|
folded_inputs(c)
|
|
budget = c["budget"]
|
|
gpu = c["gpu_uuid"]
|
|
charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0.,
|
|
screen_gpu_seconds=0., finalization_gpu_seconds=0.)
|
|
receipts = []
|
|
status = "STOPPED"
|
|
trigger = "STOP_BEFORE_DISPATCH"
|
|
try:
|
|
if shutil.disk_usage(root).free < c["disk_bytes"]:
|
|
raise RuntimeError("Insufficient free disk for the fixed attempt")
|
|
began_wait = time.monotonic()
|
|
try:
|
|
idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot)
|
|
except BaseException:
|
|
charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait
|
|
raise
|
|
charges["queue_wait_gpu_seconds"] = idle["wall_seconds"]
|
|
save(root / "idle-gate.json", idle)
|
|
trigger = idle["status"]
|
|
if idle["status"] != "IDLE_CONFIRMED" or not snapshot(gpu)["idle"]:
|
|
if idle["status"] == "IDLE_CONFIRMED":
|
|
trigger = "SELECTED_GPU_BUSY_AFTER_IDLE_GATE"
|
|
return 1
|
|
(root / "jobs/0").mkdir(parents=True, exist_ok=False)
|
|
for mode, cap_key, charge_key, output in (
|
|
("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"),
|
|
("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"),
|
|
):
|
|
began = time.monotonic()
|
|
if not snapshot(gpu)["idle"]:
|
|
trigger = "SELECTED_GPU_BUSY_BEFORE_" + mode.upper()
|
|
charges[charge_key] = time.monotonic() - began
|
|
return 1
|
|
command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)]
|
|
tracker = PeakRecords()
|
|
done = threading.Event()
|
|
reasons = []
|
|
|
|
def watch():
|
|
while not done.is_set():
|
|
try:
|
|
tracker.scan(output)
|
|
state = snapshot(gpu)
|
|
pids = set(active_command_pids(command))
|
|
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
|
|
reasons.append("RECORDED_VRAM_LIMIT")
|
|
if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]:
|
|
reasons.append("DEVICE_VRAM_LIMIT")
|
|
if any(p["pid"] not in pids for p in state["occupants"]):
|
|
reasons.append("FOREIGN_GPU_OCCUPANT")
|
|
except Exception as exc:
|
|
reasons.append(f"MONITOR_FAILURE: {exc}")
|
|
if reasons:
|
|
for pid in active_command_pids(command):
|
|
stop_group(pid)
|
|
# Also catch a child spawned just after the failing
|
|
# sample while the supervisor was taking its lock.
|
|
done.wait(.1)
|
|
continue
|
|
done.wait(.5)
|
|
|
|
monitor = threading.Thread(target=watch, daemon=True)
|
|
monitor.start()
|
|
try:
|
|
if reasons:
|
|
raise RuntimeError(reasons[0])
|
|
cap = budget[cap_key] - (time.monotonic() - began) - 10
|
|
receipt = supervise(command, root, mode, dict(
|
|
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key],
|
|
job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"],
|
|
disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]),
|
|
poll=.2, gpu_uuid=gpu)
|
|
finally:
|
|
done.set()
|
|
monitor.join(timeout=8)
|
|
charges[charge_key] = time.monotonic() - began
|
|
if monitor.is_alive():
|
|
reasons.append("MONITOR_DID_NOT_EXIT")
|
|
else:
|
|
tracker.scan(output)
|
|
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
|
|
reasons.append("RECORDED_VRAM_LIMIT")
|
|
if set(tracker.witnesses) != set(tracker.maxima):
|
|
reasons.append("MISSING_RECORDED_VRAM_PEAKS")
|
|
receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima))
|
|
if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0
|
|
or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]):
|
|
worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT"
|
|
trigger = reasons[0] if reasons else worker_reason
|
|
return 1
|
|
result_path = output / "result.json"
|
|
result = read(result_path)
|
|
if result["status"] != "COMPLETE" or result["test_targets_read"] is not False:
|
|
raise RuntimeError("Unexpected worker result")
|
|
if mode == "train":
|
|
if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"]
|
|
or result["optimizer_steps"] != c["job"]["target_steps"]):
|
|
raise RuntimeError("Fold-0 final checkpoint identity differs")
|
|
checked(result["checkpoint"], result["checkpoint_sha256"])
|
|
save(root / "training-adopted.json", dict(result_sha256=sha(result_path)))
|
|
elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]:
|
|
raise RuntimeError("Validation screen scope differs")
|
|
overhead = time.monotonic() - began_entry - sum(charges.values())
|
|
if overhead > budget["finalization_gpu_seconds"]:
|
|
raise RuntimeError("Setup and finalization cost exceeded the fixed allowance")
|
|
status, trigger = "COMPLETE", result["decision"]
|
|
return 0
|
|
except Exception as exc:
|
|
trigger = f"{type(exc).__name__}: {exc}"
|
|
return 1
|
|
except KeyboardInterrupt:
|
|
trigger = "COORDINATOR_INTERRUPTED"
|
|
return 1
|
|
finally:
|
|
charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values()))
|
|
new_gpu = sum(charges.values())
|
|
save(root / "complete.json", dict(
|
|
status=status, trigger=trigger, receipts=receipts, **charges,
|
|
cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu,
|
|
cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu,
|
|
no_retry=True, test_targets_read=False, automatic_continuation=False))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("mode", choices=("run", "train", "screen"))
|
|
parser.add_argument("root", type=Path)
|
|
args = parser.parse_args()
|
|
root = args.root.resolve()
|
|
if root != ROOT:
|
|
raise PermissionError("Only the new fixed fold-0 root is accepted")
|
|
expected_python = str(ROOT.parents[1] / "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python")
|
|
if sys.executable != expected_python:
|
|
raise PermissionError("Use the fixed original H2 environment")
|
|
validate_review(root)
|
|
contract_path = root / "contract.json"
|
|
if args.mode != "run":
|
|
coordinator = read(root / "started.json")
|
|
if process_identity(os.getppid()) != coordinator["coordinator"]:
|
|
raise PermissionError("Worker requires the live guard as its parent")
|
|
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0:
|
|
raise OSError(ctypes.get_errno(), "Cannot bind worker lifetime to guard")
|
|
if os.getppid() != coordinator["coordinator"]["pid"]:
|
|
raise RuntimeError("Guard disappeared before worker setup")
|
|
return globals()[args.mode](root, contract(contract_path))
|
|
if os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR:
|
|
raise PermissionError("Explicit original CUDA allocator setup required")
|
|
with (root / "entry.lock").open("a") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (root / "started.json").exists():
|
|
raise FileExistsError("Fold-0 attempt already started")
|
|
save(root / "started.json", dict(coordinator=process_identity(os.getpid()),
|
|
contract_sha256=sha(contract_path), unix=time.time()))
|
|
def request_stop(signum, _frame):
|
|
raise KeyboardInterrupt(signal.Signals(signum).name)
|
|
|
|
old_handler = signal.signal(signal.SIGTERM, request_stop)
|
|
try:
|
|
return dispatch_guard(root, contract_path)
|
|
finally:
|
|
signal.signal(signal.SIGTERM, old_handler)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|