Freeze Phase 3 item 1b fold-zero review entry
This commit is contained in:
1 parent
52c74d25aa
commit
a92d0cf774
13 files changed
+1721
No files matched your search
@@ -0,0 +1,390 @@
|
||||
"""One reviewed fold-0 attempt and its validation-only screen."""
|
||||
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import ctypes
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from types import FunctionType
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase3_item1b_common import fold_view, read
|
||||
from phase3_shared_capacity import process_identity
|
||||
from s2_gpu_idle_gate import snapshot, wait_idle
|
||||
from s2_main_memory_guard import PeakRecords, active_command_pids
|
||||
from s2_recovery_gpu_supervisor import stop_group, supervise
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] / "artifacts/phase3-item1b-fold0-screen-2026-11-19"
|
||||
RULING = "3a48b123da54b0bcff0e7fa672fc74e8920235a0"
|
||||
ALLOCATOR = "expandable_segments:True"
|
||||
GIB = 2**30
|
||||
ARMS = ("short_seed0", "long_seed0", "long_seed1")
|
||||
|
||||
|
||||
def contract(path):
|
||||
c = read(path)
|
||||
if (c["ruling"] != RULING or c["fold"] != 0 or c["screen_fold"] != 0
|
||||
or c["epochs"] != 100 or c["seed"] != 0 or c["max_fits"] != 1
|
||||
or c["retry"] is not False or c["test_targets_allowed"] is not False):
|
||||
raise PermissionError("Only the new fold-0 validation scope is authorized")
|
||||
for p, h in (c["code_sha256"] | c["source_sha256"]).items():
|
||||
checked(p, h)
|
||||
comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"]))
|
||||
if comparison["validation_cases"] != c["job"]["validation"]:
|
||||
raise ValueError("Fold-0 validation identity differs from comparison")
|
||||
budget = c["budget"]
|
||||
reserves = sum(budget[k] for k in ("train_gpu_seconds", "screen_gpu_seconds",
|
||||
"idle_wait_seconds", "finalization_gpu_seconds"))
|
||||
if (budget["projected_new_gpu_seconds"] != reserves
|
||||
or budget["prior_item_gpu_seconds"] + reserves > budget["item_gpu_limit_seconds"]
|
||||
or budget["prior_phase3_gpu_seconds"] + reserves > budget["phase3_gpu_limit_seconds"]):
|
||||
raise PermissionError("Phase 3 or 1b remaining budget exceeded")
|
||||
return c
|
||||
|
||||
|
||||
def folded_inputs(c):
|
||||
pilot = read(c["pilot_contract"])
|
||||
view = fold_view(pilot, 0)
|
||||
original = view["folds"][0]
|
||||
job = c["job"]
|
||||
for k in ("learning", "validation", "test", "c", "clock_sha256", "learning_nodes"):
|
||||
if job[k] != original[k]:
|
||||
raise ValueError(f"Original fold-0 identity changed: {k}")
|
||||
if job["id"] != 0 or job["target_steps"] != 100 * len(original["learning"]):
|
||||
raise ValueError("Full fold-0 learning schedule required")
|
||||
if set(original["validation"]) & set(original["test"]):
|
||||
raise ValueError("Validation and test overlap")
|
||||
return pilot, view
|
||||
|
||||
|
||||
def train(root, c):
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase3_item1b_worker import run as frozen_worker
|
||||
|
||||
pilot, _ = folded_inputs(c)
|
||||
if torch.__version__ != pilot["versions"]["torch"] or np.__version__ != pilot["versions"]["numpy"]:
|
||||
raise ValueError("Original array and Torch versions required")
|
||||
job = dict(c["job"], gpu_uuid=c["gpu_uuid"], worker_cpus=c["worker_cpus"])
|
||||
digest = sha(root / "contract.json")
|
||||
|
||||
def first_epoch_release(queue, fold):
|
||||
if queue != root or fold != 0:
|
||||
raise PermissionError("Wrong worker route")
|
||||
row = read(root / "jobs/0/worker/history/001.json")
|
||||
budget = c["budget"]
|
||||
epoch_seconds = max(row["train_epoch_seconds"], budget["train_epoch_seconds_basis"])
|
||||
projection = (row["wall_seconds"] + 99 * epoch_seconds
|
||||
+ 10 * budget["validation_seconds_basis"] + budget["training_setup_tail_seconds"])
|
||||
passed = math.isfinite(projection) and projection <= budget["train_gpu_seconds"]
|
||||
save(root / "first-epoch-projection.json", dict(
|
||||
status="PASS" if passed else "STOP_BUDGET", contract_sha256=digest,
|
||||
measured_epoch_seconds=row["train_epoch_seconds"], projected_train_gpu_seconds=projection))
|
||||
if not passed:
|
||||
raise RuntimeError("First-epoch projection exceeds fold-0 training cap")
|
||||
return True
|
||||
|
||||
# Reuse the immutable training body. The private namespace changes only
|
||||
# its root/assignment adapters and the one-fold budget gate.
|
||||
namespace = dict(frozen_worker.__globals__, config=lambda _: (c, pilot),
|
||||
assigned_job=lambda *_: job, first_epoch_release=first_epoch_release)
|
||||
runner = FunctionType(frozen_worker.__code__, namespace, frozen_worker.__name__)
|
||||
runner(root, 0, digest)
|
||||
|
||||
|
||||
def screen(root, c):
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_data import arguments, cached, to_device
|
||||
from phase2_h2_model import H2Geo
|
||||
from phase2_h2_pilot_worker import peaks
|
||||
from phase2_residual_math import decode_seconds
|
||||
from phase3_item1_postfit_data import load_evaluation_part
|
||||
from s2_custody_common import prediction_inputs
|
||||
from s2_models.objectives import relative_l2
|
||||
|
||||
pilot, view = folded_inputs(c)
|
||||
comparison = read(c["comparison_contract"])
|
||||
fold = view["folds"][0]
|
||||
cases = comparison["validation_cases"]
|
||||
if (cases != fold["validation"] or torch.__version__ != pilot["versions"]["torch"]
|
||||
or np.__version__ != pilot["versions"]["numpy"]
|
||||
or os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"]
|
||||
or os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR):
|
||||
raise PermissionError("Screen scope or environment differs")
|
||||
train_path = root / "jobs/0/worker/result.json"
|
||||
adopted = read(root / "training-adopted.json")
|
||||
short = read(checked(train_path, adopted["result_sha256"]))
|
||||
models = dict(c["references"])
|
||||
models["short_seed0"] = dict(
|
||||
checkpoint=short["checkpoint"], checkpoint_sha256=short["checkpoint_sha256"],
|
||||
standardization=str(train_path.with_name("standardization.json")),
|
||||
standardization_sha256=short["standardization_sha256"], seed=0, epochs=100,
|
||||
training_contract_sha256=sha(root / "contract.json"))
|
||||
roster = {r["case"]: r for r in read(c["query_roster_contract"])["rows"] if r["case"] in cases}
|
||||
if set(roster) != set(cases) or any(
|
||||
roster[case]["component_id"] != pilot["parts"][case]["clock_group"]
|
||||
or roster[case]["v5_cache"]["output_sha256"] != pilot["parts"][case]["output_sha256"]
|
||||
for case in cases
|
||||
):
|
||||
raise ValueError("Frozen validation query-to-cache identity differs")
|
||||
|
||||
os.sched_setaffinity(0, c["worker_cpus"])
|
||||
torch.set_num_threads(1)
|
||||
torch.set_num_interop_threads(1)
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.backends.cudnn.allow_tf32 = False
|
||||
torch.cuda.set_device(0)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
out = root / "screen"
|
||||
out.mkdir(exist_ok=False)
|
||||
predictions = {}
|
||||
with torch.no_grad():
|
||||
for arm in ARMS:
|
||||
ref = models[arm]
|
||||
stats = read(checked(ref["standardization"], ref["standardization_sha256"]))
|
||||
blob = torch.load(checked(ref["checkpoint"], ref["checkpoint_sha256"]),
|
||||
map_location="cpu", weights_only=True)
|
||||
required = dict(completed_epochs=ref["epochs"], fold=0, seed=ref["seed"],
|
||||
role="last_epoch_only", contract_sha256=ref["training_contract_sha256"],
|
||||
optimizer_steps=ref["epochs"] * len(fold["learning"]))
|
||||
if any(blob.get(k) != v for k, v in required.items()):
|
||||
raise ValueError(f"Final checkpoint identity: {arm}")
|
||||
if stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]:
|
||||
raise ValueError("Screen learning-only normalization mismatch")
|
||||
model = H2Geo().cuda()
|
||||
model.load_state_dict(blob["model"], strict=True)
|
||||
del blob
|
||||
model.eval()
|
||||
folder = out / arm
|
||||
folder.mkdir()
|
||||
for case in cases:
|
||||
data = to_device(load_evaluation_part(view, case, stats, "validation"), "cuda")
|
||||
values = model(*arguments(data), checkpoint_blocks=False).cpu().numpy()
|
||||
seconds = decode_seconds(values, data["F"], fold["c"], data["t_set"])
|
||||
if seconds.shape != (data["nodes"],) or not np.isfinite(seconds).all():
|
||||
raise ValueError("Full validation prediction is missing or nonfinite")
|
||||
path = folder / (case + ".npy")
|
||||
np.save(path, seconds, allow_pickle=False)
|
||||
predictions.setdefault(case, {})[arm] = {"file": str(path), "sha256": sha(path)}
|
||||
save(out / "progress.json", dict(stage="VALIDATION_PREDICT", arm=arm, case=case, **peaks()))
|
||||
del data
|
||||
del model
|
||||
manifest = out / "predictions-frozen.json"
|
||||
save(manifest, predictions)
|
||||
manifest_hash = sha(manifest)
|
||||
predictions = read(checked(manifest, manifest_hash))
|
||||
rows = []
|
||||
grouped = defaultdict(list)
|
||||
for case in cases:
|
||||
q = prediction_inputs(roster[case])
|
||||
ids = cached(view, case, "raw_ids")
|
||||
ix = np.searchsorted(ids, q["raw_ids"])
|
||||
if np.any(ix >= len(ids)) or not np.array_equal(ids[ix], q["raw_ids"]):
|
||||
raise ValueError("Validation query does not map to original raw IDs")
|
||||
with (out / "validation-target-access.jsonl").open("a") as log:
|
||||
log.write(json.dumps(dict(case=case, role="validation", predictions_sha256=manifest_hash,
|
||||
target_sha256=pilot["parts"][case]["output_sha256"]["seconds.npy"])) + "\n")
|
||||
truth = cached(view, case, "seconds")[ix]
|
||||
scores = {}
|
||||
for arm in ARMS:
|
||||
row = predictions[case][arm]
|
||||
pred = np.load(checked(row["file"], row["sha256"]), allow_pickle=False)[ix]
|
||||
value = relative_l2(pred, truth, q["counts"], np.ones(len(ix), dtype=bool))["value"]
|
||||
if value is None or not math.isfinite(value):
|
||||
raise ValueError("Undefined validation part metric")
|
||||
scores[arm] = value
|
||||
component = roster[case]["component_id"]
|
||||
grouped[component].append([scores[arm] for arm in ARMS])
|
||||
rows.append(dict(case=case, component_id=component, relative_l2=scores))
|
||||
means = dict(zip(ARMS, np.mean([np.mean(a, axis=0) for a in grouped.values()], axis=0).tolist()))
|
||||
delta = means["short_seed0"] - means["long_seed0"]
|
||||
seed_delta = means["long_seed1"] - means["long_seed0"]
|
||||
passed = delta <= comparison["delta_short_at_most"] and abs(delta) > abs(seed_delta)
|
||||
save(out / "result.json", dict(
|
||||
status="COMPLETE", decision="SCREEN_PASS" if passed else "SCREEN_FAIL",
|
||||
parts=len(rows), components=len(grouped), component_equal_relative_l2=means,
|
||||
delta_short=delta, delta_seed=seed_delta, rows=rows, predictions_sha256=manifest_hash,
|
||||
test_targets_read=False, automatic_continuation=False, **peaks()))
|
||||
|
||||
|
||||
def dispatch_guard(root, contract_path):
|
||||
"""The same entry consumes the generated contract in production and preflight."""
|
||||
began_entry = time.monotonic()
|
||||
c = contract(contract_path)
|
||||
folded_inputs(c)
|
||||
budget = c["budget"]
|
||||
gpu = c["gpu_uuid"]
|
||||
charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0.,
|
||||
screen_gpu_seconds=0., finalization_gpu_seconds=0.)
|
||||
receipts = []
|
||||
status = "STOPPED"
|
||||
trigger = "STOP_BEFORE_DISPATCH"
|
||||
try:
|
||||
if shutil.disk_usage(root).free < c["disk_bytes"]:
|
||||
raise RuntimeError("Insufficient free disk for the fixed attempt")
|
||||
began_wait = time.monotonic()
|
||||
try:
|
||||
idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot)
|
||||
except BaseException:
|
||||
charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait
|
||||
raise
|
||||
charges["queue_wait_gpu_seconds"] = idle["wall_seconds"]
|
||||
save(root / "idle-gate.json", idle)
|
||||
trigger = idle["status"]
|
||||
if idle["status"] != "IDLE_CONFIRMED" or not snapshot(gpu)["idle"]:
|
||||
if idle["status"] == "IDLE_CONFIRMED":
|
||||
trigger = "SELECTED_GPU_BUSY_AFTER_IDLE_GATE"
|
||||
return 1
|
||||
(root / "jobs/0").mkdir(parents=True, exist_ok=False)
|
||||
for mode, cap_key, charge_key, output in (
|
||||
("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"),
|
||||
("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"),
|
||||
):
|
||||
began = time.monotonic()
|
||||
if not snapshot(gpu)["idle"]:
|
||||
trigger = "SELECTED_GPU_BUSY_BEFORE_" + mode.upper()
|
||||
charges[charge_key] = time.monotonic() - began
|
||||
return 1
|
||||
command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)]
|
||||
tracker = PeakRecords()
|
||||
done = threading.Event()
|
||||
reasons = []
|
||||
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
tracker.scan(output)
|
||||
state = snapshot(gpu)
|
||||
pids = set(active_command_pids(command))
|
||||
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
|
||||
reasons.append("RECORDED_VRAM_LIMIT")
|
||||
if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]:
|
||||
reasons.append("DEVICE_VRAM_LIMIT")
|
||||
if any(p["pid"] not in pids for p in state["occupants"]):
|
||||
reasons.append("FOREIGN_GPU_OCCUPANT")
|
||||
except Exception as exc:
|
||||
reasons.append(f"MONITOR_FAILURE: {exc}")
|
||||
if reasons:
|
||||
for pid in active_command_pids(command):
|
||||
stop_group(pid)
|
||||
# Also catch a child spawned just after the failing
|
||||
# sample while the supervisor was taking its lock.
|
||||
done.wait(.1)
|
||||
continue
|
||||
done.wait(.5)
|
||||
|
||||
monitor = threading.Thread(target=watch, daemon=True)
|
||||
monitor.start()
|
||||
try:
|
||||
if reasons:
|
||||
raise RuntimeError(reasons[0])
|
||||
cap = budget[cap_key] - (time.monotonic() - began) - 10
|
||||
receipt = supervise(command, root, mode, dict(
|
||||
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key],
|
||||
job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"],
|
||||
disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]),
|
||||
poll=.2, gpu_uuid=gpu)
|
||||
finally:
|
||||
done.set()
|
||||
monitor.join(timeout=8)
|
||||
charges[charge_key] = time.monotonic() - began
|
||||
if monitor.is_alive():
|
||||
reasons.append("MONITOR_DID_NOT_EXIT")
|
||||
else:
|
||||
tracker.scan(output)
|
||||
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
|
||||
reasons.append("RECORDED_VRAM_LIMIT")
|
||||
if set(tracker.witnesses) != set(tracker.maxima):
|
||||
reasons.append("MISSING_RECORDED_VRAM_PEAKS")
|
||||
receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima))
|
||||
if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0
|
||||
or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]):
|
||||
worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT"
|
||||
trigger = reasons[0] if reasons else worker_reason
|
||||
return 1
|
||||
result_path = output / "result.json"
|
||||
result = read(result_path)
|
||||
if result["status"] != "COMPLETE" or result["test_targets_read"] is not False:
|
||||
raise RuntimeError("Unexpected worker result")
|
||||
if mode == "train":
|
||||
if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"]
|
||||
or result["optimizer_steps"] != c["job"]["target_steps"]):
|
||||
raise RuntimeError("Fold-0 final checkpoint identity differs")
|
||||
checked(result["checkpoint"], result["checkpoint_sha256"])
|
||||
save(root / "training-adopted.json", dict(result_sha256=sha(result_path)))
|
||||
elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]:
|
||||
raise RuntimeError("Validation screen scope differs")
|
||||
overhead = time.monotonic() - began_entry - sum(charges.values())
|
||||
if overhead > budget["finalization_gpu_seconds"]:
|
||||
raise RuntimeError("Setup and finalization cost exceeded the fixed allowance")
|
||||
status, trigger = "COMPLETE", result["decision"]
|
||||
return 0
|
||||
except Exception as exc:
|
||||
trigger = f"{type(exc).__name__}: {exc}"
|
||||
return 1
|
||||
except KeyboardInterrupt:
|
||||
trigger = "COORDINATOR_INTERRUPTED"
|
||||
return 1
|
||||
finally:
|
||||
charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values()))
|
||||
new_gpu = sum(charges.values())
|
||||
save(root / "complete.json", dict(
|
||||
status=status, trigger=trigger, receipts=receipts, **charges,
|
||||
cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu,
|
||||
cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu,
|
||||
no_retry=True, test_targets_read=False, automatic_continuation=False))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("mode", choices=("run", "train", "screen"))
|
||||
parser.add_argument("root", type=Path)
|
||||
args = parser.parse_args()
|
||||
root = args.root.resolve()
|
||||
if root != ROOT:
|
||||
raise PermissionError("Only the new fixed fold-0 root is accepted")
|
||||
expected_python = str(ROOT.parents[1] / "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python")
|
||||
if sys.executable != expected_python:
|
||||
raise PermissionError("Use the fixed original H2 environment")
|
||||
validate_review(root)
|
||||
contract_path = root / "contract.json"
|
||||
if args.mode != "run":
|
||||
coordinator = read(root / "started.json")
|
||||
if process_identity(os.getppid()) != coordinator["coordinator"]:
|
||||
raise PermissionError("Worker requires the live guard as its parent")
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), "Cannot bind worker lifetime to guard")
|
||||
if os.getppid() != coordinator["coordinator"]["pid"]:
|
||||
raise RuntimeError("Guard disappeared before worker setup")
|
||||
return globals()[args.mode](root, contract(contract_path))
|
||||
if os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR:
|
||||
raise PermissionError("Explicit original CUDA allocator setup required")
|
||||
with (root / "entry.lock").open("a") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root / "started.json").exists():
|
||||
raise FileExistsError("Fold-0 attempt already started")
|
||||
save(root / "started.json", dict(coordinator=process_identity(os.getpid()),
|
||||
contract_sha256=sha(contract_path), unix=time.time()))
|
||||
def request_stop(signum, _frame):
|
||||
raise KeyboardInterrupt(signal.Signals(signum).name)
|
||||
|
||||
old_handler = signal.signal(signal.SIGTERM, request_stop)
|
||||
try:
|
||||
return dispatch_guard(root, contract_path)
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, old_handler)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,234 @@
|
||||
"""Freeze the new fold-0 contract and exercise its actual guard on CPU."""
|
||||
|
||||
from functools import partial
|
||||
import math
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase3_item1b_common import ROOT as OLD_ROOT, read
|
||||
import phase3_item1b_fold0 as entry
|
||||
from s2_gpu_idle_gate import wait_idle
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
|
||||
|
||||
BASE = entry.ROOT.parents[1]
|
||||
RUNS = OLD_ROOT.parent
|
||||
SEED1 = RUNS / "phase3-seed1-v2-2026-11-18"
|
||||
|
||||
|
||||
def prepare():
|
||||
root = entry.ROOT
|
||||
if root.exists():
|
||||
raise FileExistsError("New fold-0 preparation already exists")
|
||||
pins = {}
|
||||
|
||||
def pin(path):
|
||||
p = Path(path)
|
||||
pins[str(p)] = sha(p)
|
||||
return read(p)
|
||||
|
||||
old = pin(OLD_ROOT / "contract.json")
|
||||
stop = pin(OLD_ROOT / "complete.json")
|
||||
settled = pin(OLD_ROOT / "post-terminal-cost.json")
|
||||
if (stop["failure"] != "RuntimeError: TOTAL_PROJECTION_OR_ACTUAL_CAP"
|
||||
or settled["training_complete_sha256"] != pins[str(OLD_ROOT / "complete.json")]
|
||||
or settled["postfit_started"] is not False or not settled["freeze_tail_charged_once"]):
|
||||
raise ValueError("Historical budget-stop record differs")
|
||||
totals = {
|
||||
"item1": pin(RUNS / "phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json")[
|
||||
"cumulative_item_gpu_seconds"],
|
||||
"seed1": pin(RUNS / "phase3-seed1-postfit-2026-11-18/post-terminal-cost.json")[
|
||||
"cumulative_seed1_gpu_seconds"],
|
||||
"item1b": settled["cumulative_gpu_seconds"],
|
||||
"c": pin(RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
|
||||
"cumulative_item_gpu_seconds"],
|
||||
"b": pin(RUNS / "phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
|
||||
"cumulative_item_gpu_seconds"],
|
||||
}
|
||||
history_path = OLD_ROOT / "jobs/0/worker/history/001.json"
|
||||
epoch = pin(history_path)
|
||||
assignments = {i: pin(OLD_ROOT / f"jobs/{i}/assignment.json") for i in (0, 2)}
|
||||
identities = {i: pin(OLD_ROOT / f"jobs/{i}/worker/worker-identity.json") for i in (0, 2)}
|
||||
start = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.started.json")["started_unix"] for i in (0, 2)}
|
||||
elapsed = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.receipt.json")["wall_seconds"] for i in (0, 2)}
|
||||
gpu = assignments[0]["gpu_uuid"]
|
||||
sample = pin(OLD_ROOT / "jobs/0/gpu-sample.json")
|
||||
observed = {row["pid"] for row in sample["rows"]}
|
||||
if (assignments[2]["gpu_uuid"] != gpu or len(observed) != 3
|
||||
or not {p["pid"] for p in identities.values()} <= observed
|
||||
or not max(start.values()) < sample["unix"] < min(start[i] + elapsed[i] for i in start)
|
||||
or epoch["optimizer_steps"] != len(old["jobs"][0]["learning"])
|
||||
or epoch["stage"] != "EPOCH_COMPLETE"):
|
||||
raise ValueError("Missing same-card fold-0 timing witness")
|
||||
solo = []
|
||||
for obs_path in sorted((SEED1 / "jobs/0/epoch-host-observations").glob("*.json")):
|
||||
obs = read(obs_path)
|
||||
on_card = [p for p in obs["gpu_processes"] if p["on_assigned_gpu"]]
|
||||
if (obs["status"] != "OBSERVED" or obs["errors"] or obs["assigned_gpu_uuid"] != gpu
|
||||
or len(on_card) != 1 or not on_card[0]["own_v2_worker"]):
|
||||
continue
|
||||
row_path = Path(obs["receipt_path"])
|
||||
row = read(checked(row_path, obs["receipt_sha256"]))
|
||||
solo.append((obs_path, row_path, row))
|
||||
if not solo:
|
||||
raise ValueError("Missing fold-0 diagnostic duration and comparison envelope")
|
||||
longest_train = max(solo, key=lambda x: x[2]["train_epoch_seconds"])
|
||||
longest_val = max(solo, key=lambda x: x[2].get("validation_seconds", 0))
|
||||
for observation, receipt, _ in (longest_train, longest_val):
|
||||
pin(observation)
|
||||
pin(receipt)
|
||||
train_basis = max(epoch["train_epoch_seconds"], longest_train[2]["train_epoch_seconds"])
|
||||
val_basis = max(old["jobs"][0]["validation_seconds_bound"], longest_val[2]["validation_seconds"])
|
||||
# The explicit compute margin applies once. Setup, bounded waiting and the
|
||||
# terminal allowance do not inflate an individual training-epoch time.
|
||||
train_setup = 120
|
||||
train_cap = math.ceil(1.25 * (100 * train_basis + 10 * val_basis)) + train_setup
|
||||
screen_cap = math.ceil(1.25 * 3 * val_basis) + 120
|
||||
budget = dict(
|
||||
item_gpu_limit_seconds=90000, phase3_gpu_limit_seconds=720000,
|
||||
prior_item_gpu_seconds=totals["item1b"], prior_phase3_gpu_seconds=sum(totals.values()),
|
||||
train_epoch_seconds_basis=train_basis, validation_seconds_basis=val_basis,
|
||||
train_gpu_seconds=train_cap, screen_gpu_seconds=screen_cap,
|
||||
training_setup_tail_seconds=train_setup, idle_wait_seconds=60, finalization_gpu_seconds=120,
|
||||
projected_new_gpu_seconds=train_cap + screen_cap + 60 + 120,
|
||||
)
|
||||
if budget["projected_new_gpu_seconds"] > 90000 - totals["item1b"]:
|
||||
raise ValueError(f"Fold-0 cap exceeds remaining authorization: {budget}")
|
||||
|
||||
pilot = pin(old["pilot_contract"])
|
||||
roster = pin(old["query_roster_contract"])
|
||||
fold = pilot["folds"][0]
|
||||
rows = {r["case"]: r for r in roster["rows"] if r["case"] in fold["validation"]}
|
||||
if set(rows) != set(fold["validation"]):
|
||||
raise ValueError("Fold-0 validation roster incomplete")
|
||||
references = {}
|
||||
baseline = old["baseline_models"][0]
|
||||
workers = [Path(baseline["result"]).parent, SEED1 / "jobs/0/worker"]
|
||||
roots = [workers[0].parent, SEED1]
|
||||
for seed, worker in enumerate(workers):
|
||||
run_contract = roots[seed] / "contract.json"
|
||||
run_meta = pin(run_contract)
|
||||
result = pin(worker / "result.json")
|
||||
stats_path = worker / "standardization.json"
|
||||
stats = pin(stats_path)
|
||||
checked(stats_path, result["standardization_sha256"])
|
||||
checkpoint = checked(result["checkpoint"], result["checkpoint_sha256"])
|
||||
pins[str(checkpoint)] = result["checkpoint_sha256"]
|
||||
if (run_meta["seed"] != seed or result["completed_epochs"] != 200
|
||||
or result["optimizer_steps"] != 200 * len(fold["learning"])
|
||||
or stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]):
|
||||
raise ValueError("Reference final checkpoint scope differs")
|
||||
references[f"long_seed{seed}"] = dict(
|
||||
checkpoint=str(checkpoint), checkpoint_sha256=result["checkpoint_sha256"],
|
||||
standardization=str(stats_path), standardization_sha256=result["standardization_sha256"],
|
||||
seed=seed, epochs=200, training_contract_sha256=pins[str(run_contract)])
|
||||
|
||||
root.mkdir(exist_ok=False)
|
||||
comparison = root / "comparison-contract.json"
|
||||
save(comparison, dict(
|
||||
ruling=entry.RULING, fold=0, validation_cases=fold["validation"],
|
||||
validation_components=len({row["component_id"] for row in rows.values()}),
|
||||
score="Relative L2 on the frozen validation query with original PPS multiplicity, then "
|
||||
"equal means by component. Negative predictions remain visible.",
|
||||
paired_references=list(references), delta_short_at_most=-.001,
|
||||
effect_strictly_greater_than_abs_seed_delta=True, duration="100 complete epochs, final epoch only",
|
||||
warmup="Original 5-epoch warmup and full cosine decay over 95 epochs",
|
||||
train_fields="Original seed0, order, 19 inputs, optimizer, MSE and numerical recipe",
|
||||
test_allowed=False, automatic_continuation=False))
|
||||
codes = local_code_closure([Path(__file__), Path(entry.__file__)] +
|
||||
list((BASE / "tools/s2_models").glob("*.py")))
|
||||
for path, digest in old["code_sha256"].items():
|
||||
checked(path, digest)
|
||||
codes.setdefault(path, digest)
|
||||
source = root / "timing-and-budget.json"
|
||||
save(source, dict(
|
||||
prior_phase3_costs=totals, budget=budget, remaining_item_gpu_seconds=90000 - totals["item1b"],
|
||||
same_card_fold0_epoch_seconds=epoch["train_epoch_seconds"], gpu_uuid=gpu,
|
||||
same_card_train_peers_instantiated={str(i): p["pid"] for i, p in identities.items()},
|
||||
sampled_cuda_pids=sorted(observed), sample_unix=sample["unix"], worker_intervals=[
|
||||
dict(fold=i, started_unix=start[i], ended_unix=start[i] + elapsed[i]) for i in start],
|
||||
same_fold_solo_epoch_records=len(solo), longest_solo_train_seconds=longest_train[2]["train_epoch_seconds"],
|
||||
longest_solo_validation_seconds=longest_val[2]["validation_seconds"],
|
||||
limitation="The two workers started together. Shared occupancy is sampled near epoch "
|
||||
"completion. The third CUDA context is observed only at that sample.",
|
||||
target_admission="Selected V100 starts idle, with stop on foreign occupancy. "
|
||||
"The co-located trace is a conservative budget input."))
|
||||
pins[str(source)] = sha(source)
|
||||
contract = root / "contract.json"
|
||||
save(contract, dict(
|
||||
version="phase3-item1b-fold0-screen-v1", ruling=entry.RULING, fold=0, screen_fold=0, seed=0,
|
||||
epochs=100, max_fits=1, retry=False, test_targets_allowed=False, gpu_uuid=gpu,
|
||||
worker_cpus=assignments[0]["worker_cpus"], total_cpu_seconds=30000, max_processes=1,
|
||||
rss_bytes=4 * entry.GIB, disk_bytes=8 * entry.GIB, device_vram_bytes=4 * entry.GIB,
|
||||
recorded_vram_bytes=2 * entry.GIB, job=old["jobs"][0], budget=budget, references=references,
|
||||
pilot_contract=old["pilot_contract"], query_roster_contract=old["query_roster_contract"],
|
||||
comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison),
|
||||
source_sha256=pins, code_sha256=codes))
|
||||
return contract
|
||||
|
||||
|
||||
def preflight(contract):
|
||||
c = entry.contract(contract)
|
||||
results = []
|
||||
for busy in (False, True):
|
||||
simulated_time = 0.
|
||||
|
||||
def sleep(seconds):
|
||||
nonlocal simulated_time
|
||||
simulated_time += seconds
|
||||
|
||||
def snapshot(uuid):
|
||||
if uuid != c["gpu_uuid"]:
|
||||
raise ValueError("Incorrect GPU identity")
|
||||
return dict(uuid=uuid, idle=not busy, memory_MiB=1500 if busy else 5, occupants=[])
|
||||
|
||||
def simulate(command, guard_root, mode, limits, *, poll, gpu_uuid):
|
||||
if (guard_root != root or command[2:] != [mode, str(root)] or gpu_uuid != c["gpu_uuid"]
|
||||
or not 0 < limits["job_wall_seconds"] < c["budget"][f"{mode}_gpu_seconds"]
|
||||
or limits["tree_rss_bytes"] != c["rss_bytes"] or limits["disk_bytes"] != c["disk_bytes"]
|
||||
or limits["max_processes"] != c["max_processes"]
|
||||
or limits["total_cpu_seconds"] != c["total_cpu_seconds"]):
|
||||
raise ValueError("Actual guard did not propagate frozen fields")
|
||||
record = dict(status="COMPLETE", test_targets_read=False, peak_allocated_bytes=1024,
|
||||
peak_reserved_bytes=2048)
|
||||
if mode == "train":
|
||||
out = root / "jobs/0/worker"
|
||||
out.mkdir()
|
||||
checkpoint = out / "simulated-checkpoint"
|
||||
checkpoint.write_bytes(b"CPU mock only")
|
||||
record.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"],
|
||||
checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint))
|
||||
else:
|
||||
out = root / "screen"
|
||||
out.mkdir()
|
||||
record.update(parts=len(c["job"]["validation"]), automatic_continuation=False,
|
||||
decision="SCREEN_PASS")
|
||||
save(out / "result.json", record)
|
||||
return dict(status="PASS", returncode=0, monitor_errors=0)
|
||||
|
||||
with TemporaryDirectory(prefix="meltmind-fold0-guard-") as folder:
|
||||
root = Path(folder)
|
||||
with (patch.object(entry, "snapshot", side_effect=snapshot),
|
||||
patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: simulated_time, sleep=sleep)),
|
||||
patch.object(entry, "supervise", side_effect=simulate) as dispatch):
|
||||
exit_code = entry.dispatch_guard(root, contract)
|
||||
terminal = read(root / "complete.json")
|
||||
gate = read(root / "idle-gate.json")
|
||||
expected = (1, "IDLE_TIMEOUT", []) if busy else (0, "SCREEN_PASS", ["train", "screen"])
|
||||
actual = (exit_code, terminal["trigger"], [call.args[2] for call in dispatch.call_args_list])
|
||||
if actual != expected:
|
||||
raise RuntimeError(f"Actual guard CPU simulation differed: {actual}")
|
||||
results.append(dict(case="busy-timeout" if busy else "idle-dispatch",
|
||||
exit_code=exit_code, trigger=terminal["trigger"], mock_dispatches=actual[2],
|
||||
gate_wait_seconds=gate["wall_seconds"]))
|
||||
return dict(status="PASS", actual_contract_sha256=sha(contract), cases=results,
|
||||
real_gpu_or_worker_started=False, scientific_arrays_read=False, review_status="PENDING")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
path = prepare()
|
||||
receipt = preflight(path)
|
||||
save(path.parent / "cpu-guard-preflight.json", receipt)
|
||||
print(receipt)
|
||||
Reference in new issue
Block a user