Freeze Phase 3 item 1b fold-zero review entry

This commit is contained in:
NewMeltAssist committed 2026-11-19 12:15:05 +08:00
1 parent 52c74d25aa
commit a92d0cf774
13 files changed
+1721

No files matched your search

+390
View File
@@ -0,0 +1,390 @@
"""One reviewed fold-0 attempt and its validation-only screen."""
import argparse
from collections import defaultdict
import ctypes
import fcntl
import json
import math
import os
from pathlib import Path
import shutil
import signal
import sys
import threading
import time
from types import FunctionType
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase3_item1b_common import fold_view, read
from phase3_shared_capacity import process_identity
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords, active_command_pids
from s2_recovery_gpu_supervisor import stop_group, supervise
ROOT = Path(__file__).resolve().parents[1] / "artifacts/phase3-item1b-fold0-screen-2026-11-19"
RULING = "3a48b123da54b0bcff0e7fa672fc74e8920235a0"
ALLOCATOR = "expandable_segments:True"
GIB = 2**30
ARMS = ("short_seed0", "long_seed0", "long_seed1")
def contract(path):
c = read(path)
if (c["ruling"] != RULING or c["fold"] != 0 or c["screen_fold"] != 0
or c["epochs"] != 100 or c["seed"] != 0 or c["max_fits"] != 1
or c["retry"] is not False or c["test_targets_allowed"] is not False):
raise PermissionError("Only the new fold-0 validation scope is authorized")
for p, h in (c["code_sha256"] | c["source_sha256"]).items():
checked(p, h)
comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"]))
if comparison["validation_cases"] != c["job"]["validation"]:
raise ValueError("Fold-0 validation identity differs from comparison")
budget = c["budget"]
reserves = sum(budget[k] for k in ("train_gpu_seconds", "screen_gpu_seconds",
"idle_wait_seconds", "finalization_gpu_seconds"))
if (budget["projected_new_gpu_seconds"] != reserves
or budget["prior_item_gpu_seconds"] + reserves > budget["item_gpu_limit_seconds"]
or budget["prior_phase3_gpu_seconds"] + reserves > budget["phase3_gpu_limit_seconds"]):
raise PermissionError("Phase 3 or 1b remaining budget exceeded")
return c
def folded_inputs(c):
pilot = read(c["pilot_contract"])
view = fold_view(pilot, 0)
original = view["folds"][0]
job = c["job"]
for k in ("learning", "validation", "test", "c", "clock_sha256", "learning_nodes"):
if job[k] != original[k]:
raise ValueError(f"Original fold-0 identity changed: {k}")
if job["id"] != 0 or job["target_steps"] != 100 * len(original["learning"]):
raise ValueError("Full fold-0 learning schedule required")
if set(original["validation"]) & set(original["test"]):
raise ValueError("Validation and test overlap")
return pilot, view
def train(root, c):
import numpy as np
import torch
from phase3_item1b_worker import run as frozen_worker
pilot, _ = folded_inputs(c)
if torch.__version__ != pilot["versions"]["torch"] or np.__version__ != pilot["versions"]["numpy"]:
raise ValueError("Original array and Torch versions required")
job = dict(c["job"], gpu_uuid=c["gpu_uuid"], worker_cpus=c["worker_cpus"])
digest = sha(root / "contract.json")
def first_epoch_release(queue, fold):
if queue != root or fold != 0:
raise PermissionError("Wrong worker route")
row = read(root / "jobs/0/worker/history/001.json")
budget = c["budget"]
epoch_seconds = max(row["train_epoch_seconds"], budget["train_epoch_seconds_basis"])
projection = (row["wall_seconds"] + 99 * epoch_seconds
+ 10 * budget["validation_seconds_basis"] + budget["training_setup_tail_seconds"])
passed = math.isfinite(projection) and projection <= budget["train_gpu_seconds"]
save(root / "first-epoch-projection.json", dict(
status="PASS" if passed else "STOP_BUDGET", contract_sha256=digest,
measured_epoch_seconds=row["train_epoch_seconds"], projected_train_gpu_seconds=projection))
if not passed:
raise RuntimeError("First-epoch projection exceeds fold-0 training cap")
return True
# Reuse the immutable training body. The private namespace changes only
# its root/assignment adapters and the one-fold budget gate.
namespace = dict(frozen_worker.__globals__, config=lambda _: (c, pilot),
assigned_job=lambda *_: job, first_epoch_release=first_epoch_release)
runner = FunctionType(frozen_worker.__code__, namespace, frozen_worker.__name__)
runner(root, 0, digest)
def screen(root, c):
import numpy as np
import torch
from phase2_h2_data import arguments, cached, to_device
from phase2_h2_model import H2Geo
from phase2_h2_pilot_worker import peaks
from phase2_residual_math import decode_seconds
from phase3_item1_postfit_data import load_evaluation_part
from s2_custody_common import prediction_inputs
from s2_models.objectives import relative_l2
pilot, view = folded_inputs(c)
comparison = read(c["comparison_contract"])
fold = view["folds"][0]
cases = comparison["validation_cases"]
if (cases != fold["validation"] or torch.__version__ != pilot["versions"]["torch"]
or np.__version__ != pilot["versions"]["numpy"]
or os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"]
or os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR):
raise PermissionError("Screen scope or environment differs")
train_path = root / "jobs/0/worker/result.json"
adopted = read(root / "training-adopted.json")
short = read(checked(train_path, adopted["result_sha256"]))
models = dict(c["references"])
models["short_seed0"] = dict(
checkpoint=short["checkpoint"], checkpoint_sha256=short["checkpoint_sha256"],
standardization=str(train_path.with_name("standardization.json")),
standardization_sha256=short["standardization_sha256"], seed=0, epochs=100,
training_contract_sha256=sha(root / "contract.json"))
roster = {r["case"]: r for r in read(c["query_roster_contract"])["rows"] if r["case"] in cases}
if set(roster) != set(cases) or any(
roster[case]["component_id"] != pilot["parts"][case]["clock_group"]
or roster[case]["v5_cache"]["output_sha256"] != pilot["parts"][case]["output_sha256"]
for case in cases
):
raise ValueError("Frozen validation query-to-cache identity differs")
os.sched_setaffinity(0, c["worker_cpus"])
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
torch.cuda.set_device(0)
torch.cuda.reset_peak_memory_stats()
out = root / "screen"
out.mkdir(exist_ok=False)
predictions = {}
with torch.no_grad():
for arm in ARMS:
ref = models[arm]
stats = read(checked(ref["standardization"], ref["standardization_sha256"]))
blob = torch.load(checked(ref["checkpoint"], ref["checkpoint_sha256"]),
map_location="cpu", weights_only=True)
required = dict(completed_epochs=ref["epochs"], fold=0, seed=ref["seed"],
role="last_epoch_only", contract_sha256=ref["training_contract_sha256"],
optimizer_steps=ref["epochs"] * len(fold["learning"]))
if any(blob.get(k) != v for k, v in required.items()):
raise ValueError(f"Final checkpoint identity: {arm}")
if stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]:
raise ValueError("Screen learning-only normalization mismatch")
model = H2Geo().cuda()
model.load_state_dict(blob["model"], strict=True)
del blob
model.eval()
folder = out / arm
folder.mkdir()
for case in cases:
data = to_device(load_evaluation_part(view, case, stats, "validation"), "cuda")
values = model(*arguments(data), checkpoint_blocks=False).cpu().numpy()
seconds = decode_seconds(values, data["F"], fold["c"], data["t_set"])
if seconds.shape != (data["nodes"],) or not np.isfinite(seconds).all():
raise ValueError("Full validation prediction is missing or nonfinite")
path = folder / (case + ".npy")
np.save(path, seconds, allow_pickle=False)
predictions.setdefault(case, {})[arm] = {"file": str(path), "sha256": sha(path)}
save(out / "progress.json", dict(stage="VALIDATION_PREDICT", arm=arm, case=case, **peaks()))
del data
del model
manifest = out / "predictions-frozen.json"
save(manifest, predictions)
manifest_hash = sha(manifest)
predictions = read(checked(manifest, manifest_hash))
rows = []
grouped = defaultdict(list)
for case in cases:
q = prediction_inputs(roster[case])
ids = cached(view, case, "raw_ids")
ix = np.searchsorted(ids, q["raw_ids"])
if np.any(ix >= len(ids)) or not np.array_equal(ids[ix], q["raw_ids"]):
raise ValueError("Validation query does not map to original raw IDs")
with (out / "validation-target-access.jsonl").open("a") as log:
log.write(json.dumps(dict(case=case, role="validation", predictions_sha256=manifest_hash,
target_sha256=pilot["parts"][case]["output_sha256"]["seconds.npy"])) + "\n")
truth = cached(view, case, "seconds")[ix]
scores = {}
for arm in ARMS:
row = predictions[case][arm]
pred = np.load(checked(row["file"], row["sha256"]), allow_pickle=False)[ix]
value = relative_l2(pred, truth, q["counts"], np.ones(len(ix), dtype=bool))["value"]
if value is None or not math.isfinite(value):
raise ValueError("Undefined validation part metric")
scores[arm] = value
component = roster[case]["component_id"]
grouped[component].append([scores[arm] for arm in ARMS])
rows.append(dict(case=case, component_id=component, relative_l2=scores))
means = dict(zip(ARMS, np.mean([np.mean(a, axis=0) for a in grouped.values()], axis=0).tolist()))
delta = means["short_seed0"] - means["long_seed0"]
seed_delta = means["long_seed1"] - means["long_seed0"]
passed = delta <= comparison["delta_short_at_most"] and abs(delta) > abs(seed_delta)
save(out / "result.json", dict(
status="COMPLETE", decision="SCREEN_PASS" if passed else "SCREEN_FAIL",
parts=len(rows), components=len(grouped), component_equal_relative_l2=means,
delta_short=delta, delta_seed=seed_delta, rows=rows, predictions_sha256=manifest_hash,
test_targets_read=False, automatic_continuation=False, **peaks()))
def dispatch_guard(root, contract_path):
"""The same entry consumes the generated contract in production and preflight."""
began_entry = time.monotonic()
c = contract(contract_path)
folded_inputs(c)
budget = c["budget"]
gpu = c["gpu_uuid"]
charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0.,
screen_gpu_seconds=0., finalization_gpu_seconds=0.)
receipts = []
status = "STOPPED"
trigger = "STOP_BEFORE_DISPATCH"
try:
if shutil.disk_usage(root).free < c["disk_bytes"]:
raise RuntimeError("Insufficient free disk for the fixed attempt")
began_wait = time.monotonic()
try:
idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot)
except BaseException:
charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait
raise
charges["queue_wait_gpu_seconds"] = idle["wall_seconds"]
save(root / "idle-gate.json", idle)
trigger = idle["status"]
if idle["status"] != "IDLE_CONFIRMED" or not snapshot(gpu)["idle"]:
if idle["status"] == "IDLE_CONFIRMED":
trigger = "SELECTED_GPU_BUSY_AFTER_IDLE_GATE"
return 1
(root / "jobs/0").mkdir(parents=True, exist_ok=False)
for mode, cap_key, charge_key, output in (
("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"),
("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"),
):
began = time.monotonic()
if not snapshot(gpu)["idle"]:
trigger = "SELECTED_GPU_BUSY_BEFORE_" + mode.upper()
charges[charge_key] = time.monotonic() - began
return 1
command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)]
tracker = PeakRecords()
done = threading.Event()
reasons = []
def watch():
while not done.is_set():
try:
tracker.scan(output)
state = snapshot(gpu)
pids = set(active_command_pids(command))
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]:
reasons.append("DEVICE_VRAM_LIMIT")
if any(p["pid"] not in pids for p in state["occupants"]):
reasons.append("FOREIGN_GPU_OCCUPANT")
except Exception as exc:
reasons.append(f"MONITOR_FAILURE: {exc}")
if reasons:
for pid in active_command_pids(command):
stop_group(pid)
# Also catch a child spawned just after the failing
# sample while the supervisor was taking its lock.
done.wait(.1)
continue
done.wait(.5)
monitor = threading.Thread(target=watch, daemon=True)
monitor.start()
try:
if reasons:
raise RuntimeError(reasons[0])
cap = budget[cap_key] - (time.monotonic() - began) - 10
receipt = supervise(command, root, mode, dict(
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key],
job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"],
disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]),
poll=.2, gpu_uuid=gpu)
finally:
done.set()
monitor.join(timeout=8)
charges[charge_key] = time.monotonic() - began
if monitor.is_alive():
reasons.append("MONITOR_DID_NOT_EXIT")
else:
tracker.scan(output)
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if set(tracker.witnesses) != set(tracker.maxima):
reasons.append("MISSING_RECORDED_VRAM_PEAKS")
receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima))
if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0
or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]):
worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT"
trigger = reasons[0] if reasons else worker_reason
return 1
result_path = output / "result.json"
result = read(result_path)
if result["status"] != "COMPLETE" or result["test_targets_read"] is not False:
raise RuntimeError("Unexpected worker result")
if mode == "train":
if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"]
or result["optimizer_steps"] != c["job"]["target_steps"]):
raise RuntimeError("Fold-0 final checkpoint identity differs")
checked(result["checkpoint"], result["checkpoint_sha256"])
save(root / "training-adopted.json", dict(result_sha256=sha(result_path)))
elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]:
raise RuntimeError("Validation screen scope differs")
overhead = time.monotonic() - began_entry - sum(charges.values())
if overhead > budget["finalization_gpu_seconds"]:
raise RuntimeError("Setup and finalization cost exceeded the fixed allowance")
status, trigger = "COMPLETE", result["decision"]
return 0
except Exception as exc:
trigger = f"{type(exc).__name__}: {exc}"
return 1
except KeyboardInterrupt:
trigger = "COORDINATOR_INTERRUPTED"
return 1
finally:
charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values()))
new_gpu = sum(charges.values())
save(root / "complete.json", dict(
status=status, trigger=trigger, receipts=receipts, **charges,
cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu,
cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu,
no_retry=True, test_targets_read=False, automatic_continuation=False))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("run", "train", "screen"))
parser.add_argument("root", type=Path)
args = parser.parse_args()
root = args.root.resolve()
if root != ROOT:
raise PermissionError("Only the new fixed fold-0 root is accepted")
expected_python = str(ROOT.parents[1] / "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python")
if sys.executable != expected_python:
raise PermissionError("Use the fixed original H2 environment")
validate_review(root)
contract_path = root / "contract.json"
if args.mode != "run":
coordinator = read(root / "started.json")
if process_identity(os.getppid()) != coordinator["coordinator"]:
raise PermissionError("Worker requires the live guard as its parent")
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), "Cannot bind worker lifetime to guard")
if os.getppid() != coordinator["coordinator"]["pid"]:
raise RuntimeError("Guard disappeared before worker setup")
return globals()[args.mode](root, contract(contract_path))
if os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR:
raise PermissionError("Explicit original CUDA allocator setup required")
with (root / "entry.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / "started.json").exists():
raise FileExistsError("Fold-0 attempt already started")
save(root / "started.json", dict(coordinator=process_identity(os.getpid()),
contract_sha256=sha(contract_path), unix=time.time()))
def request_stop(signum, _frame):
raise KeyboardInterrupt(signal.Signals(signum).name)
old_handler = signal.signal(signal.SIGTERM, request_stop)
try:
return dispatch_guard(root, contract_path)
finally:
signal.signal(signal.SIGTERM, old_handler)
if __name__ == "__main__":
raise SystemExit(main())
+234
View File
@@ -0,0 +1,234 @@
"""Freeze the new fold-0 contract and exercise its actual guard on CPU."""
from functools import partial
import math
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
from phase2_h1_worker_v2 import checked, save, sha
from phase3_item1b_common import ROOT as OLD_ROOT, read
import phase3_item1b_fold0 as entry
from s2_gpu_idle_gate import wait_idle
from track_a_h2_prepare import local_code_closure
BASE = entry.ROOT.parents[1]
RUNS = OLD_ROOT.parent
SEED1 = RUNS / "phase3-seed1-v2-2026-11-18"
def prepare():
root = entry.ROOT
if root.exists():
raise FileExistsError("New fold-0 preparation already exists")
pins = {}
def pin(path):
p = Path(path)
pins[str(p)] = sha(p)
return read(p)
old = pin(OLD_ROOT / "contract.json")
stop = pin(OLD_ROOT / "complete.json")
settled = pin(OLD_ROOT / "post-terminal-cost.json")
if (stop["failure"] != "RuntimeError: TOTAL_PROJECTION_OR_ACTUAL_CAP"
or settled["training_complete_sha256"] != pins[str(OLD_ROOT / "complete.json")]
or settled["postfit_started"] is not False or not settled["freeze_tail_charged_once"]):
raise ValueError("Historical budget-stop record differs")
totals = {
"item1": pin(RUNS / "phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json")[
"cumulative_item_gpu_seconds"],
"seed1": pin(RUNS / "phase3-seed1-postfit-2026-11-18/post-terminal-cost.json")[
"cumulative_seed1_gpu_seconds"],
"item1b": settled["cumulative_gpu_seconds"],
"c": pin(RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
"cumulative_item_gpu_seconds"],
"b": pin(RUNS / "phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
"cumulative_item_gpu_seconds"],
}
history_path = OLD_ROOT / "jobs/0/worker/history/001.json"
epoch = pin(history_path)
assignments = {i: pin(OLD_ROOT / f"jobs/{i}/assignment.json") for i in (0, 2)}
identities = {i: pin(OLD_ROOT / f"jobs/{i}/worker/worker-identity.json") for i in (0, 2)}
start = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.started.json")["started_unix"] for i in (0, 2)}
elapsed = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.receipt.json")["wall_seconds"] for i in (0, 2)}
gpu = assignments[0]["gpu_uuid"]
sample = pin(OLD_ROOT / "jobs/0/gpu-sample.json")
observed = {row["pid"] for row in sample["rows"]}
if (assignments[2]["gpu_uuid"] != gpu or len(observed) != 3
or not {p["pid"] for p in identities.values()} <= observed
or not max(start.values()) < sample["unix"] < min(start[i] + elapsed[i] for i in start)
or epoch["optimizer_steps"] != len(old["jobs"][0]["learning"])
or epoch["stage"] != "EPOCH_COMPLETE"):
raise ValueError("Missing same-card fold-0 timing witness")
solo = []
for obs_path in sorted((SEED1 / "jobs/0/epoch-host-observations").glob("*.json")):
obs = read(obs_path)
on_card = [p for p in obs["gpu_processes"] if p["on_assigned_gpu"]]
if (obs["status"] != "OBSERVED" or obs["errors"] or obs["assigned_gpu_uuid"] != gpu
or len(on_card) != 1 or not on_card[0]["own_v2_worker"]):
continue
row_path = Path(obs["receipt_path"])
row = read(checked(row_path, obs["receipt_sha256"]))
solo.append((obs_path, row_path, row))
if not solo:
raise ValueError("Missing fold-0 diagnostic duration and comparison envelope")
longest_train = max(solo, key=lambda x: x[2]["train_epoch_seconds"])
longest_val = max(solo, key=lambda x: x[2].get("validation_seconds", 0))
for observation, receipt, _ in (longest_train, longest_val):
pin(observation)
pin(receipt)
train_basis = max(epoch["train_epoch_seconds"], longest_train[2]["train_epoch_seconds"])
val_basis = max(old["jobs"][0]["validation_seconds_bound"], longest_val[2]["validation_seconds"])
# The explicit compute margin applies once. Setup, bounded waiting and the
# terminal allowance do not inflate an individual training-epoch time.
train_setup = 120
train_cap = math.ceil(1.25 * (100 * train_basis + 10 * val_basis)) + train_setup
screen_cap = math.ceil(1.25 * 3 * val_basis) + 120
budget = dict(
item_gpu_limit_seconds=90000, phase3_gpu_limit_seconds=720000,
prior_item_gpu_seconds=totals["item1b"], prior_phase3_gpu_seconds=sum(totals.values()),
train_epoch_seconds_basis=train_basis, validation_seconds_basis=val_basis,
train_gpu_seconds=train_cap, screen_gpu_seconds=screen_cap,
training_setup_tail_seconds=train_setup, idle_wait_seconds=60, finalization_gpu_seconds=120,
projected_new_gpu_seconds=train_cap + screen_cap + 60 + 120,
)
if budget["projected_new_gpu_seconds"] > 90000 - totals["item1b"]:
raise ValueError(f"Fold-0 cap exceeds remaining authorization: {budget}")
pilot = pin(old["pilot_contract"])
roster = pin(old["query_roster_contract"])
fold = pilot["folds"][0]
rows = {r["case"]: r for r in roster["rows"] if r["case"] in fold["validation"]}
if set(rows) != set(fold["validation"]):
raise ValueError("Fold-0 validation roster incomplete")
references = {}
baseline = old["baseline_models"][0]
workers = [Path(baseline["result"]).parent, SEED1 / "jobs/0/worker"]
roots = [workers[0].parent, SEED1]
for seed, worker in enumerate(workers):
run_contract = roots[seed] / "contract.json"
run_meta = pin(run_contract)
result = pin(worker / "result.json")
stats_path = worker / "standardization.json"
stats = pin(stats_path)
checked(stats_path, result["standardization_sha256"])
checkpoint = checked(result["checkpoint"], result["checkpoint_sha256"])
pins[str(checkpoint)] = result["checkpoint_sha256"]
if (run_meta["seed"] != seed or result["completed_epochs"] != 200
or result["optimizer_steps"] != 200 * len(fold["learning"])
or stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]):
raise ValueError("Reference final checkpoint scope differs")
references[f"long_seed{seed}"] = dict(
checkpoint=str(checkpoint), checkpoint_sha256=result["checkpoint_sha256"],
standardization=str(stats_path), standardization_sha256=result["standardization_sha256"],
seed=seed, epochs=200, training_contract_sha256=pins[str(run_contract)])
root.mkdir(exist_ok=False)
comparison = root / "comparison-contract.json"
save(comparison, dict(
ruling=entry.RULING, fold=0, validation_cases=fold["validation"],
validation_components=len({row["component_id"] for row in rows.values()}),
score="Relative L2 on the frozen validation query with original PPS multiplicity, then "
"equal means by component. Negative predictions remain visible.",
paired_references=list(references), delta_short_at_most=-.001,
effect_strictly_greater_than_abs_seed_delta=True, duration="100 complete epochs, final epoch only",
warmup="Original 5-epoch warmup and full cosine decay over 95 epochs",
train_fields="Original seed0, order, 19 inputs, optimizer, MSE and numerical recipe",
test_allowed=False, automatic_continuation=False))
codes = local_code_closure([Path(__file__), Path(entry.__file__)] +
list((BASE / "tools/s2_models").glob("*.py")))
for path, digest in old["code_sha256"].items():
checked(path, digest)
codes.setdefault(path, digest)
source = root / "timing-and-budget.json"
save(source, dict(
prior_phase3_costs=totals, budget=budget, remaining_item_gpu_seconds=90000 - totals["item1b"],
same_card_fold0_epoch_seconds=epoch["train_epoch_seconds"], gpu_uuid=gpu,
same_card_train_peers_instantiated={str(i): p["pid"] for i, p in identities.items()},
sampled_cuda_pids=sorted(observed), sample_unix=sample["unix"], worker_intervals=[
dict(fold=i, started_unix=start[i], ended_unix=start[i] + elapsed[i]) for i in start],
same_fold_solo_epoch_records=len(solo), longest_solo_train_seconds=longest_train[2]["train_epoch_seconds"],
longest_solo_validation_seconds=longest_val[2]["validation_seconds"],
limitation="The two workers started together. Shared occupancy is sampled near epoch "
"completion. The third CUDA context is observed only at that sample.",
target_admission="Selected V100 starts idle, with stop on foreign occupancy. "
"The co-located trace is a conservative budget input."))
pins[str(source)] = sha(source)
contract = root / "contract.json"
save(contract, dict(
version="phase3-item1b-fold0-screen-v1", ruling=entry.RULING, fold=0, screen_fold=0, seed=0,
epochs=100, max_fits=1, retry=False, test_targets_allowed=False, gpu_uuid=gpu,
worker_cpus=assignments[0]["worker_cpus"], total_cpu_seconds=30000, max_processes=1,
rss_bytes=4 * entry.GIB, disk_bytes=8 * entry.GIB, device_vram_bytes=4 * entry.GIB,
recorded_vram_bytes=2 * entry.GIB, job=old["jobs"][0], budget=budget, references=references,
pilot_contract=old["pilot_contract"], query_roster_contract=old["query_roster_contract"],
comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison),
source_sha256=pins, code_sha256=codes))
return contract
def preflight(contract):
c = entry.contract(contract)
results = []
for busy in (False, True):
simulated_time = 0.
def sleep(seconds):
nonlocal simulated_time
simulated_time += seconds
def snapshot(uuid):
if uuid != c["gpu_uuid"]:
raise ValueError("Incorrect GPU identity")
return dict(uuid=uuid, idle=not busy, memory_MiB=1500 if busy else 5, occupants=[])
def simulate(command, guard_root, mode, limits, *, poll, gpu_uuid):
if (guard_root != root or command[2:] != [mode, str(root)] or gpu_uuid != c["gpu_uuid"]
or not 0 < limits["job_wall_seconds"] < c["budget"][f"{mode}_gpu_seconds"]
or limits["tree_rss_bytes"] != c["rss_bytes"] or limits["disk_bytes"] != c["disk_bytes"]
or limits["max_processes"] != c["max_processes"]
or limits["total_cpu_seconds"] != c["total_cpu_seconds"]):
raise ValueError("Actual guard did not propagate frozen fields")
record = dict(status="COMPLETE", test_targets_read=False, peak_allocated_bytes=1024,
peak_reserved_bytes=2048)
if mode == "train":
out = root / "jobs/0/worker"
out.mkdir()
checkpoint = out / "simulated-checkpoint"
checkpoint.write_bytes(b"CPU mock only")
record.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"],
checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint))
else:
out = root / "screen"
out.mkdir()
record.update(parts=len(c["job"]["validation"]), automatic_continuation=False,
decision="SCREEN_PASS")
save(out / "result.json", record)
return dict(status="PASS", returncode=0, monitor_errors=0)
with TemporaryDirectory(prefix="meltmind-fold0-guard-") as folder:
root = Path(folder)
with (patch.object(entry, "snapshot", side_effect=snapshot),
patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: simulated_time, sleep=sleep)),
patch.object(entry, "supervise", side_effect=simulate) as dispatch):
exit_code = entry.dispatch_guard(root, contract)
terminal = read(root / "complete.json")
gate = read(root / "idle-gate.json")
expected = (1, "IDLE_TIMEOUT", []) if busy else (0, "SCREEN_PASS", ["train", "screen"])
actual = (exit_code, terminal["trigger"], [call.args[2] for call in dispatch.call_args_list])
if actual != expected:
raise RuntimeError(f"Actual guard CPU simulation differed: {actual}")
results.append(dict(case="busy-timeout" if busy else "idle-dispatch",
exit_code=exit_code, trigger=terminal["trigger"], mock_dispatches=actual[2],
gate_wait_seconds=gate["wall_seconds"]))
return dict(status="PASS", actual_contract_sha256=sha(contract), cases=results,
real_gpu_or_worker_started=False, scientific_arrays_read=False, review_status="PENDING")
if __name__ == "__main__":
path = prepare()
receipt = preflight(path)
save(path.parent / "cpu-guard-preflight.json", receipt)
print(receipt)