"""Freeze the new fold-0 contract and exercise its actual guard on CPU.""" from functools import partial import math from pathlib import Path from tempfile import TemporaryDirectory from unittest.mock import patch from phase2_h1_worker_v2 import checked, save, sha from phase3_item1b_common import ROOT as OLD_ROOT, read import phase3_item1b_fold0 as entry from s2_gpu_idle_gate import wait_idle from track_a_h2_prepare import local_code_closure BASE = entry.ROOT.parents[1] RUNS = OLD_ROOT.parent SEED1 = RUNS / "phase3-seed1-v2-2026-11-18" def prepare(): root = entry.ROOT if root.exists(): raise FileExistsError("New fold-0 preparation already exists") pins = {} def pin(path): p = Path(path) pins[str(p)] = sha(p) return read(p) old = pin(OLD_ROOT / "contract.json") stop = pin(OLD_ROOT / "complete.json") settled = pin(OLD_ROOT / "post-terminal-cost.json") if (stop["failure"] != "RuntimeError: TOTAL_PROJECTION_OR_ACTUAL_CAP" or settled["training_complete_sha256"] != pins[str(OLD_ROOT / "complete.json")] or settled["postfit_started"] is not False or not settled["freeze_tail_charged_once"]): raise ValueError("Historical budget-stop record differs") totals = { "item1": pin(RUNS / "phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json")[ "cumulative_item_gpu_seconds"], "seed1": pin(RUNS / "phase3-seed1-postfit-2026-11-18/post-terminal-cost.json")[ "cumulative_seed1_gpu_seconds"], "item1b": settled["cumulative_gpu_seconds"], "c": pin(RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[ "cumulative_item_gpu_seconds"], "b": pin(RUNS / "phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[ "cumulative_item_gpu_seconds"], } history_path = OLD_ROOT / "jobs/0/worker/history/001.json" epoch = pin(history_path) assignments = {i: pin(OLD_ROOT / f"jobs/{i}/assignment.json") for i in (0, 2)} identities = {i: pin(OLD_ROOT / f"jobs/{i}/worker/worker-identity.json") for i in (0, 2)} start = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.started.json")["started_unix"] for i in (0, 2)} elapsed = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.receipt.json")["wall_seconds"] for i in (0, 2)} gpu = assignments[0]["gpu_uuid"] sample = pin(OLD_ROOT / "jobs/0/gpu-sample.json") observed = {row["pid"] for row in sample["rows"]} if (assignments[2]["gpu_uuid"] != gpu or len(observed) != 3 or not {p["pid"] for p in identities.values()} <= observed or not max(start.values()) < sample["unix"] < min(start[i] + elapsed[i] for i in start) or epoch["optimizer_steps"] != len(old["jobs"][0]["learning"]) or epoch["stage"] != "EPOCH_COMPLETE"): raise ValueError("Missing same-card fold-0 timing witness") solo = [] for obs_path in sorted((SEED1 / "jobs/0/epoch-host-observations").glob("*.json")): obs = read(obs_path) on_card = [p for p in obs["gpu_processes"] if p["on_assigned_gpu"]] if (obs["status"] != "OBSERVED" or obs["errors"] or obs["assigned_gpu_uuid"] != gpu or len(on_card) != 1 or not on_card[0]["own_v2_worker"]): continue row_path = Path(obs["receipt_path"]) row = read(checked(row_path, obs["receipt_sha256"])) solo.append((obs_path, row_path, row)) if not solo: raise ValueError("Missing fold-0 diagnostic duration and comparison envelope") longest_train = max(solo, key=lambda x: x[2]["train_epoch_seconds"]) longest_val = max(solo, key=lambda x: x[2].get("validation_seconds", 0)) for observation, receipt, _ in (longest_train, longest_val): pin(observation) pin(receipt) train_basis = max(epoch["train_epoch_seconds"], longest_train[2]["train_epoch_seconds"]) val_basis = max(old["jobs"][0]["validation_seconds_bound"], longest_val[2]["validation_seconds"]) # The explicit compute margin applies once. Setup, bounded waiting and the # terminal allowance do not inflate an individual training-epoch time. train_setup = 120 train_cap = math.ceil(1.25 * (100 * train_basis + 10 * val_basis)) + train_setup screen_cap = math.ceil(1.25 * 3 * val_basis) + 120 budget = dict( item_gpu_limit_seconds=90000, phase3_gpu_limit_seconds=720000, prior_item_gpu_seconds=totals["item1b"], prior_phase3_gpu_seconds=sum(totals.values()), train_epoch_seconds_basis=train_basis, validation_seconds_basis=val_basis, train_gpu_seconds=train_cap, screen_gpu_seconds=screen_cap, training_setup_tail_seconds=train_setup, idle_wait_seconds=60, finalization_gpu_seconds=120, projected_new_gpu_seconds=train_cap + screen_cap + 60 + 120, ) if budget["projected_new_gpu_seconds"] > 90000 - totals["item1b"]: raise ValueError(f"Fold-0 cap exceeds remaining authorization: {budget}") pilot = pin(old["pilot_contract"]) roster = pin(old["query_roster_contract"]) fold = pilot["folds"][0] rows = {r["case"]: r for r in roster["rows"] if r["case"] in fold["validation"]} if set(rows) != set(fold["validation"]): raise ValueError("Fold-0 validation roster incomplete") references = {} baseline = old["baseline_models"][0] workers = [Path(baseline["result"]).parent, SEED1 / "jobs/0/worker"] roots = [workers[0].parent, SEED1] for seed, worker in enumerate(workers): run_contract = roots[seed] / "contract.json" run_meta = pin(run_contract) result = pin(worker / "result.json") stats_path = worker / "standardization.json" stats = pin(stats_path) checked(stats_path, result["standardization_sha256"]) checkpoint = checked(result["checkpoint"], result["checkpoint_sha256"]) pins[str(checkpoint)] = result["checkpoint_sha256"] if (run_meta["seed"] != seed or result["completed_epochs"] != 200 or result["optimizer_steps"] != 200 * len(fold["learning"]) or stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]): raise ValueError("Reference final checkpoint scope differs") references[f"long_seed{seed}"] = dict( checkpoint=str(checkpoint), checkpoint_sha256=result["checkpoint_sha256"], standardization=str(stats_path), standardization_sha256=result["standardization_sha256"], seed=seed, epochs=200, training_contract_sha256=pins[str(run_contract)]) root.mkdir(exist_ok=False) comparison = root / "comparison-contract.json" save(comparison, dict( ruling=entry.RULING, fold=0, validation_cases=fold["validation"], validation_components=len({row["component_id"] for row in rows.values()}), score="Relative L2 on the frozen validation query with original PPS multiplicity, then " "equal means by component. Negative predictions remain visible.", paired_references=list(references), delta_short_at_most=-.001, effect_strictly_greater_than_abs_seed_delta=True, duration="100 complete epochs, final epoch only", warmup="Original 5-epoch warmup and full cosine decay over 95 epochs", train_fields="Original seed0, order, 19 inputs, optimizer, MSE and numerical recipe", test_allowed=False, automatic_continuation=False)) codes = local_code_closure([Path(__file__), Path(entry.__file__)] + list((BASE / "tools/s2_models").glob("*.py"))) for path, digest in old["code_sha256"].items(): checked(path, digest) codes.setdefault(path, digest) source = root / "timing-and-budget.json" save(source, dict( prior_phase3_costs=totals, budget=budget, remaining_item_gpu_seconds=90000 - totals["item1b"], same_card_fold0_epoch_seconds=epoch["train_epoch_seconds"], gpu_uuid=gpu, same_card_train_peers_instantiated={str(i): p["pid"] for i, p in identities.items()}, sampled_cuda_pids=sorted(observed), sample_unix=sample["unix"], worker_intervals=[ dict(fold=i, started_unix=start[i], ended_unix=start[i] + elapsed[i]) for i in start], same_fold_solo_epoch_records=len(solo), longest_solo_train_seconds=longest_train[2]["train_epoch_seconds"], longest_solo_validation_seconds=longest_val[2]["validation_seconds"], limitation="The two workers started together. Shared occupancy is sampled near epoch " "completion. The third CUDA context is observed only at that sample.", target_admission="Selected V100 starts idle, with stop on foreign occupancy. " "The co-located trace is a conservative budget input.")) pins[str(source)] = sha(source) contract = root / "contract.json" save(contract, dict( version="phase3-item1b-fold0-screen-v1", ruling=entry.RULING, fold=0, screen_fold=0, seed=0, epochs=100, max_fits=1, retry=False, test_targets_allowed=False, gpu_uuid=gpu, worker_cpus=assignments[0]["worker_cpus"], total_cpu_seconds=30000, max_processes=1, rss_bytes=4 * entry.GIB, disk_bytes=8 * entry.GIB, device_vram_bytes=4 * entry.GIB, recorded_vram_bytes=2 * entry.GIB, job=old["jobs"][0], budget=budget, references=references, pilot_contract=old["pilot_contract"], query_roster_contract=old["query_roster_contract"], comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison), source_sha256=pins, code_sha256=codes)) return contract def preflight(contract): c = entry.contract(contract) results = [] for busy in (False, True): simulated_time = 0. def sleep(seconds): nonlocal simulated_time simulated_time += seconds def snapshot(uuid): if uuid != c["gpu_uuid"]: raise ValueError("Incorrect GPU identity") return dict(uuid=uuid, idle=not busy, memory_MiB=1500 if busy else 5, occupants=[]) def simulate(command, guard_root, mode, limits, *, poll, gpu_uuid): if (guard_root != root or command[2:] != [mode, str(root)] or gpu_uuid != c["gpu_uuid"] or not 0 < limits["job_wall_seconds"] < c["budget"][f"{mode}_gpu_seconds"] or limits["tree_rss_bytes"] != c["rss_bytes"] or limits["disk_bytes"] != c["disk_bytes"] or limits["max_processes"] != c["max_processes"] or limits["total_cpu_seconds"] != c["total_cpu_seconds"]): raise ValueError("Actual guard did not propagate frozen fields") record = dict(status="COMPLETE", test_targets_read=False, peak_allocated_bytes=1024, peak_reserved_bytes=2048) if mode == "train": out = root / "jobs/0/worker" out.mkdir() checkpoint = out / "simulated-checkpoint" checkpoint.write_bytes(b"CPU mock only") record.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"], checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint)) else: out = root / "screen" out.mkdir() record.update(parts=len(c["job"]["validation"]), automatic_continuation=False, decision="SCREEN_PASS") save(out / "result.json", record) return dict(status="PASS", returncode=0, monitor_errors=0) with TemporaryDirectory(prefix="meltmind-fold0-guard-") as folder: root = Path(folder) with (patch.object(entry, "snapshot", side_effect=snapshot), patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: simulated_time, sleep=sleep)), patch.object(entry, "supervise", side_effect=simulate) as dispatch): exit_code = entry.dispatch_guard(root, contract) terminal = read(root / "complete.json") gate = read(root / "idle-gate.json") expected = (1, "IDLE_TIMEOUT", []) if busy else (0, "SCREEN_PASS", ["train", "screen"]) actual = (exit_code, terminal["trigger"], [call.args[2] for call in dispatch.call_args_list]) if actual != expected: raise RuntimeError(f"Actual guard CPU simulation differed: {actual}") results.append(dict(case="busy-timeout" if busy else "idle-dispatch", exit_code=exit_code, trigger=terminal["trigger"], mock_dispatches=actual[2], gate_wait_seconds=gate["wall_seconds"])) return dict(status="PASS", actual_contract_sha256=sha(contract), cases=results, real_gpu_or_worker_started=False, scientific_arrays_read=False, review_status="PENDING") if __name__ == "__main__": path = prepare() receipt = preflight(path) save(path.parent / "cpu-guard-preflight.json", receipt) print(receipt)