Files
MeltMind/tools/prepare_phase3_item1b_fold0.py
T

235 lines
13 KiB
Python

"""Freeze the new fold-0 contract and exercise its actual guard on CPU."""
from functools import partial
import math
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
from phase2_h1_worker_v2 import checked, save, sha
from phase3_item1b_common import ROOT as OLD_ROOT, read
import phase3_item1b_fold0 as entry
from s2_gpu_idle_gate import wait_idle
from track_a_h2_prepare import local_code_closure
BASE = entry.ROOT.parents[1]
RUNS = OLD_ROOT.parent
SEED1 = RUNS / "phase3-seed1-v2-2026-11-18"
def prepare():
root = entry.ROOT
if root.exists():
raise FileExistsError("New fold-0 preparation already exists")
pins = {}
def pin(path):
p = Path(path)
pins[str(p)] = sha(p)
return read(p)
old = pin(OLD_ROOT / "contract.json")
stop = pin(OLD_ROOT / "complete.json")
settled = pin(OLD_ROOT / "post-terminal-cost.json")
if (stop["failure"] != "RuntimeError: TOTAL_PROJECTION_OR_ACTUAL_CAP"
or settled["training_complete_sha256"] != pins[str(OLD_ROOT / "complete.json")]
or settled["postfit_started"] is not False or not settled["freeze_tail_charged_once"]):
raise ValueError("Historical budget-stop record differs")
totals = {
"item1": pin(RUNS / "phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json")[
"cumulative_item_gpu_seconds"],
"seed1": pin(RUNS / "phase3-seed1-postfit-2026-11-18/post-terminal-cost.json")[
"cumulative_seed1_gpu_seconds"],
"item1b": settled["cumulative_gpu_seconds"],
"c": pin(RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
"cumulative_item_gpu_seconds"],
"b": pin(RUNS / "phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
"cumulative_item_gpu_seconds"],
}
history_path = OLD_ROOT / "jobs/0/worker/history/001.json"
epoch = pin(history_path)
assignments = {i: pin(OLD_ROOT / f"jobs/{i}/assignment.json") for i in (0, 2)}
identities = {i: pin(OLD_ROOT / f"jobs/{i}/worker/worker-identity.json") for i in (0, 2)}
start = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.started.json")["started_unix"] for i in (0, 2)}
elapsed = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.receipt.json")["wall_seconds"] for i in (0, 2)}
gpu = assignments[0]["gpu_uuid"]
sample = pin(OLD_ROOT / "jobs/0/gpu-sample.json")
observed = {row["pid"] for row in sample["rows"]}
if (assignments[2]["gpu_uuid"] != gpu or len(observed) != 3
or not {p["pid"] for p in identities.values()} <= observed
or not max(start.values()) < sample["unix"] < min(start[i] + elapsed[i] for i in start)
or epoch["optimizer_steps"] != len(old["jobs"][0]["learning"])
or epoch["stage"] != "EPOCH_COMPLETE"):
raise ValueError("Missing same-card fold-0 timing witness")
solo = []
for obs_path in sorted((SEED1 / "jobs/0/epoch-host-observations").glob("*.json")):
obs = read(obs_path)
on_card = [p for p in obs["gpu_processes"] if p["on_assigned_gpu"]]
if (obs["status"] != "OBSERVED" or obs["errors"] or obs["assigned_gpu_uuid"] != gpu
or len(on_card) != 1 or not on_card[0]["own_v2_worker"]):
continue
row_path = Path(obs["receipt_path"])
row = read(checked(row_path, obs["receipt_sha256"]))
solo.append((obs_path, row_path, row))
if not solo:
raise ValueError("Missing fold-0 diagnostic duration and comparison envelope")
longest_train = max(solo, key=lambda x: x[2]["train_epoch_seconds"])
longest_val = max(solo, key=lambda x: x[2].get("validation_seconds", 0))
for observation, receipt, _ in (longest_train, longest_val):
pin(observation)
pin(receipt)
train_basis = max(epoch["train_epoch_seconds"], longest_train[2]["train_epoch_seconds"])
val_basis = max(old["jobs"][0]["validation_seconds_bound"], longest_val[2]["validation_seconds"])
# The explicit compute margin applies once. Setup, bounded waiting and the
# terminal allowance do not inflate an individual training-epoch time.
train_setup = 120
train_cap = math.ceil(1.25 * (100 * train_basis + 10 * val_basis)) + train_setup
screen_cap = math.ceil(1.25 * 3 * val_basis) + 120
budget = dict(
item_gpu_limit_seconds=90000, phase3_gpu_limit_seconds=720000,
prior_item_gpu_seconds=totals["item1b"], prior_phase3_gpu_seconds=sum(totals.values()),
train_epoch_seconds_basis=train_basis, validation_seconds_basis=val_basis,
train_gpu_seconds=train_cap, screen_gpu_seconds=screen_cap,
training_setup_tail_seconds=train_setup, idle_wait_seconds=60, finalization_gpu_seconds=120,
projected_new_gpu_seconds=train_cap + screen_cap + 60 + 120,
)
if budget["projected_new_gpu_seconds"] > 90000 - totals["item1b"]:
raise ValueError(f"Fold-0 cap exceeds remaining authorization: {budget}")
pilot = pin(old["pilot_contract"])
roster = pin(old["query_roster_contract"])
fold = pilot["folds"][0]
rows = {r["case"]: r for r in roster["rows"] if r["case"] in fold["validation"]}
if set(rows) != set(fold["validation"]):
raise ValueError("Fold-0 validation roster incomplete")
references = {}
baseline = old["baseline_models"][0]
workers = [Path(baseline["result"]).parent, SEED1 / "jobs/0/worker"]
roots = [workers[0].parent, SEED1]
for seed, worker in enumerate(workers):
run_contract = roots[seed] / "contract.json"
run_meta = pin(run_contract)
result = pin(worker / "result.json")
stats_path = worker / "standardization.json"
stats = pin(stats_path)
checked(stats_path, result["standardization_sha256"])
checkpoint = checked(result["checkpoint"], result["checkpoint_sha256"])
pins[str(checkpoint)] = result["checkpoint_sha256"]
if (run_meta["seed"] != seed or result["completed_epochs"] != 200
or result["optimizer_steps"] != 200 * len(fold["learning"])
or stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]):
raise ValueError("Reference final checkpoint scope differs")
references[f"long_seed{seed}"] = dict(
checkpoint=str(checkpoint), checkpoint_sha256=result["checkpoint_sha256"],
standardization=str(stats_path), standardization_sha256=result["standardization_sha256"],
seed=seed, epochs=200, training_contract_sha256=pins[str(run_contract)])
root.mkdir(exist_ok=False)
comparison = root / "comparison-contract.json"
save(comparison, dict(
ruling=entry.RULING, fold=0, validation_cases=fold["validation"],
validation_components=len({row["component_id"] for row in rows.values()}),
score="Relative L2 on the frozen validation query with original PPS multiplicity, then "
"equal means by component. Negative predictions remain visible.",
paired_references=list(references), delta_short_at_most=-.001,
effect_strictly_greater_than_abs_seed_delta=True, duration="100 complete epochs, final epoch only",
warmup="Original 5-epoch warmup and full cosine decay over 95 epochs",
train_fields="Original seed0, order, 19 inputs, optimizer, MSE and numerical recipe",
test_allowed=False, automatic_continuation=False))
codes = local_code_closure([Path(__file__), Path(entry.__file__)] +
list((BASE / "tools/s2_models").glob("*.py")))
for path, digest in old["code_sha256"].items():
checked(path, digest)
codes.setdefault(path, digest)
source = root / "timing-and-budget.json"
save(source, dict(
prior_phase3_costs=totals, budget=budget, remaining_item_gpu_seconds=90000 - totals["item1b"],
same_card_fold0_epoch_seconds=epoch["train_epoch_seconds"], gpu_uuid=gpu,
same_card_train_peers_instantiated={str(i): p["pid"] for i, p in identities.items()},
sampled_cuda_pids=sorted(observed), sample_unix=sample["unix"], worker_intervals=[
dict(fold=i, started_unix=start[i], ended_unix=start[i] + elapsed[i]) for i in start],
same_fold_solo_epoch_records=len(solo), longest_solo_train_seconds=longest_train[2]["train_epoch_seconds"],
longest_solo_validation_seconds=longest_val[2]["validation_seconds"],
limitation="The two workers started together. Shared occupancy is sampled near epoch "
"completion. The third CUDA context is observed only at that sample.",
target_admission="Selected V100 starts idle, with stop on foreign occupancy. "
"The co-located trace is a conservative budget input."))
pins[str(source)] = sha(source)
contract = root / "contract.json"
save(contract, dict(
version="phase3-item1b-fold0-screen-v1", ruling=entry.RULING, fold=0, screen_fold=0, seed=0,
epochs=100, max_fits=1, retry=False, test_targets_allowed=False, gpu_uuid=gpu,
worker_cpus=assignments[0]["worker_cpus"], total_cpu_seconds=30000, max_processes=1,
rss_bytes=4 * entry.GIB, disk_bytes=8 * entry.GIB, device_vram_bytes=4 * entry.GIB,
recorded_vram_bytes=2 * entry.GIB, job=old["jobs"][0], budget=budget, references=references,
pilot_contract=old["pilot_contract"], query_roster_contract=old["query_roster_contract"],
comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison),
source_sha256=pins, code_sha256=codes))
return contract
def preflight(contract):
c = entry.contract(contract)
results = []
for busy in (False, True):
simulated_time = 0.
def sleep(seconds):
nonlocal simulated_time
simulated_time += seconds
def snapshot(uuid):
if uuid != c["gpu_uuid"]:
raise ValueError("Incorrect GPU identity")
return dict(uuid=uuid, idle=not busy, memory_MiB=1500 if busy else 5, occupants=[])
def simulate(command, guard_root, mode, limits, *, poll, gpu_uuid):
if (guard_root != root or command[2:] != [mode, str(root)] or gpu_uuid != c["gpu_uuid"]
or not 0 < limits["job_wall_seconds"] < c["budget"][f"{mode}_gpu_seconds"]
or limits["tree_rss_bytes"] != c["rss_bytes"] or limits["disk_bytes"] != c["disk_bytes"]
or limits["max_processes"] != c["max_processes"]
or limits["total_cpu_seconds"] != c["total_cpu_seconds"]):
raise ValueError("Actual guard did not propagate frozen fields")
record = dict(status="COMPLETE", test_targets_read=False, peak_allocated_bytes=1024,
peak_reserved_bytes=2048)
if mode == "train":
out = root / "jobs/0/worker"
out.mkdir()
checkpoint = out / "simulated-checkpoint"
checkpoint.write_bytes(b"CPU mock only")
record.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"],
checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint))
else:
out = root / "screen"
out.mkdir()
record.update(parts=len(c["job"]["validation"]), automatic_continuation=False,
decision="SCREEN_PASS")
save(out / "result.json", record)
return dict(status="PASS", returncode=0, monitor_errors=0)
with TemporaryDirectory(prefix="meltmind-fold0-guard-") as folder:
root = Path(folder)
with (patch.object(entry, "snapshot", side_effect=snapshot),
patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: simulated_time, sleep=sleep)),
patch.object(entry, "supervise", side_effect=simulate) as dispatch):
exit_code = entry.dispatch_guard(root, contract)
terminal = read(root / "complete.json")
gate = read(root / "idle-gate.json")
expected = (1, "IDLE_TIMEOUT", []) if busy else (0, "SCREEN_PASS", ["train", "screen"])
actual = (exit_code, terminal["trigger"], [call.args[2] for call in dispatch.call_args_list])
if actual != expected:
raise RuntimeError(f"Actual guard CPU simulation differed: {actual}")
results.append(dict(case="busy-timeout" if busy else "idle-dispatch",
exit_code=exit_code, trigger=terminal["trigger"], mock_dispatches=actual[2],
gate_wait_seconds=gate["wall_seconds"]))
return dict(status="PASS", actual_contract_sha256=sha(contract), cases=results,
real_gpu_or_worker_started=False, scientific_arrays_read=False, review_status="PENDING")
if __name__ == "__main__":
path = prepare()
receipt = preflight(path)
save(path.parent / "cpu-guard-preflight.json", receipt)
print(receipt)