235 lines
13 KiB
Python
235 lines
13 KiB
Python
"""Freeze the new fold-0 contract and exercise its actual guard on CPU."""
|
|
|
|
from functools import partial
|
|
import math
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from unittest.mock import patch
|
|
|
|
from phase2_h1_worker_v2 import checked, save, sha
|
|
from phase3_item1b_common import ROOT as OLD_ROOT, read
|
|
import phase3_item1b_fold0 as entry
|
|
from s2_gpu_idle_gate import wait_idle
|
|
from track_a_h2_prepare import local_code_closure
|
|
|
|
|
|
BASE = entry.ROOT.parents[1]
|
|
RUNS = OLD_ROOT.parent
|
|
SEED1 = RUNS / "phase3-seed1-v2-2026-11-18"
|
|
|
|
|
|
def prepare():
|
|
root = entry.ROOT
|
|
if root.exists():
|
|
raise FileExistsError("New fold-0 preparation already exists")
|
|
pins = {}
|
|
|
|
def pin(path):
|
|
p = Path(path)
|
|
pins[str(p)] = sha(p)
|
|
return read(p)
|
|
|
|
old = pin(OLD_ROOT / "contract.json")
|
|
stop = pin(OLD_ROOT / "complete.json")
|
|
settled = pin(OLD_ROOT / "post-terminal-cost.json")
|
|
if (stop["failure"] != "RuntimeError: TOTAL_PROJECTION_OR_ACTUAL_CAP"
|
|
or settled["training_complete_sha256"] != pins[str(OLD_ROOT / "complete.json")]
|
|
or settled["postfit_started"] is not False or not settled["freeze_tail_charged_once"]):
|
|
raise ValueError("Historical budget-stop record differs")
|
|
totals = {
|
|
"item1": pin(RUNS / "phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json")[
|
|
"cumulative_item_gpu_seconds"],
|
|
"seed1": pin(RUNS / "phase3-seed1-postfit-2026-11-18/post-terminal-cost.json")[
|
|
"cumulative_seed1_gpu_seconds"],
|
|
"item1b": settled["cumulative_gpu_seconds"],
|
|
"c": pin(RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
|
|
"cumulative_item_gpu_seconds"],
|
|
"b": pin(RUNS / "phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
|
|
"cumulative_item_gpu_seconds"],
|
|
}
|
|
history_path = OLD_ROOT / "jobs/0/worker/history/001.json"
|
|
epoch = pin(history_path)
|
|
assignments = {i: pin(OLD_ROOT / f"jobs/{i}/assignment.json") for i in (0, 2)}
|
|
identities = {i: pin(OLD_ROOT / f"jobs/{i}/worker/worker-identity.json") for i in (0, 2)}
|
|
start = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.started.json")["started_unix"] for i in (0, 2)}
|
|
elapsed = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.receipt.json")["wall_seconds"] for i in (0, 2)}
|
|
gpu = assignments[0]["gpu_uuid"]
|
|
sample = pin(OLD_ROOT / "jobs/0/gpu-sample.json")
|
|
observed = {row["pid"] for row in sample["rows"]}
|
|
if (assignments[2]["gpu_uuid"] != gpu or len(observed) != 3
|
|
or not {p["pid"] for p in identities.values()} <= observed
|
|
or not max(start.values()) < sample["unix"] < min(start[i] + elapsed[i] for i in start)
|
|
or epoch["optimizer_steps"] != len(old["jobs"][0]["learning"])
|
|
or epoch["stage"] != "EPOCH_COMPLETE"):
|
|
raise ValueError("Missing same-card fold-0 timing witness")
|
|
solo = []
|
|
for obs_path in sorted((SEED1 / "jobs/0/epoch-host-observations").glob("*.json")):
|
|
obs = read(obs_path)
|
|
on_card = [p for p in obs["gpu_processes"] if p["on_assigned_gpu"]]
|
|
if (obs["status"] != "OBSERVED" or obs["errors"] or obs["assigned_gpu_uuid"] != gpu
|
|
or len(on_card) != 1 or not on_card[0]["own_v2_worker"]):
|
|
continue
|
|
row_path = Path(obs["receipt_path"])
|
|
row = read(checked(row_path, obs["receipt_sha256"]))
|
|
solo.append((obs_path, row_path, row))
|
|
if not solo:
|
|
raise ValueError("Missing fold-0 diagnostic duration and comparison envelope")
|
|
longest_train = max(solo, key=lambda x: x[2]["train_epoch_seconds"])
|
|
longest_val = max(solo, key=lambda x: x[2].get("validation_seconds", 0))
|
|
for observation, receipt, _ in (longest_train, longest_val):
|
|
pin(observation)
|
|
pin(receipt)
|
|
train_basis = max(epoch["train_epoch_seconds"], longest_train[2]["train_epoch_seconds"])
|
|
val_basis = max(old["jobs"][0]["validation_seconds_bound"], longest_val[2]["validation_seconds"])
|
|
# The explicit compute margin applies once. Setup, bounded waiting and the
|
|
# terminal allowance do not inflate an individual training-epoch time.
|
|
train_setup = 120
|
|
train_cap = math.ceil(1.25 * (100 * train_basis + 10 * val_basis)) + train_setup
|
|
screen_cap = math.ceil(1.25 * 3 * val_basis) + 120
|
|
budget = dict(
|
|
item_gpu_limit_seconds=90000, phase3_gpu_limit_seconds=720000,
|
|
prior_item_gpu_seconds=totals["item1b"], prior_phase3_gpu_seconds=sum(totals.values()),
|
|
train_epoch_seconds_basis=train_basis, validation_seconds_basis=val_basis,
|
|
train_gpu_seconds=train_cap, screen_gpu_seconds=screen_cap,
|
|
training_setup_tail_seconds=train_setup, idle_wait_seconds=60, finalization_gpu_seconds=120,
|
|
projected_new_gpu_seconds=train_cap + screen_cap + 60 + 120,
|
|
)
|
|
if budget["projected_new_gpu_seconds"] > 90000 - totals["item1b"]:
|
|
raise ValueError(f"Fold-0 cap exceeds remaining authorization: {budget}")
|
|
|
|
pilot = pin(old["pilot_contract"])
|
|
roster = pin(old["query_roster_contract"])
|
|
fold = pilot["folds"][0]
|
|
rows = {r["case"]: r for r in roster["rows"] if r["case"] in fold["validation"]}
|
|
if set(rows) != set(fold["validation"]):
|
|
raise ValueError("Fold-0 validation roster incomplete")
|
|
references = {}
|
|
baseline = old["baseline_models"][0]
|
|
workers = [Path(baseline["result"]).parent, SEED1 / "jobs/0/worker"]
|
|
roots = [workers[0].parent, SEED1]
|
|
for seed, worker in enumerate(workers):
|
|
run_contract = roots[seed] / "contract.json"
|
|
run_meta = pin(run_contract)
|
|
result = pin(worker / "result.json")
|
|
stats_path = worker / "standardization.json"
|
|
stats = pin(stats_path)
|
|
checked(stats_path, result["standardization_sha256"])
|
|
checkpoint = checked(result["checkpoint"], result["checkpoint_sha256"])
|
|
pins[str(checkpoint)] = result["checkpoint_sha256"]
|
|
if (run_meta["seed"] != seed or result["completed_epochs"] != 200
|
|
or result["optimizer_steps"] != 200 * len(fold["learning"])
|
|
or stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]):
|
|
raise ValueError("Reference final checkpoint scope differs")
|
|
references[f"long_seed{seed}"] = dict(
|
|
checkpoint=str(checkpoint), checkpoint_sha256=result["checkpoint_sha256"],
|
|
standardization=str(stats_path), standardization_sha256=result["standardization_sha256"],
|
|
seed=seed, epochs=200, training_contract_sha256=pins[str(run_contract)])
|
|
|
|
root.mkdir(exist_ok=False)
|
|
comparison = root / "comparison-contract.json"
|
|
save(comparison, dict(
|
|
ruling=entry.RULING, fold=0, validation_cases=fold["validation"],
|
|
validation_components=len({row["component_id"] for row in rows.values()}),
|
|
score="Relative L2 on the frozen validation query with original PPS multiplicity, then "
|
|
"equal means by component. Negative predictions remain visible.",
|
|
paired_references=list(references), delta_short_at_most=-.001,
|
|
effect_strictly_greater_than_abs_seed_delta=True, duration="100 complete epochs, final epoch only",
|
|
warmup="Original 5-epoch warmup and full cosine decay over 95 epochs",
|
|
train_fields="Original seed0, order, 19 inputs, optimizer, MSE and numerical recipe",
|
|
test_allowed=False, automatic_continuation=False))
|
|
codes = local_code_closure([Path(__file__), Path(entry.__file__)] +
|
|
list((BASE / "tools/s2_models").glob("*.py")))
|
|
for path, digest in old["code_sha256"].items():
|
|
checked(path, digest)
|
|
codes.setdefault(path, digest)
|
|
source = root / "timing-and-budget.json"
|
|
save(source, dict(
|
|
prior_phase3_costs=totals, budget=budget, remaining_item_gpu_seconds=90000 - totals["item1b"],
|
|
same_card_fold0_epoch_seconds=epoch["train_epoch_seconds"], gpu_uuid=gpu,
|
|
same_card_train_peers_instantiated={str(i): p["pid"] for i, p in identities.items()},
|
|
sampled_cuda_pids=sorted(observed), sample_unix=sample["unix"], worker_intervals=[
|
|
dict(fold=i, started_unix=start[i], ended_unix=start[i] + elapsed[i]) for i in start],
|
|
same_fold_solo_epoch_records=len(solo), longest_solo_train_seconds=longest_train[2]["train_epoch_seconds"],
|
|
longest_solo_validation_seconds=longest_val[2]["validation_seconds"],
|
|
limitation="The two workers started together. Shared occupancy is sampled near epoch "
|
|
"completion. The third CUDA context is observed only at that sample.",
|
|
target_admission="Selected V100 starts idle, with stop on foreign occupancy. "
|
|
"The co-located trace is a conservative budget input."))
|
|
pins[str(source)] = sha(source)
|
|
contract = root / "contract.json"
|
|
save(contract, dict(
|
|
version="phase3-item1b-fold0-screen-v1", ruling=entry.RULING, fold=0, screen_fold=0, seed=0,
|
|
epochs=100, max_fits=1, retry=False, test_targets_allowed=False, gpu_uuid=gpu,
|
|
worker_cpus=assignments[0]["worker_cpus"], total_cpu_seconds=30000, max_processes=1,
|
|
rss_bytes=4 * entry.GIB, disk_bytes=8 * entry.GIB, device_vram_bytes=4 * entry.GIB,
|
|
recorded_vram_bytes=2 * entry.GIB, job=old["jobs"][0], budget=budget, references=references,
|
|
pilot_contract=old["pilot_contract"], query_roster_contract=old["query_roster_contract"],
|
|
comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison),
|
|
source_sha256=pins, code_sha256=codes))
|
|
return contract
|
|
|
|
|
|
def preflight(contract):
|
|
c = entry.contract(contract)
|
|
results = []
|
|
for busy in (False, True):
|
|
simulated_time = 0.
|
|
|
|
def sleep(seconds):
|
|
nonlocal simulated_time
|
|
simulated_time += seconds
|
|
|
|
def snapshot(uuid):
|
|
if uuid != c["gpu_uuid"]:
|
|
raise ValueError("Incorrect GPU identity")
|
|
return dict(uuid=uuid, idle=not busy, memory_MiB=1500 if busy else 5, occupants=[])
|
|
|
|
def simulate(command, guard_root, mode, limits, *, poll, gpu_uuid):
|
|
if (guard_root != root or command[2:] != [mode, str(root)] or gpu_uuid != c["gpu_uuid"]
|
|
or not 0 < limits["job_wall_seconds"] < c["budget"][f"{mode}_gpu_seconds"]
|
|
or limits["tree_rss_bytes"] != c["rss_bytes"] or limits["disk_bytes"] != c["disk_bytes"]
|
|
or limits["max_processes"] != c["max_processes"]
|
|
or limits["total_cpu_seconds"] != c["total_cpu_seconds"]):
|
|
raise ValueError("Actual guard did not propagate frozen fields")
|
|
record = dict(status="COMPLETE", test_targets_read=False, peak_allocated_bytes=1024,
|
|
peak_reserved_bytes=2048)
|
|
if mode == "train":
|
|
out = root / "jobs/0/worker"
|
|
out.mkdir()
|
|
checkpoint = out / "simulated-checkpoint"
|
|
checkpoint.write_bytes(b"CPU mock only")
|
|
record.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"],
|
|
checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint))
|
|
else:
|
|
out = root / "screen"
|
|
out.mkdir()
|
|
record.update(parts=len(c["job"]["validation"]), automatic_continuation=False,
|
|
decision="SCREEN_PASS")
|
|
save(out / "result.json", record)
|
|
return dict(status="PASS", returncode=0, monitor_errors=0)
|
|
|
|
with TemporaryDirectory(prefix="meltmind-fold0-guard-") as folder:
|
|
root = Path(folder)
|
|
with (patch.object(entry, "snapshot", side_effect=snapshot),
|
|
patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: simulated_time, sleep=sleep)),
|
|
patch.object(entry, "supervise", side_effect=simulate) as dispatch):
|
|
exit_code = entry.dispatch_guard(root, contract)
|
|
terminal = read(root / "complete.json")
|
|
gate = read(root / "idle-gate.json")
|
|
expected = (1, "IDLE_TIMEOUT", []) if busy else (0, "SCREEN_PASS", ["train", "screen"])
|
|
actual = (exit_code, terminal["trigger"], [call.args[2] for call in dispatch.call_args_list])
|
|
if actual != expected:
|
|
raise RuntimeError(f"Actual guard CPU simulation differed: {actual}")
|
|
results.append(dict(case="busy-timeout" if busy else "idle-dispatch",
|
|
exit_code=exit_code, trigger=terminal["trigger"], mock_dispatches=actual[2],
|
|
gate_wait_seconds=gate["wall_seconds"]))
|
|
return dict(status="PASS", actual_contract_sha256=sha(contract), cases=results,
|
|
real_gpu_or_worker_started=False, scientific_arrays_read=False, review_status="PENDING")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
path = prepare()
|
|
receipt = preflight(path)
|
|
save(path.parent / "cpu-guard-preflight.json", receipt)
|
|
print(receipt)
|