Files
MeltMind/tools/phase3_item1b_fold0.py
T

391 lines
19 KiB
Python

"""One reviewed fold-0 attempt and its validation-only screen."""
import argparse
from collections import defaultdict
import ctypes
import fcntl
import json
import math
import os
from pathlib import Path
import shutil
import signal
import sys
import threading
import time
from types import FunctionType
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase3_item1b_common import fold_view, read
from phase3_shared_capacity import process_identity
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords, active_command_pids
from s2_recovery_gpu_supervisor import stop_group, supervise
ROOT = Path(__file__).resolve().parents[1] / "artifacts/phase3-item1b-fold0-screen-2026-11-19"
RULING = "3a48b123da54b0bcff0e7fa672fc74e8920235a0"
ALLOCATOR = "expandable_segments:True"
GIB = 2**30
ARMS = ("short_seed0", "long_seed0", "long_seed1")
def contract(path):
c = read(path)
if (c["ruling"] != RULING or c["fold"] != 0 or c["screen_fold"] != 0
or c["epochs"] != 100 or c["seed"] != 0 or c["max_fits"] != 1
or c["retry"] is not False or c["test_targets_allowed"] is not False):
raise PermissionError("Only the new fold-0 validation scope is authorized")
for p, h in (c["code_sha256"] | c["source_sha256"]).items():
checked(p, h)
comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"]))
if comparison["validation_cases"] != c["job"]["validation"]:
raise ValueError("Fold-0 validation identity differs from comparison")
budget = c["budget"]
reserves = sum(budget[k] for k in ("train_gpu_seconds", "screen_gpu_seconds",
"idle_wait_seconds", "finalization_gpu_seconds"))
if (budget["projected_new_gpu_seconds"] != reserves
or budget["prior_item_gpu_seconds"] + reserves > budget["item_gpu_limit_seconds"]
or budget["prior_phase3_gpu_seconds"] + reserves > budget["phase3_gpu_limit_seconds"]):
raise PermissionError("Phase 3 or 1b remaining budget exceeded")
return c
def folded_inputs(c):
pilot = read(c["pilot_contract"])
view = fold_view(pilot, 0)
original = view["folds"][0]
job = c["job"]
for k in ("learning", "validation", "test", "c", "clock_sha256", "learning_nodes"):
if job[k] != original[k]:
raise ValueError(f"Original fold-0 identity changed: {k}")
if job["id"] != 0 or job["target_steps"] != 100 * len(original["learning"]):
raise ValueError("Full fold-0 learning schedule required")
if set(original["validation"]) & set(original["test"]):
raise ValueError("Validation and test overlap")
return pilot, view
def train(root, c):
import numpy as np
import torch
from phase3_item1b_worker import run as frozen_worker
pilot, _ = folded_inputs(c)
if torch.__version__ != pilot["versions"]["torch"] or np.__version__ != pilot["versions"]["numpy"]:
raise ValueError("Original array and Torch versions required")
job = dict(c["job"], gpu_uuid=c["gpu_uuid"], worker_cpus=c["worker_cpus"])
digest = sha(root / "contract.json")
def first_epoch_release(queue, fold):
if queue != root or fold != 0:
raise PermissionError("Wrong worker route")
row = read(root / "jobs/0/worker/history/001.json")
budget = c["budget"]
epoch_seconds = max(row["train_epoch_seconds"], budget["train_epoch_seconds_basis"])
projection = (row["wall_seconds"] + 99 * epoch_seconds
+ 10 * budget["validation_seconds_basis"] + budget["training_setup_tail_seconds"])
passed = math.isfinite(projection) and projection <= budget["train_gpu_seconds"]
save(root / "first-epoch-projection.json", dict(
status="PASS" if passed else "STOP_BUDGET", contract_sha256=digest,
measured_epoch_seconds=row["train_epoch_seconds"], projected_train_gpu_seconds=projection))
if not passed:
raise RuntimeError("First-epoch projection exceeds fold-0 training cap")
return True
# Reuse the immutable training body. The private namespace changes only
# its root/assignment adapters and the one-fold budget gate.
namespace = dict(frozen_worker.__globals__, config=lambda _: (c, pilot),
assigned_job=lambda *_: job, first_epoch_release=first_epoch_release)
runner = FunctionType(frozen_worker.__code__, namespace, frozen_worker.__name__)
runner(root, 0, digest)
def screen(root, c):
import numpy as np
import torch
from phase2_h2_data import arguments, cached, to_device
from phase2_h2_model import H2Geo
from phase2_h2_pilot_worker import peaks
from phase2_residual_math import decode_seconds
from phase3_item1_postfit_data import load_evaluation_part
from s2_custody_common import prediction_inputs
from s2_models.objectives import relative_l2
pilot, view = folded_inputs(c)
comparison = read(c["comparison_contract"])
fold = view["folds"][0]
cases = comparison["validation_cases"]
if (cases != fold["validation"] or torch.__version__ != pilot["versions"]["torch"]
or np.__version__ != pilot["versions"]["numpy"]
or os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"]
or os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR):
raise PermissionError("Screen scope or environment differs")
train_path = root / "jobs/0/worker/result.json"
adopted = read(root / "training-adopted.json")
short = read(checked(train_path, adopted["result_sha256"]))
models = dict(c["references"])
models["short_seed0"] = dict(
checkpoint=short["checkpoint"], checkpoint_sha256=short["checkpoint_sha256"],
standardization=str(train_path.with_name("standardization.json")),
standardization_sha256=short["standardization_sha256"], seed=0, epochs=100,
training_contract_sha256=sha(root / "contract.json"))
roster = {r["case"]: r for r in read(c["query_roster_contract"])["rows"] if r["case"] in cases}
if set(roster) != set(cases) or any(
roster[case]["component_id"] != pilot["parts"][case]["clock_group"]
or roster[case]["v5_cache"]["output_sha256"] != pilot["parts"][case]["output_sha256"]
for case in cases
):
raise ValueError("Frozen validation query-to-cache identity differs")
os.sched_setaffinity(0, c["worker_cpus"])
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
torch.cuda.set_device(0)
torch.cuda.reset_peak_memory_stats()
out = root / "screen"
out.mkdir(exist_ok=False)
predictions = {}
with torch.no_grad():
for arm in ARMS:
ref = models[arm]
stats = read(checked(ref["standardization"], ref["standardization_sha256"]))
blob = torch.load(checked(ref["checkpoint"], ref["checkpoint_sha256"]),
map_location="cpu", weights_only=True)
required = dict(completed_epochs=ref["epochs"], fold=0, seed=ref["seed"],
role="last_epoch_only", contract_sha256=ref["training_contract_sha256"],
optimizer_steps=ref["epochs"] * len(fold["learning"]))
if any(blob.get(k) != v for k, v in required.items()):
raise ValueError(f"Final checkpoint identity: {arm}")
if stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]:
raise ValueError("Screen learning-only normalization mismatch")
model = H2Geo().cuda()
model.load_state_dict(blob["model"], strict=True)
del blob
model.eval()
folder = out / arm
folder.mkdir()
for case in cases:
data = to_device(load_evaluation_part(view, case, stats, "validation"), "cuda")
values = model(*arguments(data), checkpoint_blocks=False).cpu().numpy()
seconds = decode_seconds(values, data["F"], fold["c"], data["t_set"])
if seconds.shape != (data["nodes"],) or not np.isfinite(seconds).all():
raise ValueError("Full validation prediction is missing or nonfinite")
path = folder / (case + ".npy")
np.save(path, seconds, allow_pickle=False)
predictions.setdefault(case, {})[arm] = {"file": str(path), "sha256": sha(path)}
save(out / "progress.json", dict(stage="VALIDATION_PREDICT", arm=arm, case=case, **peaks()))
del data
del model
manifest = out / "predictions-frozen.json"
save(manifest, predictions)
manifest_hash = sha(manifest)
predictions = read(checked(manifest, manifest_hash))
rows = []
grouped = defaultdict(list)
for case in cases:
q = prediction_inputs(roster[case])
ids = cached(view, case, "raw_ids")
ix = np.searchsorted(ids, q["raw_ids"])
if np.any(ix >= len(ids)) or not np.array_equal(ids[ix], q["raw_ids"]):
raise ValueError("Validation query does not map to original raw IDs")
with (out / "validation-target-access.jsonl").open("a") as log:
log.write(json.dumps(dict(case=case, role="validation", predictions_sha256=manifest_hash,
target_sha256=pilot["parts"][case]["output_sha256"]["seconds.npy"])) + "\n")
truth = cached(view, case, "seconds")[ix]
scores = {}
for arm in ARMS:
row = predictions[case][arm]
pred = np.load(checked(row["file"], row["sha256"]), allow_pickle=False)[ix]
value = relative_l2(pred, truth, q["counts"], np.ones(len(ix), dtype=bool))["value"]
if value is None or not math.isfinite(value):
raise ValueError("Undefined validation part metric")
scores[arm] = value
component = roster[case]["component_id"]
grouped[component].append([scores[arm] for arm in ARMS])
rows.append(dict(case=case, component_id=component, relative_l2=scores))
means = dict(zip(ARMS, np.mean([np.mean(a, axis=0) for a in grouped.values()], axis=0).tolist()))
delta = means["short_seed0"] - means["long_seed0"]
seed_delta = means["long_seed1"] - means["long_seed0"]
passed = delta <= comparison["delta_short_at_most"] and abs(delta) > abs(seed_delta)
save(out / "result.json", dict(
status="COMPLETE", decision="SCREEN_PASS" if passed else "SCREEN_FAIL",
parts=len(rows), components=len(grouped), component_equal_relative_l2=means,
delta_short=delta, delta_seed=seed_delta, rows=rows, predictions_sha256=manifest_hash,
test_targets_read=False, automatic_continuation=False, **peaks()))
def dispatch_guard(root, contract_path):
"""The same entry consumes the generated contract in production and preflight."""
began_entry = time.monotonic()
c = contract(contract_path)
folded_inputs(c)
budget = c["budget"]
gpu = c["gpu_uuid"]
charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0.,
screen_gpu_seconds=0., finalization_gpu_seconds=0.)
receipts = []
status = "STOPPED"
trigger = "STOP_BEFORE_DISPATCH"
try:
if shutil.disk_usage(root).free < c["disk_bytes"]:
raise RuntimeError("Insufficient free disk for the fixed attempt")
began_wait = time.monotonic()
try:
idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot)
except BaseException:
charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait
raise
charges["queue_wait_gpu_seconds"] = idle["wall_seconds"]
save(root / "idle-gate.json", idle)
trigger = idle["status"]
if idle["status"] != "IDLE_CONFIRMED" or not snapshot(gpu)["idle"]:
if idle["status"] == "IDLE_CONFIRMED":
trigger = "SELECTED_GPU_BUSY_AFTER_IDLE_GATE"
return 1
(root / "jobs/0").mkdir(parents=True, exist_ok=False)
for mode, cap_key, charge_key, output in (
("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"),
("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"),
):
began = time.monotonic()
if not snapshot(gpu)["idle"]:
trigger = "SELECTED_GPU_BUSY_BEFORE_" + mode.upper()
charges[charge_key] = time.monotonic() - began
return 1
command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)]
tracker = PeakRecords()
done = threading.Event()
reasons = []
def watch():
while not done.is_set():
try:
tracker.scan(output)
state = snapshot(gpu)
pids = set(active_command_pids(command))
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]:
reasons.append("DEVICE_VRAM_LIMIT")
if any(p["pid"] not in pids for p in state["occupants"]):
reasons.append("FOREIGN_GPU_OCCUPANT")
except Exception as exc:
reasons.append(f"MONITOR_FAILURE: {exc}")
if reasons:
for pid in active_command_pids(command):
stop_group(pid)
# Also catch a child spawned just after the failing
# sample while the supervisor was taking its lock.
done.wait(.1)
continue
done.wait(.5)
monitor = threading.Thread(target=watch, daemon=True)
monitor.start()
try:
if reasons:
raise RuntimeError(reasons[0])
cap = budget[cap_key] - (time.monotonic() - began) - 10
receipt = supervise(command, root, mode, dict(
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key],
job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"],
disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]),
poll=.2, gpu_uuid=gpu)
finally:
done.set()
monitor.join(timeout=8)
charges[charge_key] = time.monotonic() - began
if monitor.is_alive():
reasons.append("MONITOR_DID_NOT_EXIT")
else:
tracker.scan(output)
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if set(tracker.witnesses) != set(tracker.maxima):
reasons.append("MISSING_RECORDED_VRAM_PEAKS")
receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima))
if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0
or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]):
worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT"
trigger = reasons[0] if reasons else worker_reason
return 1
result_path = output / "result.json"
result = read(result_path)
if result["status"] != "COMPLETE" or result["test_targets_read"] is not False:
raise RuntimeError("Unexpected worker result")
if mode == "train":
if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"]
or result["optimizer_steps"] != c["job"]["target_steps"]):
raise RuntimeError("Fold-0 final checkpoint identity differs")
checked(result["checkpoint"], result["checkpoint_sha256"])
save(root / "training-adopted.json", dict(result_sha256=sha(result_path)))
elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]:
raise RuntimeError("Validation screen scope differs")
overhead = time.monotonic() - began_entry - sum(charges.values())
if overhead > budget["finalization_gpu_seconds"]:
raise RuntimeError("Setup and finalization cost exceeded the fixed allowance")
status, trigger = "COMPLETE", result["decision"]
return 0
except Exception as exc:
trigger = f"{type(exc).__name__}: {exc}"
return 1
except KeyboardInterrupt:
trigger = "COORDINATOR_INTERRUPTED"
return 1
finally:
charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values()))
new_gpu = sum(charges.values())
save(root / "complete.json", dict(
status=status, trigger=trigger, receipts=receipts, **charges,
cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu,
cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu,
no_retry=True, test_targets_read=False, automatic_continuation=False))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("run", "train", "screen"))
parser.add_argument("root", type=Path)
args = parser.parse_args()
root = args.root.resolve()
if root != ROOT:
raise PermissionError("Only the new fixed fold-0 root is accepted")
expected_python = str(ROOT.parents[1] / "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python")
if sys.executable != expected_python:
raise PermissionError("Use the fixed original H2 environment")
validate_review(root)
contract_path = root / "contract.json"
if args.mode != "run":
coordinator = read(root / "started.json")
if process_identity(os.getppid()) != coordinator["coordinator"]:
raise PermissionError("Worker requires the live guard as its parent")
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), "Cannot bind worker lifetime to guard")
if os.getppid() != coordinator["coordinator"]["pid"]:
raise RuntimeError("Guard disappeared before worker setup")
return globals()[args.mode](root, contract(contract_path))
if os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR:
raise PermissionError("Explicit original CUDA allocator setup required")
with (root / "entry.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / "started.json").exists():
raise FileExistsError("Fold-0 attempt already started")
save(root / "started.json", dict(coordinator=process_identity(os.getpid()),
contract_sha256=sha(contract_path), unix=time.time()))
def request_stop(signum, _frame):
raise KeyboardInterrupt(signal.Signals(signum).name)
old_handler = signal.signal(signal.SIGTERM, request_stop)
try:
return dispatch_guard(root, contract_path)
finally:
signal.signal(signal.SIGTERM, old_handler)
if __name__ == "__main__":
raise SystemExit(main())