Files
MeltMind/tools/phase3_item1b_fold0_deadline.py
T

216 lines
10 KiB
Python

"""Fold-0 1b recovery scoped to dispatch deadline and terminal settlement."""
import ctypes
import shutil
import subprocess
import sys
import threading
import time
from pathlib import Path
from types import FunctionType, SimpleNamespace
import phase3_item1b_fold0 as legacy
from phase3_item1b_fold0 import contract, folded_inputs, read, checked, save, sha
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords, active_command_pids
from s2_recovery_gpu_supervisor import stop_group, supervise as shared_supervise
ROOT = legacy.ROOT.with_name("phase3-item1b-fold0-deadline-2026-11-19")
class GuardStop(RuntimeError):
"""A named terminal stop that retains the actual accounting."""
def require_training_window(deadline):
if time.monotonic() >= deadline:
raise GuardStop("TRAIN_DISPATCH_DEADLINE")
def supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline=None):
if dispatch_deadline is None:
return shared_supervise(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
require_training_window(dispatch_deadline)
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), "Cannot read subreaper state")
def spawn(*args, **kwargs):
require_training_window(dispatch_deadline)
return subprocess.Popen(*args, **kwargs)
# Keep the published shared supervisor intact. This entry checks again
# after its internal locking, disk check and receipt, right before spawn.
namespace = dict(shared_supervise.__globals__,
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
supervisor = FunctionType(shared_supervise.__code__, namespace, shared_supervise.__name__)
try:
return supervisor(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
except GuardStop:
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), "Cannot restore subreaper state")
save(root / "train-spawn-blocked.json", dict(
trigger="TRAIN_DISPATCH_DEADLINE", worker_started=False,
dispatch_deadline_monotonic=dispatch_deadline, rejected_monotonic=time.monotonic()))
raise
def recheck_idle(gpu, busy_trigger, deadline=None):
if deadline is not None:
require_training_window(deadline)
idle = snapshot(gpu)["idle"]
if deadline is not None:
require_training_window(deadline)
if not idle:
raise GuardStop(busy_trigger)
def dispatch_guard(root, contract_path):
"""The same entry consumes the generated contract in production and preflight."""
began_entry = time.monotonic()
c = contract(contract_path)
folded_inputs(c)
budget = c["budget"]
gpu = c["gpu_uuid"]
charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0.,
screen_gpu_seconds=0., finalization_gpu_seconds=0.)
receipts = []
status = "STOPPED"
trigger = "STOP_BEFORE_DISPATCH"
try:
if shutil.disk_usage(root).free < c["disk_bytes"]:
raise RuntimeError("Insufficient free disk for the fixed attempt")
began_wait = time.monotonic()
dispatch_deadline = began_wait + budget["idle_wait_seconds"]
try:
idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot)
except BaseException:
charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait
raise
charges["queue_wait_gpu_seconds"] = idle["wall_seconds"]
save(root / "idle-gate.json", idle)
if idle["status"] != "IDLE_CONFIRMED":
raise GuardStop(idle["status"])
recheck_idle(gpu, "SELECTED_GPU_BUSY_AFTER_IDLE_GATE", dispatch_deadline)
(root / "jobs/0").mkdir(parents=True, exist_ok=False)
for mode, cap_key, charge_key, output in (
("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"),
("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"),
):
began = time.monotonic()
deadline = dispatch_deadline if mode == "train" else None
try:
recheck_idle(gpu, "SELECTED_GPU_BUSY_BEFORE_" + mode.upper(), deadline)
except GuardStop:
charges[charge_key] = time.monotonic() - began
raise
command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)]
tracker = PeakRecords()
done = threading.Event()
reasons = []
def watch():
while not done.is_set():
try:
tracker.scan(output)
state = snapshot(gpu)
pids = set(active_command_pids(command))
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]:
reasons.append("DEVICE_VRAM_LIMIT")
if any(p["pid"] not in pids for p in state["occupants"]):
reasons.append("FOREIGN_GPU_OCCUPANT")
except Exception as exc:
reasons.append(f"MONITOR_FAILURE: {exc}")
if reasons:
for pid in active_command_pids(command):
stop_group(pid)
# Also catch a child spawned just after the failing
# sample while the supervisor was taking its lock.
done.wait(.1)
continue
done.wait(.5)
monitor = threading.Thread(target=watch, daemon=True)
monitor.start()
try:
if reasons:
raise RuntimeError(reasons[0])
cap = budget[cap_key] - (time.monotonic() - began) - 10
if deadline is not None:
require_training_window(deadline)
receipt = supervise(command, root, mode, dict(
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key],
job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"],
disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]),
poll=.2, gpu_uuid=gpu, dispatch_deadline=deadline)
finally:
done.set()
monitor.join(timeout=8)
charges[charge_key] = time.monotonic() - began
if monitor.is_alive():
reasons.append("MONITOR_DID_NOT_EXIT")
else:
tracker.scan(output)
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if set(tracker.witnesses) != set(tracker.maxima):
reasons.append("MISSING_RECORDED_VRAM_PEAKS")
receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima))
if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0
or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]):
worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT"
raise GuardStop(reasons[0] if reasons else worker_reason)
result_path = output / "result.json"
result = read(result_path)
if result["status"] != "COMPLETE" or result["test_targets_read"] is not False:
raise RuntimeError("Unexpected worker result")
if mode == "train":
if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"]
or result["optimizer_steps"] != c["job"]["target_steps"]):
raise RuntimeError("Fold-0 final checkpoint identity differs")
checked(result["checkpoint"], result["checkpoint_sha256"])
save(root / "training-adopted.json", dict(result_sha256=sha(result_path)))
elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]:
raise RuntimeError("Validation screen scope differs")
overhead = time.monotonic() - began_entry - sum(charges.values())
if overhead > budget["finalization_gpu_seconds"]:
raise RuntimeError("Setup and finalization cost exceeded the fixed allowance")
status, trigger = "COMPLETE", result["decision"]
except GuardStop as exc:
trigger = str(exc)
except Exception as exc:
trigger = f"{type(exc).__name__}: {exc}"
except KeyboardInterrupt:
trigger = "COORDINATOR_INTERRUPTED"
finally:
charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values()))
new_gpu = sum(charges.values())
limit = budget["projected_new_gpu_seconds"]
overage = max(0., new_gpu - limit)
underlying_trigger = trigger
if overage > 0:
status, trigger = "STOPPED", "TOTAL_GPU_CAP_AT_SETTLEMENT"
save(root / "complete.json", dict(
status=status, trigger=trigger, underlying_trigger=underlying_trigger, receipts=receipts, **charges,
new_gpu_seconds=new_gpu, new_gpu_cap_seconds=limit, cap_overage_gpu_seconds=overage,
cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu,
cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu,
no_retry=True, test_targets_read=False, automatic_continuation=False))
return 0 if status == "COMPLETE" else 1
def main():
# The original entry and worker functions enforce the same exact review,
# process lifetime, environment and one-attempt rules at the new root.
namespace = dict(legacy.main.__globals__, ROOT=ROOT, dispatch_guard=dispatch_guard)
return FunctionType(legacy.main.__code__, namespace, legacy.main.__name__)()
if __name__ == "__main__":
raise SystemExit(main())