216 lines
10 KiB
Python
216 lines
10 KiB
Python
"""Fold-0 1b recovery scoped to dispatch deadline and terminal settlement."""
|
|
|
|
import ctypes
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from types import FunctionType, SimpleNamespace
|
|
|
|
import phase3_item1b_fold0 as legacy
|
|
from phase3_item1b_fold0 import contract, folded_inputs, read, checked, save, sha
|
|
from s2_gpu_idle_gate import snapshot, wait_idle
|
|
from s2_main_memory_guard import PeakRecords, active_command_pids
|
|
from s2_recovery_gpu_supervisor import stop_group, supervise as shared_supervise
|
|
|
|
|
|
ROOT = legacy.ROOT.with_name("phase3-item1b-fold0-deadline-2026-11-19")
|
|
|
|
|
|
class GuardStop(RuntimeError):
|
|
"""A named terminal stop that retains the actual accounting."""
|
|
|
|
|
|
def require_training_window(deadline):
|
|
if time.monotonic() >= deadline:
|
|
raise GuardStop("TRAIN_DISPATCH_DEADLINE")
|
|
|
|
|
|
def supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline=None):
|
|
if dispatch_deadline is None:
|
|
return shared_supervise(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
|
|
|
|
require_training_window(dispatch_deadline)
|
|
libc = ctypes.CDLL(None, use_errno=True)
|
|
previous_subreaper = ctypes.c_int()
|
|
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0:
|
|
raise OSError(ctypes.get_errno(), "Cannot read subreaper state")
|
|
|
|
def spawn(*args, **kwargs):
|
|
require_training_window(dispatch_deadline)
|
|
return subprocess.Popen(*args, **kwargs)
|
|
|
|
# Keep the published shared supervisor intact. This entry checks again
|
|
# after its internal locking, disk check and receipt, right before spawn.
|
|
namespace = dict(shared_supervise.__globals__,
|
|
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
|
supervisor = FunctionType(shared_supervise.__code__, namespace, shared_supervise.__name__)
|
|
try:
|
|
return supervisor(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
|
|
except GuardStop:
|
|
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
|
raise OSError(ctypes.get_errno(), "Cannot restore subreaper state")
|
|
save(root / "train-spawn-blocked.json", dict(
|
|
trigger="TRAIN_DISPATCH_DEADLINE", worker_started=False,
|
|
dispatch_deadline_monotonic=dispatch_deadline, rejected_monotonic=time.monotonic()))
|
|
raise
|
|
|
|
|
|
def recheck_idle(gpu, busy_trigger, deadline=None):
|
|
if deadline is not None:
|
|
require_training_window(deadline)
|
|
idle = snapshot(gpu)["idle"]
|
|
if deadline is not None:
|
|
require_training_window(deadline)
|
|
if not idle:
|
|
raise GuardStop(busy_trigger)
|
|
|
|
|
|
def dispatch_guard(root, contract_path):
|
|
"""The same entry consumes the generated contract in production and preflight."""
|
|
began_entry = time.monotonic()
|
|
c = contract(contract_path)
|
|
folded_inputs(c)
|
|
budget = c["budget"]
|
|
gpu = c["gpu_uuid"]
|
|
charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0.,
|
|
screen_gpu_seconds=0., finalization_gpu_seconds=0.)
|
|
receipts = []
|
|
status = "STOPPED"
|
|
trigger = "STOP_BEFORE_DISPATCH"
|
|
try:
|
|
if shutil.disk_usage(root).free < c["disk_bytes"]:
|
|
raise RuntimeError("Insufficient free disk for the fixed attempt")
|
|
began_wait = time.monotonic()
|
|
dispatch_deadline = began_wait + budget["idle_wait_seconds"]
|
|
try:
|
|
idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot)
|
|
except BaseException:
|
|
charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait
|
|
raise
|
|
charges["queue_wait_gpu_seconds"] = idle["wall_seconds"]
|
|
save(root / "idle-gate.json", idle)
|
|
if idle["status"] != "IDLE_CONFIRMED":
|
|
raise GuardStop(idle["status"])
|
|
recheck_idle(gpu, "SELECTED_GPU_BUSY_AFTER_IDLE_GATE", dispatch_deadline)
|
|
(root / "jobs/0").mkdir(parents=True, exist_ok=False)
|
|
for mode, cap_key, charge_key, output in (
|
|
("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"),
|
|
("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"),
|
|
):
|
|
began = time.monotonic()
|
|
deadline = dispatch_deadline if mode == "train" else None
|
|
try:
|
|
recheck_idle(gpu, "SELECTED_GPU_BUSY_BEFORE_" + mode.upper(), deadline)
|
|
except GuardStop:
|
|
charges[charge_key] = time.monotonic() - began
|
|
raise
|
|
command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)]
|
|
tracker = PeakRecords()
|
|
done = threading.Event()
|
|
reasons = []
|
|
|
|
def watch():
|
|
while not done.is_set():
|
|
try:
|
|
tracker.scan(output)
|
|
state = snapshot(gpu)
|
|
pids = set(active_command_pids(command))
|
|
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
|
|
reasons.append("RECORDED_VRAM_LIMIT")
|
|
if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]:
|
|
reasons.append("DEVICE_VRAM_LIMIT")
|
|
if any(p["pid"] not in pids for p in state["occupants"]):
|
|
reasons.append("FOREIGN_GPU_OCCUPANT")
|
|
except Exception as exc:
|
|
reasons.append(f"MONITOR_FAILURE: {exc}")
|
|
if reasons:
|
|
for pid in active_command_pids(command):
|
|
stop_group(pid)
|
|
# Also catch a child spawned just after the failing
|
|
# sample while the supervisor was taking its lock.
|
|
done.wait(.1)
|
|
continue
|
|
done.wait(.5)
|
|
|
|
monitor = threading.Thread(target=watch, daemon=True)
|
|
monitor.start()
|
|
try:
|
|
if reasons:
|
|
raise RuntimeError(reasons[0])
|
|
cap = budget[cap_key] - (time.monotonic() - began) - 10
|
|
if deadline is not None:
|
|
require_training_window(deadline)
|
|
receipt = supervise(command, root, mode, dict(
|
|
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key],
|
|
job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"],
|
|
disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]),
|
|
poll=.2, gpu_uuid=gpu, dispatch_deadline=deadline)
|
|
finally:
|
|
done.set()
|
|
monitor.join(timeout=8)
|
|
charges[charge_key] = time.monotonic() - began
|
|
if monitor.is_alive():
|
|
reasons.append("MONITOR_DID_NOT_EXIT")
|
|
else:
|
|
tracker.scan(output)
|
|
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
|
|
reasons.append("RECORDED_VRAM_LIMIT")
|
|
if set(tracker.witnesses) != set(tracker.maxima):
|
|
reasons.append("MISSING_RECORDED_VRAM_PEAKS")
|
|
receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima))
|
|
if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0
|
|
or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]):
|
|
worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT"
|
|
raise GuardStop(reasons[0] if reasons else worker_reason)
|
|
result_path = output / "result.json"
|
|
result = read(result_path)
|
|
if result["status"] != "COMPLETE" or result["test_targets_read"] is not False:
|
|
raise RuntimeError("Unexpected worker result")
|
|
if mode == "train":
|
|
if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"]
|
|
or result["optimizer_steps"] != c["job"]["target_steps"]):
|
|
raise RuntimeError("Fold-0 final checkpoint identity differs")
|
|
checked(result["checkpoint"], result["checkpoint_sha256"])
|
|
save(root / "training-adopted.json", dict(result_sha256=sha(result_path)))
|
|
elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]:
|
|
raise RuntimeError("Validation screen scope differs")
|
|
overhead = time.monotonic() - began_entry - sum(charges.values())
|
|
if overhead > budget["finalization_gpu_seconds"]:
|
|
raise RuntimeError("Setup and finalization cost exceeded the fixed allowance")
|
|
status, trigger = "COMPLETE", result["decision"]
|
|
except GuardStop as exc:
|
|
trigger = str(exc)
|
|
except Exception as exc:
|
|
trigger = f"{type(exc).__name__}: {exc}"
|
|
except KeyboardInterrupt:
|
|
trigger = "COORDINATOR_INTERRUPTED"
|
|
finally:
|
|
charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values()))
|
|
new_gpu = sum(charges.values())
|
|
limit = budget["projected_new_gpu_seconds"]
|
|
overage = max(0., new_gpu - limit)
|
|
underlying_trigger = trigger
|
|
if overage > 0:
|
|
status, trigger = "STOPPED", "TOTAL_GPU_CAP_AT_SETTLEMENT"
|
|
save(root / "complete.json", dict(
|
|
status=status, trigger=trigger, underlying_trigger=underlying_trigger, receipts=receipts, **charges,
|
|
new_gpu_seconds=new_gpu, new_gpu_cap_seconds=limit, cap_overage_gpu_seconds=overage,
|
|
cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu,
|
|
cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu,
|
|
no_retry=True, test_targets_read=False, automatic_continuation=False))
|
|
return 0 if status == "COMPLETE" else 1
|
|
|
|
|
|
def main():
|
|
# The original entry and worker functions enforce the same exact review,
|
|
# process lifetime, environment and one-attempt rules at the new root.
|
|
namespace = dict(legacy.main.__globals__, ROOT=ROOT, dispatch_guard=dispatch_guard)
|
|
return FunctionType(legacy.main.__code__, namespace, legacy.main.__name__)()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|