Freeze 1b fold-zero dispatch deadline and settlement repair
This commit is contained in:
1 parent
cc19b47ea9
commit
7f6476d941
5 files changed
+1321
No files matched your search
@@ -0,0 +1,215 @@
|
||||
"""Fold-0 1b recovery scoped to dispatch deadline and terminal settlement."""
|
||||
|
||||
import ctypes
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import FunctionType, SimpleNamespace
|
||||
|
||||
import phase3_item1b_fold0 as legacy
|
||||
from phase3_item1b_fold0 import contract, folded_inputs, read, checked, save, sha
|
||||
from s2_gpu_idle_gate import snapshot, wait_idle
|
||||
from s2_main_memory_guard import PeakRecords, active_command_pids
|
||||
from s2_recovery_gpu_supervisor import stop_group, supervise as shared_supervise
|
||||
|
||||
|
||||
ROOT = legacy.ROOT.with_name("phase3-item1b-fold0-deadline-2026-11-19")
|
||||
|
||||
|
||||
class GuardStop(RuntimeError):
|
||||
"""A named terminal stop that retains the actual accounting."""
|
||||
|
||||
|
||||
def require_training_window(deadline):
|
||||
if time.monotonic() >= deadline:
|
||||
raise GuardStop("TRAIN_DISPATCH_DEADLINE")
|
||||
|
||||
|
||||
def supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline=None):
|
||||
if dispatch_deadline is None:
|
||||
return shared_supervise(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
|
||||
|
||||
require_training_window(dispatch_deadline)
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), "Cannot read subreaper state")
|
||||
|
||||
def spawn(*args, **kwargs):
|
||||
require_training_window(dispatch_deadline)
|
||||
return subprocess.Popen(*args, **kwargs)
|
||||
|
||||
# Keep the published shared supervisor intact. This entry checks again
|
||||
# after its internal locking, disk check and receipt, right before spawn.
|
||||
namespace = dict(shared_supervise.__globals__,
|
||||
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
||||
supervisor = FunctionType(shared_supervise.__code__, namespace, shared_supervise.__name__)
|
||||
try:
|
||||
return supervisor(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
|
||||
except GuardStop:
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), "Cannot restore subreaper state")
|
||||
save(root / "train-spawn-blocked.json", dict(
|
||||
trigger="TRAIN_DISPATCH_DEADLINE", worker_started=False,
|
||||
dispatch_deadline_monotonic=dispatch_deadline, rejected_monotonic=time.monotonic()))
|
||||
raise
|
||||
|
||||
|
||||
def recheck_idle(gpu, busy_trigger, deadline=None):
|
||||
if deadline is not None:
|
||||
require_training_window(deadline)
|
||||
idle = snapshot(gpu)["idle"]
|
||||
if deadline is not None:
|
||||
require_training_window(deadline)
|
||||
if not idle:
|
||||
raise GuardStop(busy_trigger)
|
||||
|
||||
|
||||
def dispatch_guard(root, contract_path):
|
||||
"""The same entry consumes the generated contract in production and preflight."""
|
||||
began_entry = time.monotonic()
|
||||
c = contract(contract_path)
|
||||
folded_inputs(c)
|
||||
budget = c["budget"]
|
||||
gpu = c["gpu_uuid"]
|
||||
charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0.,
|
||||
screen_gpu_seconds=0., finalization_gpu_seconds=0.)
|
||||
receipts = []
|
||||
status = "STOPPED"
|
||||
trigger = "STOP_BEFORE_DISPATCH"
|
||||
try:
|
||||
if shutil.disk_usage(root).free < c["disk_bytes"]:
|
||||
raise RuntimeError("Insufficient free disk for the fixed attempt")
|
||||
began_wait = time.monotonic()
|
||||
dispatch_deadline = began_wait + budget["idle_wait_seconds"]
|
||||
try:
|
||||
idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot)
|
||||
except BaseException:
|
||||
charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait
|
||||
raise
|
||||
charges["queue_wait_gpu_seconds"] = idle["wall_seconds"]
|
||||
save(root / "idle-gate.json", idle)
|
||||
if idle["status"] != "IDLE_CONFIRMED":
|
||||
raise GuardStop(idle["status"])
|
||||
recheck_idle(gpu, "SELECTED_GPU_BUSY_AFTER_IDLE_GATE", dispatch_deadline)
|
||||
(root / "jobs/0").mkdir(parents=True, exist_ok=False)
|
||||
for mode, cap_key, charge_key, output in (
|
||||
("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"),
|
||||
("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"),
|
||||
):
|
||||
began = time.monotonic()
|
||||
deadline = dispatch_deadline if mode == "train" else None
|
||||
try:
|
||||
recheck_idle(gpu, "SELECTED_GPU_BUSY_BEFORE_" + mode.upper(), deadline)
|
||||
except GuardStop:
|
||||
charges[charge_key] = time.monotonic() - began
|
||||
raise
|
||||
command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)]
|
||||
tracker = PeakRecords()
|
||||
done = threading.Event()
|
||||
reasons = []
|
||||
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
tracker.scan(output)
|
||||
state = snapshot(gpu)
|
||||
pids = set(active_command_pids(command))
|
||||
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
|
||||
reasons.append("RECORDED_VRAM_LIMIT")
|
||||
if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]:
|
||||
reasons.append("DEVICE_VRAM_LIMIT")
|
||||
if any(p["pid"] not in pids for p in state["occupants"]):
|
||||
reasons.append("FOREIGN_GPU_OCCUPANT")
|
||||
except Exception as exc:
|
||||
reasons.append(f"MONITOR_FAILURE: {exc}")
|
||||
if reasons:
|
||||
for pid in active_command_pids(command):
|
||||
stop_group(pid)
|
||||
# Also catch a child spawned just after the failing
|
||||
# sample while the supervisor was taking its lock.
|
||||
done.wait(.1)
|
||||
continue
|
||||
done.wait(.5)
|
||||
|
||||
monitor = threading.Thread(target=watch, daemon=True)
|
||||
monitor.start()
|
||||
try:
|
||||
if reasons:
|
||||
raise RuntimeError(reasons[0])
|
||||
cap = budget[cap_key] - (time.monotonic() - began) - 10
|
||||
if deadline is not None:
|
||||
require_training_window(deadline)
|
||||
receipt = supervise(command, root, mode, dict(
|
||||
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key],
|
||||
job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"],
|
||||
disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]),
|
||||
poll=.2, gpu_uuid=gpu, dispatch_deadline=deadline)
|
||||
finally:
|
||||
done.set()
|
||||
monitor.join(timeout=8)
|
||||
charges[charge_key] = time.monotonic() - began
|
||||
if monitor.is_alive():
|
||||
reasons.append("MONITOR_DID_NOT_EXIT")
|
||||
else:
|
||||
tracker.scan(output)
|
||||
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
|
||||
reasons.append("RECORDED_VRAM_LIMIT")
|
||||
if set(tracker.witnesses) != set(tracker.maxima):
|
||||
reasons.append("MISSING_RECORDED_VRAM_PEAKS")
|
||||
receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima))
|
||||
if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0
|
||||
or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]):
|
||||
worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT"
|
||||
raise GuardStop(reasons[0] if reasons else worker_reason)
|
||||
result_path = output / "result.json"
|
||||
result = read(result_path)
|
||||
if result["status"] != "COMPLETE" or result["test_targets_read"] is not False:
|
||||
raise RuntimeError("Unexpected worker result")
|
||||
if mode == "train":
|
||||
if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"]
|
||||
or result["optimizer_steps"] != c["job"]["target_steps"]):
|
||||
raise RuntimeError("Fold-0 final checkpoint identity differs")
|
||||
checked(result["checkpoint"], result["checkpoint_sha256"])
|
||||
save(root / "training-adopted.json", dict(result_sha256=sha(result_path)))
|
||||
elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]:
|
||||
raise RuntimeError("Validation screen scope differs")
|
||||
overhead = time.monotonic() - began_entry - sum(charges.values())
|
||||
if overhead > budget["finalization_gpu_seconds"]:
|
||||
raise RuntimeError("Setup and finalization cost exceeded the fixed allowance")
|
||||
status, trigger = "COMPLETE", result["decision"]
|
||||
except GuardStop as exc:
|
||||
trigger = str(exc)
|
||||
except Exception as exc:
|
||||
trigger = f"{type(exc).__name__}: {exc}"
|
||||
except KeyboardInterrupt:
|
||||
trigger = "COORDINATOR_INTERRUPTED"
|
||||
finally:
|
||||
charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values()))
|
||||
new_gpu = sum(charges.values())
|
||||
limit = budget["projected_new_gpu_seconds"]
|
||||
overage = max(0., new_gpu - limit)
|
||||
underlying_trigger = trigger
|
||||
if overage > 0:
|
||||
status, trigger = "STOPPED", "TOTAL_GPU_CAP_AT_SETTLEMENT"
|
||||
save(root / "complete.json", dict(
|
||||
status=status, trigger=trigger, underlying_trigger=underlying_trigger, receipts=receipts, **charges,
|
||||
new_gpu_seconds=new_gpu, new_gpu_cap_seconds=limit, cap_overage_gpu_seconds=overage,
|
||||
cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu,
|
||||
cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu,
|
||||
no_retry=True, test_targets_read=False, automatic_continuation=False))
|
||||
return 0 if status == "COMPLETE" else 1
|
||||
|
||||
|
||||
def main():
|
||||
# The original entry and worker functions enforce the same exact review,
|
||||
# process lifetime, environment and one-attempt rules at the new root.
|
||||
namespace = dict(legacy.main.__globals__, ROOT=ROOT, dispatch_guard=dispatch_guard)
|
||||
return FunctionType(legacy.main.__code__, namespace, legacy.main.__name__)()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Freeze the narrow 1b repair and replay the independent review's boundary cases."""
|
||||
|
||||
from copy import deepcopy
|
||||
from functools import partial
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import phase3_item1b_fold0 as legacy
|
||||
import phase3_item1b_fold0_deadline as entry
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from s2_gpu_idle_gate import wait_idle
|
||||
|
||||
|
||||
OLD_BINDING = "ed6bfe113f9c0bf925684a62cf0c1bb09bfc7b1277e06fad628fc0e9e5bcd859"
|
||||
OLD_CONTRACT = "942323ddfceecf47515eb48628c45157073cbb909867a606ef4d3764a13be7d4"
|
||||
ORIGINAL_BLOCKS = {
|
||||
"meltassist-item1b-fold0-screen-review-2026-11-19": "0fadd70d365e1015f835fb953691797811a032766d3a9c1b75f3b8e4678faf61",
|
||||
"meltassist-item1b-fold0-final-deadline-review-2026-11-19": "d45ac966f612236656f9b1a0d59ed2f60493c2da0b3c31279e189d7dfa217e35",
|
||||
}
|
||||
|
||||
|
||||
class InertMonitor:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def start(self):
|
||||
pass
|
||||
|
||||
def join(self, **kwargs):
|
||||
pass
|
||||
|
||||
def is_alive(self):
|
||||
return False
|
||||
|
||||
|
||||
def prepare():
|
||||
old_path = checked(legacy.ROOT / "contract.json", OLD_CONTRACT)
|
||||
binding = checked(legacy.ROOT / "expected-review-binding.json", OLD_BINDING)
|
||||
old = legacy.contract(old_path)
|
||||
if (legacy.ROOT / "started.json").exists():
|
||||
raise PermissionError("The original fold-0 entry already has a start record")
|
||||
sources = {str(old_path): OLD_CONTRACT, str(binding): OLD_BINDING}
|
||||
for name, digest in ORIGINAL_BLOCKS.items():
|
||||
path = checked(entry.ROOT.parent / name / "review.json", digest)
|
||||
if entry.read(path)["verdict"] != "BLOCKED":
|
||||
raise ValueError("Original blocking review must be preserved")
|
||||
sources[str(path)] = digest
|
||||
new = deepcopy(old)
|
||||
new["version"] = "phase3-item1b-fold0-deadline-settlement-v2"
|
||||
new["engineering_ruling"] = "h-5064d81d"
|
||||
new["source_sha256"].update(sources)
|
||||
for path in (Path(entry.__file__), Path(__file__).resolve()):
|
||||
new["code_sha256"][str(path)] = sha(path)
|
||||
if new["budget"] != old["budget"] or new["budget"]["projected_new_gpu_seconds"] != 13180:
|
||||
raise ValueError("Original budget must remain 13,180 GPU seconds")
|
||||
entry.ROOT.mkdir(exist_ok=False)
|
||||
path = entry.ROOT / "contract.json"
|
||||
save(path, new)
|
||||
return path
|
||||
|
||||
|
||||
def simulate(path, *, latency, recheck_latency, settle_delay=None):
|
||||
c = entry.contract(path)
|
||||
now = 0.
|
||||
samples = []
|
||||
dispatches = []
|
||||
source, first = inspect.getsourcelines(entry.dispatch_guard)
|
||||
settle_line = next(first + i for i, line in enumerate(source)
|
||||
if 'charges["finalization_gpu_seconds"] = max(' in line)
|
||||
|
||||
def sleep(seconds):
|
||||
nonlocal now
|
||||
now += seconds
|
||||
|
||||
def clock():
|
||||
if settle_delay is not None and inspect.currentframe().f_back.f_lineno == settle_line:
|
||||
sleep(settle_delay)
|
||||
return now
|
||||
|
||||
def sample(uuid):
|
||||
index = len(samples)
|
||||
samples.append(now)
|
||||
sleep(latency if index < 6 else recheck_latency)
|
||||
return dict(uuid=uuid, idle=index >= 3, memory_MiB=5 if index >= 3 else 1500, occupants=[])
|
||||
|
||||
def record(path, data):
|
||||
save(path, data)
|
||||
if settle_delay is not None and path.name == "training-adopted.json":
|
||||
sleep(119.)
|
||||
|
||||
def mock_supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline):
|
||||
dispatches.append(dict(mode=mode, seconds=now, dispatch_deadline=dispatch_deadline))
|
||||
if settle_delay is None:
|
||||
raise AssertionError("Late-admission case must stop before the supervisor")
|
||||
if dispatch_deadline != (60. if mode == "train" else None):
|
||||
raise AssertionError("The original deadline must follow training only")
|
||||
sleep(limits["job_wall_seconds"] - .1)
|
||||
output = root / ("jobs/0/worker" if mode == "train" else "screen")
|
||||
output.mkdir()
|
||||
result = dict(status="COMPLETE", test_targets_read=False,
|
||||
peak_allocated_bytes=32, peak_reserved_bytes=64)
|
||||
if mode == "train":
|
||||
checkpoint = output / "mock.pt"
|
||||
checkpoint.write_bytes(b"synthetic")
|
||||
result.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"],
|
||||
checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint))
|
||||
else:
|
||||
result.update(parts=len(c["job"]["validation"]), automatic_continuation=False, decision="SCREEN_PASS")
|
||||
save(output / "result.json", result)
|
||||
return dict(status="PASS", returncode=0, monitor_errors=0)
|
||||
|
||||
with TemporaryDirectory(prefix="meltmind-1b-deadline-replay-") as scratch:
|
||||
root = Path(scratch)
|
||||
with (patch.object(entry, "snapshot", side_effect=sample),
|
||||
patch.object(entry, "wait_idle", partial(wait_idle, clock=clock, sleep=sleep)),
|
||||
patch.object(entry, "time", SimpleNamespace(monotonic=clock)),
|
||||
patch.object(entry, "threading", SimpleNamespace(Thread=InertMonitor, Event=entry.threading.Event)),
|
||||
patch.object(entry, "supervise", side_effect=mock_supervise),
|
||||
patch.object(entry, "save", side_effect=record)):
|
||||
returned = entry.dispatch_guard(root, path)
|
||||
terminal = entry.read(root / "complete.json")
|
||||
gate = entry.read(root / "idle-gate.json")
|
||||
terminal.pop("receipts")
|
||||
return dict(gate=gate["status"], gate_seconds=gate["wall_seconds"],
|
||||
snapshot_start_seconds=samples, mock_dispatches=dispatches,
|
||||
returned=returned, terminal=terminal)
|
||||
|
||||
|
||||
def supervisor_boundary(path):
|
||||
c = entry.contract(path)
|
||||
now = 59.5
|
||||
|
||||
def disk_check(_):
|
||||
nonlocal now
|
||||
now = 60.5
|
||||
return 0
|
||||
|
||||
blocked_spawn = Mock(side_effect=AssertionError("Popen invoked after the training deadline"))
|
||||
with TemporaryDirectory(prefix="meltmind-1b-supervisor-deadline-") as scratch:
|
||||
root = Path(scratch)
|
||||
with (patch.object(entry, "time", SimpleNamespace(monotonic=lambda: now)),
|
||||
patch.dict(entry.shared_supervise.__globals__, {"disk_usage": disk_check}),
|
||||
patch.object(entry, "subprocess", SimpleNamespace(Popen=blocked_spawn, STDOUT=entry.subprocess.STDOUT))):
|
||||
try:
|
||||
entry.supervise(["synthetic-no-spawn"], root, "train", dict(
|
||||
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=c["budget"]["train_gpu_seconds"],
|
||||
job_wall_seconds=10., tree_rss_bytes=c["rss_bytes"], disk_bytes=c["disk_bytes"],
|
||||
max_processes=c["max_processes"]), poll=.2, gpu_uuid=c["gpu_uuid"], dispatch_deadline=60.)
|
||||
except entry.GuardStop as exc:
|
||||
assert str(exc) == "TRAIN_DISPATCH_DEADLINE"
|
||||
else:
|
||||
raise AssertionError("The actual supervisor did not stop before spawn")
|
||||
marker = entry.read(root / "train-spawn-blocked.json")
|
||||
assert blocked_spawn.call_count == 0 and not marker["worker_started"]
|
||||
return dict(marker=marker, real_popen_calls=blocked_spawn.call_count)
|
||||
|
||||
|
||||
def preflight(path):
|
||||
late = simulate(path, latency=1.5, recheck_latency=1.5)
|
||||
cap = simulate(path, latency=1.5, recheck_latency=0., settle_delay=23.)
|
||||
baseline = simulate(path, latency=1.5, recheck_latency=0., settle_delay=0.)
|
||||
busy = simulate(path, latency=2., recheck_latency=2.)
|
||||
assert late["gate_seconds"] == 59. and late["terminal"]["trigger"] == "TRAIN_DISPATCH_DEADLINE"
|
||||
assert late["mock_dispatches"] == [] and late["returned"] == 1
|
||||
assert round(cap["terminal"]["new_gpu_seconds"], 1) == 13180.8
|
||||
assert cap["terminal"]["status"] == "STOPPED" and cap["returned"] == 1
|
||||
assert cap["terminal"]["trigger"] == "TOTAL_GPU_CAP_AT_SETTLEMENT"
|
||||
assert baseline["returned"] == 0 and baseline["terminal"]["status"] == "COMPLETE"
|
||||
assert busy["terminal"]["trigger"] == "IDLE_TIMEOUT" and busy["mock_dispatches"] == []
|
||||
return dict(status="PASS", contract_sha256=sha(path), old_binding_sha256=OLD_BINDING,
|
||||
late_recheck=late, final_settlement=cap, timely_control=baseline, busy_control=busy,
|
||||
in_supervisor_deadline=supervisor_boundary(path), real_gpu_queries=0, real_workers=0,
|
||||
targets_read=False, independent_review="PENDING")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
path = prepare()
|
||||
receipt = preflight(path)
|
||||
save(path.parent / "actual-guard-cpu-replay.json", receipt)
|
||||
print(receipt)
|
||||
Reference in new issue
Block a user