Freeze 1b fold-zero dispatch deadline and settlement repair

This commit is contained in:
NewMeltAssist committed 2026-11-19 12:54:12 +08:00
1 parent cc19b47ea9
commit 7f6476d941
5 files changed
+1321

No files matched your search

+215
View File
@@ -0,0 +1,215 @@
"""Fold-0 1b recovery scoped to dispatch deadline and terminal settlement."""
import ctypes
import shutil
import subprocess
import sys
import threading
import time
from pathlib import Path
from types import FunctionType, SimpleNamespace
import phase3_item1b_fold0 as legacy
from phase3_item1b_fold0 import contract, folded_inputs, read, checked, save, sha
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords, active_command_pids
from s2_recovery_gpu_supervisor import stop_group, supervise as shared_supervise
ROOT = legacy.ROOT.with_name("phase3-item1b-fold0-deadline-2026-11-19")
class GuardStop(RuntimeError):
"""A named terminal stop that retains the actual accounting."""
def require_training_window(deadline):
if time.monotonic() >= deadline:
raise GuardStop("TRAIN_DISPATCH_DEADLINE")
def supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline=None):
if dispatch_deadline is None:
return shared_supervise(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
require_training_window(dispatch_deadline)
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), "Cannot read subreaper state")
def spawn(*args, **kwargs):
require_training_window(dispatch_deadline)
return subprocess.Popen(*args, **kwargs)
# Keep the published shared supervisor intact. This entry checks again
# after its internal locking, disk check and receipt, right before spawn.
namespace = dict(shared_supervise.__globals__,
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
supervisor = FunctionType(shared_supervise.__code__, namespace, shared_supervise.__name__)
try:
return supervisor(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
except GuardStop:
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), "Cannot restore subreaper state")
save(root / "train-spawn-blocked.json", dict(
trigger="TRAIN_DISPATCH_DEADLINE", worker_started=False,
dispatch_deadline_monotonic=dispatch_deadline, rejected_monotonic=time.monotonic()))
raise
def recheck_idle(gpu, busy_trigger, deadline=None):
if deadline is not None:
require_training_window(deadline)
idle = snapshot(gpu)["idle"]
if deadline is not None:
require_training_window(deadline)
if not idle:
raise GuardStop(busy_trigger)
def dispatch_guard(root, contract_path):
"""The same entry consumes the generated contract in production and preflight."""
began_entry = time.monotonic()
c = contract(contract_path)
folded_inputs(c)
budget = c["budget"]
gpu = c["gpu_uuid"]
charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0.,
screen_gpu_seconds=0., finalization_gpu_seconds=0.)
receipts = []
status = "STOPPED"
trigger = "STOP_BEFORE_DISPATCH"
try:
if shutil.disk_usage(root).free < c["disk_bytes"]:
raise RuntimeError("Insufficient free disk for the fixed attempt")
began_wait = time.monotonic()
dispatch_deadline = began_wait + budget["idle_wait_seconds"]
try:
idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot)
except BaseException:
charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait
raise
charges["queue_wait_gpu_seconds"] = idle["wall_seconds"]
save(root / "idle-gate.json", idle)
if idle["status"] != "IDLE_CONFIRMED":
raise GuardStop(idle["status"])
recheck_idle(gpu, "SELECTED_GPU_BUSY_AFTER_IDLE_GATE", dispatch_deadline)
(root / "jobs/0").mkdir(parents=True, exist_ok=False)
for mode, cap_key, charge_key, output in (
("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"),
("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"),
):
began = time.monotonic()
deadline = dispatch_deadline if mode == "train" else None
try:
recheck_idle(gpu, "SELECTED_GPU_BUSY_BEFORE_" + mode.upper(), deadline)
except GuardStop:
charges[charge_key] = time.monotonic() - began
raise
command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)]
tracker = PeakRecords()
done = threading.Event()
reasons = []
def watch():
while not done.is_set():
try:
tracker.scan(output)
state = snapshot(gpu)
pids = set(active_command_pids(command))
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]:
reasons.append("DEVICE_VRAM_LIMIT")
if any(p["pid"] not in pids for p in state["occupants"]):
reasons.append("FOREIGN_GPU_OCCUPANT")
except Exception as exc:
reasons.append(f"MONITOR_FAILURE: {exc}")
if reasons:
for pid in active_command_pids(command):
stop_group(pid)
# Also catch a child spawned just after the failing
# sample while the supervisor was taking its lock.
done.wait(.1)
continue
done.wait(.5)
monitor = threading.Thread(target=watch, daemon=True)
monitor.start()
try:
if reasons:
raise RuntimeError(reasons[0])
cap = budget[cap_key] - (time.monotonic() - began) - 10
if deadline is not None:
require_training_window(deadline)
receipt = supervise(command, root, mode, dict(
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key],
job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"],
disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]),
poll=.2, gpu_uuid=gpu, dispatch_deadline=deadline)
finally:
done.set()
monitor.join(timeout=8)
charges[charge_key] = time.monotonic() - began
if monitor.is_alive():
reasons.append("MONITOR_DID_NOT_EXIT")
else:
tracker.scan(output)
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if set(tracker.witnesses) != set(tracker.maxima):
reasons.append("MISSING_RECORDED_VRAM_PEAKS")
receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima))
if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0
or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]):
worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT"
raise GuardStop(reasons[0] if reasons else worker_reason)
result_path = output / "result.json"
result = read(result_path)
if result["status"] != "COMPLETE" or result["test_targets_read"] is not False:
raise RuntimeError("Unexpected worker result")
if mode == "train":
if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"]
or result["optimizer_steps"] != c["job"]["target_steps"]):
raise RuntimeError("Fold-0 final checkpoint identity differs")
checked(result["checkpoint"], result["checkpoint_sha256"])
save(root / "training-adopted.json", dict(result_sha256=sha(result_path)))
elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]:
raise RuntimeError("Validation screen scope differs")
overhead = time.monotonic() - began_entry - sum(charges.values())
if overhead > budget["finalization_gpu_seconds"]:
raise RuntimeError("Setup and finalization cost exceeded the fixed allowance")
status, trigger = "COMPLETE", result["decision"]
except GuardStop as exc:
trigger = str(exc)
except Exception as exc:
trigger = f"{type(exc).__name__}: {exc}"
except KeyboardInterrupt:
trigger = "COORDINATOR_INTERRUPTED"
finally:
charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values()))
new_gpu = sum(charges.values())
limit = budget["projected_new_gpu_seconds"]
overage = max(0., new_gpu - limit)
underlying_trigger = trigger
if overage > 0:
status, trigger = "STOPPED", "TOTAL_GPU_CAP_AT_SETTLEMENT"
save(root / "complete.json", dict(
status=status, trigger=trigger, underlying_trigger=underlying_trigger, receipts=receipts, **charges,
new_gpu_seconds=new_gpu, new_gpu_cap_seconds=limit, cap_overage_gpu_seconds=overage,
cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu,
cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu,
no_retry=True, test_targets_read=False, automatic_continuation=False))
return 0 if status == "COMPLETE" else 1
def main():
# The original entry and worker functions enforce the same exact review,
# process lifetime, environment and one-attempt rules at the new root.
namespace = dict(legacy.main.__globals__, ROOT=ROOT, dispatch_guard=dispatch_guard)
return FunctionType(legacy.main.__code__, namespace, legacy.main.__name__)()
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,183 @@
"""Freeze the narrow 1b repair and replay the independent review's boundary cases."""
from copy import deepcopy
from functools import partial
import inspect
from pathlib import Path
from tempfile import TemporaryDirectory
from types import SimpleNamespace
from unittest.mock import Mock, patch
import phase3_item1b_fold0 as legacy
import phase3_item1b_fold0_deadline as entry
from phase2_h1_worker_v2 import checked, save, sha
from s2_gpu_idle_gate import wait_idle
OLD_BINDING = "ed6bfe113f9c0bf925684a62cf0c1bb09bfc7b1277e06fad628fc0e9e5bcd859"
OLD_CONTRACT = "942323ddfceecf47515eb48628c45157073cbb909867a606ef4d3764a13be7d4"
ORIGINAL_BLOCKS = {
"meltassist-item1b-fold0-screen-review-2026-11-19": "0fadd70d365e1015f835fb953691797811a032766d3a9c1b75f3b8e4678faf61",
"meltassist-item1b-fold0-final-deadline-review-2026-11-19": "d45ac966f612236656f9b1a0d59ed2f60493c2da0b3c31279e189d7dfa217e35",
}
class InertMonitor:
def __init__(self, **kwargs):
pass
def start(self):
pass
def join(self, **kwargs):
pass
def is_alive(self):
return False
def prepare():
old_path = checked(legacy.ROOT / "contract.json", OLD_CONTRACT)
binding = checked(legacy.ROOT / "expected-review-binding.json", OLD_BINDING)
old = legacy.contract(old_path)
if (legacy.ROOT / "started.json").exists():
raise PermissionError("The original fold-0 entry already has a start record")
sources = {str(old_path): OLD_CONTRACT, str(binding): OLD_BINDING}
for name, digest in ORIGINAL_BLOCKS.items():
path = checked(entry.ROOT.parent / name / "review.json", digest)
if entry.read(path)["verdict"] != "BLOCKED":
raise ValueError("Original blocking review must be preserved")
sources[str(path)] = digest
new = deepcopy(old)
new["version"] = "phase3-item1b-fold0-deadline-settlement-v2"
new["engineering_ruling"] = "h-5064d81d"
new["source_sha256"].update(sources)
for path in (Path(entry.__file__), Path(__file__).resolve()):
new["code_sha256"][str(path)] = sha(path)
if new["budget"] != old["budget"] or new["budget"]["projected_new_gpu_seconds"] != 13180:
raise ValueError("Original budget must remain 13,180 GPU seconds")
entry.ROOT.mkdir(exist_ok=False)
path = entry.ROOT / "contract.json"
save(path, new)
return path
def simulate(path, *, latency, recheck_latency, settle_delay=None):
c = entry.contract(path)
now = 0.
samples = []
dispatches = []
source, first = inspect.getsourcelines(entry.dispatch_guard)
settle_line = next(first + i for i, line in enumerate(source)
if 'charges["finalization_gpu_seconds"] = max(' in line)
def sleep(seconds):
nonlocal now
now += seconds
def clock():
if settle_delay is not None and inspect.currentframe().f_back.f_lineno == settle_line:
sleep(settle_delay)
return now
def sample(uuid):
index = len(samples)
samples.append(now)
sleep(latency if index < 6 else recheck_latency)
return dict(uuid=uuid, idle=index >= 3, memory_MiB=5 if index >= 3 else 1500, occupants=[])
def record(path, data):
save(path, data)
if settle_delay is not None and path.name == "training-adopted.json":
sleep(119.)
def mock_supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline):
dispatches.append(dict(mode=mode, seconds=now, dispatch_deadline=dispatch_deadline))
if settle_delay is None:
raise AssertionError("Late-admission case must stop before the supervisor")
if dispatch_deadline != (60. if mode == "train" else None):
raise AssertionError("The original deadline must follow training only")
sleep(limits["job_wall_seconds"] - .1)
output = root / ("jobs/0/worker" if mode == "train" else "screen")
output.mkdir()
result = dict(status="COMPLETE", test_targets_read=False,
peak_allocated_bytes=32, peak_reserved_bytes=64)
if mode == "train":
checkpoint = output / "mock.pt"
checkpoint.write_bytes(b"synthetic")
result.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"],
checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint))
else:
result.update(parts=len(c["job"]["validation"]), automatic_continuation=False, decision="SCREEN_PASS")
save(output / "result.json", result)
return dict(status="PASS", returncode=0, monitor_errors=0)
with TemporaryDirectory(prefix="meltmind-1b-deadline-replay-") as scratch:
root = Path(scratch)
with (patch.object(entry, "snapshot", side_effect=sample),
patch.object(entry, "wait_idle", partial(wait_idle, clock=clock, sleep=sleep)),
patch.object(entry, "time", SimpleNamespace(monotonic=clock)),
patch.object(entry, "threading", SimpleNamespace(Thread=InertMonitor, Event=entry.threading.Event)),
patch.object(entry, "supervise", side_effect=mock_supervise),
patch.object(entry, "save", side_effect=record)):
returned = entry.dispatch_guard(root, path)
terminal = entry.read(root / "complete.json")
gate = entry.read(root / "idle-gate.json")
terminal.pop("receipts")
return dict(gate=gate["status"], gate_seconds=gate["wall_seconds"],
snapshot_start_seconds=samples, mock_dispatches=dispatches,
returned=returned, terminal=terminal)
def supervisor_boundary(path):
c = entry.contract(path)
now = 59.5
def disk_check(_):
nonlocal now
now = 60.5
return 0
blocked_spawn = Mock(side_effect=AssertionError("Popen invoked after the training deadline"))
with TemporaryDirectory(prefix="meltmind-1b-supervisor-deadline-") as scratch:
root = Path(scratch)
with (patch.object(entry, "time", SimpleNamespace(monotonic=lambda: now)),
patch.dict(entry.shared_supervise.__globals__, {"disk_usage": disk_check}),
patch.object(entry, "subprocess", SimpleNamespace(Popen=blocked_spawn, STDOUT=entry.subprocess.STDOUT))):
try:
entry.supervise(["synthetic-no-spawn"], root, "train", dict(
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=c["budget"]["train_gpu_seconds"],
job_wall_seconds=10., tree_rss_bytes=c["rss_bytes"], disk_bytes=c["disk_bytes"],
max_processes=c["max_processes"]), poll=.2, gpu_uuid=c["gpu_uuid"], dispatch_deadline=60.)
except entry.GuardStop as exc:
assert str(exc) == "TRAIN_DISPATCH_DEADLINE"
else:
raise AssertionError("The actual supervisor did not stop before spawn")
marker = entry.read(root / "train-spawn-blocked.json")
assert blocked_spawn.call_count == 0 and not marker["worker_started"]
return dict(marker=marker, real_popen_calls=blocked_spawn.call_count)
def preflight(path):
late = simulate(path, latency=1.5, recheck_latency=1.5)
cap = simulate(path, latency=1.5, recheck_latency=0., settle_delay=23.)
baseline = simulate(path, latency=1.5, recheck_latency=0., settle_delay=0.)
busy = simulate(path, latency=2., recheck_latency=2.)
assert late["gate_seconds"] == 59. and late["terminal"]["trigger"] == "TRAIN_DISPATCH_DEADLINE"
assert late["mock_dispatches"] == [] and late["returned"] == 1
assert round(cap["terminal"]["new_gpu_seconds"], 1) == 13180.8
assert cap["terminal"]["status"] == "STOPPED" and cap["returned"] == 1
assert cap["terminal"]["trigger"] == "TOTAL_GPU_CAP_AT_SETTLEMENT"
assert baseline["returned"] == 0 and baseline["terminal"]["status"] == "COMPLETE"
assert busy["terminal"]["trigger"] == "IDLE_TIMEOUT" and busy["mock_dispatches"] == []
return dict(status="PASS", contract_sha256=sha(path), old_binding_sha256=OLD_BINDING,
late_recheck=late, final_settlement=cap, timely_control=baseline, busy_control=busy,
in_supervisor_deadline=supervisor_boundary(path), real_gpu_queries=0, real_workers=0,
targets_read=False, independent_review="PENDING")
if __name__ == "__main__":
path = prepare()
receipt = preflight(path)
save(path.parent / "actual-guard-cpu-replay.json", receipt)
print(receipt)