Files
MeltMind/tools/prepare_phase3_item1b_fold0_deadline.py

184 lines
8.3 KiB
Python

"""Freeze the narrow 1b repair and replay the independent review's boundary cases."""
from copy import deepcopy
from functools import partial
import inspect
from pathlib import Path
from tempfile import TemporaryDirectory
from types import SimpleNamespace
from unittest.mock import Mock, patch
import phase3_item1b_fold0 as legacy
import phase3_item1b_fold0_deadline as entry
from phase2_h1_worker_v2 import checked, save, sha
from s2_gpu_idle_gate import wait_idle
OLD_BINDING = "ed6bfe113f9c0bf925684a62cf0c1bb09bfc7b1277e06fad628fc0e9e5bcd859"
OLD_CONTRACT = "942323ddfceecf47515eb48628c45157073cbb909867a606ef4d3764a13be7d4"
ORIGINAL_BLOCKS = {
"meltassist-item1b-fold0-screen-review-2026-11-19": "0fadd70d365e1015f835fb953691797811a032766d3a9c1b75f3b8e4678faf61",
"meltassist-item1b-fold0-final-deadline-review-2026-11-19": "d45ac966f612236656f9b1a0d59ed2f60493c2da0b3c31279e189d7dfa217e35",
}
class InertMonitor:
def __init__(self, **kwargs):
pass
def start(self):
pass
def join(self, **kwargs):
pass
def is_alive(self):
return False
def prepare():
old_path = checked(legacy.ROOT / "contract.json", OLD_CONTRACT)
binding = checked(legacy.ROOT / "expected-review-binding.json", OLD_BINDING)
old = legacy.contract(old_path)
if (legacy.ROOT / "started.json").exists():
raise PermissionError("The original fold-0 entry already has a start record")
sources = {str(old_path): OLD_CONTRACT, str(binding): OLD_BINDING}
for name, digest in ORIGINAL_BLOCKS.items():
path = checked(entry.ROOT.parent / name / "review.json", digest)
if entry.read(path)["verdict"] != "BLOCKED":
raise ValueError("Original blocking review must be preserved")
sources[str(path)] = digest
new = deepcopy(old)
new["version"] = "phase3-item1b-fold0-deadline-settlement-v2"
new["engineering_ruling"] = "h-5064d81d"
new["source_sha256"].update(sources)
for path in (Path(entry.__file__), Path(__file__).resolve()):
new["code_sha256"][str(path)] = sha(path)
if new["budget"] != old["budget"] or new["budget"]["projected_new_gpu_seconds"] != 13180:
raise ValueError("Original budget must remain 13,180 GPU seconds")
entry.ROOT.mkdir(exist_ok=False)
path = entry.ROOT / "contract.json"
save(path, new)
return path
def simulate(path, *, latency, recheck_latency, settle_delay=None):
c = entry.contract(path)
now = 0.
samples = []
dispatches = []
source, first = inspect.getsourcelines(entry.dispatch_guard)
settle_line = next(first + i for i, line in enumerate(source)
if 'charges["finalization_gpu_seconds"] = max(' in line)
def sleep(seconds):
nonlocal now
now += seconds
def clock():
if settle_delay is not None and inspect.currentframe().f_back.f_lineno == settle_line:
sleep(settle_delay)
return now
def sample(uuid):
index = len(samples)
samples.append(now)
sleep(latency if index < 6 else recheck_latency)
return dict(uuid=uuid, idle=index >= 3, memory_MiB=5 if index >= 3 else 1500, occupants=[])
def record(path, data):
save(path, data)
if settle_delay is not None and path.name == "training-adopted.json":
sleep(119.)
def mock_supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline):
dispatches.append(dict(mode=mode, seconds=now, dispatch_deadline=dispatch_deadline))
if settle_delay is None:
raise AssertionError("Late-admission case must stop before the supervisor")
if dispatch_deadline != (60. if mode == "train" else None):
raise AssertionError("The original deadline must follow training only")
sleep(limits["job_wall_seconds"] - .1)
output = root / ("jobs/0/worker" if mode == "train" else "screen")
output.mkdir()
result = dict(status="COMPLETE", test_targets_read=False,
peak_allocated_bytes=32, peak_reserved_bytes=64)
if mode == "train":
checkpoint = output / "mock.pt"
checkpoint.write_bytes(b"synthetic")
result.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"],
checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint))
else:
result.update(parts=len(c["job"]["validation"]), automatic_continuation=False, decision="SCREEN_PASS")
save(output / "result.json", result)
return dict(status="PASS", returncode=0, monitor_errors=0)
with TemporaryDirectory(prefix="meltmind-1b-deadline-replay-") as scratch:
root = Path(scratch)
with (patch.object(entry, "snapshot", side_effect=sample),
patch.object(entry, "wait_idle", partial(wait_idle, clock=clock, sleep=sleep)),
patch.object(entry, "time", SimpleNamespace(monotonic=clock)),
patch.object(entry, "threading", SimpleNamespace(Thread=InertMonitor, Event=entry.threading.Event)),
patch.object(entry, "supervise", side_effect=mock_supervise),
patch.object(entry, "save", side_effect=record)):
returned = entry.dispatch_guard(root, path)
terminal = entry.read(root / "complete.json")
gate = entry.read(root / "idle-gate.json")
terminal.pop("receipts")
return dict(gate=gate["status"], gate_seconds=gate["wall_seconds"],
snapshot_start_seconds=samples, mock_dispatches=dispatches,
returned=returned, terminal=terminal)
def supervisor_boundary(path):
c = entry.contract(path)
now = 59.5
def disk_check(_):
nonlocal now
now = 60.5
return 0
blocked_spawn = Mock(side_effect=AssertionError("Popen invoked after the training deadline"))
with TemporaryDirectory(prefix="meltmind-1b-supervisor-deadline-") as scratch:
root = Path(scratch)
with (patch.object(entry, "time", SimpleNamespace(monotonic=lambda: now)),
patch.dict(entry.shared_supervise.__globals__, {"disk_usage": disk_check}),
patch.object(entry, "subprocess", SimpleNamespace(Popen=blocked_spawn, STDOUT=entry.subprocess.STDOUT))):
try:
entry.supervise(["synthetic-no-spawn"], root, "train", dict(
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=c["budget"]["train_gpu_seconds"],
job_wall_seconds=10., tree_rss_bytes=c["rss_bytes"], disk_bytes=c["disk_bytes"],
max_processes=c["max_processes"]), poll=.2, gpu_uuid=c["gpu_uuid"], dispatch_deadline=60.)
except entry.GuardStop as exc:
assert str(exc) == "TRAIN_DISPATCH_DEADLINE"
else:
raise AssertionError("The actual supervisor did not stop before spawn")
marker = entry.read(root / "train-spawn-blocked.json")
assert blocked_spawn.call_count == 0 and not marker["worker_started"]
return dict(marker=marker, real_popen_calls=blocked_spawn.call_count)
def preflight(path):
late = simulate(path, latency=1.5, recheck_latency=1.5)
cap = simulate(path, latency=1.5, recheck_latency=0., settle_delay=23.)
baseline = simulate(path, latency=1.5, recheck_latency=0., settle_delay=0.)
busy = simulate(path, latency=2., recheck_latency=2.)
assert late["gate_seconds"] == 59. and late["terminal"]["trigger"] == "TRAIN_DISPATCH_DEADLINE"
assert late["mock_dispatches"] == [] and late["returned"] == 1
assert round(cap["terminal"]["new_gpu_seconds"], 1) == 13180.8
assert cap["terminal"]["status"] == "STOPPED" and cap["returned"] == 1
assert cap["terminal"]["trigger"] == "TOTAL_GPU_CAP_AT_SETTLEMENT"
assert baseline["returned"] == 0 and baseline["terminal"]["status"] == "COMPLETE"
assert busy["terminal"]["trigger"] == "IDLE_TIMEOUT" and busy["mock_dispatches"] == []
return dict(status="PASS", contract_sha256=sha(path), old_binding_sha256=OLD_BINDING,
late_recheck=late, final_settlement=cap, timely_control=baseline, busy_control=busy,
in_supervisor_deadline=supervisor_boundary(path), real_gpu_queries=0, real_workers=0,
targets_read=False, independent_review="PENDING")
if __name__ == "__main__":
path = prepare()
receipt = preflight(path)
save(path.parent / "actual-guard-cpu-replay.json", receipt)
print(receipt)