184 lines
8.3 KiB
Python
184 lines
8.3 KiB
Python
"""Freeze the narrow 1b repair and replay the independent review's boundary cases."""
|
|
|
|
from copy import deepcopy
|
|
from functools import partial
|
|
import inspect
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock, patch
|
|
|
|
import phase3_item1b_fold0 as legacy
|
|
import phase3_item1b_fold0_deadline as entry
|
|
from phase2_h1_worker_v2 import checked, save, sha
|
|
from s2_gpu_idle_gate import wait_idle
|
|
|
|
|
|
OLD_BINDING = "ed6bfe113f9c0bf925684a62cf0c1bb09bfc7b1277e06fad628fc0e9e5bcd859"
|
|
OLD_CONTRACT = "942323ddfceecf47515eb48628c45157073cbb909867a606ef4d3764a13be7d4"
|
|
ORIGINAL_BLOCKS = {
|
|
"meltassist-item1b-fold0-screen-review-2026-11-19": "0fadd70d365e1015f835fb953691797811a032766d3a9c1b75f3b8e4678faf61",
|
|
"meltassist-item1b-fold0-final-deadline-review-2026-11-19": "d45ac966f612236656f9b1a0d59ed2f60493c2da0b3c31279e189d7dfa217e35",
|
|
}
|
|
|
|
|
|
class InertMonitor:
|
|
def __init__(self, **kwargs):
|
|
pass
|
|
|
|
def start(self):
|
|
pass
|
|
|
|
def join(self, **kwargs):
|
|
pass
|
|
|
|
def is_alive(self):
|
|
return False
|
|
|
|
|
|
def prepare():
|
|
old_path = checked(legacy.ROOT / "contract.json", OLD_CONTRACT)
|
|
binding = checked(legacy.ROOT / "expected-review-binding.json", OLD_BINDING)
|
|
old = legacy.contract(old_path)
|
|
if (legacy.ROOT / "started.json").exists():
|
|
raise PermissionError("The original fold-0 entry already has a start record")
|
|
sources = {str(old_path): OLD_CONTRACT, str(binding): OLD_BINDING}
|
|
for name, digest in ORIGINAL_BLOCKS.items():
|
|
path = checked(entry.ROOT.parent / name / "review.json", digest)
|
|
if entry.read(path)["verdict"] != "BLOCKED":
|
|
raise ValueError("Original blocking review must be preserved")
|
|
sources[str(path)] = digest
|
|
new = deepcopy(old)
|
|
new["version"] = "phase3-item1b-fold0-deadline-settlement-v2"
|
|
new["engineering_ruling"] = "h-5064d81d"
|
|
new["source_sha256"].update(sources)
|
|
for path in (Path(entry.__file__), Path(__file__).resolve()):
|
|
new["code_sha256"][str(path)] = sha(path)
|
|
if new["budget"] != old["budget"] or new["budget"]["projected_new_gpu_seconds"] != 13180:
|
|
raise ValueError("Original budget must remain 13,180 GPU seconds")
|
|
entry.ROOT.mkdir(exist_ok=False)
|
|
path = entry.ROOT / "contract.json"
|
|
save(path, new)
|
|
return path
|
|
|
|
|
|
def simulate(path, *, latency, recheck_latency, settle_delay=None):
|
|
c = entry.contract(path)
|
|
now = 0.
|
|
samples = []
|
|
dispatches = []
|
|
source, first = inspect.getsourcelines(entry.dispatch_guard)
|
|
settle_line = next(first + i for i, line in enumerate(source)
|
|
if 'charges["finalization_gpu_seconds"] = max(' in line)
|
|
|
|
def sleep(seconds):
|
|
nonlocal now
|
|
now += seconds
|
|
|
|
def clock():
|
|
if settle_delay is not None and inspect.currentframe().f_back.f_lineno == settle_line:
|
|
sleep(settle_delay)
|
|
return now
|
|
|
|
def sample(uuid):
|
|
index = len(samples)
|
|
samples.append(now)
|
|
sleep(latency if index < 6 else recheck_latency)
|
|
return dict(uuid=uuid, idle=index >= 3, memory_MiB=5 if index >= 3 else 1500, occupants=[])
|
|
|
|
def record(path, data):
|
|
save(path, data)
|
|
if settle_delay is not None and path.name == "training-adopted.json":
|
|
sleep(119.)
|
|
|
|
def mock_supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline):
|
|
dispatches.append(dict(mode=mode, seconds=now, dispatch_deadline=dispatch_deadline))
|
|
if settle_delay is None:
|
|
raise AssertionError("Late-admission case must stop before the supervisor")
|
|
if dispatch_deadline != (60. if mode == "train" else None):
|
|
raise AssertionError("The original deadline must follow training only")
|
|
sleep(limits["job_wall_seconds"] - .1)
|
|
output = root / ("jobs/0/worker" if mode == "train" else "screen")
|
|
output.mkdir()
|
|
result = dict(status="COMPLETE", test_targets_read=False,
|
|
peak_allocated_bytes=32, peak_reserved_bytes=64)
|
|
if mode == "train":
|
|
checkpoint = output / "mock.pt"
|
|
checkpoint.write_bytes(b"synthetic")
|
|
result.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"],
|
|
checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint))
|
|
else:
|
|
result.update(parts=len(c["job"]["validation"]), automatic_continuation=False, decision="SCREEN_PASS")
|
|
save(output / "result.json", result)
|
|
return dict(status="PASS", returncode=0, monitor_errors=0)
|
|
|
|
with TemporaryDirectory(prefix="meltmind-1b-deadline-replay-") as scratch:
|
|
root = Path(scratch)
|
|
with (patch.object(entry, "snapshot", side_effect=sample),
|
|
patch.object(entry, "wait_idle", partial(wait_idle, clock=clock, sleep=sleep)),
|
|
patch.object(entry, "time", SimpleNamespace(monotonic=clock)),
|
|
patch.object(entry, "threading", SimpleNamespace(Thread=InertMonitor, Event=entry.threading.Event)),
|
|
patch.object(entry, "supervise", side_effect=mock_supervise),
|
|
patch.object(entry, "save", side_effect=record)):
|
|
returned = entry.dispatch_guard(root, path)
|
|
terminal = entry.read(root / "complete.json")
|
|
gate = entry.read(root / "idle-gate.json")
|
|
terminal.pop("receipts")
|
|
return dict(gate=gate["status"], gate_seconds=gate["wall_seconds"],
|
|
snapshot_start_seconds=samples, mock_dispatches=dispatches,
|
|
returned=returned, terminal=terminal)
|
|
|
|
|
|
def supervisor_boundary(path):
|
|
c = entry.contract(path)
|
|
now = 59.5
|
|
|
|
def disk_check(_):
|
|
nonlocal now
|
|
now = 60.5
|
|
return 0
|
|
|
|
blocked_spawn = Mock(side_effect=AssertionError("Popen invoked after the training deadline"))
|
|
with TemporaryDirectory(prefix="meltmind-1b-supervisor-deadline-") as scratch:
|
|
root = Path(scratch)
|
|
with (patch.object(entry, "time", SimpleNamespace(monotonic=lambda: now)),
|
|
patch.dict(entry.shared_supervise.__globals__, {"disk_usage": disk_check}),
|
|
patch.object(entry, "subprocess", SimpleNamespace(Popen=blocked_spawn, STDOUT=entry.subprocess.STDOUT))):
|
|
try:
|
|
entry.supervise(["synthetic-no-spawn"], root, "train", dict(
|
|
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=c["budget"]["train_gpu_seconds"],
|
|
job_wall_seconds=10., tree_rss_bytes=c["rss_bytes"], disk_bytes=c["disk_bytes"],
|
|
max_processes=c["max_processes"]), poll=.2, gpu_uuid=c["gpu_uuid"], dispatch_deadline=60.)
|
|
except entry.GuardStop as exc:
|
|
assert str(exc) == "TRAIN_DISPATCH_DEADLINE"
|
|
else:
|
|
raise AssertionError("The actual supervisor did not stop before spawn")
|
|
marker = entry.read(root / "train-spawn-blocked.json")
|
|
assert blocked_spawn.call_count == 0 and not marker["worker_started"]
|
|
return dict(marker=marker, real_popen_calls=blocked_spawn.call_count)
|
|
|
|
|
|
def preflight(path):
|
|
late = simulate(path, latency=1.5, recheck_latency=1.5)
|
|
cap = simulate(path, latency=1.5, recheck_latency=0., settle_delay=23.)
|
|
baseline = simulate(path, latency=1.5, recheck_latency=0., settle_delay=0.)
|
|
busy = simulate(path, latency=2., recheck_latency=2.)
|
|
assert late["gate_seconds"] == 59. and late["terminal"]["trigger"] == "TRAIN_DISPATCH_DEADLINE"
|
|
assert late["mock_dispatches"] == [] and late["returned"] == 1
|
|
assert round(cap["terminal"]["new_gpu_seconds"], 1) == 13180.8
|
|
assert cap["terminal"]["status"] == "STOPPED" and cap["returned"] == 1
|
|
assert cap["terminal"]["trigger"] == "TOTAL_GPU_CAP_AT_SETTLEMENT"
|
|
assert baseline["returned"] == 0 and baseline["terminal"]["status"] == "COMPLETE"
|
|
assert busy["terminal"]["trigger"] == "IDLE_TIMEOUT" and busy["mock_dispatches"] == []
|
|
return dict(status="PASS", contract_sha256=sha(path), old_binding_sha256=OLD_BINDING,
|
|
late_recheck=late, final_settlement=cap, timely_control=baseline, busy_control=busy,
|
|
in_supervisor_deadline=supervisor_boundary(path), real_gpu_queries=0, real_workers=0,
|
|
targets_read=False, independent_review="PENDING")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
path = prepare()
|
|
receipt = preflight(path)
|
|
save(path.parent / "actual-guard-cpu-replay.json", receipt)
|
|
print(receipt)
|