183 lines
10 KiB
Python
183 lines
10 KiB
Python
"""Freeze the paper protocol and CPU-test the measurement dispatch contract."""
|
|
|
|
from functools import partial
|
|
from pathlib import Path
|
|
from tempfile import TemporaryDirectory
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
from phase2_h1_worker_v2 import checked, save, sha
|
|
from track_a_h2_prepare import local_code_closure
|
|
import track_b_gino_shapenet_entry as entry
|
|
from s2_gpu_idle_gate import wait_idle
|
|
|
|
|
|
BASE = entry.BASE
|
|
DATA = BASE / "data/track-b-gino-shapenet611-2026-11-19"
|
|
AUDIT = BASE / "artifacts/track-b-gino-shapenet611-2026-11-19"
|
|
UPSTREAM = Path("/tmp/meltmind-track-b-shapenet611-20261119")
|
|
NEURALOP = "fbdc7f872792b75b84545c2f374e7a881c72a718"
|
|
GPU = "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36"
|
|
|
|
|
|
def prepare():
|
|
root = entry.MEASUREMENT_ROOT
|
|
if root.exists():
|
|
raise FileExistsError("The Track B measurement preparation already exists")
|
|
split_path = AUDIT / "paper-split.json"
|
|
split = entry.read(split_path)
|
|
mapping = AUDIT / "paper-mapping-audit.json"
|
|
audit = entry.read(mapping)
|
|
if (audit["status"] != "PAPER_ARCHIVE_SPLIT_AND_MAPPING_AUDITED"
|
|
or audit["manifest_sha256"] != split["manifest_sha256"]
|
|
or audit["global_bounds_sha256"] != split["raw_global_bounds_sha256"]):
|
|
raise ValueError("Paper data not verified")
|
|
for partition in ("train", "test"):
|
|
if [r["case"] for r in audit["rows"] if r["split"] == partition] != split[partition]:
|
|
raise ValueError("Audit sample order differs")
|
|
pressure = checked(DATA / "paper-car-pressure-data.zip", audit["archive_sha256"])
|
|
supplement = checked(DATA / "paper-neurips-supplement.zip", split["paper_supplement_zip_sha256"])
|
|
paper_code = split["paper_code_zip_sha256"]
|
|
pin_names = {"utils.py": "legacy-utils.py",
|
|
"models/__init__.py": "legacy-__init__.py",
|
|
"models/tfno.py": "legacy-tfno.py",
|
|
"models/spectral_convolution.py": "legacy-spectral_convolution.py"}
|
|
upstream_hashes = {key: sha(UPSTREAM / name) for key, name in pin_names.items()}
|
|
environment = dict(
|
|
python=str(BASE / "data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python"),
|
|
python_version="3.10", torch_cuda="11.6", neuralop_commit=NEURALOP, neuralop_file_sha256=upstream_hashes,
|
|
package_versions={
|
|
"torch": "1.13.1+cu116", "open3d": "0.17.0", "numpy": "1.24.3",
|
|
"torch-scatter": "2.1.1+pt113cu116", "torch-sparse": "0.6.17+pt113cu116",
|
|
"torch-geometric": "2.3.1", "tensorly": "0.8.1", "tensorly-torch": "0.4.0",
|
|
})
|
|
protocol = dict(
|
|
model="NeurIPS 2023 supplement SDFFNOGNO decoder", seeds=[0, 1, 2],
|
|
grid=[64, 64, 64], modes=[24, 24, 24], hidden_channels=32, radius=.05,
|
|
epochs_per_seed=100, batch_size=1, learning_rate=.001, bounds_expansion=.01,
|
|
pressure_epsilon=1.e-6, optimizer="Adam", weight_decay=1.e-4,
|
|
lr_step_epochs=50, lr_gamma=.5, pressure_training_transform="scalar Gaussian on the 500 learning cases",
|
|
metric="per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)",
|
|
report="equal mean over 111 cases for each seed, then equal mean over the three seeds",
|
|
acceptance_percent=[6.62, 7.62], paper_reference_percent=7.12,
|
|
checkpoint_epoch=100, heldout_access_after_all_checkpoints_frozen=True)
|
|
root.mkdir(exist_ok=False)
|
|
comparison = root / "comparison-contract.json"
|
|
save(comparison, dict(
|
|
ruling=entry.RULING, protocol=protocol, paper_code_zip_sha256=paper_code,
|
|
source_protocol_commit="8d5c498e3be25580cf022890add6b32d7d6b8a93",
|
|
split_sha256=sha(split_path), audited_mapping_sha256=sha(mapping),
|
|
environment=environment, compute_regime="one V100, one serial process, no other GPU occupants",
|
|
opens_only_original_paper_data=True, acceptance_interval_provenance="team preregistered tolerance",
|
|
exact_original_published_cli_overrides_known=False))
|
|
code_paths = [Path(__file__), Path(entry.__file__), BASE / "tools/track_b_gino_shapenet_worker.py"]
|
|
codes = local_code_closure(code_paths)
|
|
limit = dict(gpu_wall_seconds=3600, cleanup_seconds=60, idle_wait_seconds=60, cpu_seconds=3500,
|
|
rss_bytes=12 * 2**30, recorded_vram_bytes=14 * 2**30, device_vram_bytes=15 * 2**30,
|
|
disk_bytes=2 * 2**30, torch_threads=1, max_processes=1)
|
|
path = root / "contract.json"
|
|
def pin(p):
|
|
return dict(path=str(p), sha256=sha(p))
|
|
|
|
save(path, dict(mode="measurement", ruling=entry.RULING, retry=False, evaluate_other_models=False,
|
|
measurement_allowed_heldout_labels=False, gpu_uuid=GPU, limits=limit,
|
|
protocol=protocol, environment=environment, comparison_contract=str(comparison),
|
|
comparison_contract_sha256=sha(comparison), code_sha256=codes,
|
|
paper_data=pin(pressure), paper_supplement=pin(supplement),
|
|
paper_code_zip_sha256=paper_code, split_receipt=pin(split_path), mapping_audit=pin(mapping),
|
|
split={key: split[key] for key in ("train", "test", "raw_global_bounds")}))
|
|
save(root / "measurement-gate.json", dict(
|
|
status="NOT_AUTHORIZED_FOR_EXECUTION", candidate_contract_sha256=sha(path),
|
|
runtime_installed_and_import_verified=False, measured_epoch_exists=False,
|
|
requested_candidate_limit=dict(gpu_wall_seconds=3600, cpu_seconds=3500, device_count=1),
|
|
requires=["install and CPU import-verify the isolated pinned environment",
|
|
"Claude stage/budget authorization binding the measurement contract",
|
|
"independent exact PASS on the measurement contract and CPU mock receipt"],
|
|
measurement="one complete 500-case seed-0 training epoch plus preproc and a 111-case training-only forward proxy",
|
|
heldout_labels_read=False, training_weights_reused=False,
|
|
benchmark_contract_created=False,
|
|
benchmark_cap_formula="ceil(1.25*(prep_seconds+300*epoch_seconds+proxy_prep_seconds+3*proxy_inference_seconds)) + wait + cleanup",
|
|
benchmark_requires="fixed same-regime measurement receipts, new exact cap and separate review"))
|
|
return path
|
|
|
|
|
|
def preflight(path):
|
|
c = entry.contract(path)
|
|
gpu = c["gpu_uuid"]
|
|
clock = 0.
|
|
mock_cases = []
|
|
for scenario in ("idle", "busy", "late_final_check", "total_cap"):
|
|
clock = 0.
|
|
polls = 0
|
|
|
|
def sleep(seconds):
|
|
nonlocal clock
|
|
clock += seconds
|
|
|
|
def sample(uuid):
|
|
nonlocal polls
|
|
if uuid != gpu:
|
|
raise ValueError("Actual contract GPU identity lost")
|
|
polls += 1
|
|
idle = scenario != "busy"
|
|
if scenario == "late_final_check":
|
|
early_busy = polls <= 3
|
|
duration = 2. if early_busy else 1. if polls <= 6 else 3.
|
|
idle = not early_busy
|
|
sleep(duration)
|
|
elif scenario == "busy":
|
|
sleep(2.)
|
|
return dict(uuid=uuid, idle=idle, memory_MiB=5 if idle else 1600, occupants=[])
|
|
|
|
def worker(command, actual_root, name, limits, *, poll, gpu_uuid):
|
|
if (actual_root != root or name != "gino" or command[0] != c["environment"]["python"]
|
|
or Path(command[1]).name != "track_b_gino_shapenet_worker.py" or command[2:] != [str(root)]
|
|
or gpu_uuid != gpu or limits["total_cpu_seconds"] != c["limits"]["cpu_seconds"]
|
|
or limits["tree_rss_bytes"] != c["limits"]["rss_bytes"]
|
|
or limits["disk_bytes"] != c["limits"]["disk_bytes"]
|
|
or limits["max_processes"] != c["limits"]["max_processes"]
|
|
or not 0 < limits["job_wall_seconds"] < c["limits"]["gpu_wall_seconds"]):
|
|
raise ValueError("Modified fields at actual dispatch")
|
|
records = root / "worker-records"
|
|
records.mkdir()
|
|
save(records / "result.json", dict(
|
|
status="COMPLETE", mode="measurement", heldout_labels_read=False, epochs=1,
|
|
training_examples=500, proxy_examples=111, reuse_measurement_weights=False,
|
|
preparation_seconds=1., epoch_seconds=2., proxy_preparation_seconds=1.,
|
|
proxy_inference_seconds=1., peak_allocated_bytes=100, peak_reserved_bytes=200))
|
|
sleep(c["limits"]["gpu_wall_seconds"] - clock + .8 if scenario == "total_cap" else 2.)
|
|
return dict(status="PASS", returncode=0, monitor_errors=0)
|
|
|
|
expected = {
|
|
"idle": ("COMPLETE", None, 1),
|
|
"busy": ("STOPPED", "IDLE_DEADLINE", 0),
|
|
"late_final_check": ("STOPPED", "IDLE_DEADLINE_AFTER_FINAL_CHECK", 0),
|
|
"total_cap": ("STOPPED", "TOTAL_GPU_WALL_CAP", 1),
|
|
}[scenario]
|
|
with TemporaryDirectory(prefix="track-b-gino-cpu-guard-") as scratch:
|
|
root = Path(scratch)
|
|
with (patch.object(entry, "snapshot", side_effect=sample),
|
|
patch.object(entry, "time", SimpleNamespace(monotonic=lambda: clock)),
|
|
patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: clock, sleep=sleep)),
|
|
patch.object(entry, "supervise", side_effect=worker) as dispatch):
|
|
code = entry.dispatch_guard(root, path)
|
|
terminal = entry.read(root / "complete.json")
|
|
if (terminal["status"], terminal["trigger"], dispatch.call_count) != expected:
|
|
raise RuntimeError(f"Actual guard differed for {scenario}: {terminal}")
|
|
if code != (0 if scenario == "idle" else 1):
|
|
raise RuntimeError("Terminal return code differs")
|
|
mock_cases.append(dict(scenario=scenario, synthetic_worker_dispatches=dispatch.call_count,
|
|
status=terminal["status"], trigger=terminal["trigger"],
|
|
booked_idle_seconds=terminal["booked_idle_seconds"],
|
|
charged_gpu_wall_seconds=terminal["cumulative_gpu_wall_seconds"]))
|
|
return dict(status="PASS", actual_contract_sha256=sha(path), cases=mock_cases,
|
|
real_gpu_queries=0, real_workers=0, scientific_arrays_read=False,
|
|
does_not_supply_measured_epoch_seconds=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
path = prepare()
|
|
proof = preflight(path)
|
|
save(path.parent / "actual-guard-cpu-preflight.json", proof)
|
|
print(proof)
|