Prepare original ShapeNet GINO measurement entry and record 1b gate blocker
This commit is contained in:
1 parent
8d5c498e3b
commit
cc19b47ea9
17 files changed
+2439
No files matched your search
@@ -0,0 +1,182 @@
|
||||
"""Freeze the paper protocol and CPU-test the measurement dispatch contract."""
|
||||
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
import track_b_gino_shapenet_entry as entry
|
||||
from s2_gpu_idle_gate import wait_idle
|
||||
|
||||
|
||||
BASE = entry.BASE
|
||||
DATA = BASE / "data/track-b-gino-shapenet611-2026-11-19"
|
||||
AUDIT = BASE / "artifacts/track-b-gino-shapenet611-2026-11-19"
|
||||
UPSTREAM = Path("/tmp/meltmind-track-b-shapenet611-20261119")
|
||||
NEURALOP = "fbdc7f872792b75b84545c2f374e7a881c72a718"
|
||||
GPU = "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36"
|
||||
|
||||
|
||||
def prepare():
|
||||
root = entry.MEASUREMENT_ROOT
|
||||
if root.exists():
|
||||
raise FileExistsError("The Track B measurement preparation already exists")
|
||||
split_path = AUDIT / "paper-split.json"
|
||||
split = entry.read(split_path)
|
||||
mapping = AUDIT / "paper-mapping-audit.json"
|
||||
audit = entry.read(mapping)
|
||||
if (audit["status"] != "PAPER_ARCHIVE_SPLIT_AND_MAPPING_AUDITED"
|
||||
or audit["manifest_sha256"] != split["manifest_sha256"]
|
||||
or audit["global_bounds_sha256"] != split["raw_global_bounds_sha256"]):
|
||||
raise ValueError("Paper data not verified")
|
||||
for partition in ("train", "test"):
|
||||
if [r["case"] for r in audit["rows"] if r["split"] == partition] != split[partition]:
|
||||
raise ValueError("Audit sample order differs")
|
||||
pressure = checked(DATA / "paper-car-pressure-data.zip", audit["archive_sha256"])
|
||||
supplement = checked(DATA / "paper-neurips-supplement.zip", split["paper_supplement_zip_sha256"])
|
||||
paper_code = split["paper_code_zip_sha256"]
|
||||
pin_names = {"utils.py": "legacy-utils.py",
|
||||
"models/__init__.py": "legacy-__init__.py",
|
||||
"models/tfno.py": "legacy-tfno.py",
|
||||
"models/spectral_convolution.py": "legacy-spectral_convolution.py"}
|
||||
upstream_hashes = {key: sha(UPSTREAM / name) for key, name in pin_names.items()}
|
||||
environment = dict(
|
||||
python=str(BASE / "data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python"),
|
||||
python_version="3.10", torch_cuda="11.6", neuralop_commit=NEURALOP, neuralop_file_sha256=upstream_hashes,
|
||||
package_versions={
|
||||
"torch": "1.13.1+cu116", "open3d": "0.17.0", "numpy": "1.24.3",
|
||||
"torch-scatter": "2.1.1+pt113cu116", "torch-sparse": "0.6.17+pt113cu116",
|
||||
"torch-geometric": "2.3.1", "tensorly": "0.8.1", "tensorly-torch": "0.4.0",
|
||||
})
|
||||
protocol = dict(
|
||||
model="NeurIPS 2023 supplement SDFFNOGNO decoder", seeds=[0, 1, 2],
|
||||
grid=[64, 64, 64], modes=[24, 24, 24], hidden_channels=32, radius=.05,
|
||||
epochs_per_seed=100, batch_size=1, learning_rate=.001, bounds_expansion=.01,
|
||||
pressure_epsilon=1.e-6, optimizer="Adam", weight_decay=1.e-4,
|
||||
lr_step_epochs=50, lr_gamma=.5, pressure_training_transform="scalar Gaussian on the 500 learning cases",
|
||||
metric="per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)",
|
||||
report="equal mean over 111 cases for each seed, then equal mean over the three seeds",
|
||||
acceptance_percent=[6.62, 7.62], paper_reference_percent=7.12,
|
||||
checkpoint_epoch=100, heldout_access_after_all_checkpoints_frozen=True)
|
||||
root.mkdir(exist_ok=False)
|
||||
comparison = root / "comparison-contract.json"
|
||||
save(comparison, dict(
|
||||
ruling=entry.RULING, protocol=protocol, paper_code_zip_sha256=paper_code,
|
||||
source_protocol_commit="8d5c498e3be25580cf022890add6b32d7d6b8a93",
|
||||
split_sha256=sha(split_path), audited_mapping_sha256=sha(mapping),
|
||||
environment=environment, compute_regime="one V100, one serial process, no other GPU occupants",
|
||||
opens_only_original_paper_data=True, acceptance_interval_provenance="team preregistered tolerance",
|
||||
exact_original_published_cli_overrides_known=False))
|
||||
code_paths = [Path(__file__), Path(entry.__file__), BASE / "tools/track_b_gino_shapenet_worker.py"]
|
||||
codes = local_code_closure(code_paths)
|
||||
limit = dict(gpu_wall_seconds=3600, cleanup_seconds=60, idle_wait_seconds=60, cpu_seconds=3500,
|
||||
rss_bytes=12 * 2**30, recorded_vram_bytes=14 * 2**30, device_vram_bytes=15 * 2**30,
|
||||
disk_bytes=2 * 2**30, torch_threads=1, max_processes=1)
|
||||
path = root / "contract.json"
|
||||
def pin(p):
|
||||
return dict(path=str(p), sha256=sha(p))
|
||||
|
||||
save(path, dict(mode="measurement", ruling=entry.RULING, retry=False, evaluate_other_models=False,
|
||||
measurement_allowed_heldout_labels=False, gpu_uuid=GPU, limits=limit,
|
||||
protocol=protocol, environment=environment, comparison_contract=str(comparison),
|
||||
comparison_contract_sha256=sha(comparison), code_sha256=codes,
|
||||
paper_data=pin(pressure), paper_supplement=pin(supplement),
|
||||
paper_code_zip_sha256=paper_code, split_receipt=pin(split_path), mapping_audit=pin(mapping),
|
||||
split={key: split[key] for key in ("train", "test", "raw_global_bounds")}))
|
||||
save(root / "measurement-gate.json", dict(
|
||||
status="NOT_AUTHORIZED_FOR_EXECUTION", candidate_contract_sha256=sha(path),
|
||||
runtime_installed_and_import_verified=False, measured_epoch_exists=False,
|
||||
requested_candidate_limit=dict(gpu_wall_seconds=3600, cpu_seconds=3500, device_count=1),
|
||||
requires=["install and CPU import-verify the isolated pinned environment",
|
||||
"Claude stage/budget authorization binding the measurement contract",
|
||||
"independent exact PASS on the measurement contract and CPU mock receipt"],
|
||||
measurement="one complete 500-case seed-0 training epoch plus preproc and a 111-case training-only forward proxy",
|
||||
heldout_labels_read=False, training_weights_reused=False,
|
||||
benchmark_contract_created=False,
|
||||
benchmark_cap_formula="ceil(1.25*(prep_seconds+300*epoch_seconds+proxy_prep_seconds+3*proxy_inference_seconds)) + wait + cleanup",
|
||||
benchmark_requires="fixed same-regime measurement receipts, new exact cap and separate review"))
|
||||
return path
|
||||
|
||||
|
||||
def preflight(path):
|
||||
c = entry.contract(path)
|
||||
gpu = c["gpu_uuid"]
|
||||
clock = 0.
|
||||
mock_cases = []
|
||||
for scenario in ("idle", "busy", "late_final_check", "total_cap"):
|
||||
clock = 0.
|
||||
polls = 0
|
||||
|
||||
def sleep(seconds):
|
||||
nonlocal clock
|
||||
clock += seconds
|
||||
|
||||
def sample(uuid):
|
||||
nonlocal polls
|
||||
if uuid != gpu:
|
||||
raise ValueError("Actual contract GPU identity lost")
|
||||
polls += 1
|
||||
idle = scenario != "busy"
|
||||
if scenario == "late_final_check":
|
||||
early_busy = polls <= 3
|
||||
duration = 2. if early_busy else 1. if polls <= 6 else 3.
|
||||
idle = not early_busy
|
||||
sleep(duration)
|
||||
elif scenario == "busy":
|
||||
sleep(2.)
|
||||
return dict(uuid=uuid, idle=idle, memory_MiB=5 if idle else 1600, occupants=[])
|
||||
|
||||
def worker(command, actual_root, name, limits, *, poll, gpu_uuid):
|
||||
if (actual_root != root or name != "gino" or command[0] != c["environment"]["python"]
|
||||
or Path(command[1]).name != "track_b_gino_shapenet_worker.py" or command[2:] != [str(root)]
|
||||
or gpu_uuid != gpu or limits["total_cpu_seconds"] != c["limits"]["cpu_seconds"]
|
||||
or limits["tree_rss_bytes"] != c["limits"]["rss_bytes"]
|
||||
or limits["disk_bytes"] != c["limits"]["disk_bytes"]
|
||||
or limits["max_processes"] != c["limits"]["max_processes"]
|
||||
or not 0 < limits["job_wall_seconds"] < c["limits"]["gpu_wall_seconds"]):
|
||||
raise ValueError("Modified fields at actual dispatch")
|
||||
records = root / "worker-records"
|
||||
records.mkdir()
|
||||
save(records / "result.json", dict(
|
||||
status="COMPLETE", mode="measurement", heldout_labels_read=False, epochs=1,
|
||||
training_examples=500, proxy_examples=111, reuse_measurement_weights=False,
|
||||
preparation_seconds=1., epoch_seconds=2., proxy_preparation_seconds=1.,
|
||||
proxy_inference_seconds=1., peak_allocated_bytes=100, peak_reserved_bytes=200))
|
||||
sleep(c["limits"]["gpu_wall_seconds"] - clock + .8 if scenario == "total_cap" else 2.)
|
||||
return dict(status="PASS", returncode=0, monitor_errors=0)
|
||||
|
||||
expected = {
|
||||
"idle": ("COMPLETE", None, 1),
|
||||
"busy": ("STOPPED", "IDLE_DEADLINE", 0),
|
||||
"late_final_check": ("STOPPED", "IDLE_DEADLINE_AFTER_FINAL_CHECK", 0),
|
||||
"total_cap": ("STOPPED", "TOTAL_GPU_WALL_CAP", 1),
|
||||
}[scenario]
|
||||
with TemporaryDirectory(prefix="track-b-gino-cpu-guard-") as scratch:
|
||||
root = Path(scratch)
|
||||
with (patch.object(entry, "snapshot", side_effect=sample),
|
||||
patch.object(entry, "time", SimpleNamespace(monotonic=lambda: clock)),
|
||||
patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: clock, sleep=sleep)),
|
||||
patch.object(entry, "supervise", side_effect=worker) as dispatch):
|
||||
code = entry.dispatch_guard(root, path)
|
||||
terminal = entry.read(root / "complete.json")
|
||||
if (terminal["status"], terminal["trigger"], dispatch.call_count) != expected:
|
||||
raise RuntimeError(f"Actual guard differed for {scenario}: {terminal}")
|
||||
if code != (0 if scenario == "idle" else 1):
|
||||
raise RuntimeError("Terminal return code differs")
|
||||
mock_cases.append(dict(scenario=scenario, synthetic_worker_dispatches=dispatch.call_count,
|
||||
status=terminal["status"], trigger=terminal["trigger"],
|
||||
booked_idle_seconds=terminal["booked_idle_seconds"],
|
||||
charged_gpu_wall_seconds=terminal["cumulative_gpu_wall_seconds"]))
|
||||
return dict(status="PASS", actual_contract_sha256=sha(path), cases=mock_cases,
|
||||
real_gpu_queries=0, real_workers=0, scientific_arrays_read=False,
|
||||
does_not_supply_measured_epoch_seconds=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
path = prepare()
|
||||
proof = preflight(path)
|
||||
save(path.parent / "actual-guard-cpu-preflight.json", proof)
|
||||
print(proof)
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Review-gated dispatch for the original ShapeNet GINO reproduction."""
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import fcntl
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase3_item1b_common import read
|
||||
from phase3_shared_capacity import process_identity
|
||||
from s2_gpu_idle_gate import snapshot, wait_idle
|
||||
from s2_main_memory_guard import PeakRecords, active_command_pids
|
||||
from s2_recovery_gpu_supervisor import stop_group, supervise
|
||||
|
||||
|
||||
BASE = Path(__file__).resolve().parents[1]
|
||||
MEASUREMENT_ROOT = BASE / "artifacts/track-b-gino-shapenet611-measurement-2026-11-19"
|
||||
BENCHMARK_ROOT = BASE / "artifacts/track-b-gino-shapenet611-benchmark-2026-11-19"
|
||||
RULING = "h-f84547d7"
|
||||
|
||||
|
||||
def projected_budget(measurement, limits):
|
||||
prep = measurement["preparation_seconds"]
|
||||
train = measurement["epoch_seconds"]
|
||||
test_prep = measurement["proxy_preparation_seconds"]
|
||||
inference = measurement["proxy_inference_seconds"]
|
||||
if any(not math.isfinite(x) or x <= 0 for x in (prep, train, test_prep, inference)):
|
||||
raise ValueError("Incomplete empirical timing components")
|
||||
if (measurement["epochs"], measurement["training_examples"], measurement["proxy_examples"],
|
||||
measurement["heldout_labels_read"], measurement["reuse_measurement_weights"]) != (1, 500, 111, False, False):
|
||||
raise ValueError("Timing does not represent the reviewed learning-only measurement")
|
||||
# One serial worker, one preprocessing pass per domain, and three fresh
|
||||
# serial 100-epoch fits. The fixed wait and cleanup are charged separately.
|
||||
return math.ceil(1.25 * (prep + 3 * 100 * train + test_prep + 3 * inference)) + (
|
||||
limits["idle_wait_seconds"] + limits["cleanup_seconds"])
|
||||
|
||||
|
||||
def contract(path):
|
||||
c = read(path)
|
||||
if c["ruling"] != RULING or c["mode"] not in ("measurement", "benchmark"):
|
||||
raise PermissionError("Only the paper GINO execution scope is supported")
|
||||
if c["retry"] is not False or c["evaluate_other_models"] is not False:
|
||||
raise PermissionError("Single reviewed GINO attempt required")
|
||||
for path, digest in c["code_sha256"].items():
|
||||
checked(path, digest)
|
||||
sources = ("paper_data", "paper_supplement", "split_receipt", "mapping_audit")
|
||||
for name in sources:
|
||||
pin = c[name]
|
||||
checked(pin["path"], pin["sha256"])
|
||||
comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"]))
|
||||
if c["protocol"] != comparison["protocol"]:
|
||||
raise ValueError("Model/score protocol differs from the comparison contract")
|
||||
split = c["split"]
|
||||
original = read(c["split_receipt"]["path"])
|
||||
if any(split[key] != original[key] for key in ("train", "test", "raw_global_bounds")):
|
||||
raise ValueError("Original ShapeNet split or bounds changed")
|
||||
if len(set(split["train"])) != 500 or len(set(split["test"])) != 111 or set(split["train"]) & set(split["test"]):
|
||||
raise ValueError("Expected disjoint original 500/111 split")
|
||||
p = c["protocol"]
|
||||
if p["grid"] != [64, 64, 64] or p["epochs_per_seed"] != 100 or p["seeds"] != [0, 1, 2]:
|
||||
raise ValueError("Fixed decoder resolution, epochs or seeds changed")
|
||||
if p["acceptance_percent"] != [6.62, 7.62]:
|
||||
raise ValueError("Original pre-registered team interval changed")
|
||||
limit = c["limits"]
|
||||
caps = ("gpu_wall_seconds", "cleanup_seconds", "idle_wait_seconds", "cpu_seconds", "rss_bytes",
|
||||
"recorded_vram_bytes", "device_vram_bytes", "disk_bytes", "torch_threads", "max_processes")
|
||||
if any(type(limit[k]) is not int or limit[k] <= 0 for k in caps):
|
||||
raise ValueError("All runtime limits must be positive integers")
|
||||
if limit["idle_wait_seconds"] + limit["cleanup_seconds"] >= limit["gpu_wall_seconds"]:
|
||||
raise ValueError("No runtime after booking admission and cleanup")
|
||||
if c["mode"] == "measurement":
|
||||
if c["measurement_allowed_heldout_labels"] is not False:
|
||||
raise PermissionError("Measurement may only use the learning split")
|
||||
else:
|
||||
timed = c["measured_timing"]
|
||||
expected_paths = {
|
||||
"path": MEASUREMENT_ROOT / "worker-records/result.json",
|
||||
"guard_path": MEASUREMENT_ROOT / "complete.json",
|
||||
"contract_path": MEASUREMENT_ROOT / "contract.json",
|
||||
}
|
||||
if any(Path(timed[key]).resolve() != expected for key, expected in expected_paths.items()):
|
||||
raise ValueError("Timing must come from the sole reviewed measurement root")
|
||||
timing = read(checked(timed["path"], timed["sha256"]))
|
||||
witness = read(checked(timed["guard_path"], timed["guard_sha256"]))
|
||||
if (witness["status"] != "COMPLETE" or witness["contract_sha256"] != timed["contract_sha256"]
|
||||
or witness["result_sha256"] != timed["sha256"] or timing["mode"] != "measurement"
|
||||
or timing["status"] != "COMPLETE"):
|
||||
raise ValueError("Benchmark requires the authentic measured timing and guard result")
|
||||
measured = contract(Path(checked(timed["contract_path"], timed["contract_sha256"])))
|
||||
for key in ("gpu_uuid", "paper_data", "paper_code_zip_sha256", "comparison_contract_sha256", "environment"):
|
||||
if measured[key] != c[key]:
|
||||
raise ValueError(f"Timing and benchmark execution regime differs: {key}")
|
||||
if measured["mode"] != "measurement" or measured["limits"]["torch_threads"] != limit["torch_threads"]:
|
||||
raise ValueError("Timing is not from the same worker/CPU configuration")
|
||||
validate_review(MEASUREMENT_ROOT)
|
||||
if limit["gpu_wall_seconds"] != projected_budget(timing, limit):
|
||||
raise ValueError("Benchmark cap must come from the measured +25% projection")
|
||||
return c
|
||||
|
||||
|
||||
def dispatch_guard(root, contract_path):
|
||||
started = time.monotonic()
|
||||
c = contract(contract_path)
|
||||
cap = c["limits"]
|
||||
deadline = started + cap["gpu_wall_seconds"]
|
||||
wait_deadline = started + cap["idle_wait_seconds"]
|
||||
gpu = c["gpu_uuid"]
|
||||
status, reason = "STOPPED", None
|
||||
receipt = None
|
||||
waiting = 0.
|
||||
output = None
|
||||
tracker = PeakRecords()
|
||||
command = [c["environment"]["python"], str(Path(__file__).with_name("track_b_gino_shapenet_worker.py")), str(root)]
|
||||
try:
|
||||
if shutil.disk_usage(root).free < cap["disk_bytes"]:
|
||||
raise RuntimeError("DISK_HEADROOM")
|
||||
# Use the shared poller, then look at wall time again after the last
|
||||
# independent check and the dispatch receipt write.
|
||||
begin_wait = time.monotonic()
|
||||
remaining_window = max(0., wait_deadline - begin_wait)
|
||||
idle = wait_idle(gpu, remaining_window, lambda _: None, sample=snapshot)
|
||||
waiting = time.monotonic() - begin_wait
|
||||
save(root / "idle-gate.json", idle)
|
||||
if idle["status"] != "IDLE_CONFIRMED" or time.monotonic() >= wait_deadline:
|
||||
raise RuntimeError("IDLE_DEADLINE")
|
||||
if not snapshot(gpu)["idle"]:
|
||||
raise RuntimeError("SELECTED_GPU_BECAME_BUSY")
|
||||
save(root / "dispatch.json", dict(contract_sha256=sha(contract_path),
|
||||
coordinator=process_identity(os.getpid()),
|
||||
start_by_monotonic=wait_deadline))
|
||||
now = time.monotonic()
|
||||
waiting = now - begin_wait
|
||||
if now >= wait_deadline:
|
||||
raise RuntimeError("IDLE_DEADLINE_AFTER_FINAL_CHECK")
|
||||
remaining = deadline - now - cap["cleanup_seconds"]
|
||||
if remaining <= 0:
|
||||
raise RuntimeError("INSUFFICIENT_REMAINING_BUDGET")
|
||||
reasons = []
|
||||
done = threading.Event()
|
||||
|
||||
def monitor():
|
||||
while not done.is_set():
|
||||
try:
|
||||
tracker.scan(root / "worker-records")
|
||||
state = snapshot(gpu)
|
||||
allowed = set(active_command_pids(command))
|
||||
if max(tracker.maxima.values()) > cap["recorded_vram_bytes"]:
|
||||
reasons.append("RECORDED_VRAM_LIMIT")
|
||||
if state["memory_MiB"] * 2**20 > cap["device_vram_bytes"]:
|
||||
reasons.append("DEVICE_VRAM_LIMIT")
|
||||
if any(p["pid"] not in allowed for p in state["occupants"]):
|
||||
reasons.append("FOREIGN_GPU_OCCUPANT")
|
||||
except Exception as exc:
|
||||
reasons.append("MONITOR_FAILURE: " + str(exc))
|
||||
if reasons:
|
||||
for pid in active_command_pids(command):
|
||||
stop_group(pid)
|
||||
done.wait(.5)
|
||||
|
||||
watcher = threading.Thread(target=monitor, daemon=True)
|
||||
watcher.start()
|
||||
try:
|
||||
receipt = supervise(command, root, "gino", dict(
|
||||
total_cpu_seconds=cap["cpu_seconds"], job_cpu_seconds=cap["cpu_seconds"],
|
||||
job_wall_seconds=remaining, tree_rss_bytes=cap["rss_bytes"],
|
||||
disk_bytes=cap["disk_bytes"], max_processes=cap["max_processes"]), poll=.2, gpu_uuid=gpu)
|
||||
finally:
|
||||
done.set()
|
||||
watcher.join(timeout=8)
|
||||
if watcher.is_alive():
|
||||
raise RuntimeError("MONITOR_DID_NOT_EXIT")
|
||||
tracker.scan(root / "worker-records")
|
||||
if max(tracker.maxima.values()) > cap["recorded_vram_bytes"]:
|
||||
reasons.append("RECORDED_VRAM_LIMIT")
|
||||
if reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0 or receipt["monitor_errors"] != 0:
|
||||
raise RuntimeError(reasons[0] if reasons else receipt.get("stop_reason") or "WORKER_FAILED")
|
||||
output = root / "worker-records/result.json"
|
||||
result = read(output)
|
||||
if (result["status"] != "COMPLETE" or result["mode"] != c["mode"]
|
||||
or set(tracker.witnesses) != set(tracker.maxima)):
|
||||
raise RuntimeError("INCOMPLETE_WORKER_RESULT")
|
||||
if c["mode"] == "measurement" and (result["heldout_labels_read"] or result["epochs"] != 1):
|
||||
raise RuntimeError("MEASUREMENT_SCOPE_MISMATCH")
|
||||
if c["mode"] == "benchmark" and (not result["heldout_labels_read"]
|
||||
or set(result["seeds"]) != {str(x) for x in c["protocol"]["seeds"]}
|
||||
or any([item["case"] for item in seed["cases"]] != c["split"]["test"] for seed in result["seeds"].values())):
|
||||
raise RuntimeError("BENCHMARK_SUPPORT_MISMATCH")
|
||||
status = "COMPLETE"
|
||||
except (Exception, KeyboardInterrupt) as exc:
|
||||
reason = str(exc) or type(exc).__name__
|
||||
result_hash = sha(output) if output is not None and output.is_file() else None
|
||||
contract_hash = sha(contract_path)
|
||||
measured = time.monotonic() - started
|
||||
if measured > cap["gpu_wall_seconds"]:
|
||||
status, reason = "STOPPED", "TOTAL_GPU_WALL_CAP"
|
||||
save(root / "complete.json", dict(
|
||||
status=status, trigger=reason, contract_sha256=contract_hash, result_sha256=result_hash,
|
||||
worker=receipt, booked_idle_seconds=waiting, cumulative_gpu_wall_seconds=measured,
|
||||
gpu_wall_cap_seconds=cap["gpu_wall_seconds"], recorded_peaks=tracker.maxima, no_retry=True))
|
||||
return 0 if status == "COMPLETE" else 1
|
||||
|
||||
|
||||
def authorized_child(root):
|
||||
root = Path(root).resolve()
|
||||
if root not in (MEASUREMENT_ROOT, BENCHMARK_ROOT):
|
||||
raise PermissionError("Unrecognized Track B run root")
|
||||
ticket = read(root / "dispatch.json")
|
||||
parent = os.getppid()
|
||||
if (ticket["coordinator"] != process_identity(parent)
|
||||
or ticket["contract_sha256"] != sha(root / "contract.json")
|
||||
or time.monotonic() >= ticket["start_by_monotonic"]):
|
||||
raise PermissionError("Child is not attached to this reviewed guard")
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0 or os.getppid() != parent:
|
||||
raise RuntimeError("Could not attach child lifetime to its coordinator")
|
||||
validate_review(root)
|
||||
return contract(root / "contract.json")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("root", type=Path)
|
||||
args = parser.parse_args()
|
||||
root = args.root.resolve()
|
||||
if root not in (MEASUREMENT_ROOT, BENCHMARK_ROOT):
|
||||
raise PermissionError("Unrecognized Track B run root")
|
||||
validate_review(root)
|
||||
path = root / "contract.json"
|
||||
c = contract(path)
|
||||
if (root == MEASUREMENT_ROOT) != (c["mode"] == "measurement"):
|
||||
raise PermissionError("Execution root is tied to its reviewed stage")
|
||||
# Separate authority for the costed measurement or benchmark.
|
||||
authorization = read(root / "stage-authorization.json")
|
||||
if (authorization["contract_sha256"] != sha(path)
|
||||
or authorization["decision"] != "APPROVED"
|
||||
or authorization["mode"] != c["mode"]):
|
||||
raise PermissionError("Exact stage authorization is required")
|
||||
if not os.access(c["environment"]["python"], os.X_OK):
|
||||
raise FileNotFoundError("The isolated pinned GINO runtime is not available")
|
||||
with (root / "entry.lock").open("a") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root / "started.json").exists():
|
||||
raise FileExistsError("One attempt already recorded")
|
||||
save(root / "started.json", dict(coordinator=process_identity(os.getpid()), contract_sha256=sha(path)))
|
||||
def request_stop(signum, _frame):
|
||||
raise KeyboardInterrupt(signal.Signals(signum).name)
|
||||
|
||||
handler = signal.signal(signal.SIGTERM, request_stop)
|
||||
try:
|
||||
return dispatch_guard(root, path)
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, handler)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,315 @@
|
||||
"""Paper GINO-decoder worker with a measurement-only stage and held-out scoring."""
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import sys
|
||||
from tempfile import TemporaryDirectory
|
||||
import time
|
||||
from zipfile import ZipFile
|
||||
|
||||
from track_b_gino_shapenet_entry import authorized_child, checked, save, sha
|
||||
|
||||
|
||||
def extract_paper_code(c, directory):
|
||||
with ZipFile(c["paper_supplement"]["path"]) as outer:
|
||||
archive = outer.read("GINO-supp/code.zip")
|
||||
import hashlib
|
||||
if hashlib.sha256(archive).hexdigest() != c["paper_code_zip_sha256"]:
|
||||
raise ValueError("NeurIPS code archive identity mismatch")
|
||||
directory = Path(directory)
|
||||
with ZipFile(io.BytesIO(archive)) as code:
|
||||
for member in code.infolist():
|
||||
path = PurePosixPath(member.filename)
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
raise ValueError("Invalid archive member path")
|
||||
if not member.is_dir() and len(path.parts) >= 3 and path.parts[:2] == ("GINO-submission", "src"):
|
||||
if path.suffix == ".py":
|
||||
output = directory.joinpath(*path.parts[1:])
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with output.open("xb") as f:
|
||||
f.write(code.read(member))
|
||||
sys.path.insert(0, str(directory))
|
||||
|
||||
|
||||
class OriginalData:
|
||||
def __init__(self, c, workspace):
|
||||
import numpy as np
|
||||
import torch
|
||||
from src.data.cfd_datamodule import CFDSDFDataModule, DictDatasetWithConstant
|
||||
from neuralop.utils import UnitGaussianNormalizer
|
||||
|
||||
self.np = np
|
||||
self.torch = torch
|
||||
self.dataset_class = DictDatasetWithConstant
|
||||
self.normalizer_class = UnitGaussianNormalizer
|
||||
self.c = c
|
||||
self.workspace = Path(workspace)
|
||||
self.paper = object.__new__(CFDSDFDataModule)
|
||||
self.rows = {r["case"]: r for r in json.loads(Path(c["mapping_audit"]["path"]).read_bytes())["rows"]}
|
||||
self.train_ids = c["split"]["train"]
|
||||
self.test_ids = c["split"]["test"]
|
||||
resolution = c["protocol"]["grid"]
|
||||
raw_bounds = c["split"]["raw_global_bounds"]
|
||||
epsilon = c["protocol"]["bounds_expansion"]
|
||||
# Identical physical grid and coordinate transforms to the attached loader.
|
||||
lo = [v - epsilon for v in raw_bounds[0]]
|
||||
hi = [v + epsilon for v in raw_bounds[1]]
|
||||
axis = [np.linspace(a, b, n) for a, b, n in zip(lo, hi, resolution)]
|
||||
self.xyz = np.stack(np.meshgrid(*axis, indexing="ij"), axis=-1).astype(np.float32)
|
||||
self.bounds = torch.tensor(lo), torch.tensor(hi)
|
||||
self.grid = self.paper.location_normalization(
|
||||
torch.from_numpy(self.xyz), *self.bounds).permute(3, 0, 1, 2)
|
||||
self.normalizer = None
|
||||
|
||||
def data(self, ids, role, target_log=None):
|
||||
import hashlib
|
||||
|
||||
if role == "train":
|
||||
if ids != self.train_ids or self.normalizer is not None:
|
||||
raise PermissionError("Only first-pass training normalization")
|
||||
elif role == "train_proxy":
|
||||
if ids != self.train_ids[:111] or self.normalizer is None:
|
||||
raise PermissionError("Only the fixed learning-only timing proxy")
|
||||
elif role == "heldout":
|
||||
if ids != self.test_ids or target_log is None or self.normalizer is None:
|
||||
raise PermissionError("Held-out arrays require a completed checkpoint freeze")
|
||||
else:
|
||||
raise PermissionError("Unexpected data role")
|
||||
torch = self.torch
|
||||
folder = self.workspace / role
|
||||
folder.mkdir(exist_ok=False)
|
||||
sdfs, vertices, pressures = [], [], []
|
||||
with ZipFile(self.c["paper_data"]["path"]) as archive:
|
||||
for case in ids:
|
||||
row = self.rows[case]
|
||||
mesh = archive.read(f"car-pressure-data/data/mesh_{case:03d}.ply")
|
||||
if hashlib.sha256(mesh).hexdigest() != row["mesh_sha256"]:
|
||||
raise ValueError(f"Mesh hash mismatch: {case}")
|
||||
path = folder / f"mesh_{case:03d}.ply"
|
||||
path.write_bytes(mesh)
|
||||
sdf, xyz, _ = self.paper.sdf_vertices_closest_from_mesh(path, self.xyz, False)
|
||||
path.unlink()
|
||||
if len(xyz) != row["mesh_vertices"] or sdf.shape != tuple(self.c["protocol"]["grid"]):
|
||||
raise ValueError(f"Unexpected geometry dimensions: {case}")
|
||||
sdfs.append(torch.as_tensor(sdf, dtype=torch.float32))
|
||||
vertices.append(self.paper.location_normalization(
|
||||
torch.as_tensor(xyz, dtype=torch.float32), *self.bounds))
|
||||
if role == "heldout":
|
||||
target_log.write(json.dumps(dict(case=case, role=role, sha256=row["pressure_sha256"])) + "\n")
|
||||
target_log.flush()
|
||||
pressure = archive.read(f"car-pressure-data/data/press_{case:03d}.npy")
|
||||
if hashlib.sha256(pressure).hexdigest() != row["pressure_sha256"]:
|
||||
raise ValueError(f"Pressure hash mismatch: {case}")
|
||||
raw = self.np.load(io.BytesIO(pressure), allow_pickle=False).reshape(-1).astype(self.np.float32)
|
||||
retained = self.np.concatenate((raw[:16], raw[112:]))
|
||||
if len(retained) != len(xyz) or not self.np.isfinite(retained).all():
|
||||
raise ValueError(f"Pressure mapping mismatch: {case}")
|
||||
pressures.append(torch.from_numpy(retained))
|
||||
p = torch.stack(pressures)
|
||||
if role == "train":
|
||||
self.normalizer = self.normalizer_class(
|
||||
p, eps=self.c["protocol"]["pressure_epsilon"], reduce_dim=[0, 1], verbose=False)
|
||||
p = self.normalizer.encode(p)
|
||||
return self.dataset_class(
|
||||
{"sdf": torch.stack(sdfs), "vertices": torch.stack(vertices), "pressure": p},
|
||||
{"sdf_query_points": self.grid})
|
||||
|
||||
|
||||
def paper_model(c):
|
||||
# Use the same factory call schema as the paper. The attached factory is
|
||||
# missing this symbol import, so the class is imported directly.
|
||||
from src.networks.SDFFNOGNO import SDFFNOGNO
|
||||
|
||||
p = c["protocol"]
|
||||
return SDFFNOGNO(n_modes=p["modes"], hidden_channels=p["hidden_channels"],
|
||||
norm="group_norm", use_mlp=True, mlp={"expansion": 1., "dropout": 0.},
|
||||
domain_padding=.125, factorization="tucker", rank=.4, out_channels=1,
|
||||
r=p["radius"], resolution=p["grid"], gno_implementation="torch_scatter")
|
||||
|
||||
|
||||
def cuda_peaks(torch):
|
||||
return dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
||||
|
||||
|
||||
def train_seed(c, dataset, seed, epochs, progress):
|
||||
import numpy as np
|
||||
import torch
|
||||
from src.losses import LpLoss
|
||||
|
||||
torch.manual_seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.cuda.manual_seed(seed)
|
||||
torch.backends.cudnn.deterministic = True
|
||||
model = paper_model(c).to("cuda")
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=c["protocol"]["learning_rate"], weight_decay=1e-4)
|
||||
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=.5)
|
||||
loss_fn = LpLoss(size_average=True)
|
||||
loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=True, num_workers=0)
|
||||
times = []
|
||||
for epoch in range(1, epochs + 1):
|
||||
model.train()
|
||||
began = time.monotonic()
|
||||
examples = 0
|
||||
for item in loader:
|
||||
optimizer.zero_grad()
|
||||
loss = model.loss_dict(item, loss_fn=loss_fn)["loss"].mean()
|
||||
if not bool(torch.isfinite(loss)):
|
||||
raise ValueError("Nonfinite training loss")
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
examples += 1
|
||||
scheduler.step()
|
||||
torch.cuda.synchronize()
|
||||
if examples != 500:
|
||||
raise ValueError("Incomplete full training epoch")
|
||||
seconds = time.monotonic() - began
|
||||
times.append(seconds)
|
||||
save(progress, dict(stage="EPOCH_COMPLETE", seed=seed, epoch=epoch, examples=examples,
|
||||
seconds=seconds, **cuda_peaks(torch)))
|
||||
return model, times
|
||||
|
||||
|
||||
def case_scores(model, data, cases, normalizer):
|
||||
import torch
|
||||
|
||||
loader = torch.utils.data.DataLoader(data, batch_size=1, shuffle=False, num_workers=0)
|
||||
model.eval()
|
||||
normalizer.to("cuda")
|
||||
rows = []
|
||||
with torch.no_grad():
|
||||
for case, item in zip(cases, loader):
|
||||
raw_input, output_vertices = model.data_dict_to_input(item)
|
||||
prediction = normalizer.decode(model(raw_input, output_vertices)).reshape(-1)
|
||||
truth = normalizer.decode(item["pressure"].to("cuda")).reshape(-1)
|
||||
if prediction.numel() != 3586 or prediction.shape != truth.shape:
|
||||
raise ValueError(f"Incomplete pressure vector: {case}")
|
||||
denominator = torch.linalg.vector_norm(truth)
|
||||
value = (torch.linalg.vector_norm(prediction - truth) / denominator).item()
|
||||
if not math.isfinite(value) or denominator.item() <= 0:
|
||||
raise ValueError(f"Undefined decoded relative L2: {case}")
|
||||
rows.append(dict(case=case, relative_l2=value))
|
||||
normalizer.to("cpu")
|
||||
if len(rows) != len(cases):
|
||||
raise ValueError("Incomplete case coverage")
|
||||
return rows
|
||||
|
||||
|
||||
def test_environment(c):
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
import platform
|
||||
|
||||
if ".".join(platform.python_version_tuple()[:2]) != c["environment"]["python_version"]:
|
||||
raise RuntimeError("Pinned Python minor release required")
|
||||
|
||||
for name, expected in c["environment"]["package_versions"].items():
|
||||
if importlib.metadata.version(name) != expected:
|
||||
raise RuntimeError(f"Runtime distribution mismatch: {name}")
|
||||
direct_url = importlib.metadata.distribution("neuraloperator").read_text("direct_url.json")
|
||||
if not direct_url or json.loads(direct_url).get("vcs_info", {}).get("commit_id") != c["environment"]["neuralop_commit"]:
|
||||
raise RuntimeError("Historical NeuralOperator commit not installed from its reviewed VCS pin")
|
||||
module = importlib.util.find_spec("neuralop")
|
||||
if module is None or module.origin is None:
|
||||
raise RuntimeError("Historical NeuralOperator import missing")
|
||||
root = Path(module.origin).parent
|
||||
for relative, digest in c["environment"]["neuralop_file_sha256"].items():
|
||||
checked(root / relative, digest)
|
||||
|
||||
|
||||
def work(root, c):
|
||||
began = time.monotonic()
|
||||
test_environment(c)
|
||||
import torch
|
||||
|
||||
if (os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"]
|
||||
or torch.version.cuda != c["environment"]["torch_cuda"]):
|
||||
raise PermissionError("Runtime CUDA binding mismatch")
|
||||
torch.set_num_threads(c["limits"]["torch_threads"])
|
||||
torch.set_num_interop_threads(1)
|
||||
torch.cuda.set_device(0)
|
||||
if "V100" not in torch.cuda.get_device_name(0):
|
||||
raise ValueError("Selected device is not the fixed V100")
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.backends.cudnn.allow_tf32 = False
|
||||
records = root / "worker-records"
|
||||
records.mkdir(exist_ok=False)
|
||||
with TemporaryDirectory(prefix="original-gino-", dir=root) as scratch:
|
||||
extract_paper_code(c, scratch)
|
||||
data = OriginalData(c, scratch)
|
||||
train = data.data(data.train_ids, "train")
|
||||
preparation_seconds = time.monotonic() - began
|
||||
if c["mode"] == "measurement":
|
||||
model, epoch_times = train_seed(c, train, c["protocol"]["seeds"][0], 1, records / "progress.json")
|
||||
began_proxy = time.monotonic()
|
||||
ids = data.train_ids[:111]
|
||||
proxy = data.data(ids, "train_proxy")
|
||||
proxy_preparation_seconds = time.monotonic() - began_proxy
|
||||
began_eval = time.monotonic()
|
||||
case_scores(model, proxy, ids, data.normalizer)
|
||||
torch.cuda.synchronize()
|
||||
result = dict(status="COMPLETE", mode="measurement", epochs=1, training_examples=500,
|
||||
heldout_labels_read=False, proxy_examples=111,
|
||||
preparation_seconds=preparation_seconds, epoch_seconds=epoch_times[0],
|
||||
proxy_preparation_seconds=proxy_preparation_seconds,
|
||||
proxy_inference_seconds=time.monotonic() - began_eval,
|
||||
reuse_measurement_weights=False, **cuda_peaks(torch))
|
||||
else:
|
||||
checkpoints = {}
|
||||
for seed in c["protocol"]["seeds"]:
|
||||
model, _ = train_seed(c, train, seed, 100, records / "progress.json")
|
||||
file = root / f"seed-{seed}-epoch-100.pt"
|
||||
torch.save(model.state_dict(), file)
|
||||
checkpoints[str(seed)] = dict(path=str(file), sha256=sha(file))
|
||||
del model
|
||||
predictions = root / "checkpoints-frozen.json"
|
||||
save(predictions, checkpoints)
|
||||
freeze_hash = sha(predictions)
|
||||
checkpoints = json.loads(checked(predictions, freeze_hash).read_bytes())
|
||||
for item in checkpoints.values():
|
||||
checked(item["path"], item["sha256"])
|
||||
del train
|
||||
with (root / "heldout-target-access.jsonl").open("x") as log:
|
||||
heldout = data.data(data.test_ids, "heldout", log)
|
||||
rows = {}
|
||||
for seed, item in checkpoints.items():
|
||||
model = paper_model(c).to("cuda")
|
||||
model.load_state_dict(torch.load(checked(item["path"], item["sha256"]),
|
||||
map_location="cpu"), strict=True)
|
||||
scores = case_scores(model, heldout, data.test_ids, data.normalizer)
|
||||
rows[seed] = dict(cases=scores, mean_percent=100 * math.fsum(x["relative_l2"] for x in scores) / len(scores))
|
||||
del model
|
||||
mean = math.fsum(seed["mean_percent"] for seed in rows.values()) / len(rows)
|
||||
lower, upper = c["protocol"]["acceptance_percent"]
|
||||
result = dict(status="COMPLETE", mode="benchmark", seeds=rows, mean_percent=mean,
|
||||
within_registered_range=lower <= mean <= upper,
|
||||
heldout_labels_read=True, split_sha256=c["split_receipt"]["sha256"],
|
||||
checkpoints_manifest_sha256=freeze_hash, **cuda_peaks(torch))
|
||||
save(records / "result.json", result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("root", type=Path)
|
||||
parser.add_argument("--check-environment", action="store_true")
|
||||
arg = parser.parse_args()
|
||||
if arg.check_environment:
|
||||
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
|
||||
raise PermissionError("Set CUDA_VISIBLE_DEVICES empty for the isolated import check")
|
||||
from track_b_gino_shapenet_entry import contract
|
||||
c = contract(arg.root / "contract.json")
|
||||
test_environment(c)
|
||||
with TemporaryDirectory(prefix="track-b-gino-import-") as directory:
|
||||
extract_paper_code(c, directory)
|
||||
from src.data.cfd_datamodule import CFDSDFDataModule
|
||||
from src.networks.SDFFNOGNO import SDFFNOGNO
|
||||
print(json.dumps(dict(status="PASS", imports=[CFDSDFDataModule.__name__, SDFFNOGNO.__name__],
|
||||
models_created=False, arrays_loaded=False)))
|
||||
else:
|
||||
c = authorized_child(arg.root)
|
||||
work(arg.root, c)
|
||||
Reference in new issue
Block a user