Frank14f/CelerisLab
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: apply timing harness review caveats

Browse Source 
Explicit stream synchronization before/after each timed loop (refined
FixedPatchStepper stream, uniform Simulation stream, in their contexts);
required --max-wall-seconds outer timeout checked between variants and in
per-step loops; untimed finite leaf/fluid health checks; force notes for
B/C/E' (no host reads; final base interval only; whole-window average with
different summation order). Parity stays A/B/C vs A and E/E' vs E only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ClaudeandClaude Opus 5.5 committed 2026-10-18 12:27:16 +00:00
1 parent 15fb58e36f
commit 77bab1c4ea
3 files changed
+76 -24
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/validation/time_refined_dfg.py
+57 -16
View File
@@ -2,15 +2,19 @@
Variants over EQUAL simulated duration (one refined base step = two uniform D80 steps):
A refined: step(1) + read_raw_exchange per base step (as run_refined_dfg)
B refined: step(1) only -- timing-only, no force history
C refined: one step(N) call -- timing-only, no force history
B refined: step(1) only, no host reads -- timing-only, no force history
C refined: one step(N) call -- timing-only; ledger keeps final base interval only
E uniform: sim.run(<=100) + read_force (as run_refined_dfg)
E' uniform: one sim.run(2N) call -- timing-only, no force history
E' uniform: one sim.run(2N) call -- timing-only; at most a whole-window force
average (different summation order), no history
observe_raw_exchange stays True for A/B/C: B/C omit host reads only.
Parity compares A/B/C with each other and E/E' with each other, never uniform with refined.
A-B and B-C are measured comparative differences, not exact causal decompositions.
Each repeat starts from the same fresh initial state with the same warm-up; construction,
compilation and parity checks are outside timed regions. FixedPatchStepper.step() synchronizes
its stream after every base interval and Simulation.run() synchronizes before returning, so the
timer starts and stops with no GPU work in flight. Wall time (perf_counter) is the only source.
compilation, parity and health checks are outside timed regions. The actual stepping stream is
synchronized explicitly immediately before the timer starts and after the timed loop. Wall time
(perf_counter) is the only timing source; no CUDA-event span is reported as kernel time. An outer
--max-wall-seconds deadline is checked between variants and inside per-step loops.
"""
import argparse
@@ -31,10 +35,13 @@ import run_refined_dfg as r # noqa: E402 (published runner: same setup functio
VARIANTS = {
"A": dict(case="refined", force_history_valid=True),
"B": dict(case="refined", force_history_valid=False, timing_only=True),
"C": dict(case="refined", force_history_valid=False, timing_only=True),
"B": dict(case="refined", force_history_valid=False, timing_only=True,
force_note="no host reads"),
"C": dict(case="refined", force_history_valid=False, timing_only=True,
force_note="raw ledger retains only the final base interval"),
"E": dict(case="uniform", force_history_valid=True),
"E'": dict(case="uniform", force_history_valid=False, timing_only=True),
"E'": dict(case="uniform", force_history_valid=False, timing_only=True,
force_note="at most a whole-window accumulated average; different summation order"),
}
@@ -60,17 +67,34 @@ def same_state(reference, values):
return None
def refined_loop(backend, variant, n):
def check_deadline(deadline):
if time.monotonic() >= deadline:
raise TimeoutError("timing harness exceeded --max-wall-seconds")
def sync_refined(backend):
with backend.ctx.activate():
backend._stream.synchronize()
def sync_uniform(sim):
with sim.ctx.activate():
sim.stream.synchronize()
def refined_loop(backend, variant, n, deadline):
if variant == "C":
backend.step(n)
return
for _ in range(n):
for i in range(n):
backend.step(1)
if variant == "A":
backend.read_raw_exchange(0, level=1, window="last_base_step")
if i % 100 == 99:
check_deadline(deadline)
def uniform_loop(sim, variant, n):
def uniform_loop(sim, variant, n, deadline):
if variant == "E'":
sim.run(n)
return
@@ -80,6 +104,7 @@ def uniform_loop(sim, variant, n):
sim.run(batch)
sim.read_force(0)
done += batch
check_deadline(deadline)
def time_refined(a, h, sched):
@@ -100,12 +125,17 @@ def time_refined(a, h, sched):
print(json.dumps(dict(status="READY", case="refined")), flush=True)
for repeat in range(a.repeats):
for variant in ("A", "B", "C"):
check_deadline(a.deadline)
backend.initialize(*fresh) # resets step count; same fresh state every time
refined_loop(backend, variant, warm)
refined_loop(backend, variant, warm, a.deadline)
sync_refined(backend)
start = time.perf_counter()
refined_loop(backend, variant, n)
refined_loop(backend, variant, n, a.deadline)
sync_refined(backend)
seconds = time.perf_counter() - start
values = [np.array(v, copy=True) for v in backend._download_raw().values]
if not all(np.isfinite(v[:, m]).all() for v, m in zip(values, backend._state)):
raise RuntimeError(f"refined {variant} repeat {repeat}: nonfinite leaf populations")
mismatch = None
if repeat == 0 and variant == "A":
reference["A"] = values
@@ -121,6 +151,7 @@ def time_refined(a, h, sched):
def time_uniform(a, h, sched):
from CelerisLab.simulation import Simulation
from CelerisLab.lbm.descriptors import FLUID
warm, n = sched["uniform"]
rows, reference = [], {}
@@ -135,14 +166,20 @@ def time_uniform(a, h, sched):
print(json.dumps(dict(status="READY", case="uniform")), flush=True)
for repeat in range(a.repeats):
for variant in ("E", "E'"):
check_deadline(a.deadline)
sim.restore()
if sim.stepper.step_count % 2:
raise RuntimeError("uniform restore parity changed between variants")
uniform_loop(sim, variant, warm)
uniform_loop(sim, variant, warm, a.deadline)
sync_uniform(sim)
start = time.perf_counter()
uniform_loop(sim, variant, n)
uniform_loop(sim, variant, n, a.deadline)
sync_uniform(sim)
seconds = time.perf_counter() - start
values = [sim.get_ddf()]
fluid = (sim.get_flags().reshape(values[0].shape[1:]) & FLUID) != 0
if not np.isfinite(values[0][:, fluid]).all():
raise RuntimeError(f"uniform {variant} repeat {repeat}: nonfinite fluid populations")
mismatch = None
if repeat == 0 and variant == "E":
reference["E"] = values
@@ -184,6 +221,7 @@ def main():
p.add_argument("--base-steps", type=int, default=2000)
p.add_argument("--warmup-base-steps", type=int, default=200)
p.add_argument("--repeats", type=int, default=3)
p.add_argument("--max-wall-seconds", type=float, help="Required outer timeout for GPU runs")
p.add_argument("--plan", action="store_true")
a = p.parse_args()
if a.repeats < 1 or a.base_steps > 20000:
@@ -197,6 +235,9 @@ def main():
if a.plan:
print(json.dumps(plan, indent=2))
return 0
if a.max_wall_seconds is None or not math.isfinite(a.max_wall_seconds) or a.max_wall_seconds <= 0:
p.error("GPU timing requires a finite positive --max-wall-seconds")
a.deadline = time.monotonic() + a.max_wall_seconds
if not os.environ.get("CUDA_VISIBLE_DEVICES") or "," in os.environ["CUDA_VISIBLE_DEVICES"]:
p.error("Root must select exactly one verified idle GPU via CUDA_VISIBLE_DEVICES")
if a.output.exists() and any(a.output.iterdir()):