212 lines
7.5 KiB
Python
212 lines
7.5 KiB
Python
"""One bounded GPU case, with explicit opt-in experimental explosion."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import numpy as np
|
|
from oracle import U, TIMES, LIMITS, initial, metrics, rectangles
|
|
|
|
|
|
def provenance(repo):
|
|
git = lambda *a: subprocess.check_output(
|
|
["git", "-C", str(repo), *a], text=True
|
|
).strip()
|
|
source_hash = hashlib.sha256()
|
|
for path in sorted((repo / "src").rglob("*")):
|
|
if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc":
|
|
source_hash.update(
|
|
str(path.relative_to(repo)).encode() + b"\0" + path.read_bytes() + b"\0"
|
|
)
|
|
return dict(
|
|
revision=git("rev-parse", "HEAD"),
|
|
dirty=bool(git("status", "--porcelain")),
|
|
source_tree_sha256=source_hash.hexdigest(),
|
|
harness_sha256={
|
|
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in Path(__file__).parent.glob("*.py")
|
|
},
|
|
)
|
|
|
|
|
|
def config(n, re):
|
|
from CelerisLab.config import LBMConfig
|
|
|
|
cfg = LBMConfig(
|
|
nx=n,
|
|
ny=n,
|
|
viscosity=U * n / re,
|
|
velocity=0.0,
|
|
rho=1.0,
|
|
lattice_model="D2Q9",
|
|
collision="SRT",
|
|
streaming="double_buffer",
|
|
store_precision="FP32",
|
|
ddf_shifting=True,
|
|
les_enabled=False,
|
|
)
|
|
cfg.validate()
|
|
return cfg
|
|
|
|
|
|
def workload(masks, base_steps):
|
|
cells = [int(np.count_nonzero(m)) for m in masks]
|
|
hops = [base_steps * 2**level for level in range(len(masks))]
|
|
updates = [count * hop for count, hop in zip(cells, hops)]
|
|
return dict(
|
|
active_cells_by_level=cells,
|
|
level_hops=hops,
|
|
active_collision_updates_by_level=updates,
|
|
active_collision_updates=sum(updates),
|
|
root_intervals=base_steps,
|
|
excludes="ghost streaming, halo copies, transfers, initialization and diagnostics",
|
|
)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--repo", type=Path, required=True)
|
|
p.add_argument("--re", type=int, choices=[100, 1000], required=True)
|
|
p.add_argument("--n", type=int, choices=[32, 64, 128, 256], required=True)
|
|
p.add_argument("--depth", type=int, choices=[1, 2, 3], required=True)
|
|
p.add_argument("--output", type=Path, required=True)
|
|
p.add_argument("--explosion", choices=["constant", "projected_linear"], default="constant")
|
|
args = p.parse_args()
|
|
if args.explosion != "constant" and args.depth != 2:
|
|
p.error("projected_linear requires depth 2")
|
|
if not os.environ.get("CUDA_VISIBLE_DEVICES"):
|
|
p.error("Explicit physical CUDA_VISIBLE_DEVICES selection required")
|
|
repo = args.repo.resolve()
|
|
sys.path.insert(0, str(repo / "src"))
|
|
import CelerisLab
|
|
|
|
if Path(CelerisLab.__file__).resolve() != repo / "src/CelerisLab/__init__.py":
|
|
raise RuntimeError("Wrong imported source checkout")
|
|
from CelerisLab.cuda.context import CudaContext
|
|
from CelerisLab.lbm.blocks import BlockLayout2D
|
|
from CelerisLab.lbm.block_stepper import PeriodicBlockStepper
|
|
from CelerisLab.lbm.refinement_stepper import FixedPatchStepper
|
|
from CelerisLab.lbm.refinement import ACTIVE
|
|
from CelerisLab.lbm.hierarchy import build_hierarchy, leaf_partition
|
|
|
|
cfg = config(args.n, args.re)
|
|
refinements = [] if args.depth == 1 else rectangles(args.n, args.depth)
|
|
h = build_hierarchy(cfg, refinements)
|
|
np.testing.assert_array_equal(leaf_partition(h), np.ones((args.n, args.n)))
|
|
masks = [k == ACTIVE for k in h.kinds]
|
|
report = dict(
|
|
schema=1,
|
|
source=provenance(repo),
|
|
historical_baseline_revision="9805383f9c50d867e8555471baf5dbda8333a652",
|
|
explosion=args.explosion,
|
|
n=args.n,
|
|
re_L=args.re,
|
|
depth=args.depth,
|
|
refinements=refinements,
|
|
U=U,
|
|
mach=U * np.sqrt(3),
|
|
viscosity=cfg.viscosity,
|
|
tau_by_level=[0.5 + 3 * c.viscosity for c in h.configs],
|
|
initialization="physical FP64 equilibrium G at actual cell centers; rho=1+3p",
|
|
cuda_visible_devices=os.environ["CUDA_VISIBLE_DEVICES"],
|
|
device_id=0,
|
|
limits=LIMITS,
|
|
workload=workload(masks, round(TIMES[-1] * args.n / U)),
|
|
rows=[],
|
|
completed=False,
|
|
)
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
save = lambda: args.output.write_text(
|
|
json.dumps(report, indent=2, allow_nan=False) + "\n"
|
|
)
|
|
save()
|
|
started = time.monotonic()
|
|
with CudaContext(0) as ctx:
|
|
if args.depth == 1:
|
|
layout = BlockLayout2D.build(args.n, args.n, 8, 8, periodic=(True, True))
|
|
run = PeriodicBlockStepper(cfg, layout, context=ctx)
|
|
else:
|
|
run = FixedPatchStepper(
|
|
cfg,
|
|
context=ctx,
|
|
coarse_boundary="periodic",
|
|
refinements=refinements,
|
|
level_blocks=[(8, 8)] * args.depth,
|
|
explosion=args.explosion,
|
|
)
|
|
with run:
|
|
run.initialize(
|
|
*[
|
|
initial(m.shape, level, args.n, cfg.viscosity)
|
|
for level, m in enumerate(masks)
|
|
]
|
|
)
|
|
initial_mass = None
|
|
for tstar in TIMES:
|
|
step = round(tstar * args.n / U)
|
|
run.step(step - run.step_count)
|
|
arrays = (run.download(),) if args.depth == 1 else run.download()
|
|
row = metrics(arrays, masks, step, args.n, cfg.viscosity, initial_mass)
|
|
row["step"] = step
|
|
if initial_mass is None:
|
|
initial_mass = row["mass"]
|
|
report["rows"].append(row)
|
|
save()
|
|
print(
|
|
json.dumps(
|
|
dict(
|
|
event="PROGRESS",
|
|
n=args.n,
|
|
re_L=args.re,
|
|
depth=args.depth,
|
|
**row,
|
|
)
|
|
),
|
|
flush=True,
|
|
)
|
|
if step == 0:
|
|
# Exercise real kernels/halos/transfers on every level before READY.
|
|
# This is the first measured trajectory step, not discarded warmup.
|
|
run.step(1)
|
|
probe = (run.download(),) if args.depth == 1 else run.download()
|
|
metrics(probe, masks, 1, args.n, cfg.viscosity, initial_mass)
|
|
report["ready_after_step"] = 1
|
|
save()
|
|
print(
|
|
json.dumps(
|
|
dict(
|
|
event="READY",
|
|
step=1,
|
|
n=args.n,
|
|
re_L=args.re,
|
|
depth=args.depth,
|
|
workload=report["workload"],
|
|
)
|
|
),
|
|
flush=True,
|
|
)
|
|
report.update(
|
|
completed=True,
|
|
seconds=time.monotonic() - started,
|
|
screening_pass=all(
|
|
row[key] <= limit for row in report["rows"] for key, limit in LIMITS.items()
|
|
),
|
|
)
|
|
save()
|
|
print(
|
|
json.dumps(
|
|
{k: report[k] for k in ("n", "re_L", "depth", "screening_pass", "seconds")}
|
|
),
|
|
flush=True,
|
|
)
|
|
return 0 if report["screening_pass"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|