improve LES stability

This commit is contained in:
Frank14f committed 2026-03-29 22:16:20 +08:00
1 parent f3e2e557d4
commit 5076e5d789
13 files changed
+1163 -141

No files matched your search

+7 -1
View File
@@ -95,7 +95,9 @@ def config_kernal_v2(config_cuda: CudaConfig, config_field: FlowFieldConfig,
collision_model: int = 2,
streaming_model: int = 0,
store_precision: int = 0,
use_ddf_shifting: int = 0):
use_ddf_shifting: int = 0,
use_les: int = 0,
les_cs: float = 0.16):
"""Configure macros.h for the new modular kernel architecture.
Args:
@@ -103,6 +105,8 @@ def config_kernal_v2(config_cuda: CudaConfig, config_field: FlowFieldConfig,
streaming_model: 0=double-buffer (default), 1=Esoteric-Pull
store_precision: 0=FP32 (default), 1=FP16S, 2=FP16C
use_ddf_shifting: 0=off (default), 1=on
use_les: 0=off (default), 1=Smagorinsky LES
les_cs: Smagorinsky constant C_s
"""
# First apply legacy config
config_kernal(config_cuda, config_field)
@@ -113,6 +117,8 @@ def config_kernal_v2(config_cuda: CudaConfig, config_field: FlowFieldConfig,
lines = modify_macro(lines, "STREAMING_MODEL", streaming_model)
lines = modify_macro(lines, "STORE_PRECISION", store_precision)
lines = modify_macro(lines, "USE_DDF_SHIFTING", use_ddf_shifting)
lines = modify_macro(lines, "USE_LES", use_les)
lines = modify_macro(lines, "LES_CS", f"{les_cs:.6f}f")
write_lines(kernel_path("macros.h"), lines)
def config_object(n_obj: int):
+66 -19
View File
@@ -23,6 +23,13 @@ class FlowField:
field_config: utils.FlowFieldConfig,
cuda_config: utils.CudaConfig,
device_id: Union[int, List[int]] = None,
use_kernel_v2: bool = True,
collision_model: int = 0,
streaming_model: int = 0,
store_precision: int = 0,
use_ddf_shifting: int = 0,
use_les: int = 0,
les_cs: float = 0.16,
):
self.field_config = field_config
self.cuda_config = cuda_config
@@ -49,10 +56,26 @@ class FlowField:
utils.check_cuda_capability(field_config, cuda_config, device_id)
# Config kernel
compiler.config_kernal(cuda_config, field_config)
compiler.config_object(int(0))
# compiler.config_sensor(int(0))
self.use_kernel_v2 = bool(use_kernel_v2)
self.collision_model = int(collision_model)
self.streaming_model = int(streaming_model)
self.store_precision = int(store_precision)
self.use_ddf_shifting = int(use_ddf_shifting)
self.use_les = int(use_les)
self.les_cs = float(les_cs)
if self.collision_model not in (0, 1, 2):
raise ValueError("collision_model must be 0(SRT), 1(TRT), or 2(MRT).")
if self.streaming_model not in (0, 1):
raise ValueError("streaming_model must be 0(double-buffer) or 1(esopull).")
if self.store_precision not in (0, 1, 2):
raise ValueError("store_precision must be 0(FP32), 1(FP16S), or 2(FP16C).")
if self.use_ddf_shifting not in (0, 1):
raise ValueError("use_ddf_shifting must be 0 or 1.")
if self.use_les not in (0, 1):
raise ValueError("use_les must be 0 or 1.")
if not (0.0 < self.les_cs < 1.0):
raise ValueError("les_cs must be in (0, 1).")
# Set constants
if field_config.data_type == "FP32":
@@ -84,11 +107,10 @@ class FlowField:
f"Unsupported lattice type {field_config.lattice} in {field_config.dimensionality} dimensions."
)
# Compile kernel
compiler.compile_kernel()
self.ptx = cuda.module_from_file(compiler.kernel_path("kernel.ptx"))
self.step = self.ptx.get_function("OneStep")
initflow = self.ptx.get_function("InitTubeFlow")
self.objects = {}
# Compile and load kernel
self._rebuild_kernel()
# Initialize memory
self.ddf = np.zeros(self.FIELD_SIZE * self.LATTICE, dtype=self.DATA_TYPE)
@@ -105,11 +127,10 @@ class FlowField:
self.delta_gpu = cuda.mem_alloc(1)
self.vortex_gpu = cuda.mem_alloc(self.vortex_config.nbytes)
self.objects = {}
self.action = np.zeros(0, dtype=self.DATA_TYPE)
self.obs = np.zeros(0, dtype=self.DATA_TYPE)
initflow(
self.initflow(
self.flag_gpu,
self.ddf_gpu,
block=(self.cuda_config.threads_per_block, 1, 1),
@@ -122,6 +143,38 @@ class FlowField:
cuda.memcpy_dtoh(self.flag, self.flag_gpu)
cuda.memcpy_dtoh(self.ddf, self.ddf_gpu)
def _configure_kernel(self):
if self.use_kernel_v2:
compiler.config_kernal_v2(
self.cuda_config,
self.field_config,
collision_model=self.collision_model,
streaming_model=self.streaming_model,
store_precision=self.store_precision,
use_ddf_shifting=self.use_ddf_shifting,
use_les=self.use_les,
les_cs=self.les_cs,
)
else:
compiler.config_kernal(self.cuda_config, self.field_config)
def _compile_and_load_kernel(self):
if self.use_kernel_v2:
compiler.compile_kernel_v2()
self.ptx = cuda.module_from_file(compiler.kernel_path("kernel_v2.ptx"))
self.step = self.ptx.get_function("OneStep")
self.initflow = self.ptx.get_function("InitTubeFlow_v2")
else:
compiler.compile_kernel()
self.ptx = cuda.module_from_file(compiler.kernel_path("kernel.ptx"))
self.step = self.ptx.get_function("OneStep")
self.initflow = self.ptx.get_function("InitTubeFlow")
def _rebuild_kernel(self):
self._configure_kernel()
compiler.config_object(len(self.objects))
self._compile_and_load_kernel()
def add_cylinder(self, center: Tuple[float, float, float], radius: float, id_obj: Optional[int] = None):
x_c, y_c, z_c = center
@@ -193,10 +246,7 @@ class FlowField:
cuda.memcpy_htod(self.flag_gpu, self.flag)
cuda.memcpy_htod(self.indx_gpu, self.indx)
compiler.config_object(len(self.objects))
compiler.compile_kernel()
self.ptx = cuda.module_from_file(compiler.kernel_path("kernel.ptx"))
self.step = self.ptx.get_function("OneStep")
self._rebuild_kernel()
def add_sensor(self, center: Tuple[float, float, float], radius: float):
x_c, y_c, z_c = center
@@ -235,10 +285,7 @@ class FlowField:
cuda.memcpy_htod(self.flag_gpu, self.flag)
cuda.memcpy_htod(self.indx_gpu, self.indx)
compiler.config_object(len(self.objects))
compiler.compile_kernel()
self.ptx = cuda.module_from_file(compiler.kernel_path("kernel.ptx"))
self.step = self.ptx.get_function("OneStep")
self._rebuild_kernel()
def add_vortex(self, center: Tuple[float, float, float], radius: float, strength: float, direction: float, type: str):
x_c, y_c, z_c = center
@@ -15,6 +15,31 @@
#ifndef CELERIS_BOUNDARY_INLET_OUTLET_CUH
#define CELERIS_BOUNDARY_INLET_OUTLET_CUH
#ifndef INLET_PROFILE
#define INLET_PROFILE 1
#endif
#ifndef OUTLET_MODE
#define OUTLET_MODE 0
#endif
#ifndef OUTLET_BACKFLOW_CLAMP
#define OUTLET_BACKFLOW_CLAMP 1
#endif
#ifndef OUTLET_BLEND_ALPHA
#define OUTLET_BLEND_ALPHA 0.70f
#endif
__device__ __forceinline__ float inlet_target_u(float y_coord) {
#if INLET_PROFILE == 0
return U0;
#else
float yy = (y_coord - 0.5f * (NY - 1)) / (NY - 2.0f);
return U0 * 1.5f * (1.0f - 4.0f * yy * yy);
#endif
}
#if NQ == 9
// ---------------------------------------------------------------------------
@@ -33,37 +58,21 @@ __device__ inline void apply_parabolic_inlet(float* __restrict__ f,
float y_coord)
{
// Neighbor macros
float p_neb = (f_neb[0]+f_neb[1]+f_neb[2]+f_neb[3]+f_neb[4]
+f_neb[5]+f_neb[6]+f_neb[7]+f_neb[8]) / 3.0f;
float rho_neb, u_neb, v_neb;
compute_rho_u(f_neb, rho_neb, u_neb, v_neb);
// Target velocity (parabolic profile)
float yy = (y_coord - 0.5f * (NY - 1)) / (NY - 2.0f);
float u_target = U0 * 1.5f * (1.0f - 4.0f * yy * yy);
float u_target = inlet_target_u(y_coord);
float v_target = 0.0f;
// Neighbor velocity
float u_neb = (f_neb[1]-f_neb[2]+f_neb[5]-f_neb[6]+f_neb[7]-f_neb[8]) / RHO;
float v_neb = (f_neb[3]-f_neb[4]+f_neb[5]-f_neb[6]-f_neb[7]+f_neb[8]) / RHO;
// feq for direction i=1 (cx=1, cy=0), w=1/9:
// feq = (2p + RHO*(2u² + 2u - v²)) / 6
float feq1_target = (2.0f*p_neb + RHO*(2.0f*u_target*u_target + 2.0f*u_target - v_target*v_target)) / 6.0f;
float feq1_neb = (2.0f*p_neb + RHO*(2.0f*u_neb*u_neb + 2.0f*u_neb - v_neb*v_neb)) / 6.0f;
// feq for direction i=5 (cx=1, cy=1), w=1/36:
// feq = (p + RHO*(u² + 3uv + u + v² + v)) / 12
float feq5_target = (p_neb + RHO*(u_target*u_target + 3.0f*u_target*v_target + u_target + v_target*v_target + v_target)) / 12.0f;
float feq5_neb = (p_neb + RHO*(u_neb*u_neb + 3.0f*u_neb*v_neb + u_neb + v_neb*v_neb + v_neb)) / 12.0f;
// feq for direction i=7 (cx=1, cy=-1), w=1/36:
// feq = (p + RHO*(u² - 3uv + u + v² - v)) / 12
float feq7_target = (p_neb + RHO*(u_target*u_target - 3.0f*u_target*v_target + u_target + v_target*v_target - v_target)) / 12.0f;
float feq7_neb = (p_neb + RHO*(u_neb*u_neb - 3.0f*u_neb*v_neb + u_neb + v_neb*v_neb - v_neb)) / 12.0f;
float feq_tar[9], feq_neb[9];
compute_feq(rho_neb, u_target, v_target, feq_tar);
compute_feq(rho_neb, u_neb, v_neb, feq_neb);
// Non-equilibrium extrapolation
f[1] = f_neb[1] - feq1_neb + feq1_target;
f[5] = f_neb[5] - feq5_neb + feq5_target;
f[7] = f_neb[7] - feq7_neb + feq7_target;
f[1] = f_neb[1] - feq_neb[1] + feq_tar[1];
f[5] = f_neb[5] - feq_neb[5] + feq_tar[5];
f[7] = f_neb[7] - feq_neb[7] + feq_tar[7];
}
// ---------------------------------------------------------------------------
@@ -76,35 +85,39 @@ __device__ inline void apply_pressure_outlet(float* __restrict__ f,
const float* __restrict__ f_neb,
float y_coord)
{
float p_out = 0.0f;
(void)y_coord;
// Target velocity (parabolic, same as inlet for consistency)
float yy = (y_coord - 0.5f * (NY - 1)) / (NY - 2.0f);
float u_target = U0 * 1.5f * (1.0f - 4.0f * yy * yy);
float v_target = 0.0f;
#if OUTLET_MODE == 1
// Simple zero-gradient copy for unknown incoming directions at outlet.
f[2] = f_neb[2];
f[8] = f_neb[8];
f[6] = f_neb[6];
#else
// Neighbor velocity
float u_neb = (f_neb[1]-f_neb[2]+f_neb[5]-f_neb[6]+f_neb[7]-f_neb[8]) / RHO;
float v_neb = (f_neb[3]-f_neb[4]+f_neb[5]-f_neb[6]-f_neb[7]+f_neb[8]) / RHO;
// Convective non-reflecting style: extrapolate outlet density from neighbor
// and keep neighbor velocity.
float rho_neb, u_neb, v_neb;
compute_rho_u(f_neb, rho_neb, u_neb, v_neb);
#if OUTLET_BACKFLOW_CLAMP
u_neb = fmaxf(u_neb, 0.0f);
#endif
float rho_out = rho_neb;
// feq for direction i=2 (cx=-1, cy=0), w=1/9:
// feq = (2p - RHO*(-2u² + 2u + v²)) / 6
float feq2_target = (2.0f*p_out - RHO*(-2.0f*u_target*u_target + 2.0f*u_target + v_target*v_target)) / 6.0f;
float feq2_neb = (2.0f*p_out - RHO*(-2.0f*u_neb*u_neb + 2.0f*u_neb + v_neb*v_neb)) / 6.0f;
float feq_tar[9], feq_neb[9];
compute_feq(rho_out, u_neb, v_neb, feq_tar);
compute_feq(rho_neb, u_neb, v_neb, feq_neb);
// feq for direction i=8 (cx=-1, cy=1), w=1/36:
// feq = (p + RHO*(u² - 3uv - u + v² + v)) / 12
float feq8_target = (p_out + RHO*(u_target*u_target - 3.0f*u_target*v_target - u_target + v_target*v_target + v_target)) / 12.0f;
float feq8_neb = (p_out + RHO*(u_neb*u_neb - 3.0f*u_neb*v_neb - u_neb + v_neb*v_neb + v_neb)) / 12.0f;
// feq for direction i=6 (cx=-1, cy=-1), w=1/36:
// feq = (p + RHO*(u² + 3uv - u + v² - v)) / 12
float feq6_target = (p_out + RHO*(u_target*u_target + 3.0f*u_target*v_target - u_target + v_target*v_target - v_target)) / 12.0f;
float feq6_neb = (p_out + RHO*(u_neb*u_neb + 3.0f*u_neb*v_neb - u_neb + v_neb*v_neb - v_neb)) / 12.0f;
f[2] = f_neb[2] - feq2_neb + feq2_target;
f[8] = f_neb[8] - feq8_neb + feq8_target;
f[6] = f_neb[6] - feq6_neb + feq6_target;
#if OUTLET_MODE == 2
const float a = OUTLET_BLEND_ALPHA;
f[2] = a * (f_neb[2] - feq_neb[2] + feq_tar[2]) + (1.0f - a) * f_neb[2];
f[8] = a * (f_neb[8] - feq_neb[8] + feq_tar[8]) + (1.0f - a) * f_neb[8];
f[6] = a * (f_neb[6] - feq_neb[6] + feq_tar[6]) + (1.0f - a) * f_neb[6];
#else
f[2] = f_neb[2] - feq_neb[2] + feq_tar[2];
f[8] = f_neb[8] - feq_neb[8] + feq_tar[8];
f[6] = f_neb[6] - feq_neb[6] + feq_tar[6];
#endif
#endif
}
#endif // NQ == 9
@@ -128,8 +141,7 @@ __device__ inline void apply_parabolic_inlet_3d(float* __restrict__ f,
compute_rho_u(f_neb, rho_neb, un, vn, wn);
// Target velocity (parabolic in y, uniform in z)
float yy = (y_coord - 0.5f * (NY - 1)) / (NY - 2.0f);
float u_tar = U0 * 1.5f * (1.0f - 4.0f * yy * yy);
float u_tar = inlet_target_u(y_coord);
// feq arrays
float feq_tar[19], feq_neb[19];
@@ -148,24 +160,47 @@ __device__ inline void apply_pressure_outlet_3d(float* __restrict__ f,
const float* __restrict__ f_neb,
float y_coord)
{
(void)y_coord;
#if OUTLET_MODE == 1
// Simple zero-gradient copy for unknown incoming directions at outlet.
f[2] = f_neb[2];
f[8] = f_neb[8];
f[10] = f_neb[10];
f[14] = f_neb[14];
f[16] = f_neb[16];
#else
// Neighbor macros
float rho_neb, un, vn, wn;
compute_rho_u(f_neb, rho_neb, un, vn, wn);
#if OUTLET_BACKFLOW_CLAMP
un = fmaxf(un, 0.0f);
#endif
// Target: p_out = 0 gauge → rho = 1.0 (using neighbor velocity)
float yy = (y_coord - 0.5f * (NY - 1)) / (NY - 2.0f);
float u_tar = U0 * 1.5f * (1.0f - 4.0f * yy * yy);
// Convective non-reflecting style: extrapolate outlet density from neighbor
// and keep neighbor velocity.
float rho_out = rho_neb;
float feq_tar[19], feq_neb[19];
compute_feq(RHO, u_tar, 0.0f, 0.0f, feq_tar);
compute_feq(RHO, un, vn, wn, feq_neb);
compute_feq(rho_out, un, vn, wn, feq_tar);
compute_feq(rho_neb, un, vn, wn, feq_neb);
// Reconstruct cx<0 directions: i = 2, 8, 10, 14, 16
#if OUTLET_MODE == 2
const float a = OUTLET_BLEND_ALPHA;
f[2] = a * (f_neb[2] - feq_neb[2] + feq_tar[2]) + (1.0f - a) * f_neb[2];
f[8] = a * (f_neb[8] - feq_neb[8] + feq_tar[8]) + (1.0f - a) * f_neb[8];
f[10] = a * (f_neb[10] - feq_neb[10] + feq_tar[10]) + (1.0f - a) * f_neb[10];
f[14] = a * (f_neb[14] - feq_neb[14] + feq_tar[14]) + (1.0f - a) * f_neb[14];
f[16] = a * (f_neb[16] - feq_neb[16] + feq_tar[16]) + (1.0f - a) * f_neb[16];
#else
f[2] = f_neb[2] - feq_neb[2] + feq_tar[2];
f[8] = f_neb[8] - feq_neb[8] + feq_tar[8];
f[10] = f_neb[10] - feq_neb[10] + feq_tar[10];
f[14] = f_neb[14] - feq_neb[14] + feq_tar[14];
f[16] = f_neb[16] - feq_neb[16] + feq_tar[16];
#endif
#endif
}
#endif // NQ == 19
+29 -4
View File
@@ -39,6 +39,7 @@
#include "operators/collision_trt.cuh"
#include "operators/collision_mrt.cuh"
#include "operators/forcing_guo.cuh"
#include "operators/turbulence_smag.cuh"
// ---------------------------------------------------------------------------
// Layer 3: Streaming
@@ -208,18 +209,31 @@ __global__ void OneStep(
}
}
// Obstacle nodes: full-way bounce-back (same swap as wall nodes)
if (fl & LEGACY_OBSTACLE) {
#pragma unroll
for (int i = 1; i < NQ; i += 2) {
float t = f[i]; f[i] = f[i+1]; f[i+1] = t;
}
}
// Collision
if (fl & LEGACY_FLUID) {
float feq[NQ], Fin[NQ];
compute_feq(rho_n, ux, uy, feq);
zero_forcing(Fin);
float omega_col = d_params.omega;
#if USE_LES
omega_col = compute_omega_smag(f, feq, rho_n, omega_col);
#endif
omega_col = fminf(OMEGA_COLLISION_MAX, fmaxf(OMEGA_COLLISION_MIN, omega_col));
#if COLLISION_MODEL == 0
collide_srt(f, feq, Fin, d_params.omega);
collide_srt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 1
collide_trt(f, feq, Fin, d_params.omega);
collide_trt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 2
collide_mrt(f, rho_n, ux, uy, Fin, d_params.omega);
collide_mrt(f, rho_n, ux, uy, Fin, omega_col);
#endif
}
@@ -289,7 +303,18 @@ __global__ void OneStep(
float feq[NQ], Fin[NQ];
compute_feq(rho_n, ux, uy, uz, feq);
zero_forcing(Fin);
collide_srt(f, feq, Fin, d_params.omega);
float omega_col = d_params.omega;
#if USE_LES
omega_col = compute_omega_smag(f, feq, rho_n, omega_col);
#endif
omega_col = fminf(OMEGA_COLLISION_MAX, fmaxf(OMEGA_COLLISION_MIN, omega_col));
#if COLLISION_MODEL == 0
collide_srt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 1
collide_trt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 2
collide_mrt(f, rho_n, ux, uy, uz, Fin, omega_col);
#endif
}
stream_pull_store(k, f, fi_out);
+40
View File
@@ -60,4 +60,44 @@
// DDF-shifting: 0=off, 1=on
#ifndef USE_DDF_SHIFTING
#define USE_DDF_SHIFTING 0
#endif
// LES model: 0=off, 1=Smagorinsky
#ifndef USE_LES
#define USE_LES 0
#endif
// Smagorinsky constant C_s
#ifndef LES_CS
#define LES_CS 0.16f
#endif
// Inlet profile: 1=parabolic (channel), 0=uniform (external flow)
#ifndef INLET_PROFILE
#define INLET_PROFILE 1
#endif
// Outlet mode: 0=non-equilibrium extrapolation, 1=zero-gradient copy (more dissipative)
#ifndef OUTLET_MODE
#define OUTLET_MODE 0
#endif
// Outlet blend factor for damped outlet mode (OUTLET_MODE=2):
// f_out = a*(non-eq extrapolation) + (1-a)*(zero-gradient copy)
#ifndef OUTLET_BLEND_ALPHA
#define OUTLET_BLEND_ALPHA 0.70f
#endif
// Outlet backflow clamp: 0=off, 1=force non-negative streamwise velocity at outlet target
#ifndef OUTLET_BACKFLOW_CLAMP
#define OUTLET_BACKFLOW_CLAMP 1
#endif
// Global collision omega guardrails
#ifndef OMEGA_COLLISION_MIN
#define OMEGA_COLLISION_MIN 0.01f
#endif
#ifndef OMEGA_COLLISION_MAX
#define OMEGA_COLLISION_MAX 1.999f
#endif
@@ -31,19 +31,19 @@ __device__ __forceinline__ void collide_mrt(float* __restrict__ g,
float omega)
{
// ----- Relaxation rates -----
// s_rho = s_jx = s_jy = 1.0 (conserved moments → overwrite to eq)
// s_rho = s_jx = s_jy = 0.0 (strictly conserved in collision)
// s_e = s_eps = s_q = 1.2 (kinetic transport)
// s_nu = omega (viscosity-related)
const float s_rho = 1.0f; // conserved moment relaxation
const float s_rho = 0.0f; // conserved moments
const float s_e = 1.2f;
const float s_eps = 1.2f;
const float s_jx = 1.0f;
const float s_jx = 0.0f;
const float s_q = 1.2f;
const float s_jy = 1.0f;
const float s_jy = 0.0f;
const float s_nu = omega;
// ----- Pressure (from density) -----
const float p = (g[0]+g[1]+g[2]+g[3]+g[4]+g[5]+g[6]+g[7]+g[8]) / 3.0f;
// ----- Pressure from local density -----
const float p = rho * (1.0f / 3.0f);
// ----- Forward transform: m = M * g (new paired ordering) -----
float m[9];
@@ -61,14 +61,14 @@ __device__ __forceinline__ void collide_mrt(float* __restrict__ g,
const float u2 = ux * ux + uy * uy;
float meq[9];
meq[0] = 3.0f * p; // ρ
meq[1] = -6.0f * p + 3.0f * RHO * u2; // e
meq[2] = 3.0f * p - 3.0f * RHO * u2; // ε
meq[3] = RHO * ux; // jx
meq[4] = -RHO * ux; // qx
meq[5] = RHO * uy; // jy
meq[6] = -RHO * uy; // qy
meq[7] = RHO * (ux * ux - uy * uy); // pxx
meq[8] = RHO * ux * uy; // pxy
meq[1] = -6.0f * p + 3.0f * rho * u2; // e
meq[2] = 3.0f * p - 3.0f * rho * u2; // ε
meq[3] = rho * ux; // jx
meq[4] = -rho * ux; // qx
meq[5] = rho * uy; // jy
meq[6] = -rho * uy; // qy
meq[7] = rho * (ux * ux - uy * uy); // pxx
meq[8] = rho * ux * uy; // pxy
// ----- Relaxation: delta_m[i] = s_i * (meq[i] - m[i]) -----
float dm[9];
@@ -112,21 +112,106 @@ __device__ __forceinline__ void collide_mrt_no_force(float* __restrict__ g,
#elif NQ == 19
// ---------------------------------------------------------------------------
// D3Q19 MRT (placeholder – use SRT or TRT until fully validated)
// D3Q19 MRT (tensor-projected multi-mode relaxation)
//
// This implementation performs a true multi-relaxation update by splitting
// non-equilibrium content into three subspaces:
// 1) deviatoric 2nd-order stress modes -> s_nu (shear, controlled by omega)
// 2) isotropic 2nd-order stress mode -> s_bulk (bulk, controlled by omega_bulk)
// 3) higher-order residual modes -> s_high (kinetic damping)
//
// It is formulated in Hermite/tensor space, avoiding explicit 19x19 matrices
// while still relaxing different mode families at different rates.
// ---------------------------------------------------------------------------
__device__ __forceinline__ void collide_mrt(float* __restrict__ g,
float rho, float ux, float uy, float uz,
const float* __restrict__ Fin,
float omega)
{
// TODO: implement D3Q19 MRT with 19-moment Gram–Schmidt basis
// Fall back to SRT for now
float feq[19];
compute_feq(rho, ux, uy, uz, feq);
const float c_tau = 1.0f - 0.5f * omega;
// Relaxation rates
const float s_nu = omega;
const float s_bulk = (d_params.omega_bulk > 0.0f) ? d_params.omega_bulk : 1.2f;
const float s_high = 1.4f;
const float one_minus_s_nu = 1.0f - s_nu;
const float one_minus_s_bulk = 1.0f - s_bulk;
const float one_minus_s_high = 1.0f - s_high;
// Non-equilibrium populations
float neq[19];
#pragma unroll
for (int i = 0; i < 19; i++) {
g[i] = fmaf(1.0f - omega, g[i], fmaf(omega, feq[i], c_tau * Fin[i]));
neq[i] = g[i] - feq[i];
}
// Build non-equilibrium 2nd-order tensor Πneq = Σ (ci ci) (fi-feqi)
float pixx = 0.0f, piyy = 0.0f, pizz = 0.0f;
float pixy = 0.0f, pixz = 0.0f, piyz = 0.0f;
#pragma unroll
for (int i = 0; i < 19; i++) {
const float ci_x = (float)d_cx[i];
const float ci_y = (float)d_cy[i];
const float ci_z = (float)d_cz[i];
const float fneq = neq[i];
pixx += fneq * ci_x * ci_x;
piyy += fneq * ci_y * ci_y;
pizz += fneq * ci_z * ci_z;
pixy += fneq * ci_x * ci_y;
pixz += fneq * ci_x * ci_z;
piyz += fneq * ci_y * ci_z;
}
// Isotropic / deviatoric split of Πneq
const float tr_pi = pixx + piyy + pizz;
const float tr_pi_thrd = tr_pi * (1.0f / 3.0f);
const float dev_xx = pixx - tr_pi_thrd;
const float dev_yy = piyy - tr_pi_thrd;
const float dev_zz = pizz - tr_pi_thrd;
const float dev_xy = pixy;
const float dev_xz = pixz;
const float dev_yz = piyz;
// cs^2 = 1/3 => 2*cs^4 = 2/9, inverse = 4.5
// Project neq onto second-order Hermite basis and relax mode families.
const float proj_pref = 4.5f;
const float c_tau = 1.0f - 0.5f * omega;
#pragma unroll
for (int i = 0; i < 19; i++) {
const float ci_x = (float)d_cx[i];
const float ci_y = (float)d_cy[i];
const float ci_z = (float)d_cz[i];
// Q = ci ci - cs^2 I
const float q_xx = ci_x * ci_x - CS2;
const float q_yy = ci_y * ci_y - CS2;
const float q_zz = ci_z * ci_z - CS2;
const float q_xy = ci_x * ci_y;
const float q_xz = ci_x * ci_z;
const float q_yz = ci_y * ci_z;
// Contractions Q:Πdev and Q:Πiso
const float q_dot_dev =
q_xx * dev_xx + q_yy * dev_yy + q_zz * dev_zz
+ 2.0f * (q_xy * dev_xy + q_xz * dev_xz + q_yz * dev_yz);
const float q_trace = q_xx + q_yy + q_zz; // = |ci|^2 - 1
const float q_dot_iso = q_trace * tr_pi_thrd;
const float neq_dev = d_w[i] * proj_pref * q_dot_dev;
const float neq_iso = d_w[i] * proj_pref * q_dot_iso;
const float neq_h = neq[i] - neq_dev - neq_iso;
g[i] = feq[i]
+ one_minus_s_nu * neq_dev
+ one_minus_s_bulk * neq_iso
+ one_minus_s_high * neq_h
+ c_tau * Fin[i];
}
}
@@ -0,0 +1,111 @@
// CelerisLab – operators/turbulence_smag.cuh
// Smagorinsky LES: compute effective omega from local non-equilibrium stress.
//
// Model:
// nu_t = (C_s * Delta)^2 * |S|, Delta = 1 (lattice unit)
// omega_eff = 1 / (3*(nu + nu_t) + 0.5)
//
// |S| is estimated from non-equilibrium second-order moments using tau0.
// ============================================================================
#ifndef CELERIS_OPERATORS_TURBULENCE_SMAG_CUH
#define CELERIS_OPERATORS_TURBULENCE_SMAG_CUH
#ifndef USE_LES
#define USE_LES 0
#endif
#ifndef LES_CS
#define LES_CS 0.16f
#endif
__device__ __forceinline__ float clamp_omega(float w) {
return fminf(1.999f, fmaxf(0.01f, w));
}
#if NQ == 9
__device__ __forceinline__ float compute_omega_smag(const float* __restrict__ f,
const float* __restrict__ feq,
float rho,
float omega0)
{
const float rho_safe = fmaxf(rho, 1.0e-12f);
const float tau0 = fmaxf(1.0f / fmaxf(omega0, 1.0e-6f), 0.500001f);
// Πneq = Σ (ci ci)(fi-feqi)
float pixx = 0.0f, piyy = 0.0f, pixy = 0.0f;
#pragma unroll
for (int i = 0; i < 9; i++) {
const float fneq = f[i] - feq[i];
const float cx = (float)d_cx[i];
const float cy = (float)d_cy[i];
pixx += fneq * cx * cx;
piyy += fneq * cy * cy;
pixy += fneq * cx * cy;
}
const float denom = 2.0f * rho_safe * CS2 * tau0;
const float sxx = -pixx / denom;
const float syy = -piyy / denom;
const float sxy = -pixy / denom;
// |S| = sqrt(2 Sij Sij)
const float s_mag = sqrtf(2.0f * (sxx*sxx + syy*syy + 2.0f*sxy*sxy));
const float nu0 = (tau0 - 0.5f) * (1.0f / 3.0f);
const float nut = (LES_CS * LES_CS) * s_mag;
const float nue = nu0 + nut;
return clamp_omega(1.0f / (3.0f * nue + 0.5f));
}
#elif NQ == 19
__device__ __forceinline__ float compute_omega_smag(const float* __restrict__ f,
const float* __restrict__ feq,
float rho,
float omega0)
{
const float rho_safe = fmaxf(rho, 1.0e-12f);
const float tau0 = fmaxf(1.0f / fmaxf(omega0, 1.0e-6f), 0.500001f);
// Πneq = Σ (ci ci)(fi-feqi)
float pixx = 0.0f, piyy = 0.0f, pizz = 0.0f;
float pixy = 0.0f, pixz = 0.0f, piyz = 0.0f;
#pragma unroll
for (int i = 0; i < 19; i++) {
const float fneq = f[i] - feq[i];
const float cx = (float)d_cx[i];
const float cy = (float)d_cy[i];
const float cz = (float)d_cz[i];
pixx += fneq * cx * cx;
piyy += fneq * cy * cy;
pizz += fneq * cz * cz;
pixy += fneq * cx * cy;
pixz += fneq * cx * cz;
piyz += fneq * cy * cz;
}
const float denom = 2.0f * rho_safe * CS2 * tau0;
const float sxx = -pixx / denom;
const float syy = -piyy / denom;
const float szz = -pizz / denom;
const float sxy = -pixy / denom;
const float sxz = -pixz / denom;
const float syz = -piyz / denom;
// |S| = sqrt(2 Sij Sij)
const float s_mag = sqrtf(2.0f * (sxx*sxx + syy*syy + szz*szz
+ 2.0f*(sxy*sxy + sxz*sxz + syz*syz)));
const float nu0 = (tau0 - 0.5f) * (1.0f / 3.0f);
const float nut = (LES_CS * LES_CS) * s_mag;
const float nue = nu0 + nut;
return clamp_omega(1.0f / (3.0f * nue + 0.5f));
}
#endif // NQ
#endif // CELERIS_OPERATORS_TURBULENCE_SMAG_CUH
@@ -69,7 +69,8 @@ __global__ void StreamCollideDouble(
// ----- Boundary conditions (inlet/outlet/wall) -----
#if NQ == 9
if (fl & LEGACY_SOLID) {
if (x == 0) {
bool interior_y = (y > 0u) && (y < (unsigned int)(NY - 1));
if (x == 0 && interior_y) {
float f_neb[NQ];
unsigned long k_neb = linear_index(x + 1u, y);
for (int i = 0; i < NQ; i++) {
@@ -77,18 +78,31 @@ __global__ void StreamCollideDouble(
}
apply_parabolic_inlet(f, f_neb, (float)y);
}
else if (x == NX - 1) {
else if (x == (unsigned int)(NX - 1) && interior_y) {
float f_neb[NQ];
unsigned long k_neb = linear_index(x - 1u, y);
for (int i = 0; i < NQ; i++) {
f_neb[i] = load_ddf(fi_in, index_f(k_neb, (unsigned int)i));
}
apply_pressure_outlet(f, f_neb, (float)y);
} else {
#pragma unroll
for (int i = 1; i < NQ; i += 2) {
float t = f[i]; f[i] = f[i+1]; f[i+1] = t;
}
}
}
if (fl & LEGACY_OBSTACLE) {
#pragma unroll
for (int i = 1; i < NQ; i += 2) {
float t = f[i]; f[i] = f[i+1]; f[i+1] = t;
}
}
#elif NQ == 19
if (fl & LEGACY_SOLID) {
if (x == 0) {
bool interior_y = (y > 0u) && (y < (unsigned int)(NY - 1));
if (x == 0 && interior_y) {
float f_neb[NQ];
unsigned long k_neb = linear_index(x + 1u, y, z);
for (int i = 0; i < NQ; i++) {
@@ -96,13 +110,18 @@ __global__ void StreamCollideDouble(
}
apply_parabolic_inlet_3d(f, f_neb, (float)y);
}
else if (x == NX - 1) {
else if (x == (unsigned int)(NX - 1) && interior_y) {
float f_neb[NQ];
unsigned long k_neb = linear_index(x - 1u, y, z);
for (int i = 0; i < NQ; i++) {
f_neb[i] = load_ddf(fi_in, index_f(k_neb, (unsigned int)i));
}
apply_pressure_outlet_3d(f, f_neb, (float)y);
} else {
#pragma unroll
for (int i = 1; i < NQ; i += 2) {
float t = f[i]; f[i] = f[i+1]; f[i+1] = t;
}
}
}
#endif
@@ -126,20 +145,36 @@ __global__ void StreamCollideDouble(
#if NQ == 9
compute_feq(rho_n, ux, uy, feq);
zero_forcing(Fin);
float omega_col = d_params.omega;
#if USE_LES
omega_col = compute_omega_smag(f, feq, rho_n, omega_col);
#endif
omega_col = fminf(OMEGA_COLLISION_MAX, fmaxf(OMEGA_COLLISION_MIN, omega_col));
// ----- Collision -----
#if COLLISION_MODEL == 0
collide_srt(f, feq, Fin, d_params.omega);
collide_srt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 1
collide_trt(f, feq, Fin, d_params.omega);
collide_trt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 2
collide_mrt(f, rho_n, ux, uy, Fin, d_params.omega);
collide_mrt(f, rho_n, ux, uy, Fin, omega_col);
#endif
#elif NQ == 19
compute_feq(rho_n, ux, uy, uz, feq);
zero_forcing(Fin);
collide_srt(f, feq, Fin, d_params.omega);
float omega_col = d_params.omega;
#if USE_LES
omega_col = compute_omega_smag(f, feq, rho_n, omega_col);
#endif
omega_col = fminf(OMEGA_COLLISION_MAX, fmaxf(OMEGA_COLLISION_MIN, omega_col));
#if COLLISION_MODEL == 0
collide_srt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 1
collide_trt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 2
collide_mrt(f, rho_n, ux, uy, uz, Fin, omega_col);
#endif
#endif
}
@@ -70,7 +70,8 @@ __global__ void StreamCollideEsoPull(
// ----- Boundary conditions -----
#if NQ == 9
if (fl & LEGACY_SOLID) {
if (x == 0) {
bool interior_y = (y > 0u) && (y < (unsigned int)(NY - 1));
if (x == 0 && interior_y) {
float f_neb[NQ];
unsigned long k_neb = linear_index(x + 1u, y);
load_f_esopull(k_neb, f_neb, fi, j, t); // approximate: reuse j
@@ -80,19 +81,67 @@ __global__ void StreamCollideEsoPull(
load_f_esopull(k_neb, f_neb, fi, j_neb, t);
apply_parabolic_inlet(f, f_neb, (float)y);
}
else if (x == (unsigned int)(NX - 1)) {
else if (x == (unsigned int)(NX - 1) && interior_y) {
unsigned long k_neb = linear_index(x - 1u, y);
unsigned long j_neb[NQ];
compute_neighbors(k_neb, j_neb);
float f_neb[NQ];
load_f_esopull(k_neb, f_neb, fi, j_neb, t);
apply_pressure_outlet(f, f_neb, (float)y);
} else {
#pragma unroll
for (int i = 1; i < NQ; i += 2) {
float ttmp = f[i]; f[i] = f[i+1]; f[i+1] = ttmp;
}
}
}
if (fl & LEGACY_OBSTACLE) {
#pragma unroll
for (int i = 1; i < NQ; i += 2) {
float ttmp = f[i]; f[i] = f[i+1]; f[i+1] = ttmp;
}
}
if (y == 1 || y == (unsigned int)(NY - 2)) {
apply_wall_bb_d2q9(y, f);
}
#elif NQ == 19
if (fl & LEGACY_SOLID) {
bool interior_y = (y > 0u) && (y < (unsigned int)(NY - 1));
if (x == 0 && interior_y) {
unsigned long k_neb = linear_index(x + 1u, y, z);
unsigned long j_neb[NQ];
compute_neighbors(k_neb, j_neb);
float f_neb[NQ];
load_f_esopull(k_neb, f_neb, fi, j_neb, t);
apply_parabolic_inlet_3d(f, f_neb, (float)y);
}
else if (x == (unsigned int)(NX - 1) && interior_y) {
unsigned long k_neb = linear_index(x - 1u, y, z);
unsigned long j_neb[NQ];
compute_neighbors(k_neb, j_neb);
float f_neb[NQ];
load_f_esopull(k_neb, f_neb, fi, j_neb, t);
apply_pressure_outlet_3d(f, f_neb, (float)y);
} else {
#pragma unroll
for (int i = 1; i < NQ; i += 2) {
float ttmp = f[i]; f[i] = f[i+1]; f[i+1] = ttmp;
}
}
}
if (fl & LEGACY_OBSTACLE) {
#pragma unroll
for (int i = 1; i < NQ; i += 2) {
float ttmp = f[i]; f[i] = f[i+1]; f[i+1] = ttmp;
}
}
if (y == 1 || y == (unsigned int)(NY - 2)) {
apply_wall_bb_d3q19_y(y, f);
}
#endif
// ----- Forcing -----
@@ -123,13 +172,18 @@ __global__ void StreamCollideEsoPull(
float feq[NQ];
#if NQ == 9
compute_feq(rho_n, ux, uy, feq);
float omega_col = d_params.omega;
#if USE_LES
omega_col = compute_omega_smag(f, feq, rho_n, omega_col);
#endif
omega_col = fminf(OMEGA_COLLISION_MAX, fmaxf(OMEGA_COLLISION_MIN, omega_col));
#if COLLISION_MODEL == 0
collide_srt(f, feq, Fin, d_params.omega);
collide_srt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 1
collide_trt(f, feq, Fin, d_params.omega);
collide_trt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 2
collide_mrt(f, rho_n, ux, uy, Fin, d_params.omega);
collide_mrt(f, rho_n, ux, uy, Fin, omega_col);
#endif
#elif NQ == 19
@@ -137,7 +191,18 @@ __global__ void StreamCollideEsoPull(
if (force_field != nullptr) fzn += force_field[2*TOTAL_CELLS + k];
apply_guo_velocity_correction(ux, uy, uz, fxn, fyn, fzn, rho_n);
compute_feq(rho_n, ux, uy, uz, feq);
collide_srt(f, feq, Fin, d_params.omega);
float omega_col = d_params.omega;
#if USE_LES
omega_col = compute_omega_smag(f, feq, rho_n, omega_col);
#endif
omega_col = fminf(OMEGA_COLLISION_MAX, fmaxf(OMEGA_COLLISION_MIN, omega_col));
#if COLLISION_MODEL == 0
collide_srt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 1
collide_trt(f, feq, Fin, omega_col);
#elif COLLISION_MODEL == 2
collide_mrt(f, rho_n, ux, uy, uz, Fin, omega_col);
#endif
#endif
}