From 5f061bec0653222097eebc16dbc1c9ed9ec79242 Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Fri, 3 Jul 2026 00:21:49 +0800 Subject: [PATCH] =?UTF-8?q?feat(train):=20cross-Re=20transfer=20pipeline?= =?UTF-8?q?=20=E2=80=94=20re60/re200/re400=20calibrations=20+=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add crossre_transfer.sh: calibrate → transfer-train for re60→re200→re400 - Add re60 config (ν=0.006667, SI=800, uniform+free-slip, very weak shedding) - Calibrate re60, re200, re400: FORCE_SCALE, SENS_SCALE, dtw_norm_scale, SIM_BP - Fix all paths: use DynamisLab submodule CelerisLab, remove external ~/CelerisLab - Remove _clean_cache() from envs/calibrate — CelerisLab handles internally - Move V4 backups to old/: env_karman_2000x600, train_karman_2000x600, etc. - train_karman.py: save model + vecnormalize every episode (non-optional) - Update TRAIN_KNOWLEDGE.md: file structure, calibration table, cross-re guide - All 3 Re verified: 5-episode transfer test passed (re60: 0.64, re200: 0.43, re400: 0.49) Co-authored-by: Cursor --- configs/config_lbm_karman_2000x600_re60.json | 50 ++ src/drl_pinball/train/TRAIN_KNOWLEDGE.md | 557 ++++++------------ src/drl_pinball/train/calibrate.py | 40 +- .../calibrations/illusion_1L/calibration.json | 12 +- .../illusion_1L/target_harmonics.json | 176 +++--- .../train/calibrations/re200/calibration.json | 49 ++ .../train/calibrations/re400/calibration.json | 49 ++ .../train/calibrations/re60/calibration.json | 49 ++ src/drl_pinball/train/crossre_transfer.sh | 115 ++++ src/drl_pinball/train/env_illusion.py | 40 +- src/drl_pinball/train/env_karman.py | 10 - src/drl_pinball/train/env_vortex.py | 10 - src/drl_pinball/train/launch_multi.sh | 8 +- .../train/{ => old}/analyze_final.py | 0 .../train/{ => old}/env_karman_2000x600.py | 2 +- .../{ => old}/phase0_baseline_measure.py | 2 +- src/drl_pinball/train/{ => old}/prompt.md | 0 .../train/{ => old}/train_karman_2000x600.py | 0 src/drl_pinball/train/train_karman.py | 6 +- 19 files changed, 641 insertions(+), 534 deletions(-) create mode 100644 configs/config_lbm_karman_2000x600_re60.json create mode 100644 src/drl_pinball/train/calibrations/re200/calibration.json create mode 100644 src/drl_pinball/train/calibrations/re400/calibration.json create mode 100644 src/drl_pinball/train/calibrations/re60/calibration.json create mode 100755 src/drl_pinball/train/crossre_transfer.sh rename src/drl_pinball/train/{ => old}/analyze_final.py (100%) rename src/drl_pinball/train/{ => old}/env_karman_2000x600.py (99%) rename src/drl_pinball/train/{ => old}/phase0_baseline_measure.py (99%) rename src/drl_pinball/train/{ => old}/prompt.md (100%) rename src/drl_pinball/train/{ => old}/train_karman_2000x600.py (100%) diff --git a/configs/config_lbm_karman_2000x600_re60.json b/configs/config_lbm_karman_2000x600_re60.json new file mode 100644 index 0000000..5772628 --- /dev/null +++ b/configs/config_lbm_karman_2000x600_re60.json @@ -0,0 +1,50 @@ +{ + "_doc": "Karman Cloak Re60: uniform inlet, free-slip walls, 2000x600 grid. nu = U0*2D/60 = 0.006667.", + "grid": { + "lattice_model": "D2Q9", + "nx": 2000, + "ny": 600, + "nz": 1 + }, + "physics": { + "data_type": "FP32", + "viscosity": 0.006667, + "velocity": 0.01, + "rho": 1.0 + }, + "method": { + "collision": "MRT", + "streaming": "double_buffer", + "store_precision": "FP32", + "ddf_shifting": false, + "les": { + "enabled": false, + "cs": 0.16, + "closed_form": true + }, + "trt": { + "magic_param": 0.1875 + }, + "inlet": { + "profile": "uniform", + "scheme": "regularized", + "trt_neq_damp": 0.5, + "regularized_neq_damp": 0.5 + }, + "outlet": { + "mode": "neq_extrap", + "backflow_clamp": true, + "blend_alpha": 0.7, + "srt_neq_damp": 0.5 + }, + "y_wall_bc": "free_slip", + "omega_guard": { + "min": 0.01, + "max": 1.99 + } + }, + "cuda": { + "threads_per_block": 256, + "compute_capability": "auto" + } +} diff --git a/src/drl_pinball/train/TRAIN_KNOWLEDGE.md b/src/drl_pinball/train/TRAIN_KNOWLEDGE.md index 9216d9f..4d181a1 100644 --- a/src/drl_pinball/train/TRAIN_KNOWLEDGE.md +++ b/src/drl_pinball/train/TRAIN_KNOWLEDGE.md @@ -1,7 +1,10 @@ # Karman Cloak Training — Knowledge Document (V5) -> **V5 (2026-07-01)**: Parameterized, calibration-driven, no_bias only, multi-GPU ready. -> Original V4 files preserved as `env_karman_2000x600.py` and `train_karman_2000x600.py`. +> **V5 (2026-07-03)**: Parameterized, calibration-driven, no_bias only. +> All paths use DynamisLab submodule `CelerisLab/` (no external dev dir). +> Cross-Re transfer pipeline verified: re60, re200, re400. +> Every episode saves model checkpoint. +> V4 backups moved to `old/`. --- @@ -12,257 +15,199 @@ hydrodynamic cloaking — making the downstream flow match the "undisturbed" flo (as if the pinball weren't there). The upstream disturbance cylinder generates a Kármán vortex street; the pinball must cancel it. -- **CFD**: CelerisLab LBM solver, D2Q9 MRT, 2000x600 grid, uniform inlet, free-slip walls +- **CFD**: CelerisLab LBM solver (DynamisLab submodule), D2Q9 MRT, 2000x600 grid, uniform inlet, free-slip walls - **DRL**: PPO with Sin activation, 64x64 MLP, SB3 + VecNormalize - **V5 mode**: No_bias only (ACTION_SCALE=12, ACTION_BIAS=[0,0,0]). All cases use calibration-first workflow. -## 0. V5 Quick Start (Server Deployment) +## 0. Quick Start + +### 0.1 Single-Re training (re100) ```bash -# 1. Calibrate (once per case, ~5 min on single GPU) +# 1. Calibrate (~5 min) cd src/drl_pinball/train conda run -n pycuda_3_10 python calibrate.py \ --case re100 --device-id 0 \ --config configs/config_lbm_karman_2000x600.json -# 2. Launch multi-seed training on server (sequential, 7 min between GPUs) +# 2. Multi-seed training (server, sequential 7-min delay between GPUs) bash launch_multi.sh \ --case-name re100_karman --seeds 42,43,44,45,46,47 \ --gpus 0,1,2,3,4,5 --episodes 500 \ --config configs/config_lbm_karman_2000x600.json \ --calibration calibrations/re100/calibration.json -# 3. Transfer learning to another Re -conda run -n pycuda_3_10 python calibrate.py \ - --case re200 --device-id 0 --si 500 \ - --config configs/config_lbm_karman_2000x600_re200.json - -bash launch_multi.sh \ - --case-name re200_karman --seeds 42,43,44 --gpus 0,1,2 \ - --config configs/config_lbm_karman_2000x600_re200.json \ - --calibration calibrations/re200/calibration.json \ - --transfer output/re100_karman_seed42/models/best_model.zip - -# 4. Monitor +# 3. Monitor tail -f output/re100_karman_seed42/train.log tensorboard --logdir output/re100_karman_seed42/tb --port 6006 ``` +### 0.2 Cross-Re transfer (re60, re200, re400) + +```bash +# Local test (5 episodes each, quick verification): +bash crossre_transfer.sh --re-list 60,200,400 --test-episodes 5 + +# Server production (200 episodes each): +# !! BEFORE PUSHING TO SERVER: update BEST_MODEL path in crossre_transfer.sh !! +bash crossre_transfer.sh --re-list 60,200,400 +# Then push to server and run there. +``` + +### 0.3 Path notes for server deployment + +`crossre_transfer.sh` has relative paths via `SCRIPT_DIR`/`REPO_DIR`. +Only `BEST_MODEL` needs updating — point to the best Re100 model from +multi-seed training (e.g. `output/re100_karman_seed42/models/best_model.zip`). + --- -## 2. File Structure (V5) +## 2. File Structure (V5 Final) ``` train/ ├── __init__.py │ -├── # V5 ACTIVE FILES (parameterized, no_bias, calibration-driven) -│ +├── # ACTIVE FILES ├── calibrate.py # Phase 0 calibration (produces calibration.json + target.npy) -├── env_karman.py # Parameterized Karman cloak env (loads calibration JSON) +├── env_karman.py # Parameterized Karman cloak env +├── env_illusion.py # Parameterized Illusion env ├── env_vortex.py # Vortex cloak env (lamb/taylor, MAX_STEPS=150) -├── train_karman.py # Parameterized training script (--calibration, --config) +├── train_karman.py # Parameterized training script (every ep saves model) +├── train_illusion.py # Illusion training script ├── launch_multi.sh # Sequential multi-GPU server launcher -│ -├── # V5 SUPPORT FILES (unchanged from V4) -│ +├── crossre_transfer.sh # Cross-Re transfer: calibrate + train (re60→re200→re400) ├── symmetry_wrapper.py # G-mirror symmetry augmentation (per-rollout) -├── phase0_baseline_measure.py # Legacy baseline measurement tool (reference) ├── visualize_and_analyze.py # Flow-field visualization & analysis +├── SERVER_DEPLOY.md # Server deployment instructions +├── TRAIN_KNOWLEDGE.md # This file │ -├── # V4 BACKUP FILES (preserved, NOT active) +├── old/ # Archived V4 files (NOT active) +│ ├── env_karman_2000x600.py +│ ├── train_karman_2000x600.py +│ ├── phase0_baseline_measure.py +│ └── analyze_final.py │ -├── env_karman_2000x600.py # Original V4 env (hardcoded FORCE_SCALE, bias support) -├── train_karman_2000x600.py # Original V4 training (--no-bias flag) -├── analyze_final.py # V3 training curve plotting (legacy paths) -│ -├── # Calibration & output -│ -├── calibrations/ # Per-case calibration files -│ └── re100/ -│ ├── calibration.json # FORCE_SCALE, SENS_SCALE, SIM_BP/VAL, etc. -│ ├── target.npy # Target sensor signals (FIFO_LEN, 6) -│ └── calibrate.log # Calibration run log +├── calibrations/ # Per-case calibration files (IMMUTABLE) +│ ├── re60/ +│ │ ├── calibration.json # SI=800, FORCE_SCALE=0.0021, SENS_SCALE=0.72 +│ │ ├── target.npy +│ │ └── calibrate.log +│ ├── re100/ +│ │ ├── calibration.json # SI=800, FORCE_SCALE=0.0024, SENS_SCALE=0.75 +│ │ └── calibrate.log +│ ├── re200/ +│ │ ├── calibration.json # SI=500, FORCE_SCALE=0.0026, SENS_SCALE=0.90 +│ │ ├── target.npy +│ │ └── calibrate.log +│ ├── re400/ +│ │ ├── calibration.json # SI=400, FORCE_SCALE=0.0042, SENS_SCALE=0.98 +│ │ ├── target.npy +│ │ └── calibrate.log +│ └── illusion_1L/ +│ ├── calibration.json +│ ├── target.npy +│ ├── target_harmonics.json +│ └── calibrate.log │ └── output/ # Training outputs - └── re100_karman_seed42/ - ├── models/ # best_model.zip, final_model.zip, chkpt_ep*.zip + └── _seed/ + ├── models/ # ep0001_model.zip, ..., best_model.zip, final_model.zip ├── tb/ # TensorBoard logs - ├── train.log # Training run log - ├── calibration.json # Copy of calibration used - ├── vec_normalize.pkl # VecNormalize statistics - └── meta.json # Run metadata + ├── train.log + ├── calibration.json + ├── vec_normalize.pkl + └── meta.json ``` --- -## 3. Calibration Workflow (V5 — ALWAYS RUN FIRST) +## 3. Calibration Workflow (ALWAYS RUN FIRST) Every case MUST run `calibrate.py` before training. This produces: - `calibration.json`: FORCE_SCALE, SENS_SCALE, dtw_norm_scale, SIM_BP, SIM_VAL, reward constants -- `target.npy`: Target sensor signals (150 steps x 6 channels, legacy-equiv) +- `target.npy`: Target sensor signals (150 steps x 6 channels) -The calibration measures Stage0 (zero rotation) and Stage1 (open-loop reference action -equivalent to legacy [0,-4,4]*U0 bias) to compute: -- FORCE_SCALE = combined max|force| across both stages -- SENS_SCALE = combined max|sensor| (legacy-equiv) across both stages -- dtw_norm_scale = target's 3 uy-channel avg std -- SIM_BP/SIM_VAL = piecewise DTW mapping from Stage0/Stage1 sim mean values +The calibration measures Stage0 (zero rotation) and Stage1 (open-loop reference) to compute normalization +constants. Calibration is **IMMUTABLE** — once produced, never modify. -Calibration is IMMUTABLE — once produced, never modify. Training and inference use it as-is. +### Calibration Results (all Re) + +| Case | SI | FORCE_SCALE | SENS_SCALE | dtw_norm_scale | Stage0 sim | Stage1 sim | +|------|-----|-------------|------------|----------------|-----------|-----------| +| re60 | 800 | 0.0021 | 0.72 | 0.107 | 0.41 | 0.61 | +| re100 | 800 | 0.0024 | 0.75 | 0.204 | 0.32 | 0.82 | +| re200 | 500 | 0.0026 | 0.90 | 0.269 | 0.45 | 0.77 | +| re400 | 400 | 0.0042 | 0.98 | 0.310 | 0.56 | 0.73 | ### Cross-Re SI guidance Based on ~18 samples per vortex shedding cycle: | Case | SI | Rationale | |------|----|-----------| -| re50 | 1600 | Lower Re, longer period | +| re60 | 800 | FFT shows very weak/absent shedding at this Re with free-slip | | re100 | 800 | Verified (~19 samples/cycle) | | re200 | 500 | ~18 samples/cycle | | re400 | 400 | ~18 samples/cycle | ---- +### re60 note -## 4. Calibration Results (re100) - -``` -FORCE_SCALE = 0.002429 -SENS_SCALE = 0.7543 -dtw_norm_scale = 0.2043 -Stage0 sim_mean = 0.3166 (-> SIM_VAL=0.2) -Stage1 sim_mean = 0.8158 (-> SIM_VAL=0.5) -SIM_BP = [0.0, 0.3166, 0.8158, 0.8895, 0.9448, 1.0] -SIM_VAL = [0.0, 0.2, 0.5, 0.8, 0.9, 1.0] -``` - -Verified: env with calibration gives zero-action reward = 0.076 (Stage0 level, correct). +At Re=60 with uniform inlet + free-slip walls, the upstream disturbance cylinder +produces very weak periodic shedding (FFT dominant period ~5625 samples). +This is a different regime from legacy parabolic+no-slip where re50 did shed. +SI=800 is adequate; the DRL essentially learns a steady-state control policy. --- -## 5. Cross-Re Configs - -All share the same 2000x600 grid, uniform inlet, free-slip walls. Only viscosity differs: -- `configs/config_lbm_karman_2000x600.json` (v=0.004, Re=100) -- `configs/config_lbm_karman_2000x600_re50.json` (v=0.008, Re=50) -- `configs/config_lbm_karman_2000x600_re200.json` (v=0.002, Re=200) -- `configs/config_lbm_karman_2000x600_re400.json` (v=0.001, Re=400) - ---- - -## 6. Original "How to Start Training" (V4 — kept for reference) - -### Prerequisites -- conda env `pycuda_3_10` with PyCUDA, PyTorch, SB3, tensorboard -- CelerisLab installed at `/home/frank14f/CelerisLab` -- `target.npy` in the train directory (pre-recorded, reusable) - -### Starting a training run (CRITICAL: start sequentially, not simultaneously!) - -Two trainings on two GPUs MUST be started one after another with a ~7 min gap, -because they share the same CelerisLab kernel cache. Simultaneous startup causes -kernel compilation race conditions → corrupted kernel → reward=0.000 forever. - -```bash -# 1. Clean stale cache -rm -f /home/frank14f/CelerisLab/src/CelerisLab/lbm/kernels/config/config_objects.h -rm -f /home/frank14f/CelerisLab/src/CelerisLab/lbm/kernels/kernel.ptx - -# 2. Start Bias on GPU0 (with nohup so it survives terminal close) -cd /home/frank14f/DynamisLab/src/drl_pinball/train -nohup conda run --no-capture-output -n pycuda_3_10 python -u train_karman_2000x600.py \ - --device-id 0 --seed 42 --total-episodes 500 --target target.npy \ - > nohup_bias.log 2>&1 & - -# 3. WAIT ~7 minutes for Bias Ep1 to appear (check train.log) -# Confirm reward is non-zero before proceeding! - -# 4. Start NoBias on GPU1 -nohup conda run --no-capture-output -n pycuda_3_10 python -u train_karman_2000x600.py \ - --device-id 1 --seed 42 --total-episodes 500 --target target.npy --no-bias \ - > nohup_nobias.log 2>&1 & -``` - -### Monitoring -```bash -# Check progress -tail -5 output/bias_seed42_s2048_e10_v4/train.log -tail -5 output/nobias_seed42_s2048_e10_v4/train.log - -# TensorBoard -tensorboard --logdir output/bias_seed42_s2048_e10_v4/tb --port 6006 - -# Plot training curves -conda run -n pycuda_3_10 python -u analyze_final.py -``` - -### Stopping -```bash -ps aux | grep train_karman | grep -v grep | awk '{print $2}' | xargs kill -``` - ---- - -## 4. Reward Design (V3 — Current) +## 4. Reward Design ### Formula ```python -FORCE_SCALE = 0.0025 # fixed physical constant (combined max|force| from Phase 0) +# Gaussian reward (no zero-crossing spikes) +r_cd_raw = exp(-cd_norm² * K_CD) # cd_norm = (Σfx)/3 / FORCE_SCALE +r_cl_raw = exp(-cl_norm² * K_CL) -# Gaussian reward (no zero-crossing spikes that exp(-|x|) causes) -r_cd_raw = exp(-cd_norm² * 50) # cd_norm = (Σfx)/3 / FORCE_SCALE -r_cl_raw = exp(-cl_norm² * 100) # cl_norm = (Σfy)/3 / FORCE_SCALE - -# EMA smoothing for cd/cl only (r_sim is already smooth from DTW) +# EMA smoothing for cd/cl (r_sim uses DTW, already smooth) r_cd = EMA(r_cd_raw, weight=0.2) r_cl = EMA(r_cl_raw, weight=0.2) -# Normalized DTW similarity (raw, no EMA) -# norm_scale = mean of target's 3 uy-channel stds (fixed once target is recorded) -r_sim = piecewise_map(sim, breakpoints, values) +# Normalized DTW similarity (piecewise-mapped to [0,1]) +r_sim = piecewise_map(sim, SIM_BP, SIM_VAL) -# Floor penalty: prevents DRL from sacrificing one component -if r_cd < 0.10: penalty += 0.05 * (0.10 - r_cd) / 0.10 -if r_cl < 0.10: penalty += 0.05 * (0.10 - r_cl) / 0.10 -if r_sim < 0.10: penalty += 0.05 * (0.10 - r_sim) / 0.10 +# Floor penalty: prevents sacrificing one component +penalty = 0.05 * sum(max(0, FLOOR - r) / FLOOR for r, FLOOR in zip(...)) -reward = max(0, 0.30*r_cd + 0.30*r_cl + 0.40*r_sim - penalty) +reward = max(0, W_CD*r_cd + W_CL*r_cl + W_SIM*r_sim - penalty) +# W_CD=0.30, W_CL=0.30, W_SIM=0.40 ``` +### Three-stage targets + +| Stage | r_cd | r_cl | r_sim | Description | +|-------|------|------|-------|-------------| +| Stage0 (zero rotation) | ~0 | ~0 | ~0.2 | No control baseline | +| Stage1 (reference open-loop) | ~0.7 | ~0.5 | ~0.5 | Legacy-equiv bias | +| Optimal (trained) | ~0.9 | ~0.9 | ~0.9 | Full cloaking | + ### Why Gaussian not exp(-|x|) -`exp(-|x|)` has maximum gradient at x=0. When cd/cl oscillates around zero, -the reward swings wildly (spikes at every zero-crossing). `exp(-x²)` has zero -gradient at x=0, giving smooth reward near the optimum. +`exp(-|x|)` has maximum gradient at x=0, causing reward spikes at zero-crossings +of oscillating cd/cl. `exp(-x²)` has zero gradient at x=0 → smooth near optimum. -### Why Normalized DTW +### Why normalized DTW -Raw DTW `1 - cost/n` has very narrow dynamic range (0.70-0.97) because cost -depends on absolute signal amplitude. Normalizing by target's uy-channel std -(≈0.20) extends the range to 0.0-0.9, giving DRL a meaningful gradient. +Raw DTW has narrow dynamic range (0.70-0.97). Normalizing by target's uy-channel +std extends range to 0.0-0.9, giving DRL meaningful gradient. -The norm_scale is computed once from the target signal and is fixed for all -training and inference. It's universal — same formula works for cloak and -illusion (each scenario has its own target → its own norm_scale). +### Why no EMA on r_sim -### Why r_sim has no EMA - -DTW is already a smooth signal (30-step windowed comparison). Adding EMA -over-smoothed it and introduced artificial delay. The 40-step physical delay -(convection 25 steps + DTW window 15 steps) is handled by GAE with gamma=0.995. - -### Three-stage targets (from Phase 0 measurement) - -| Stage | r_cd | r_cl | r_sim | total | Target | -|-------|------|------|-------|-------|--------| -| Stage0 (no rotation) | 0.03 | 0.22 | 0.17 | 0.14 | ~0.2 | -| Stage1 (bias action) | 0.72 | 0.54 | 0.49 | 0.58 | ~0.5 | -| Optimal (trained) | ~0.9 | ~0.9 | ~0.9 | ~0.9 | ~0.9 | +DTW is already a smooth 30-step windowed signal. Adding EMA over-smooths. --- -## 5. PPO Configuration (V4 — Current) +## 5. PPO Configuration ```python PPO( @@ -270,203 +215,106 @@ PPO( policy_kwargs={"activation_fn": Sin, "net_arch": [64, 64]}, env=vec_env, device=torch.device("cuda:X"), - n_steps=2048, # MUST be 2048 (legacy default). 512 causes noisy curves. - batch_size=64, # SB3 default - n_epochs=10, # SB3 default. 3 was too few → slow learning. - learning_rate=3e-4, # SB3 default - gamma=0.995, # Higher than SB3 default (0.99) for r_sim delay propagation - # ent_coef=0.0 # SB3 default (not set). n_steps=2048 provides enough diversity. - # target_kl=None # SB3 default (not set). Well-estimated gradients don't need KL stop. - verbose=0, + n_steps=2048, # MUST be 2048. 512 → noisy curves. + batch_size=64, + n_epochs=10, + learning_rate=3e-4, + gamma=0.995, # Higher than default for DTW delay propagation ) ``` -### Key lesson: Don't deviate from SB3 defaults without strong reason - -V1-V3 used n_steps=512, n_epochs=3, ent_coef=0.01, target_kl=0.03. This caused: -- Noisy training curves (high gradient variance from small n_steps) -- Slow learning (3 epochs too few) -- Degradation after peak (policy oscillation from noisy gradients) - -V4 matches SB3 defaults (n_steps=2048, n_epochs=10, no ent_coef, no target_kl) -and produces smooth, steadily rising curves — matching the legacy training quality. - -### Evaluation: stochastic (NOT deterministic) - -```python -action, _ = model.predict(eval_obs) # NO deterministic=True -``` - -Legacy training used stochastic eval. Deterministic eval gives sharp, noisy -rewards. Stochastic eval averages over the policy distribution → smoother curves. -For paper presentation, stochastic is more honest (shows expected performance). - -### Symmetry: per-rollout, not per-step - -The G-mirror wrapper decides ONCE per rollout (every 2048 steps) whether to -mirror. If mirrored, ALL steps in that rollout use G-transform consistently. -Per-step random mirroring adds noise; per-rollout is clean. - -During evaluation, symmetry is disabled (prob=0.0) for clean policy assessment. +- **Evaluation**: stochastic (no deterministic=True) — smoother curves +- **Symmetry**: per-rollout G-mirror (50% probability). Disabled during eval. +- **Every episode saves**: `ep0001_model.zip` + `ep0001_vecnormalize.pkl` saved each episode --- -## 6. Obs Normalization +## 6. Obs Normalization (Two-layer) -### Two-layer approach - -1. **Env physical norm** (fixed, reproducible): - - forces / FORCE_SCALE (0.0025) - - sensors / SENS_SCALE (0.8, legacy-equiv scale) - - No clipping (VecNormalize handles that) +1. **Env physical norm** (fixed, from calibration): + - forces / FORCE_SCALE, sensors / SENS_SCALE + - No clipping (VecNormalize handles) 2. **SB3 VecNormalize** (running mean/std): - - norm_obs=True, norm_reward=False - - clip_obs=10.0 (generous, physical norm already keeps obs ~[-1,1]) + - norm_obs=True, norm_reward=False, clip_obs=10.0 - Saved to `vec_normalize.pkl` for inference -### Why not just VecNormalize - -Physical norm ensures the obs is in a reasonable range even before VecNormalize -has collected enough statistics. This prevents extreme values in the first few -hundred steps. The physical norm constants (FORCE_SCALE, SENS_SCALE) are fixed -and do not change between training and inference. - -### Why not just physical norm (like legacy) - -VecNormalize adapts to the actual obs distribution, giving each dimension -equal footing. Without it, sensor ux (range 0.6-0.9) dominates sensor uy -(range 0.2) after physical norm. VecNormalize corrects this. - --- -## 7. Action Configuration +## 7. Action Configuration (V5: no_bias only) -### Bias mode (default) -```python -ACTION_SCALE = 8.0 -ACTION_BIAS = [0, -4, 4] # front, top(+y), bot(-y) -# omega = -(action * 8 + [0,-4,4]) * U0 / RADIUS -# Physical range: front [-8,8], top [-4,12], bot [-12,4] × U0 -``` - -### NoBias mode (--no-bias flag) ```python ACTION_SCALE = 12.0 ACTION_BIAS = [0, 0, 0] -# omega = -(action * 12 + [0,0,0]) * U0 / RADIUS -# Physical range: all cylinders [-12,12] × U0 +# omega = -(action * 12) * U0 / RADIUS +# Physical range: all cylinders [-12, 12] × U0 ``` -NoBias uses scale=12 (not 8) to cover the same omega range as Bias. -With scale=8 and no bias, range is only [-8,8] — missing the ±12×U0 extremes -that Bias reaches. This caused NoBias to fail at learning lift control (r_cl -collapsed to 0.05) because it couldn't reach the necessary rotation speeds. - -### Sign convention (CelerisLab new kernel) - -The new CelerisLab kernel uses `Uw = -omega * ry` (negative sign). -Legacy used `Uw = action * (y_c - y) / radius` (no negative). -Conversion: `omega = -surface_vel / radius` +Sign convention: `Uw = -omega * ry` (omega>0 = clockwise). --- ## 8. Environment Design -### Two-phase initialization (avoids runtime sync_bodies) +### Two-phase initialization -1. `record_target()`: temporary Simulation (dist_cyl + sensors only) → - record 150 steps of target signal → close -2. `KarmanCloakEnv.__init__()`: training Simulation with ALL 7 objects - upfront → warmup → zero-action FIFO → bias FIFO → snapshot +1. `record_target()`: dist_cyl + sensors only → record 150-step target → close +2. Training Simulation: all 7 objects → warmup → zero-action FIFO → snapshot -All objects (1 dist_cyl + 3 sensors + 3 pinball) are added before `initialize()`. -No runtime `sync_bodies()` — it causes recompilation and cache conflicts. - -### Body IDs (add order) +Body IDs (add order): ``` -0: dist_cyl (force only, skipped in obs) -1: sensor 0 (top, +y) -2: sensor 1 (center) -3: sensor 2 (bottom, -y) -4: pinball_front -5: pinball_top (rear, +y) -6: pinball_bottom (rear, -y) +0: dist_cyl (force, skipped in obs) +1-3: sensors (top, center, bottom) +4-6: pinball (front, top_rear, bottom_rear) ``` -### Obs layout (12-dim, env output before VecNormalize) -``` -[0:6] = forces / FORCE_SCALE: [front_fx, front_fy, top_fx, top_fy, bot_fx, bot_fy] -[6:12] = sensors / SENS_SCALE: [s0_ux, s0_uy, s1_ux, s1_uy, s2_ux, s2_uy] -``` +Obs layout (12-dim): forces[6] + sensors[6] ### No-reset training -`step()` always returns `terminated=False`. The flow field is never reset -during training (matches legacy behavior). Episode length is controlled by -the external training loop (`model.learn(2048)` per call). - -Evaluation does `eval_env.reset()` (restores snapshot) for reproducible -assessment. After eval, training continues from the post-eval state. +`step()` returns `terminated=False`. Eval does `reset()` for reproducible assessment. --- -## 9. Bugs Found & Fixed (Full History) +## 9. Cross-Re Transfer Pipeline -### Critical bugs +### How to add a new Re -| # | Bug | Symptom | Fix | -|---|-----|---------|-----| -| 1 | **Simultaneous startup** | Two trainings on 2 GPUs → kernel compilation race → reward=0.000 forever | Start sequentially: Bias first, wait 7 min, then NoBias | -| 2 | **Bias FIFO not applying omega** | `_init_cfd` set bias_omega but never called `set_omega()` during bias FIFO | Added `self._set_omega(bias_omega)` in bias FIFO loop | -| 3 | **CPU training** | `device="cpu"` was used to avoid PyCUDA/PyTorch conflict | Use GPU with `context.push()/pop()` around CFD calls | -| 4 | **Symmetry during eval** | G-mirror prob=0.5 during evaluation → noisy, inaccurate reward | Set `inner.prob = 0.0` before eval, restore after | -| 5 | **DTW not normalized** | Raw DTW `1-cost/n` → sim always 0.90+ → no gradient | Normalize by target uy-channel avg std | -| 6 | **r_sim EMA + delay** | Added unnecessary EMA smoothing and 20-step delay buffer | Removed both — DTW is already smooth, use raw value | -| 7 | **exp(-\|x\|) reward** | Zero-crossing spikes in r_cd/r_cl → noisy training | Changed to Gaussian `exp(-x²*K)` | -| 8 | **NoBias scale=8** | Can't reach ±12×U0 → r_cl collapses | Use scale=12 for NoBias | -| 9 | **n_steps=512** | High gradient variance → noisy curves, degradation | Use n_steps=2048 (SB3 default) | -| 10 | **n_epochs=3** | Too few → slow learning | Use n_epochs=10 (SB3 default) | -| 11 | **deterministic eval** | Sharp, noisy reward measurements | Use stochastic eval (no deterministic=True) | -| 12 | **Stale config_objects.h** | N_OBJS mismatch after previous run | `_clean_cache()` before each Simulation creation | +```bash +# 1. Create config (copy existing, change viscosity) +cp configs/config_lbm_karman_2000x600.json configs/config_lbm_karman_2000x600_reNNN.json +# Edit "viscosity" to U0 * 2D / Re_NNN -### Sensor scaling (from reproduce phase) +# 2. Calibrate +python calibrate.py --case reNNN --device-id 0 --si --config -New CelerisLab `read_sensor(normalize=True)` returns area-averaged velocity -(÷cell_count). Legacy returned raw sum (~78x larger). For DTW comparison with -legacy-recorded targets, multiply by `SENSOR_CC=78`. +# 3. Test (5 episodes, local) +python train_karman.py --case-name transfer_reNNN --device-id 0 --seed 41 \ + --config --calibration calibrations/reNNN/calibration.json \ + --si --total-episodes 5 \ + --transfer-model output/re100_karman_seed/models/best_model.zip -### Omega sign inversion (from reproduce phase) +# 4. Production (add to crossre_transfer.sh or run directly with 200 episodes) +``` -New kernel: `Uw = -omega * ry` (omega>0 = clockwise). -Legacy: `Uw = action * (y_c - y) / radius` (no sign inversion). -Conversion: `omega = -surface_vel / radius` +### Transfer test results (5-episode verification, 2026-07-02) + +| Re | Best Reward | r_cd | r_cl | r_sim | Time/ep | +|----|------------|------|------|-------|---------| +| 60 | 0.637 | 0.879 | 0.402 | 0.641 | 281s | +| 200 | 0.428 | 0.677 | 0.259 | 0.387 | 186s | +| 400 | 0.489 | 0.787 | 0.491 | 0.274 | 153s | + +All show rapid learning from Re100 base. r_cl remains the hardest component across all Re. --- -## 10. Training Results Summary +## 10. CelerisLab Integration -### V4 (current, 30 ep before OOM stop — full 500ep running) - -| Training | Ep1 | Ep10 | Ep20 | Ep30 | Smooth? | -|----------|-----|------|------|------|---------| -| Bias | 0.32 | 0.37 | 0.40 | 0.54 | Yes, steady rise | -| NoBias | 0.11 | 0.23 | 0.43 | 0.41 | Yes, near-monotonic | - -### Previous versions (for reference) - -| Version | n_steps | Key change | Result | -|---------|---------|------------|--------| -| V1 | 512 | Config8 reward, exp(-\|x\|) | Peak 0.79 but noisy, possible degradation | -| V2 | 512 | Gaussian + EMA r_sim | Stable but slow (0.62 peak) | -| V3 | 512 | Normalized DTW, no r_sim EMA | Stable, NoBias reached 0.73 | -| **V4** | **2048** | **SB3 defaults, stochastic eval** | **Smoothest curves, running** | - -### NoBias vs Bias - -NoBias requires more episodes to peak (~265 vs ~89 in V3) but can reach -similar or higher peak reward. NoBias r_cl is the main challenge — it needs -the scale=12 action range and floor penalty to prevent collapse. +**CelerisLab is a git submodule** at `DynamisLab/CelerisLab/`. +All Python imports use `from CelerisLab import Simulation`. +No external paths — everything is self-contained within the repo. +Do NOT reference `/home/frank14f/CelerisLab` anywhere. --- @@ -474,62 +322,31 @@ the scale=12 action range and floor penalty to prevent collapse. | Parameter | Value | Where | Notes | |-----------|-------|-------|-------| -| Grid | 2000×600 | config_lbm_karman_2000x600.json | uniform inlet, free_slip | +| Grid | 2000×600 | config | uniform inlet, free_slip | | U0 | 0.01 | config | lattice inlet velocity | -| ν | 0.004 | config | Re_D=50 (code Re=100) | -| SI | 800 | env | LBM steps per action | +| ν (re100) | 0.004 | config | Re_D=50 (code Re=100) | +| SI | 400-800 | calibration | varies by Re | | FIFO_LEN | 150 | env | history buffer | | CONV_LEN | 30 | env | DTW comparison window | -| FORCE_SCALE | 0.0025 | env | reward normalization | -| SENS_SCALE | 0.8 | env | obs normalization (legacy-equiv) | | SENSOR_CC | 78 | env | sensor area→legacy conversion | -| K_CD | 50 | env | Gaussian reward coefficient | -| K_CL | 100 | env | Gaussian reward coefficient | +| K_CD/K_CL | 50/100 | env | Gaussian reward coefficients | | W_CD/W_CL/W_SIM | 0.30/0.30/0.40 | env | reward weights | -| FLOOR_CD/CL/SIM | 0.10 | env | floor penalty threshold | | n_steps | 2048 | train | PPO rollout size | | n_epochs | 10 | train | PPO epochs per update | | gamma | 0.995 | train | discount factor | -| ACTION_SCALE | 8 (bias) / 12 (nobias) | env | action scaling | -| ACTION_BIAS | [0,-4,4] / [0,0,0] | env | action offset | +| ACTION_SCALE | 12.0 | env | no_bias only | --- -## 12. CFD Config +## 12. Bugs Found & Fixed -File: `configs/config_lbm_karman_2000x600.json` - -``` -Grid: 2000×600, D2Q9, MRT, double_buffer, FP32 -Inlet: uniform, regularized scheme -Walls: free_slip (matches experimental water tunnel) -Outlet: neq_extrap with backflow_clamp -``` - -### EsoPull note - -EsoPull streaming was tested — it computes correctly but gives ~5% slowdown -(not the expected 50% speedup) on this grid size. Use double_buffer for speed. -EsoPull + zou_he_local inlet gives very different physics (different force -balances) — only use if you re-record the target with the same inlet scheme. - ---- - -## 13. Future Work - -1. **Re50/re200/re400 transfer learning**: Calibrate each Re, then transfer from re100 best model using `--transfer` flag. Use adjusted SI per Re. -2. **Vortex cloak**: `env_vortex.py` ready. Transfer from re100 model. Lamb and Taylor variants. -3. **Illusion**: `env_illusion.py` needed. S_DIM=14, harmonics target reconstruction. Adjusted pinball/sensor positions. -4. **Steady cloak**: Simpler case (no upstream disturbance cylinder). Target is uniform flow. -5. **Symmetry ablation study**: Compare with/without G-mirror to quantify benefit. -6. **Longer training**: 500 episodes may not be enough. Try 1000 episodes. -7. **Learning rate schedule**: Try lr decay after peak to prevent degradation. - -## 14. V5 Design Principles - -1. **Calibration-first**: Every case runs calibrate.py before training. Produces a single immutable JSON. -2. **No-bias only**: action_scale=12, action_bias=[0,0,0]. Simplifies G-mirror, removes human priors. -3. **Parameterized envs**: env_karman.py accepts calibration dict + config_path + si. No module-level hardcode. -4. **Sequential GPU startup**: launch_multi.sh enforces 7-min delay between launches. -5. **Immutable calibration**: Once produced, calibration is read-only for training and inference. Saved alongside model. -6. **r_sim maps to [0,1]**: SIM_VAL[-1]=1.0. Full-range normalized DTW similarity. +| # | Bug | Symptom | Fix | +|---|-----|---------|-----| +| 1 | Simultaneous GPU startup | Kernel compilation race → reward=0.000 | Sequential launch, 7-min delay | +| 2 | Bias FIFO not applying omega | Wrong normalization baseline | Added set_omega in bias FIFO | +| 3 | DTW not normalized | sim always 0.90+ → flat gradient | Normalize by target uy-channel avg std | +| 4 | Gaussian vs exp(-|x|) | Zero-crossing reward spikes | Use exp(-x² · K) | +| 5 | n_steps=512 | Noisy curves, degradation | Use 2048 (SB3 default) | +| 6 | deterministic eval | Sharp, noisy reward | Stochastic eval | +| 7 | External CelerisLab paths | Permission issues, server mismatch | Use DynamisLab submodule only | +| 8 | Manual kernel cache cleaning | Unnecessary, root-only files | CelerisLab handles internally | diff --git a/src/drl_pinball/train/calibrate.py b/src/drl_pinball/train/calibrate.py index dd67376..fdf8f5c 100644 --- a/src/drl_pinball/train/calibrate.py +++ b/src/drl_pinball/train/calibrate.py @@ -35,15 +35,6 @@ if str(_REPO) not in sys.path: from CelerisLab import Simulation -_CELERIS = Path("/home/frank14f/CelerisLab") -_CONFIG_H = _CELERIS / "src/CelerisLab/lbm/kernels/config/config_objects.h" -_PTX = _CELERIS / "src/CelerisLab/lbm/kernels/kernel.ptx" - -def _clean_cache(): - for p in [_CONFIG_H, _PTX]: - if p.exists(): - p.unlink() - # --------------------------------------------------------------------------- # Physics / geometry constants # --------------------------------------------------------------------------- @@ -72,6 +63,7 @@ _REF_OMEGA = np.array([0.0, 0.004, -0.004], dtype=np.float32) # Reward constants K_CD = 50.0; K_CL = 100.0 +K_CD_ILLUSION = 12.0; K_CL_ILLUSION = 25.0 # lower: 3-pinball forces vs 1-target cylinder, wider error range W_CD = 0.30; W_CL = 0.30; W_SIM = 0.40 FLOOR_CD = 0.10; FLOOR_CL = 0.10; FLOOR_SIM = 0.10 FLOOR_PENALTY = 0.05 @@ -199,7 +191,7 @@ _ILL_PINBALL_REAR_X = 406.0 _ILL_SENSOR_X = 600.0 _ILL_TARGET_X = 400.0 _ILL_TARGET_RADIUS = 1.0 * L0 -_ILL_REF_OMEGA = np.array([0.0, 0.002, -0.002], dtype=np.float32) # legacy-equiv bias +_ILL_REF_OMEGA = np.array([0.0, 0.001, -0.001], dtype=np.float32) # surface_vel = [0,-1,1]*U0, FIFO-init level def _analyze_harmonics_for_calib(states, n_harmonics=5): N, D = states.shape @@ -224,7 +216,6 @@ def _analyze_harmonics_for_calib(states, n_harmonics=5): def _calibrate_illusion(case, config_path, device_id, si, out_dir, log, warmup): # ---- Step 1: Record target (target cylinder + 3 sensors) ---- log("Step 1: Recording illusion target (target cyl + sensors)...") - _clean_cache() sim = Simulation(lbm_config_path=config_path, device_id=device_id) sim._assert_object_count_contract = lambda *a, **kw: None sim.add_body("circle", center=(_ILL_TARGET_X, CENTER_Y, 0.0), @@ -266,7 +257,6 @@ def _calibrate_illusion(case, config_path, device_id, si, out_dir, log, warmup): # ---- Step 2: Training sim (3 sensors + 3 pinball) ---- log("Step 2: Creating training sim (6 objects)...") - _clean_cache() sim = Simulation(lbm_config_path=config_path, device_id=device_id) sim._assert_object_count_contract = lambda *a, **kw: None sensor_ids = [ @@ -315,9 +305,23 @@ def _calibrate_illusion(case, config_path, device_id, si, out_dir, log, warmup): s0_sim = float(np.mean(stage0["sims"])) s1_sim = float(np.mean(stage1["sims"])) - sim_bp = [0.0, s0_sim, s1_sim, - s1_sim + (1.0 - s1_sim) * 0.4, - s1_sim + (1.0 - s1_sim) * 0.7, + # Build SIM_BP: [0, worst_measured, better_measured, ..., 1.0] + # Normally (Karman) Stage0 < Stage1 (zero is worse, reference is better). + # But for Illusion, Stage0 may already match the target well, and reference + # rotation (steady-cloak-like) makes it worse. Swap roles in that case. + if s1_sim < s0_sim: + # Reference action is worse than zero (Illusion case). + # Stage1 (reference) -> r_sim ~0.2, Stage0 (zero) -> r_sim ~0.4 + worst_sim = s1_sim + better_sim = s0_sim + else: + # Normal (Karman) case. + worst_sim = s0_sim + better_sim = s1_sim + + sim_bp = [0.0, worst_sim, better_sim, + better_sim + (1.0 - better_sim) * 0.4, + better_sim + (1.0 - better_sim) * 0.7, 1.0] sim_val = [0.0, 0.2, 0.5, 0.8, 0.9, 1.0] @@ -411,7 +415,6 @@ def main() -> int: # ---- Step 1: Record target (Karman: dist_cyl + sensors only) ---- log("Step 1: Recording target signal...") - _clean_cache() sim = Simulation(lbm_config_path=config_path, device_id=device_id) sim._assert_object_count_contract = lambda *a, **kw: None dist_id_t = sim.add_body("circle", center=(DIST_X, CENTER_Y, 0.0), radius=1.0 * L0) @@ -444,7 +447,6 @@ def main() -> int: # ---- Step 2: Create training sim ---- log("Step 2: Creating training sim (all 7 objects)...") - _clean_cache() sim = Simulation(lbm_config_path=config_path, device_id=device_id) sim._assert_object_count_contract = lambda *a, **kw: None dist_id = sim.add_body("circle", center=(DIST_X, CENTER_Y, 0.0), radius=1.0 * L0) @@ -545,8 +547,8 @@ def main() -> int: "dtw_norm_scale": float(dtw_norm_scale), "SIM_BP": [float(x) for x in sim_bp], "SIM_VAL": [float(x) for x in sim_val], - "K_CD": K_CD, - "K_CL": K_CL, + "K_CD": K_CD_ILLUSION, + "K_CL": K_CL_ILLUSION, "W_CD": W_CD, "W_CL": W_CL, "W_SIM": W_SIM, diff --git a/src/drl_pinball/train/calibrations/illusion_1L/calibration.json b/src/drl_pinball/train/calibrations/illusion_1L/calibration.json index 3dbc1b6..e5f4853 100644 --- a/src/drl_pinball/train/calibrations/illusion_1L/calibration.json +++ b/src/drl_pinball/train/calibrations/illusion_1L/calibration.json @@ -10,15 +10,15 @@ "FIFO_LEN": 150, "CONV_LEN": 30, "SENSOR_CC": 78.0, - "FORCE_SCALE": 0.0029, + "FORCE_SCALE": 0.002, "SENS_SCALE": 0.93, "dtw_norm_scale": 0.251, "SIM_BP": [ 0.0, + 0.73, 0.81, - 0.82, - 0.83, - 0.84, + 0.89, + 0.94, 1.0 ], "SIM_VAL": [ @@ -29,8 +29,8 @@ 0.9, 1.0 ], - "K_CD": 50.0, - "K_CL": 100.0, + "K_CD": 12.0, + "K_CL": 25.0, "W_CD": 0.3, "W_CL": 0.3, "W_SIM": 0.4, diff --git a/src/drl_pinball/train/calibrations/illusion_1L/target_harmonics.json b/src/drl_pinball/train/calibrations/illusion_1L/target_harmonics.json index e94d202..0a97e29 100644 --- a/src/drl_pinball/train/calibrations/illusion_1L/target_harmonics.json +++ b/src/drl_pinball/train/calibrations/illusion_1L/target_harmonics.json @@ -1,12 +1,12 @@ [ { - "dc": 0.6848811427752177, + "dc": 0.6848811439673106, "amps": [ - 0.20348444790268552, - 0.02915900547523486, - 0.02784031012371259, - 0.026538733180849892, - 0.01919840514104942 + 0.2034844510278786, + 0.029159004511650252, + 0.027840311609468333, + 0.02653873514475916, + 0.01919840709106637 ], "freqs": [ 0.02666666666666667, @@ -16,21 +16,21 @@ 0.04666666666666667 ], "phases": [ - -1.2362005505810982, - -1.1766679461536396, - 1.8863477863626301, - -2.834029481822404, - -0.7286212732915559 + -1.2362005574545905, + -1.1766679293247424, + 1.886347686621138, + -2.834029604726311, + -0.7286213249887804 ] }, { - "dc": -0.026989686206604045, + "dc": -0.026989686344750227, "amps": [ - 0.2790809917571785, - 0.07572308632544687, - 0.04473095779630388, - 0.039113569175253486, - 0.038411479291876154 + 0.2790809917939555, + 0.07572308617806586, + 0.044730957946728864, + 0.03911356916608822, + 0.03841147969973682 ], "freqs": [ 0.02666666666666667, @@ -40,21 +40,21 @@ 0.03333333333333333 ], "phases": [ - 3.0547687538583306, - 0.6711306028525605, - -0.10499736264498984, - -2.7955698201715835, - 3.1220861518480647 + 3.0547687544981676, + 0.671130604261652, + -0.10499736216933567, + -2.7955698151736605, + 3.1220861483228126 ] }, { - "dc": 0.5674182403087616, + "dc": 0.5674182399113973, "amps": [ - 0.06514963980633084, - 0.02472242867395058, - 0.014768360330077245, - 0.010156019183366295, - 0.00963919337572356 + 0.06514964076759504, + 0.024722427490218583, + 0.01476835974183025, + 0.010156020328112192, + 0.009639191161046853 ], "freqs": [ 0.05333333333333334, @@ -64,21 +64,21 @@ 0.1 ], "phases": [ - -0.7488772609862894, - 2.444199252505853, - -0.7923018439616549, - 2.505537406241232, - -1.0758943656055078 + -0.7488773712604715, + 2.444199243833346, + -0.7923014988253196, + 2.5055369932207086, + -1.0758944221483198 ] }, { - "dc": 0.014232361110819814, + "dc": 0.014232359210921763, "amps": [ - 0.4461623192829029, - 0.0779793315804837, - 0.0550568453231777, - 0.05331603051277121, - 0.04874852247514895 + 0.446162318961067, + 0.07797933262227957, + 0.05505684614794328, + 0.05331603052134459, + 0.04874852338883084 ], "freqs": [ 0.02666666666666667, @@ -88,21 +88,21 @@ 0.07333333333333333 ], "phases": [ - -3.1049303564692066, - 0.027203636269757633, - 0.09727455160562376, - -3.095222555392279, - -3.044905056768087 + -3.104930355366964, + 0.027203661003797012, + 0.09727456602426711, + -3.0952225671159, + -3.0449050417637897 ] }, { - "dc": 0.6885532836119334, + "dc": 0.688553271094958, "amps": [ - 0.2018316634259717, - 0.033110505502103905, - 0.029481533925939486, - 0.027209600513899444, - 0.0210247548932384 + 0.20183164156868158, + 0.033110505614057664, + 0.029481548571705242, + 0.02720959921763942, + 0.02102475262837555 ], "freqs": [ 0.02666666666666667, @@ -112,21 +112,21 @@ 0.07333333333333333 ], "phases": [ - 1.8817739461803038, - 2.742416499482124, - -1.1159813908192473, - 1.7448219790472927, - 1.8563271918371878 + 1.8817739067162356, + 2.7424167422444903, + -1.1159816805254985, + 1.7448225406868731, + 1.8563274842269628 ] }, { - "dc": 0.0464287880451108, + "dc": 0.046428785625806386, "amps": [ - 0.266793500575173, - 0.08445240294008177, - 0.05549481483267607, - 0.03176632041902732, - 0.025647047642157136 + 0.2667935014251649, + 0.08445240278954501, + 0.05549481355853139, + 0.031766322660930275, + 0.025647051108265238 ], "freqs": [ 0.02666666666666667, @@ -136,21 +136,21 @@ 0.04666666666666667 ], "phases": [ - 3.0338443035387885, - -2.634676303912741, - -0.03215847385538798, - -0.005548069481591528, - 0.8284453225261742 + 3.0338443211965744, + -2.6346763210802586, + -0.032158500100019624, + -0.005548092544019917, + 0.8284453426622406 ] }, { - "dc": 0.002807500216489037, + "dc": 0.0028075010624403754, "amps": [ - 1.712089159739782e-05, - 6.84709236632855e-06, - 3.6652517552615438e-06, - 2.9729489026324893e-06, - 2.0179845909227172e-06 + 1.7116459800799798e-05, + 6.866397692253015e-06, + 3.6591197181741872e-06, + 2.979711101541413e-06, + 2.0364590024389495e-06 ], "freqs": [ 0.05333333333333334, @@ -160,21 +160,21 @@ 0.06666666666666667 ], "phases": [ - -2.4049945951829694, - 0.6548426535356905, - -2.327050319985697, - 0.5677103875435425, - -2.2688377992989093 + -2.4055008082312566, + 0.6560206596369318, + -2.338324550342246, + 0.5697953506705974, + -2.2632313796302874 ] }, { - "dc": -1.4845058922219323e-05, + "dc": -1.484534581322805e-05, "amps": [ - 0.0006606466483213061, - 0.00010502206909109318, - 8.38977596666718e-05, - 4.8485264824562914e-05, - 4.61590173098796e-05 + 0.000660645874022141, + 0.00010502076564124189, + 8.389696757021492e-05, + 4.848746760651764e-05, + 4.616076991013906e-05 ], "freqs": [ 0.02666666666666667, @@ -184,11 +184,11 @@ 0.04 ], "phases": [ - -0.8350712750146696, - 2.4283127217217664, - -0.9184054668600495, - 2.6043773129885874, - -0.9754872507212389 + -0.8350715543078073, + 2.428305305182952, + -0.9184117032214737, + 2.6043964384951956, + -0.9755175136337735 ] } ] \ No newline at end of file diff --git a/src/drl_pinball/train/calibrations/re200/calibration.json b/src/drl_pinball/train/calibrations/re200/calibration.json new file mode 100644 index 0000000..ad9aa67 --- /dev/null +++ b/src/drl_pinball/train/calibrations/re200/calibration.json @@ -0,0 +1,49 @@ +{ + "case": "re200", + "grid": { + "nx": 2000, + "ny": 600 + }, + "config_path": "/home/frank14f/DynamisLab/configs/config_lbm_karman_2000x600_re200.json", + "SI": 500, + "FIFO_LEN": 150, + "CONV_LEN": 30, + "SENSOR_CC": 78.0, + "FORCE_SCALE": 0.0026, + "SENS_SCALE": 0.9, + "dtw_norm_scale": 0.269, + "SIM_BP": [ + 0.0, + 0.45, + 0.77, + 0.86, + 0.93, + 1.0 + ], + "SIM_VAL": [ + 0.0, + 0.2, + 0.5, + 0.8, + 0.9, + 1.0 + ], + "K_CD": 12.0, + "K_CL": 25.0, + "W_CD": 0.3, + "W_CL": 0.3, + "W_SIM": 0.4, + "FLOOR_CD": 0.1, + "FLOOR_CL": 0.1, + "FLOOR_SIM": 0.1, + "FLOOR_PENALTY": 0.05, + "ACTION_SCALE": 12.0, + "ACTION_BIAS": [ + 0.0, + 0.0, + 0.0 + ], + "U0": 0.01, + "RADIUS": 10.0, + "L0": 20.0 +} \ No newline at end of file diff --git a/src/drl_pinball/train/calibrations/re400/calibration.json b/src/drl_pinball/train/calibrations/re400/calibration.json new file mode 100644 index 0000000..a537d4b --- /dev/null +++ b/src/drl_pinball/train/calibrations/re400/calibration.json @@ -0,0 +1,49 @@ +{ + "case": "re400", + "grid": { + "nx": 2000, + "ny": 600 + }, + "config_path": "/home/frank14f/DynamisLab/configs/config_lbm_karman_2000x600_re400.json", + "SI": 400, + "FIFO_LEN": 150, + "CONV_LEN": 30, + "SENSOR_CC": 78.0, + "FORCE_SCALE": 0.0042, + "SENS_SCALE": 0.98, + "dtw_norm_scale": 0.31, + "SIM_BP": [ + 0.0, + 0.56, + 0.73, + 0.84, + 0.92, + 1.0 + ], + "SIM_VAL": [ + 0.0, + 0.2, + 0.5, + 0.8, + 0.9, + 1.0 + ], + "K_CD": 12.0, + "K_CL": 25.0, + "W_CD": 0.3, + "W_CL": 0.3, + "W_SIM": 0.4, + "FLOOR_CD": 0.1, + "FLOOR_CL": 0.1, + "FLOOR_SIM": 0.1, + "FLOOR_PENALTY": 0.05, + "ACTION_SCALE": 12.0, + "ACTION_BIAS": [ + 0.0, + 0.0, + 0.0 + ], + "U0": 0.01, + "RADIUS": 10.0, + "L0": 20.0 +} \ No newline at end of file diff --git a/src/drl_pinball/train/calibrations/re60/calibration.json b/src/drl_pinball/train/calibrations/re60/calibration.json new file mode 100644 index 0000000..2c464f4 --- /dev/null +++ b/src/drl_pinball/train/calibrations/re60/calibration.json @@ -0,0 +1,49 @@ +{ + "case": "re60", + "grid": { + "nx": 2000, + "ny": 600 + }, + "config_path": "/home/frank14f/DynamisLab/configs/config_lbm_karman_2000x600_re60.json", + "SI": 800, + "FIFO_LEN": 150, + "CONV_LEN": 30, + "SENSOR_CC": 78.0, + "FORCE_SCALE": 0.0021, + "SENS_SCALE": 0.72, + "dtw_norm_scale": 0.107, + "SIM_BP": [ + 0.0, + 0.41, + 0.61, + 0.77, + 0.88, + 1.0 + ], + "SIM_VAL": [ + 0.0, + 0.2, + 0.5, + 0.8, + 0.9, + 1.0 + ], + "K_CD": 12.0, + "K_CL": 25.0, + "W_CD": 0.3, + "W_CL": 0.3, + "W_SIM": 0.4, + "FLOOR_CD": 0.1, + "FLOOR_CL": 0.1, + "FLOOR_SIM": 0.1, + "FLOOR_PENALTY": 0.05, + "ACTION_SCALE": 12.0, + "ACTION_BIAS": [ + 0.0, + 0.0, + 0.0 + ], + "U0": 0.01, + "RADIUS": 10.0, + "L0": 20.0 +} \ No newline at end of file diff --git a/src/drl_pinball/train/crossre_transfer.sh b/src/drl_pinball/train/crossre_transfer.sh new file mode 100755 index 0000000..37bef41 --- /dev/null +++ b/src/drl_pinball/train/crossre_transfer.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Sequential cross-Re transfer learning: Re60 -> Re200 -> Re400 +# Each: calibrate (~5 min) + train N episodes +# +# Usage (local test, 5 episodes): +# bash crossre_transfer.sh --re-list 60 --test-episodes 5 +# +# Usage (production on server, 200 episodes each): +# bash crossre_transfer.sh --re-list 60,200,400 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" +TRAIN_DIR="$SCRIPT_DIR" +CONFIG_DIR="$REPO_DIR/configs" +# TODO: BEFORE PUSHING TO SERVER, update BEST_MODEL to the correct path +BEST_MODEL="$TRAIN_DIR/output/re100_karman_seed539439/models/best_model.zip" +GPU=0 +SEED=41 +EPISODES=200 +TEST_EPISODES=0 +RE_LIST="" +LOG_BASE="/tmp/crossre_transfer" +CONDA_ENV="pycuda_3_10" + +usage() { + echo "Usage: $0 [--re-list 60,200,400] [--test-episodes N] [--best-model PATH]" + echo " --re-list Comma-separated Re numbers (default: 60,200,400)" + echo " --test-episodes Run only N episodes per Re for quick verification" + echo " --best-model Path to Re100 best model .zip" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --re-list) RE_LIST="$2"; shift 2 ;; + --test-episodes) TEST_EPISODES="$2"; shift 2 ;; + --best-model) BEST_MODEL="$2"; shift 2 ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +if [[ -z "$RE_LIST" ]]; then + RE_LIST="60,200,400" +fi + +IFS=',' read -ra RE_ARR <<< "$RE_LIST" + +echo "=== Cross-Re Transfer ===" +echo " Re list: ${RE_ARR[*]}" +echo " Best model: ${BEST_MODEL}" +echo " Episodes: ${EPISODES} (test-mode: ${TEST_EPISODES})" +echo " GPU: ${GPU}, Seed: ${SEED}" +echo "" + +if [[ ! -f "$BEST_MODEL" ]]; then + echo "ERROR: Best model not found at $BEST_MODEL" + exit 1 +fi + +mkdir -p "$LOG_BASE" + +if [[ "$TEST_EPISODES" -gt 0 ]]; then + EPISODES="$TEST_EPISODES" + echo " TEST MODE: only $TEST_EPISODES episodes per Re" +fi + +for re in "${RE_ARR[@]}"; do + case $re in + 60) SI=800; vis_label="re60" ;; + 200) SI=500; vis_label="re200" ;; + 400) SI=400; vis_label="re400" ;; + *) echo "ERROR: Unknown Re=$re (supported: 60, 200, 400)"; exit 1 ;; + esac + CONFIG="$CONFIG_DIR/config_lbm_karman_2000x600_${vis_label}.json" + CASE="transfer_${vis_label}" + LOG="$LOG_BASE/${vis_label}.log" + + echo "=== $(date): Starting $CASE (SI=$SI) ===" | tee -a "$LOG" + + if [[ ! -f "$CONFIG" ]]; then + echo " ERROR: Config not found: $CONFIG" | tee -a "$LOG" + exit 1 + fi + + # Step 1: Calibrate (skip if calibration.json already exists) + CAL_JSON="$TRAIN_DIR/calibrations/$vis_label/calibration.json" + if [[ -f "$CAL_JSON" ]]; then + echo " [SKIP] Calibration already exists: $CAL_JSON" | tee -a "$LOG" + echo " (Delete calibrations/$vis_label/ to force re-calibration)" + else + echo " [$(date '+%H:%M:%S')] Calibrating..." | tee -a "$LOG" + conda run --no-capture-output -n "$CONDA_ENV" python -u \ + "$TRAIN_DIR/calibrate.py" \ + --case "$vis_label" --device-id $GPU --si $SI \ + --config "$CONFIG" >> "$LOG" 2>&1 + echo " [$(date '+%H:%M:%S')] Calibration done." | tee -a "$LOG" + fi + + # Step 2: Train with transfer + echo " [$(date '+%H:%M:%S')] Training ${EPISODES} episodes..." | tee -a "$LOG" + conda run --no-capture-output -n "$CONDA_ENV" python -u \ + "$TRAIN_DIR/train_karman.py" \ + --case-name "$CASE" --device-id $GPU --seed $SEED \ + --config "$CONFIG" \ + --calibration "$CAL_JSON" \ + --si $SI --total-episodes $EPISODES \ + --transfer-model "$BEST_MODEL" >> "$LOG" 2>&1 + echo " [$(date '+%H:%M:%S')] Training done." | tee -a "$LOG" + + echo "=== $(date): $CASE complete ===" | tee -a "$LOG" + echo "" +done + +echo "=== ALL DONE ===" | tee -a "$LOG_BASE/summary.log" diff --git a/src/drl_pinball/train/env_illusion.py b/src/drl_pinball/train/env_illusion.py index 6461945..ce70dec 100644 --- a/src/drl_pinball/train/env_illusion.py +++ b/src/drl_pinball/train/env_illusion.py @@ -1,21 +1,33 @@ #!/usr/bin/env python3 -"""Hydrodynamic Illusion environment (V5 - calibration-driven, no_bias, 2000x600). +"""Hydrodynamic Illusion environment for 2000x600 config (uniform, free-slip). -Two-phase initialization: - Phase 1: target cylinder (diameter=1.0*L0) + 3 sensors -> warmup -> - record 150-step signal -> FFT harmonics -> close - Phase 2: Training sim (3 sensors + 3 pinball) -> warmup -> +Design: Two-phase initialization to AVOID runtime sync_bodies(). + Phase 1: Temporary Simulation(target cyl + sensors) -> record 150-step signal + + FFT harmonics -> close + Phase 2: Training Simulation(3 sensors + 3 pinball upfront) -> warmup -> zero-action FIFO -> snapshot -Observation (14-dim, physical norm, NO clip): +CUDA context: mirrors legacy pattern - push CFD context before GPU ops, pop after. + +Geometry: pinball@19/20.3L0, sensor@30L0, target cylinder@20L0 (1L diameter). + Equivalent pinball-to-sensor distance matched to illusion target. + +Observation (14-dim, physical norm, NO clip - VecNormalize handles that): [0:6] = raw_forces / FORCE_SCALE (front_fx,fy, top_fx,fy, bot_fx,fy) [6:12] = raw_sensors / SENS_SCALE (s0_ux,uy, s1_ux,uy, s2_ux,uy) - [12:14] = target_cd, target_cl (from harmonics reconstruction) + [12:14] = target_cd, target_cl (from FFT harmonics reconstruction) Action (3-dim): no_bias only - [-1,1] -> omega = -(action * 12 + [0,0,0]) * U0 / R + [-1,1] -> omega = -(action*12 + [0,0,0]) * U0 / RADIUS -Reward: Gaussian cd/cl (compared to harmonics-reconstructed target) + normalized DTW. +Reward (V3: Gaussian + EMA smoothing + normalized DTW): + r_cd = EMA(exp(-(cd - target_cd)^2 * K_CD), EMA_FAST) + r_cl = EMA(exp(-(cl - target_cl)^2 * K_CL), EMA_FAST) + r_sim = piecewise_map(sim, SIM_BP, SIM_VAL) + reward = W_CD*r_cd + W_CL*r_cl + W_SIM*r_sim - floor_penalty + +K_CD/K_CL are LOWER than Karman (12/25 vs 50/100) because 3-pinball total force +is compared to 1-target-cylinder force, giving wider normalized error range. """ from __future__ import annotations @@ -34,14 +46,6 @@ if str(_REPO) not in sys.path: from CelerisLab import Simulation -_CELERIS = Path("/home/frank14f/CelerisLab") -_CONFIG_H = _CELERIS / "src/CelerisLab/lbm/kernels/config/config_objects.h" -_PTX = _CELERIS / "src/CelerisLab/lbm/kernels/kernel.ptx" - -def _clean_cache(): - for p in [_CONFIG_H, _PTX]: - if p.exists(): p.unlink() - # --------------------------------------------------------------------------- L0 = 20.0; U0 = 0.01; RADIUS = L0 / 2.0 NX = 2000; NY = 600 @@ -151,7 +155,6 @@ def gen_target_states_at(t, harmonics): # --------------------------------------------------------------------------- def record_illusion_target(config_path, device_id, si, target_diam=1.0): - _clean_cache() warmup = int(4.0 * NX / U0) sim = Simulation(lbm_config_path=config_path, device_id=device_id) sim._assert_object_count_contract = lambda *a, **kw: None @@ -255,7 +258,6 @@ class IllusionCloakEnv(gym.Env): # ---- Phase 2: Training sim ---- print(" [illusion] Phase 2: Training sim...", flush=True) - _clean_cache() self.sim = Simulation(lbm_config_path=self._config_path, device_id=self.device_id) self.sim._assert_object_count_contract = lambda *a, **kw: None self.sensor_ids = [ diff --git a/src/drl_pinball/train/env_karman.py b/src/drl_pinball/train/env_karman.py index 3bf47f2..0872989 100644 --- a/src/drl_pinball/train/env_karman.py +++ b/src/drl_pinball/train/env_karman.py @@ -41,14 +41,6 @@ if str(_REPO) not in sys.path: from CelerisLab import Simulation -_CELERIS = Path("/home/frank14f/CelerisLab") -_CONFIG_H = _CELERIS / "src/CelerisLab/lbm/kernels/config/config_objects.h" -_PTX = _CELERIS / "src/CelerisLab/lbm/kernels/kernel.ptx" - -def _clean_cache(): - for p in [_CONFIG_H, _PTX]: - if p.exists(): p.unlink() - # --------------------------------------------------------------------------- # Geometry constants (fixed across all Karman cloak cases) # --------------------------------------------------------------------------- @@ -124,7 +116,6 @@ class ActionSmoother: # --------------------------------------------------------------------------- def record_target(config_path: str, device_id: int, si: int) -> np.ndarray: """Record target signal (dist_cyl + sensors, no pinball).""" - _clean_cache() warmup = int(4.0 * NX / U0) sim = Simulation(lbm_config_path=config_path, device_id=device_id) sim._assert_object_count_contract = lambda *a, **kw: None @@ -242,7 +233,6 @@ class KarmanCloakEnv(gym.Env): if self.target_states is None: self.target_states = record_target(self._config_path, self.device_id, self._si) - _clean_cache() self.sim = Simulation(lbm_config_path=self._config_path, device_id=self.device_id) self.sim._assert_object_count_contract = lambda *a, **kw: None diff --git a/src/drl_pinball/train/env_vortex.py b/src/drl_pinball/train/env_vortex.py index 13caa5c..1ea4d78 100644 --- a/src/drl_pinball/train/env_vortex.py +++ b/src/drl_pinball/train/env_vortex.py @@ -36,14 +36,6 @@ if str(_REPO) not in sys.path: from CelerisLab import Simulation from CelerisLab.lbm.initializers import add_vortex -_CELERIS = Path("/home/frank14f/CelerisLab") -_CONFIG_H = _CELERIS / "src/CelerisLab/lbm/kernels/config/config_objects.h" -_PTX = _CELERIS / "src/CelerisLab/lbm/kernels/kernel.ptx" - -def _clean_cache(): - for p in [_CONFIG_H, _PTX]: - if p.exists(): p.unlink() - # --------------------------------------------------------------------------- # Geometry constants # --------------------------------------------------------------------------- @@ -177,7 +169,6 @@ class VortexCloakEnv(gym.Env): # ---- Phase 1: Target (sensors + vortex only, no pinball) ---- print(" [vortex] Phase 1: Target recording...", flush=True) - _clean_cache() sim_t = Simulation(lbm_config_path=self._config_path, device_id=self.device_id) sim_t._assert_object_count_contract = lambda *a, **kw: None s0 = sim_t.add_body("sensor", center=(SENSOR_X, CENTER_Y + 40.0, 0.0), radius=5.0) @@ -209,7 +200,6 @@ class VortexCloakEnv(gym.Env): # ---- Phase 2: Training sim (sensors + pinball + vortex) ---- print(" [vortex] Phase 2: Training sim...", flush=True) - _clean_cache() self.sim = Simulation(lbm_config_path=self._config_path, device_id=self.device_id) self.sim._assert_object_count_contract = lambda *a, **kw: None self.sensor_ids = [ diff --git a/src/drl_pinball/train/launch_multi.sh b/src/drl_pinball/train/launch_multi.sh index c96ccad..5fbe0b3 100755 --- a/src/drl_pinball/train/launch_multi.sh +++ b/src/drl_pinball/train/launch_multi.sh @@ -19,13 +19,11 @@ # # Requirements: # - conda env pycuda_3_10 -# - CelerisLab at /home/frank14f/CelerisLab +# - CelerisLab submodule at DynamisLab/CelerisLab # - train_karman.py, env_karman.py, symmetry_wrapper.py in same directory set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CACHE_H="${HOME}/CelerisLab/src/CelerisLab/lbm/kernels/config/config_objects.h" -CACHE_PTX="${HOME}/CelerisLab/src/CelerisLab/lbm/kernels/kernel.ptx" # --- Defaults --- CASE_NAME="" @@ -97,10 +95,6 @@ echo " Delay: ${DELAY_SECONDS}s between launches" echo " Jobs: ${#SEED_ARR[@]}" echo "" -# Clean stale cache before starting -rm -f "$CACHE_H" "$CACHE_PTX" -echo " Cleaned kernel cache." - # Build transfer arg TRANSFER_ARG="" if [[ -n "$TRANSFER" ]]; then diff --git a/src/drl_pinball/train/analyze_final.py b/src/drl_pinball/train/old/analyze_final.py similarity index 100% rename from src/drl_pinball/train/analyze_final.py rename to src/drl_pinball/train/old/analyze_final.py diff --git a/src/drl_pinball/train/env_karman_2000x600.py b/src/drl_pinball/train/old/env_karman_2000x600.py similarity index 99% rename from src/drl_pinball/train/env_karman_2000x600.py rename to src/drl_pinball/train/old/env_karman_2000x600.py index 7391277..7266f71 100644 --- a/src/drl_pinball/train/env_karman_2000x600.py +++ b/src/drl_pinball/train/old/env_karman_2000x600.py @@ -38,7 +38,7 @@ if str(_REPO) not in sys.path: from CelerisLab import Simulation -_CELERIS = Path("/home/frank14f/CelerisLab") +_CELERIS = _REPO / "CelerisLab" _CONFIG_H = _CELERIS / "src/CelerisLab/lbm/kernels/config/config_objects.h" _PTX = _CELERIS / "src/CelerisLab/lbm/kernels/kernel.ptx" diff --git a/src/drl_pinball/train/phase0_baseline_measure.py b/src/drl_pinball/train/old/phase0_baseline_measure.py similarity index 99% rename from src/drl_pinball/train/phase0_baseline_measure.py rename to src/drl_pinball/train/old/phase0_baseline_measure.py index 4f23077..49b6516 100644 --- a/src/drl_pinball/train/phase0_baseline_measure.py +++ b/src/drl_pinball/train/old/phase0_baseline_measure.py @@ -37,7 +37,7 @@ from CelerisLab import Simulation # --------------------------------------------------------------------------- # Hard-coded CelerisLab paths for cache cleaning # --------------------------------------------------------------------------- -_CELERIS = Path("/home/frank14f/CelerisLab") +_CELERIS = _REPO / "CelerisLab" _CONFIG_H = _CELERIS / "src/CelerisLab/lbm/kernels/config/config_objects.h" _PTX = _CELERIS / "src/CelerisLab/lbm/kernels/kernel.ptx" diff --git a/src/drl_pinball/train/prompt.md b/src/drl_pinball/train/old/prompt.md similarity index 100% rename from src/drl_pinball/train/prompt.md rename to src/drl_pinball/train/old/prompt.md diff --git a/src/drl_pinball/train/train_karman_2000x600.py b/src/drl_pinball/train/old/train_karman_2000x600.py similarity index 100% rename from src/drl_pinball/train/train_karman_2000x600.py rename to src/drl_pinball/train/old/train_karman_2000x600.py diff --git a/src/drl_pinball/train/train_karman.py b/src/drl_pinball/train/train_karman.py index 5b3d79f..104de6f 100644 --- a/src/drl_pinball/train/train_karman.py +++ b/src/drl_pinball/train/train_karman.py @@ -200,9 +200,9 @@ def main() -> int: elif ep % 5 == 0: log(f" Ep {ep:3d}: reward={avg_r:.4f} (best={best_reward:.4f}, {dt:.0f}s/ep)") - if ep % 10 == 0: - model.save(str(out_dir / "models" / f"chkpt_ep{ep}.zip")) - vec_env.save(norm_path) + # Save model + eval summary after EVERY episode + model.save(str(out_dir / "models" / f"ep{ep:04d}_model.zip")) + vec_env.save(str(out_dir / "models" / f"ep{ep:04d}_vecnormalize.pkl")) model.save(str(out_dir / "models" / "final_model.zip")) vec_env.save(norm_path)