From b144d629201ce57ba79d6e141e66b8fe634b312c Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Fri, 7 Aug 2026 16:41:00 +0800 Subject: [PATCH] feat(ccd): freeze dynamic-increment analysis pipeline Replace the legacy CCD workspace with acquisition, direct-dq, original and lagged CCD contracts so the DRL-versus-constant-mean mechanism is reproducible and fail-closed. Co-authored-by: Cursor --- LegacyCelerisLab/driver.py | 157 +++ src/CCD_analysis/DUAL_CLOCK_SAMPLING.md | 173 --- src/CCD_analysis/EXECUTION_CHECKPOINT.json | 1 + src/CCD_analysis/FINAL_RESULTS.md | 38 + src/CCD_analysis/Lyu23.md | 1000 ----------------- src/CCD_analysis/PIPELINE.md | 164 --- src/CCD_analysis/README.md | 137 ++- src/CCD_analysis/__init__.py | 5 + src/CCD_analysis/acquisition/__init__.py | 7 + src/CCD_analysis/acquisition/__main__.py | 2 + src/CCD_analysis/acquisition/artifacts.py | 99 ++ src/CCD_analysis/acquisition/cli.py | 58 + src/CCD_analysis/acquisition/compatibility.py | 72 ++ src/CCD_analysis/acquisition/contracts.py | 195 ++++ src/CCD_analysis/acquisition/dual_clock.py | 95 ++ src/CCD_analysis/acquisition/runtime.py | 514 +++++++++ src/CCD_analysis/acquisition/solver_state.py | 91 ++ src/CCD_analysis/acquisition/validation.py | 251 +++++ src/CCD_analysis/ccd/run_ccd.py | 346 ------ src/CCD_analysis/ccd/validate.py | 270 ----- src/CCD_analysis/ccd_knowledge.md | 509 --------- src/CCD_analysis/configs.py | 291 ----- src/CCD_analysis/configs/config_cuda.json | 9 - .../configs/config_flowfield.json | 13 - .../correction_analysis/__init__.py | 0 .../compare_dqctl_scenes.py | 188 ---- .../compute_correction_fields.py | 434 ------- .../decompose_corrections.py | 223 ---- .../diagnose_corrections.py | 377 ------- .../process_legacy_steady.py | 153 --- .../correction_analysis/run_15L_correction.py | 366 ------ .../correction_analysis/run_signature_line.py | 563 ---------- .../correction_analysis/run_steady_metrics.py | 344 ------ .../correction_analysis/run_zone_ccd.py | 365 ------ .../visualize_action_ccd.py | 126 --- src/CCD_analysis/direct_dq/__init__.py | 13 + src/CCD_analysis/direct_dq/__main__.py | 3 + src/CCD_analysis/direct_dq/analysis.py | 217 ++++ src/CCD_analysis/direct_dq/cli.py | 35 + src/CCD_analysis/direct_dq/convergence.py | 35 + src/CCD_analysis/direct_dq/io.py | 164 +++ src/CCD_analysis/direct_dq/schema.py | 136 +++ .../acquisition_contract_reviews.json | 291 +++++ .../evidence/archive_boundary_check.json | 45 + .../evidence/direct_dq_core_reviews.json | 1 + .../evidence/final_results_review.json | 1 + ...illusion-compatibility-certificate-v1.json | 1 + .../evidence/karman_dynamic_final_review.json | 1 + .../original_ccd_derivation_reviews.json | 98 ++ .../original_ccd_implementation_reviews.json | 92 ++ .../evidence/real_ccd_contract_review.json | 72 ++ .../real_ccd_karman_figures_review.json | 1 + .../real_ccd_karman_result_review.json | 1 + .../real_ccd_preflight_tests_review.json | 1 + .../evidence/real_ccd_streaming_review.json | 1 + src/CCD_analysis/karman_dynamic/__init__.py | 10 + src/CCD_analysis/karman_dynamic/__main__.py | 2 + src/CCD_analysis/karman_dynamic/artifacts.py | 33 + src/CCD_analysis/karman_dynamic/cli.py | 38 + src/CCD_analysis/karman_dynamic/contracts.py | 31 + .../karman_dynamic/dynamic_increment.py | 89 ++ .../karman_dynamic/orchestration.py | 31 + src/CCD_analysis/karman_dynamic/phase.py | 72 ++ .../karman_dynamic/phase_domain_ccd.py | 119 ++ .../karman_dynamic/publication.py | 72 ++ src/CCD_analysis/karman_dynamic/runtime.py | 46 + src/CCD_analysis/karman_dynamic/safety.py | 34 + .../karman_dynamic/temporal_ccd.py | 161 +++ .../original_ccd/ORIGINAL_CCD_MATH.md | 154 +++ src/CCD_analysis/original_ccd/README.md | 61 + .../original_ccd/REAL_CASE_CCD_CONTRACT.md | 108 ++ src/CCD_analysis/original_ccd/__init__.py | 7 + src/CCD_analysis/original_ccd/_reference.py | 169 +++ src/CCD_analysis/original_ccd/api.py | 362 ++++++ src/CCD_analysis/original_ccd/delays.py | 94 ++ src/CCD_analysis/real_ccd/__init__.py | 5 + src/CCD_analysis/real_ccd/__main__.py | 3 + src/CCD_analysis/real_ccd/cli.py | 46 + src/CCD_analysis/real_ccd/core.py | 147 +++ src/CCD_analysis/real_ccd/figures.py | 83 ++ src/CCD_analysis/real_ccd/io.py | 71 ++ src/CCD_analysis/real_ccd/preflight.py | 36 + src/CCD_analysis/real_ccd/schema.py | 46 + .../results/ccd/correction_ccd_results.json | 731 ------------ .../results/ccd/zone_metrics.json | 282 ----- .../figures/01_panorama_all_scenes.png | 1 - .../results/figures/02_cloak_comparison.png | 1 - .../figures/03_illusion_comparison.png | 1 - .../figures/04_steady_cloak_cancel.png | 1 - .../figures/05_illusion_075L_ctl_vs_tar.png | 1 - .../figures/06_illusion_10L_ctl_vs_tar.png | 1 - .../figures/07_illusion_15L_ctl_vs_tar.png | 1 - .../results/figures/08_karman_ctl_vs_tar.png | 1 - .../figures/09_vortex_lamb_ctl_vs_tar.png | 1 - .../figures/10_vortex_taylor_ctl_vs_tar.png | 1 - .../figures/11_vortex_lamb_diagnosis.png | 1 - .../figures/12_vortex_taylor_diagnosis.png | 1 - .../figures/13_vortex_lamb_vorticity.png | 1 - .../figures/14_vortex_taylor_vorticity.png | 1 - .../15_vortex_lamb_target_vorticity.png | 1 - .../16_vortex_taylor_target_vorticity.png | 1 - .../scripts/collect_empty_channel.py | 103 -- src/CCD_analysis/scripts/collect_illusion.py | 302 ----- src/CCD_analysis/scripts/collect_karman.py | 238 ---- .../scripts/collect_karman_q_blk.py | 129 --- .../scripts/collect_karman_q_in.py | 104 -- src/CCD_analysis/scripts/collect_pinball.py | 104 -- .../scripts/collect_steady_cloak.py | 126 --- .../scripts/collect_target_cylinder.py | 122 -- src/CCD_analysis/scripts/collect_vortex.py | 509 --------- src/CCD_analysis/scripts/detect_period.py | 179 --- src/CCD_analysis/scripts/replay_fields.py | 280 ----- src/CCD_analysis/scripts/resample.py | 172 --- .../scripts/sanity_check_force.py | 180 --- src/CCD_analysis/scripts/verify_cases.py | 227 ---- src/CCD_analysis/scripts/visualize_ccd.py | 605 ---------- src/CCD_analysis/tests/__init__.py | 1 + src/CCD_analysis/tests/test_acquisition.py | 579 ++++++++++ src/CCD_analysis/tests/test_direct_dq.py | 507 +++++++++ src/CCD_analysis/tests/test_dual_clock.py | 65 -- src/CCD_analysis/tests/test_karman_dynamic.py | 97 ++ .../tests/test_karman_dynamic_increment.py | 27 + .../tests/test_karman_dynamic_phase.py | 36 + .../tests/test_karman_dynamic_phase_domain.py | 40 + .../tests/test_karman_dynamic_publication.py | 10 + .../tests/test_karman_dynamic_temporal.py | 39 + src/CCD_analysis/tests/test_original_ccd.py | 347 ++++++ .../tests/test_original_ccd_derivation.py | 303 +++++ src/CCD_analysis/tests/test_real_ccd.py | 258 +++++ src/CCD_analysis/utils/__init__.py | 14 - src/CCD_analysis/utils/cfd_interface.py | 341 ------ src/CCD_analysis/utils/dual_clock.py | 101 -- src/CCD_analysis/utils/field_translate.py | 113 -- src/CCD_analysis/utils/load_vortex_fields.py | 114 -- src/CCD_analysis/utils/resampling.py | 433 ------- src/drl_pinball/acquisition.py | 575 ++++++++++ .../legacy_test/core/dtw_metrics.py | 278 ++++- tests/test_drl_pinball_acquisition.py | 345 ++++++ 138 files changed, 8443 insertions(+), 11564 deletions(-) delete mode 100644 src/CCD_analysis/DUAL_CLOCK_SAMPLING.md create mode 100644 src/CCD_analysis/EXECUTION_CHECKPOINT.json create mode 100644 src/CCD_analysis/FINAL_RESULTS.md delete mode 100644 src/CCD_analysis/Lyu23.md delete mode 100644 src/CCD_analysis/PIPELINE.md create mode 100644 src/CCD_analysis/__init__.py create mode 100644 src/CCD_analysis/acquisition/__init__.py create mode 100644 src/CCD_analysis/acquisition/__main__.py create mode 100644 src/CCD_analysis/acquisition/artifacts.py create mode 100644 src/CCD_analysis/acquisition/cli.py create mode 100644 src/CCD_analysis/acquisition/compatibility.py create mode 100644 src/CCD_analysis/acquisition/contracts.py create mode 100644 src/CCD_analysis/acquisition/dual_clock.py create mode 100644 src/CCD_analysis/acquisition/runtime.py create mode 100644 src/CCD_analysis/acquisition/solver_state.py create mode 100644 src/CCD_analysis/acquisition/validation.py delete mode 100644 src/CCD_analysis/ccd/run_ccd.py delete mode 100644 src/CCD_analysis/ccd/validate.py delete mode 100644 src/CCD_analysis/ccd_knowledge.md delete mode 100644 src/CCD_analysis/configs.py delete mode 100644 src/CCD_analysis/configs/config_cuda.json delete mode 100644 src/CCD_analysis/configs/config_flowfield.json delete mode 100644 src/CCD_analysis/correction_analysis/__init__.py delete mode 100644 src/CCD_analysis/correction_analysis/compare_dqctl_scenes.py delete mode 100644 src/CCD_analysis/correction_analysis/compute_correction_fields.py delete mode 100644 src/CCD_analysis/correction_analysis/decompose_corrections.py delete mode 100644 src/CCD_analysis/correction_analysis/diagnose_corrections.py delete mode 100644 src/CCD_analysis/correction_analysis/process_legacy_steady.py delete mode 100644 src/CCD_analysis/correction_analysis/run_15L_correction.py delete mode 100644 src/CCD_analysis/correction_analysis/run_signature_line.py delete mode 100644 src/CCD_analysis/correction_analysis/run_steady_metrics.py delete mode 100644 src/CCD_analysis/correction_analysis/run_zone_ccd.py delete mode 100644 src/CCD_analysis/correction_analysis/visualize_action_ccd.py create mode 100644 src/CCD_analysis/direct_dq/__init__.py create mode 100644 src/CCD_analysis/direct_dq/__main__.py create mode 100644 src/CCD_analysis/direct_dq/analysis.py create mode 100644 src/CCD_analysis/direct_dq/cli.py create mode 100644 src/CCD_analysis/direct_dq/convergence.py create mode 100644 src/CCD_analysis/direct_dq/io.py create mode 100644 src/CCD_analysis/direct_dq/schema.py create mode 100644 src/CCD_analysis/evidence/acquisition_contract_reviews.json create mode 100644 src/CCD_analysis/evidence/archive_boundary_check.json create mode 100644 src/CCD_analysis/evidence/direct_dq_core_reviews.json create mode 100644 src/CCD_analysis/evidence/final_results_review.json create mode 100644 src/CCD_analysis/evidence/illusion-compatibility-certificate-v1.json create mode 100644 src/CCD_analysis/evidence/karman_dynamic_final_review.json create mode 100644 src/CCD_analysis/evidence/original_ccd_derivation_reviews.json create mode 100644 src/CCD_analysis/evidence/original_ccd_implementation_reviews.json create mode 100644 src/CCD_analysis/evidence/real_ccd_contract_review.json create mode 100644 src/CCD_analysis/evidence/real_ccd_karman_figures_review.json create mode 100644 src/CCD_analysis/evidence/real_ccd_karman_result_review.json create mode 100644 src/CCD_analysis/evidence/real_ccd_preflight_tests_review.json create mode 100644 src/CCD_analysis/evidence/real_ccd_streaming_review.json create mode 100644 src/CCD_analysis/karman_dynamic/__init__.py create mode 100644 src/CCD_analysis/karman_dynamic/__main__.py create mode 100644 src/CCD_analysis/karman_dynamic/artifacts.py create mode 100644 src/CCD_analysis/karman_dynamic/cli.py create mode 100644 src/CCD_analysis/karman_dynamic/contracts.py create mode 100644 src/CCD_analysis/karman_dynamic/dynamic_increment.py create mode 100644 src/CCD_analysis/karman_dynamic/orchestration.py create mode 100644 src/CCD_analysis/karman_dynamic/phase.py create mode 100644 src/CCD_analysis/karman_dynamic/phase_domain_ccd.py create mode 100644 src/CCD_analysis/karman_dynamic/publication.py create mode 100644 src/CCD_analysis/karman_dynamic/runtime.py create mode 100644 src/CCD_analysis/karman_dynamic/safety.py create mode 100644 src/CCD_analysis/karman_dynamic/temporal_ccd.py create mode 100644 src/CCD_analysis/original_ccd/ORIGINAL_CCD_MATH.md create mode 100644 src/CCD_analysis/original_ccd/README.md create mode 100644 src/CCD_analysis/original_ccd/REAL_CASE_CCD_CONTRACT.md create mode 100644 src/CCD_analysis/original_ccd/__init__.py create mode 100644 src/CCD_analysis/original_ccd/_reference.py create mode 100644 src/CCD_analysis/original_ccd/api.py create mode 100644 src/CCD_analysis/original_ccd/delays.py create mode 100644 src/CCD_analysis/real_ccd/__init__.py create mode 100644 src/CCD_analysis/real_ccd/__main__.py create mode 100644 src/CCD_analysis/real_ccd/cli.py create mode 100644 src/CCD_analysis/real_ccd/core.py create mode 100644 src/CCD_analysis/real_ccd/figures.py create mode 100644 src/CCD_analysis/real_ccd/io.py create mode 100644 src/CCD_analysis/real_ccd/preflight.py create mode 100644 src/CCD_analysis/real_ccd/schema.py delete mode 100644 src/CCD_analysis/results/ccd/correction_ccd_results.json delete mode 100644 src/CCD_analysis/results/ccd/zone_metrics.json delete mode 120000 src/CCD_analysis/results/figures/01_panorama_all_scenes.png delete mode 120000 src/CCD_analysis/results/figures/02_cloak_comparison.png delete mode 120000 src/CCD_analysis/results/figures/03_illusion_comparison.png delete mode 120000 src/CCD_analysis/results/figures/04_steady_cloak_cancel.png delete mode 120000 src/CCD_analysis/results/figures/05_illusion_075L_ctl_vs_tar.png delete mode 120000 src/CCD_analysis/results/figures/06_illusion_10L_ctl_vs_tar.png delete mode 120000 src/CCD_analysis/results/figures/07_illusion_15L_ctl_vs_tar.png delete mode 120000 src/CCD_analysis/results/figures/08_karman_ctl_vs_tar.png delete mode 120000 src/CCD_analysis/results/figures/09_vortex_lamb_ctl_vs_tar.png delete mode 120000 src/CCD_analysis/results/figures/10_vortex_taylor_ctl_vs_tar.png delete mode 120000 src/CCD_analysis/results/figures/11_vortex_lamb_diagnosis.png delete mode 120000 src/CCD_analysis/results/figures/12_vortex_taylor_diagnosis.png delete mode 120000 src/CCD_analysis/results/figures/13_vortex_lamb_vorticity.png delete mode 120000 src/CCD_analysis/results/figures/14_vortex_taylor_vorticity.png delete mode 120000 src/CCD_analysis/results/figures/15_vortex_lamb_target_vorticity.png delete mode 120000 src/CCD_analysis/results/figures/16_vortex_taylor_target_vorticity.png delete mode 100644 src/CCD_analysis/scripts/collect_empty_channel.py delete mode 100644 src/CCD_analysis/scripts/collect_illusion.py delete mode 100644 src/CCD_analysis/scripts/collect_karman.py delete mode 100644 src/CCD_analysis/scripts/collect_karman_q_blk.py delete mode 100644 src/CCD_analysis/scripts/collect_karman_q_in.py delete mode 100644 src/CCD_analysis/scripts/collect_pinball.py delete mode 100644 src/CCD_analysis/scripts/collect_steady_cloak.py delete mode 100644 src/CCD_analysis/scripts/collect_target_cylinder.py delete mode 100644 src/CCD_analysis/scripts/collect_vortex.py delete mode 100644 src/CCD_analysis/scripts/detect_period.py delete mode 100644 src/CCD_analysis/scripts/replay_fields.py delete mode 100644 src/CCD_analysis/scripts/resample.py delete mode 100644 src/CCD_analysis/scripts/sanity_check_force.py delete mode 100644 src/CCD_analysis/scripts/verify_cases.py delete mode 100644 src/CCD_analysis/scripts/visualize_ccd.py create mode 100644 src/CCD_analysis/tests/__init__.py create mode 100644 src/CCD_analysis/tests/test_acquisition.py create mode 100644 src/CCD_analysis/tests/test_direct_dq.py delete mode 100644 src/CCD_analysis/tests/test_dual_clock.py create mode 100644 src/CCD_analysis/tests/test_karman_dynamic.py create mode 100644 src/CCD_analysis/tests/test_karman_dynamic_increment.py create mode 100644 src/CCD_analysis/tests/test_karman_dynamic_phase.py create mode 100644 src/CCD_analysis/tests/test_karman_dynamic_phase_domain.py create mode 100644 src/CCD_analysis/tests/test_karman_dynamic_publication.py create mode 100644 src/CCD_analysis/tests/test_karman_dynamic_temporal.py create mode 100644 src/CCD_analysis/tests/test_original_ccd.py create mode 100644 src/CCD_analysis/tests/test_original_ccd_derivation.py create mode 100644 src/CCD_analysis/tests/test_real_ccd.py delete mode 100644 src/CCD_analysis/utils/__init__.py delete mode 100644 src/CCD_analysis/utils/cfd_interface.py delete mode 100644 src/CCD_analysis/utils/dual_clock.py delete mode 100644 src/CCD_analysis/utils/field_translate.py delete mode 100644 src/CCD_analysis/utils/load_vortex_fields.py delete mode 100644 src/CCD_analysis/utils/resampling.py create mode 100644 src/drl_pinball/acquisition.py create mode 100644 tests/test_drl_pinball_acquisition.py diff --git a/LegacyCelerisLab/driver.py b/LegacyCelerisLab/driver.py index 4c1b921..753341f 100644 --- a/LegacyCelerisLab/driver.py +++ b/LegacyCelerisLab/driver.py @@ -9,6 +9,7 @@ from typing import List, Tuple, Union, Optional from . import utils from . import preprocess as preproc from . import compiler +from src.CCD_analysis.acquisition.solver_state import copy_ping_pong_ddf, d2q9_q_over_u0_xy FLUID = 0b00000001 SOLID = 0b00000010 @@ -112,6 +113,10 @@ class FlowField: self.action = np.zeros(0, dtype=self.DATA_TYPE) self.obs = np.zeros(0, dtype=self.DATA_TYPE) self._control_interval = None + self._last_completed_observation = None + self._last_effective_action = None + self._completed_lattice_steps = 0 + self._completed_control_intervals = 0 initflow( self.flag_gpu, @@ -126,6 +131,23 @@ class FlowField: cuda.memcpy_dtoh(self.flag, self.flag_gpu) cuda.memcpy_dtoh(self.ddf, self.ddf_gpu) + def completed_flags_xy(self) -> np.ndarray: + """Return a read-only copy of configured flags in canonical ``(NX, NY)`` order. + + The solver stores flags flat with ``k = x + y * NX``. No bit is + interpreted or rewritten here, so FLUID/SOLID and auxiliary bits are + preserved exactly. + """ + flat = np.asarray(self.flag) + expected = int(self.FIELD_SHAPE[0]) * int(self.FIELD_SHAPE[1]) * int(self.FIELD_SHAPE[2]) + if flat.dtype != np.dtype("uint8") or flat.ndim != 1 or flat.size != expected: + raise RuntimeError("configured solver flag storage is not canonical uint8 flat data") + if int(self.FIELD_SHAPE[2]) != 1: + raise RuntimeError("canonical CCD flag export supports completed D2 geometry only") + result = np.ascontiguousarray(flat.reshape((self.FIELD_SHAPE[1], self.FIELD_SHAPE[0])).T) + result.setflags(write=False) + return result + def add_cylinder(self, center: Tuple[float, float, float], radius: float, id_obj: Optional[int] = None): x_c, y_c, z_c = center @@ -224,6 +246,7 @@ class FlowField: self.objects[id_object] = { "type": "sensor", "center": center, + "radius": radius, } self.action = np.zeros(len(self.objects), dtype=self.DATA_TYPE) @@ -409,6 +432,133 @@ class FlowField: cuda.memset_d32_async(self.obs_gpu, 0, self.obs.size, stream) stream.synchronize() state["completed_steps"] += num_steps + self._completed_lattice_steps += num_steps + + def current_step_observation(self): + """Return raw telemetry for the latest completed lattice step. + + During an active split interval this is the latest synchronized step. At + a completed control boundary it is the persisted final raw step, not the + interval-averaged public ``obs``. + """ + state = self._control_interval + if state is not None and state["completed_steps"] >= 1: + return state["obs_steps"][state["completed_steps"] - 1].copy() + if self._last_completed_observation is None: + raise RuntimeError("no completed lattice step is available") + return self._last_completed_observation.copy() + + def current_effective_action(self): + """Return the latest EMA action, including at a completed boundary.""" + state = self._control_interval + if state is not None and state["completed_steps"] >= 1: + return np.asarray(state["action"]).copy() + if self._last_effective_action is None: + raise RuntimeError("no completed lattice step is available") + return self._last_effective_action.copy() + + def _require_completed_split_step(self): + state = self._control_interval + if state is None or state["completed_steps"] < 1: + raise RuntimeError("no completed step is available in the active control interval") + + def current_step_velocity_field(self): + """Return completed Legacy nondimensional velocity ``q/U0`` as ``(NX, NY)``.""" + if self._control_interval is not None: + self._require_completed_split_step() + elif self._last_completed_observation is None: + raise RuntimeError("no completed Legacy step is available") + # run_control_segment synchronizes before returning. After its pointer swap, + # ddf_gpu is the completed state and temp_gpu is the previous/work buffer. + cuda.memcpy_dtoh(self.ddf, self.ddf_gpu) + flags = self.completed_flags_xy() + return d2q9_q_over_u0_xy( + self.ddf, int(self.FIELD_SHAPE[0]), int(self.FIELD_SHAPE[1]), flags, + float(self.field_config.velocity), + ) + + def current_step_velocity_probe(self, lattice_index: Tuple[int, int]): + """Read one synchronized Legacy nondimensional ``(ux/U0, uy/U0)`` pair.""" + self._require_completed_split_step() + if (not isinstance(lattice_index, tuple) or len(lattice_index) != 2 + or any(type(value) is not int for value in lattice_index)): + raise ValueError("lattice_index must be an (x, y) integer tuple") + x, y = lattice_index + if not (0 <= x < self.FIELD_SHAPE[0] and 0 <= y < self.FIELD_SHAPE[1]): + raise ValueError("velocity probe is outside the lattice") + ux, uy = self.current_step_velocity_field() + return np.asarray([ux[x, y], uy[x, y]], dtype=self.DATA_TYPE) + + def active_step_clock_state(self): + """Return solver lineage during a split interval after a completed step. + + The control clock is the number of fully completed control intervals; it + therefore identifies the active interval's zero-based absolute index. + """ + state = self._control_interval + if state is None or state["completed_steps"] < 1: + raise RuntimeError("active-step clocks require a split interval with a completed step") + return { + "solver_absolute_lattice_clock": int(self._completed_lattice_steps), + "solver_absolute_control_clock": int(self._completed_control_intervals), + } + + def solver_clock_state(self): + """Return public absolute solver lineage clocks at the current boundary.""" + if self._control_interval is not None: + raise RuntimeError("solver clocks are boundary-safe only") + return { + "solver_absolute_lattice_clock": int(self._completed_lattice_steps), + "solver_absolute_control_clock": int(self._completed_control_intervals), + } + + def full_state_checkpoint(self): + """Capture exact restart state only at a completed control boundary.""" + if self._control_interval is not None: + raise RuntimeError("full checkpoint requires a completed control boundary") + ddf = self.current_step_ddf_checkpoint() + return { + **ddf, + "action": self.action.copy(), + "last_effective_action": None if self._last_effective_action is None else self._last_effective_action.copy(), + "raw_observation": None if self._last_completed_observation is None else self._last_completed_observation.copy(), + "boundary_observation": self.obs.copy(), + "solver_absolute_lattice_clock": int(self._completed_lattice_steps), + "solver_absolute_control_clock": int(self._completed_control_intervals), + } + + def restore_full_state(self, checkpoint): + """Restore both ping-pong DDFs and solver-side boundary lifecycle state.""" + if self._control_interval is not None: + raise RuntimeError("cannot restore during an active control interval") + current = np.asarray(checkpoint["current_ddf"]); temp = np.asarray(checkpoint["temp_ddf"]) + action = np.asarray(checkpoint["action"]); boundary = np.asarray(checkpoint["boundary_observation"]) + raw = checkpoint["raw_observation"]; effective = checkpoint["last_effective_action"] + if current.dtype != self.DATA_TYPE or temp.dtype != self.DATA_TYPE or current.shape != self.ddf.shape or temp.shape != self.ddf.shape: + raise ValueError("checkpoint DDF storage mismatch") + if action.dtype != self.DATA_TYPE or action.shape != self.action.shape or boundary.dtype != self.DATA_TYPE or boundary.shape != self.obs.shape: + raise ValueError("checkpoint action/observation mismatch") + if raw is not None and (np.asarray(raw).dtype != self.DATA_TYPE or np.asarray(raw).shape != self.obs.shape): raise ValueError("checkpoint raw observation mismatch") + if effective is not None and (np.asarray(effective).dtype != self.DATA_TYPE or np.asarray(effective).shape != self.action.shape): raise ValueError("checkpoint effective action mismatch") + cuda.memcpy_htod(self.ddf_gpu, current); cuda.memcpy_htod(self.temp_gpu, temp) + self.ddf = current.copy(); self.action = action.copy(); self.obs = boundary.copy() + self._last_completed_observation = None if raw is None else np.asarray(raw).copy() + self._last_effective_action = None if effective is None else np.asarray(effective).copy() + self._completed_lattice_steps = int(checkpoint["solver_absolute_lattice_clock"]) + self._completed_control_intervals = int(checkpoint["solver_absolute_control_clock"]) + cuda.memcpy_htod(self.action_gpu, self.action) + + def current_step_ddf_checkpoint(self): + """Return copies/hashes of current(completed) and temp(previous/work) buffers. + + Synchronous device-to-host copies make this safe both at a completed split + step and at a completed control boundary; no solver state is modified. + """ + return copy_ping_pong_ddf( + lambda host: cuda.memcpy_dtoh(host, self.ddf_gpu), + lambda host: cuda.memcpy_dtoh(host, self.temp_gpu), + int(self.FIELD_SIZE * self.LATTICE), + ) def end_control_interval(self): """Publish obs once, only at the original policy-control boundary.""" @@ -423,6 +573,13 @@ class FlowField: self.obs = (self.obs / state["total_steps"]).astype(self.DATA_TYPE) cuda.memcpy_dtoh(self.error_flag, self.error_flag_gpu) self.last_error_flag = int(self.error_flag[0]) + # Persist the final lattice-step state before clearing the split lifecycle. + # The next begin_control_interval therefore starts its EMA from this exact + # action, preserving the historical uninterrupted-run semantics. + self.action = np.asarray(state["action"], dtype=self.DATA_TYPE).copy() + self._last_effective_action = self.action.copy() + self._last_completed_observation = state["obs_steps"][-1].copy() + self._completed_control_intervals += 1 self._control_interval = None return self.obs diff --git a/src/CCD_analysis/DUAL_CLOCK_SAMPLING.md b/src/CCD_analysis/DUAL_CLOCK_SAMPLING.md deleted file mode 100644 index d94d87b..0000000 --- a/src/CCD_analysis/DUAL_CLOCK_SAMPLING.md +++ /dev/null @@ -1,173 +0,0 @@ -# Legacy 双时钟流场采样 - -## 目的 - -Legacy DRL 模型只应在固定控制周期边界接收 observation 并预测下一动作。例如： - -- Illusion：每 600 lattice steps 决策一次； -- Kármán：每 800 lattice steps 决策一次。 - -流场分析通常需要另一套保存时钟，例如每 257 或 317 lattice steps 保存一次 DDF。这两个时钟不必整除，也不应相互改变。 - -错误做法是在每个流场保存点调用普通 `ff.run(segment_steps, action)`。Legacy `run()` 每次都会： - -1. 从 `ff.action` 重新初始化 action smoothing； -2. 清空 observation 累计； -3. 将该段 observation 除以段长度并发布。 - -这样会把流场保存点错误地变成控制边界，改变闭环控制。 - -## 正确语义 - -一个 DRL 控制周期只执行一次以下生命周期： - -```python -ff.begin_control_interval(sample_interval, action_target) -ff.run_control_segment(first_length) -ff.get_ddf() # 只读流场 -ff.run_control_segment(second_length) -ff.get_ddf() -# ...累计恰好 sample_interval steps... -obs = ff.end_control_interval() -``` - -其中： - -- `begin_control_interval()` 只在真实控制边界调用一次； -- `run_control_segment()` 可以按任意正整数长度调用多次； -- segment 之间延续同一个 interval-local action smoothing； -- segment 之间延续同一个 raw observation 累计； -- `get_ddf()` 只将当前 GPU DDF 复制到 host，不更新 observation、FIFO、reward、action 或 policy； -- 只有累计推进完整控制周期后才能调用 `end_control_interval()`； -- `end_control_interval()` 发布完整控制周期的平均 observation； -- FIFO、observation normalization 和下一次 `model.predict()` 仍然只在控制边界执行一次。 - -普通 `ff.run(total_steps, action_target)` 保持兼容，它等价于 begin、一次完整 segment、end。 - -## 推荐复用接口 - -通用实现位于： - -- `LegacyCelerisLab/driver.py`：control-interval session； -- `src/CCD_analysis/utils/dual_clock.py`：绝对 field-step 调度和 `ux/uy` 保存； -- `src/CCD_analysis/scripts/collect_illusion.py`、`collect_karman.py`：实际接入示例。 - -OID 若使用同一个 Legacy `FlowField`，应直接复用 `CCD_analysis.utils.dual_clock.DualClockCollector`，不要复制一套控制循环。 - -### 固定流场间隔 - -```python -from CCD_analysis.utils.dual_clock import ( - DualClockCollector, - field_steps_from_interval, -) - -control_interval = 600 -control_count = 200 -field_interval = 257 - -collector = DualClockCollector( - ff, - control_interval=control_interval, - control_count=control_count, - u0=u0, - field_steps=field_steps_from_interval( - control_interval * control_count, - field_interval, - ), -) - -obs = initial_obs -for control_index in range(control_count): - action, _ = model.predict(obs, deterministic=True) - action_target = decode_action(action) - - # 此调用内部可能在一个控制周期内读取多次或零次 DDF， - # 但始终只结束一个真实控制周期。 - raw_obs = collector.run_interval(control_index, action_target) - - fifo.append(select_observation(raw_obs)) - obs = normalize(select_observation(raw_obs)) - -collector.save("fields.npz") -``` - -### 任意、不均匀保存位置 - -`field_steps` 使用从 rollout 开始计数的绝对 lattice step，并且必须严格递增： - -```python -collector = DualClockCollector( - ff, - control_interval=800, - control_count=200, - u0=u0, - field_steps=[137, 800, 913, 1721, 2400], -) -``` - -位于控制边界的 field step 只保存一次，同时仍只产生一次完整控制 observation。 - -## 输出合同 - -`collector.save("fields.npz")` 保存： - -- `ux`, `uy`：读取时刻的无量纲速度场； -- `lattice_steps`：每个场对应的绝对 lattice step； -- `control_indices`：所属的零基控制周期； -- `control_offsets`：在该控制周期内的位置，范围为 `1..control_interval`； -- `control_interval`：模型固定控制周期。 - -后续 CCD/OID 对齐必须使用 `lattice_steps`，不能通过数组下标猜测时间。 - -## 命令示例 - -所有 Legacy GPU 采集使用 `pycuda_3_10` 和 GPU 2： - -```bash -PYTHONPATH=src:. conda run -n pycuda_3_10 python \ - src/CCD_analysis/scripts/collect_illusion.py \ - --scene illusion_1.0L --device 2 --steps 200 \ - --field-interval 257 --output-dir /path/to/run/illusion_1.0L -``` - -```bash -PYTHONPATH=src:. conda run -n pycuda_3_10 python \ - src/CCD_analysis/scripts/collect_karman.py \ - --scene karman_re100 --device 2 --steps 200 \ - --field-interval 317 --output-dir /path/to/run/karman_re100 -``` - -不提供 `--field-interval` 时使用原来的单时钟 `ff.run()` 路径，不保存 `fields.npz`。 - -## 实际闭环验证 - -2026-07-22 在 `pycuda_3_10`、物理 GPU 2 上进行了 200 个控制周期的 A/B：baseline 不读取中间 DDF，实验组使用与控制周期不整除的流场间隔，因此读取位置持续在控制周期内漂移。 - -### Illusion 1.0L - -- 控制周期：600；流场间隔：257； -- baseline 末 100 周期 DTW similarity：0.973265； -- DDF 读取组：0.976044； -- 差值：+0.002779； -- 平均 reward 差值：-0.001633。 - -### Kármán Re=100 - -- 控制周期：800；流场间隔：317； -- baseline DTW similarity：0.954562； -- DDF 读取组：0.953442； -- 差值：-0.001119（约 -0.12%）； -- 两组末 100 周期传感器序列经相位补偿后的相互 DTW similarity：0.965680。 - -最终受控流场保持相同类型的尾流结构，没有出现控制失效。因此对同一条连续在线闭环轨迹，可以认为控制期间独立读取 DDF 不会造成有意义的控制效果下降。 - -## 边界和注意事项 - -- 该方法解决的是同一条连续闭环轨迹中的独立流场采样。 -- 它不保证不同 checkpoint restore 分支 bitwise 相同；Legacy CUDA kernel 本身存在数值非确定性。 -- 不要把两个独立 restore 分支当成逐点 bitwise 配对数据。 -- 不要在 active control interval 中调用普通 `ff.run()`。 -- 不要提前调用 `end_control_interval()`。 -- 不要在同一 control interval 内切换 action target。 -- DDF 读取后必须继续同一个 session，直至原固定控制边界。 diff --git a/src/CCD_analysis/EXECUTION_CHECKPOINT.json b/src/CCD_analysis/EXECUTION_CHECKPOINT.json new file mode 100644 index 0000000..54e0a81 --- /dev/null +++ b/src/CCD_analysis/EXECUTION_CHECKPOINT.json @@ -0,0 +1 @@ +{"aborted_attempts":[{"cause":"tempfile.mkdtemp used a validated but not-yet-created output.parent","effects":{"artifact_published":false,"cfd_executed":false,"cfd_initialized":false,"output_overwritten":false},"entry":"first-authorized-drl-smoke","remediation":"Create each missing non-symlink output-parent component under the already validated Optane backing root, re-resolve containment, recheck no-clobber, then create same-filesystem hidden staging.","status":"ABORTED_BEFORE_CFD"}],"active_campaign":"karman-dynamic-v1","artifacts_created":["src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/drl","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/drl-phase-compact-v1","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/constant_mean-phase-compact-v1","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/target-phase-compact-v1","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/zero-phase-compact-v1","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/dynamic-increment-v1","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/dynamic-increment-v2","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/drl-temporal-negative-lag-ccd-v1","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/drl-constant-phase-domain-ccd-v1","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/publication-v2"],"blockers":[],"cfd_executed":true,"commands":[{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests/test_karman_dynamic.py -q","result":"11 passed including missing-parent pre-CFD cleanup, smoke forwarding/count binding, and mode persistence"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","result":"163 passed in 51.58s"},{"command":"ReadLints changed karman_dynamic runtime/CLI/artifacts and focused tests","result":"no linter errors"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m CCD_analysis.karman_dynamic orchestrate --campaign-id karman-dynamic-v1 --root src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-smoke --warmup-intervals 1 --collect-boundaries 2 --launch-delay-seconds 120 --smoke","result":"dry-run printed DRL-first four fresh child commands; no CFD"},{"command":"phase-gate on fresh DRL production","result":"PASS; immutable compact artifact published; 19 complete cycles; dense fields retained"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests/test_karman_dynamic.py src/CCD_analysis/tests/test_karman_dynamic_phase.py -q","result":"16 passed"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","result":"168 passed"},{"command":"fresh-load all four production role artifacts","result":"all wrapper/payload hashes and semantic contracts passed"},{"command":"phase-gate constant_mean/target/zero","result":"constant_mean and target passed; zero failed closed with metrics-only artifact"},{"command":"publish dynamic-increment-v2","result":"immutable four-role mean/statistics and DRL-constant total/centered phase result published and live-provenance reloaded"},{"command":"temporal-ccd DRL primary lag grid -17..0 with neighboring/common-support windows","result":"immutable result published; live parents and full essential recomputation passed"}],"dynamic_increment":{"authoritative_result":"src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/dynamic-increment-v2","closures":{"benefit":0.0,"mean":4.76837158203125e-07,"phase":2.384185791015625e-07},"common_mask_points":649592,"mean_target_error":{"constant_mean":0.16801922402189767,"drl":0.15999457585705143,"zero":0.2177700755224745},"phase_target_error_cycle_mean":{"constant_mean":0.21264614848958815,"drl":0.18212715976743282},"superseded_partial_result":"src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/dynamic-increment-v1","target_error_reduction":{"constant_to_drl":0.008024648164846243,"zero_to_constant":0.049750851500576815}},"files_changed":["src/CCD_analysis/EXECUTION_CHECKPOINT.json","src/CCD_analysis/README.md","src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/drl-phase-compact-v1","src/CCD_analysis/karman_dynamic/__init__.py","src/CCD_analysis/karman_dynamic/__main__.py","src/CCD_analysis/karman_dynamic/artifacts.py","src/CCD_analysis/karman_dynamic/cli.py","src/CCD_analysis/karman_dynamic/contracts.py","src/CCD_analysis/karman_dynamic/orchestration.py","src/CCD_analysis/karman_dynamic/phase.py","src/CCD_analysis/karman_dynamic/runtime.py","src/CCD_analysis/karman_dynamic/safety.py","src/CCD_analysis/tests/test_karman_dynamic.py","src/CCD_analysis/tests/test_karman_dynamic_phase.py","src/CCD_analysis/karman_dynamic/temporal_ccd.py","src/CCD_analysis/tests/test_karman_dynamic_temporal.py"],"final_publication":{"authoritative_artifact":"src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/publication-v2","dense_deletion":{"constant_mean":"retain: independent source-level mean/statistics recomputation requires dense fields","decision":"RETAIN_ALL","drl":"retain: temporal CCD source-level recomputation requires dense fields","target":"retain: independent source-level reference mean/statistics recomputation requires dense fields","zero":"retain: failed phase gate produced no compact fields; dense source is sole source-level mean/statistics basis"},"figure_stems":["01_four_role_mean_performance","02_drl_constant_phase_difference","03_temporal_ccd_spectrum_sensitivity","04_temporal_ccd_leading_modes","05_temporal_ccd_left_lag_functions","06_phase_domain_ccd_downgrade"],"review":{"decision":"PASS_AFTER_ONE_REMEDIATION","initial_medium_finding":"publication-v1 mean panel omitted target reference despite four-role requirement","unresolved_high":[],"unresolved_medium":[]},"superseded_pre_review_artifact":"src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/publication-v1","verification":{"closure":true,"focused_tests":"26 passed","four_role_mean_recomputation":true,"fresh_provenance_reloads":true,"full_tests":"178 passed in 50.43s","git_diff_check":"passed","hashes":true,"lints":"no errors","old_loader_isolation":true}},"historical_evidence":{"immutable":true,"previous_checkpoint":{"authoritative_files":["src/CCD_analysis/README.md","src/CCD_analysis/EXECUTION_CHECKPOINT.json","src/CCD_analysis/direct_dq/__init__.py","src/CCD_analysis/direct_dq/__main__.py","src/CCD_analysis/direct_dq/io.py","src/CCD_analysis/direct_dq/analysis.py","src/CCD_analysis/direct_dq/convergence.py","src/CCD_analysis/direct_dq/cli.py","src/CCD_analysis/tests/test_direct_dq.py","src/CCD_analysis/evidence/direct_dq_core_reviews.json","src/CCD_analysis/acquisition/validation.py","src/CCD_analysis/direct_dq/schema.py","src/CCD_analysis/original_ccd/ORIGINAL_CCD_MATH.md","src/CCD_analysis/original_ccd/_reference.py","src/CCD_analysis/original_ccd/__init__.py","src/CCD_analysis/tests/test_original_ccd_derivation.py","src/CCD_analysis/evidence/original_ccd_derivation_reviews.json","src/CCD_analysis/original_ccd/api.py","src/CCD_analysis/original_ccd/delays.py","src/CCD_analysis/original_ccd/README.md","src/CCD_analysis/tests/test_original_ccd.py","src/CCD_analysis/evidence/original_ccd_implementation_reviews.json","LegacyCelerisLab/driver.py","src/CCD_analysis/acquisition/solver_state.py","src/CCD_analysis/tests/test_acquisition.py","src/CCD_analysis/evidence/acquisition_contract_reviews.json","src/CCD_analysis/original_ccd/REAL_CASE_CCD_CONTRACT.md","src/CCD_analysis/evidence/real_ccd_contract_review.json","src/CCD_analysis/real_ccd/__init__.py","src/CCD_analysis/real_ccd/core.py","src/CCD_analysis/real_ccd/io.py","src/CCD_analysis/real_ccd/schema.py","src/CCD_analysis/real_ccd/preflight.py","src/CCD_analysis/tests/test_real_ccd.py","src/CCD_analysis/evidence/real_ccd_preflight_tests_review.json","src/CCD_analysis/real_ccd/cli.py","src/CCD_analysis/real_ccd/__main__.py","src/CCD_analysis/evidence/real-ccd-karman-q1-tau0-burn120000-v1","src/CCD_analysis/evidence/real_ccd_karman_result_review.json","src/CCD_analysis/FINAL_RESULTS.md","src/CCD_analysis/real_ccd/figures.py","src/CCD_analysis/evidence/real-ccd-karman-figures-v1","src/CCD_analysis/evidence/real_ccd_karman_figures_review.json"],"blockers":[],"commands":[{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests/test_original_ccd.py src/CCD_analysis/tests/test_original_ccd_derivation.py -q","environment":"pinball_math","result":"42 focused production+derivation tests passed after A-F remediation"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"105 full active tests passed after A-F remediation"},{"command":"ReadLints original_ccd production and tests","environment":"Cursor IDE","result":"no linter errors"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests/test_acquisition.py -q","environment":"pinball_math","result":"30 focused acquisition tests passed after solver-mask-aware extraction remediation"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"109 full active tests passed after runtime smoke defect remediation"},{"command":"ReadLints changed runtime extraction, validation, Legacy accessor, and tests","environment":"Cursor IDE","result":"no linter errors"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"111 full active tests passed after exact coordinate-contract remediation"},{"command":"ReadLints exact coordinate contract runtime/validation files","environment":"Cursor IDE","result":"no linter errors"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests/test_acquisition.py -q","environment":"pinball_math","result":"35 focused acquisition tests passed after CPU policy-inference isolation remediation"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"114 full active tests passed after CPU policy-inference isolation remediation"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"114 full active tests passed after exact Legacy q/U0 decoder and schema-v2 remediation"},{"command":"load all six withdrawn smoke-20260804 artifacts with current loader","environment":"pinball_math","result":"all six rejected by current schema/decoder contract"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"116 full active tests passed after schema-v3 complete control-boundary lineage remediation"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"122 full active tests passed after explicit orchestration schedule remediation"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"125 full active tests passed after strict direct-dq burn-in/retained selection implementation"},{"command":"acquisition orchestrate --case karman_re100 --control-count 450 --field-interval 2000 --execute","environment":"pycuda_3_10","result":"three roles published successfully at fresh no-clobber production root"},{"command":"acquisition orchestrate --case illusion_1.0L --control-count 450 --field-interval 1250 --execute","environment":"pycuda_3_10","result":"three roles published successfully at fresh no-clobber production root"},{"command":"python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"125 passed"},{"command":"direct_dq karman production with start>120000, stations 35/40/45, windows 30/60/90/120","environment":"pinball_math","result":"published 120-field provenance-bound result; weighted vector RMS target error 0.15965716015602738"},{"command":"direct_dq illusion production with start>90000, stations 35/40/45, windows 36/72/108/144","environment":"pinball_math","result":"SUPERSEDED pre-certificate result; excluded from authoritative evidence"},{"command":"load_result on both production direct-dq roots","environment":"pinball_math","result":"both live-provenance reloads passed"},{"command":"certify-illusion-pilot on schema-v3 q_ctl-pilot150","environment":"pinball_math","result":"PASS certificate; bounded runtime/history compatibility only"},{"command":"pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"128 passed after certificate and atomic no-replace remediation"},{"command":"authorized Illusion orchestrate 450 controls field_interval=1250 with certificate","environment":"pycuda_3_10","result":"fresh three-role acquisition succeeded"},{"command":"direct_dq authorized Illusion start>90000 windows 36/72/108/144","environment":"pinball_math","result":"published 144-field result; weighted vector RMS target error 0.08031177071338107"},{"command":"load_result authorized Illusion","environment":"pinball_math","result":"live-provenance reload passed"},{"command":"strict independent final result review after certificate/no-clobber remediation","environment":"independent read-only review","result":"PASS; no high or medium findings"},{"command":"python3 contract/checkpoint invariant assertions and documentation whitespace checks","environment":"workspace Python 3","result":"passed"},{"command":"git diff --check on real-ccd-contract scoped files","environment":"git","result":"passed"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests/test_real_ccd.py -q","environment":"pinball_math","result":"22 focused real-CCD tests passed after one review remediation cycle"},{"command":"preflight_real_artifact on both authoritative direct-dq roots with chunk_size=8","environment":"pinball_math","result":"both passed mandatory live-provenance validation and explicit host RAM/scratch admission; no result directories created"},{"command":"independent code+science review and one remediation re-review","environment":"independent read-only review","result":"FINAL PASS; two high and one medium findings closed; no unresolved high findings"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"150 full active tests passed"},{"command":"ReadLints real_ccd package and focused tests","environment":"Cursor IDE","result":"no linter errors"},{"command":"git diff --check scoped todo files","environment":"git","result":"passed; full workspace check separately reports unrelated pre-existing src/steady_pinball_theory/tests/test_core.py:582 blank line at EOF"},{"command":"python -m CCD_analysis.real_ccd preflight --direct-dq-root src/CCD_analysis/evidence/direct-dq-karman-burn120000 --chunk-size 8","environment":"pinball_math","result":"PASS; live provenance validated; estimated peak 16982785740 bytes within 47083299840-byte safe host budget; zero scratch"},{"command":"python -m CCD_analysis.real_ccd run --direct-dq-root src/CCD_analysis/evidence/direct-dq-karman-burn120000 --output src/CCD_analysis/evidence/real-ccd-karman-q1-tau0-burn120000-v1 --chunk-size 8","environment":"pinball_math","result":"published immutable no-clobber Karman Q=1 tau=0 result; no CFD; Illusion not run"},{"command":"fresh-process load_result on authoritative Karman real-CCD result","environment":"pinball_math","result":"VERIFIED; live inputs reread and means, weights, cross-correlation, orthonormality, SVD, coefficients recomputed"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests/test_real_ccd.py -q","environment":"pinball_math","result":"23 passed"},{"command":"PYTHONPATH=$PWD/src conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"151 passed"},{"command":"ReadLints real_ccd package and focused tests","environment":"Cursor IDE","result":"no linter errors"},{"command":"git diff --check scoped real-ccd-karman files","environment":"git","result":"passed"},{"command":"pytest src/CCD_analysis/tests/test_real_ccd.py -q","environment":"pinball_math","result":"24 passed including artifact-only/full-resolution/labels/no-clobber plotting test"},{"command":"real_ccd.figures Karman publication","environment":"pinball_math CPU","result":"published fresh no-clobber full-resolution PNG/PDF package; no CFD or Illusion"},{"command":"pytest src/CCD_analysis/tests -q","environment":"pinball_math","result":"152 passed after Karman figures publication"},{"command":"ReadLints and scoped git diff --check","environment":"Cursor/git","result":"no lint errors; scoped diff check passed"}],"current_todo":{"completed_subset":"karman_re100","id":"real-ccd-figures-review","remaining_subset":"illusion_1.0L awaiting user authorization","status":"in_progress"},"datasets":[{"acquisition_root":"src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-karman-450-fi2000","burn_selection":"acquisition_relative_lattice_step > 120000","case_id":"karman_re100","control_count":450,"field_interval":2000,"result_manifest_sha256":"f4c835d3da947e3a675815ec9884fcaeb737552ce19406d9b9e08c36eeb77814","retained_count":120,"retained_result":"src/CCD_analysis/evidence/direct-dq-karman-burn120000","role_manifest_sha256":{"q_blk":"6b44c09e9099d3678d358616983ba68a79056e4811d12fe0ad98e6c390b8a9c3","q_ctl":"67122188832bd548d19bca8be2651317a64623d97129b865f0d1ca5d942b6e6b","q_target":"2d97d71be33235ceb79e0015571e65d17f0caafcf4f0b33dc81d87e3ebc78f8a"}},{"acquisition_root":"src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-illusion-authorized-450-fi1250","burn_selection":"acquisition_relative_lattice_step > 90000","case_id":"illusion_1.0L","compatibility_certificate":"src/CCD_analysis/evidence/illusion-compatibility-certificate-v1.json","compatibility_certificate_sha256":"557dfba465ee43ccf5ece590030f9d8609f180e13b491b35e77f1d2bc1d62e9c","control_count":450,"field_interval":1250,"result_manifest_sha256":"b0370194bcbbc52225c77742a1fb0dc765dd3a15061e2b5c5fb2074f87ec28d5","retained_count":144,"retained_result":"src/CCD_analysis/evidence/direct-dq-illusion-authorized-burn90000","role_manifest_sha256":{"q_blk":"e43e42abc89307e10a0019a28fb2ec1d4e1ba503b0d081f4da70835f8465edbf","q_ctl":"3ea081639ef245d57fc13d19fe66617194cc40fd44d7fb51c3872e3d416cc75f","q_target":"229147a740df234700632ee3dccea0ffb19e6059ad58de2dcef728cda00ed8b6"}}],"direct_dq_selection":{"cfd_run":false,"end_semantics":"optional inclusive and required terminal selected step","focused_tests":"41 passed","full_tests":"125 passed","production_preflight":{"illusion_1.0L":{"artifact_root":"src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-illusion-authorized-450-fi1250","selected_count":144,"start_after_relative_step":90000,"stations":["35","40","45"],"windows":[36,72,108,144]},"karman_re100":{"artifact_root":"src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-karman-450-fi2000","selected_count":120,"start_after_relative_step":120000,"stations":["35","40","45"],"windows":[30,60,90,120]}},"result_run":true,"schema_version":2,"start_semantics":"exclusive exact acquisition-relative physical-step inequality","status":"KARMAN_AND_CERTIFICATE_AUTHORIZED_ILLUSION_RESULTS_PUBLISHED_AND_PROVENANCE_RELOADED"},"environments":{"cfd":{"required":"pycuda_3_10","status":"Karman production and certificate-authorized fresh Illusion production succeeded in pycuda_3_10"},"cpu_verification":{"required":"pinball_math","status":"128 full active tests passed; final Karman and authorized Illusion results live-provenance validated"}},"final_results_gate":{"authoritative_illusion_only":"certificate-authorized fresh production/result; pre-certificate roots excluded","evidence":"src/CCD_analysis/evidence/final_results_review.json","high_findings":[],"medium_findings":[],"status":"PASS"},"independent_direct_dq_review":{"downstream_blocked":false,"evidence":"src/CCD_analysis/evidence/direct_dq_core_reviews.json","option2_finding":"PASS","remediation_status":"FINAL_INDEPENDENT_PASS"},"independent_original_ccd_math_gate":{"cfd_claims_authorized":false,"downstream_blocked":false,"empirical_claims_authorized":false,"evidence":"src/CCD_analysis/evidence/original_ccd_derivation_reviews.json","production_cfd_authorized":false,"production_implementation_authorized":true,"required":true,"status":"FINAL_INDEPENDENT_PASS"},"live_input_provenance":{"portability_limitation":"Standalone or moved result loading without those live inputs is metadata-only and explicitly unverified.","requirement":"Verified result loading and publication require rereading and validating the immutable live acquisition inputs at their recorded absolute paths."},"original_ccd_implementation":{"cfd_run":false,"code_review_findings":["LaggedObservables metadata/matrix consistency now fails closed before decomposition.","Diagonal and dense HPD weight encodings now share one numerical conditioning threshold."],"empirical_claim":false,"implementation_available":true,"independent_code_review":"FINAL_PASS_AFTER_REMEDIATION","independent_science_review":"FINAL_PASS_AFTER_REMEDIATION","pod_comparison_claim":false,"real_case_ccd_run":false,"science_review_findings":["Stale production-unavailable statements in authoritative contract were removed.","Independent snapshot/observable centering regimes are now explicitly part of the authoritative objective contract."],"status":"COMPLETED_FINAL_INDEPENDENT_PASS_AFTER_REMEDIATION"},"plan":"/home/frank14f/.cursor/plans/ccd_three-part_reset_15f14717.plan.md","post_todo_reviews":"src/CCD_analysis/evidence/real_ccd_preflight_tests_review.json","real_ccd_contract":{"blockers":[],"cases":["karman_re100","illusion_1.0L"],"cfd_run":false,"contract":"src/CCD_analysis/original_ccd/REAL_CASE_CCD_CONTRACT.md","field_estimand":"centered full-resolution dq_ctl=q_ctl-q_blk on authoritative persisted analysis_fluid_mask","illusion_real_ccd_run":false,"immutable_schema_contract":"future per-case atomic no-replace artifact with exact provenance, estimand/config/means/mask/weights/actions/modes/spectrum/coefficients/blocks/residuals and fresh-process reload validation","implementation_package":"src/CCD_analysis/real_ccd","karman_result":"src/CCD_analysis/evidence/real-ccd-karman-q1-tau0-burn120000-v1","karman_result_manifest_sha256":"97d5cb300d642295bd3dffedd85d940fde06c8cc1c236fade390db78c3e3810e","karman_review":"src/CCD_analysis/evidence/real_ccd_karman_result_review.json","mean_action_algebra":"constant action mean contribution is zero because U_c 1_N=0; means are reported outside CCD","next_requires_user_authorization":"real-ccd-illusion","observable_estimand":"three centered q_ctl effective_applied_action channels at exact selected field times in front/upper/lower native physical units","preflight_evidence":"src/CCD_analysis/evidence/real_ccd_preflight_tests_review.json","preprocessing":"explicit field/action centering only; no POD, whitening, standardization, nearest-time, interpolation, phase matching, or silent trim","q":1,"real_case_ccd_run":true,"real_case_preflight_run":true,"real_results_claimed":true,"review_evidence":"src/CCD_analysis/evidence/real_ccd_contract_review.json","status":"KARMAN_REAL_CCD_COMPLETED_STOP_CHECKPOINT_ILLUSION_NOT_RUN","streaming_api":["MemoryBudget","StreamingConfig","load_validated_input","decompose_streaming","ResultTransaction","load_result"],"streaming_contract":"future provenance-validated mask-compressed two/three-pass computation with conservative declared RAM/scratch budgets and fail-closed OOM preflight","streaming_implemented":true,"tau":0,"verification":{"focused_tests":"23 passed","fresh_process_reload":"essential identities recomputed and passed","full_active_tests":"151 passed in pinball_math","git_diff_check":"scoped todo diff check passed","karman_preflight":"live provenance and RAM/scratch admission passed","lints":"no errors in real_ccd package and focused tests"},"weight_and_flatten":"coordinate-quadrature cell area repeated for component-major ux-then-uy C-order mask flattening; W^(1/2) coordinates"},"real_ccd_figures":{"artifact_only_public_loader":true,"formats":["PNG","PDF"],"full_resolution":[1280,512],"governing_todo_status":"in_progress","illusion_status":"NOT_RUN_AWAITING_USER_AUTHORIZATION","karman_package":"src/CCD_analysis/evidence/real-ccd-karman-figures-v1","remediation":"snapshot diagnostics split by timestamp and include ux/uy for ranks 1/2/3; reflection metrics explicitly treated as fixed-parity mismatches","review":"PASS_AFTER_ONE_REMEDIATION","snapshot_relative_steps":[122000,242000,360000],"source_result":"src/CCD_analysis/evidence/real-ccd-karman-q1-tau0-burn120000-v1","status":"KARMAN_COMPLETE_ILLUSION_PENDING_USER_DECISION","whole_todo_completed":false},"runtime_smoke":{"aborted_wrong_launch":{"actual_incorrect_schedule":{"control_count":1,"field_interval":"case sample_interval"},"claim":"aborted wrong launch only; no production result","complete_three_role_production_result":false,"observed_paths":["src/CCD_analysis/evidence/production-20260804-q-over-u0-v3/karman_re100/q_target","src/CCD_analysis/evidence/production-20260804-q-over-u0-v3/karman_re100/q_blk","src/CCD_analysis/evidence/production-20260804-q-over-u0-v3/karman_re100/q_ctl"],"observed_state":"three complete one-field Karman artifacts exist, but all are rejected as intended production evidence because the launch used the accidental one-control schedule","partial_transaction":"none found; ArtifactTransaction automatically removes active partial directories on failure","requested_intent":{"control_count":450,"illusion_field_interval":1250,"karman_field_interval":2000}},"corrected_fresh_paths":{"illusion_1.0L":"src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-illusion-authorized-450-fi1250","karman_re100":"src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-karman-450-fi2000"},"current_contract":{"formula":"ux=(f1+f5+f8-f3-f6-f7)/u0; uy=(f2+f5+f6-f4-f7-f8)/u0; fluid cells only; nonfluid=0","quantity":"nondimensional velocity q/U0","schema_id":"legacy-d2q9-q-over-u0/v1"},"decoder_finding":"All six artifacts used momentum/density, contradicting archived verified Legacy momentum/u0 decoding.","immutability":"withdrawn artifacts remain immutable negative evidence and must not be overwritten","lineage_contract":{"artifact_schema":"ccd-acquisition-artifact/v3","cfd_executed":false,"complete_control_boundary_lineage":true,"cpu_tests":"116 passed","field_telemetry_separate":true,"v2_current_production_evidence":false},"next_path":"src/CCD_analysis/evidence/smoke-20260804-q-over-u0-lineage-v3","production_authorized":true,"readiness":"FINAL AUTHORIZED DATASETS COMPLETE; FINAL INDEPENDENT GATE RERUN PENDING","status":"FRESH_SCHEMA_V3_SMOKE_AND_BOTH_450_CONTROL_PRODUCTION_ACQUISITIONS_SUCCEEDED","withdrawn_artifacts":["src/CCD_analysis/evidence/smoke-20260804/karman_re100/q_target","src/CCD_analysis/evidence/smoke-20260804/karman_re100/q_blk","src/CCD_analysis/evidence/smoke-20260804/karman_re100/q_ctl","src/CCD_analysis/evidence/smoke-20260804/illusion_1.0L/q_target","src/CCD_analysis/evidence/smoke-20260804/illusion_1.0L/q_blk","src/CCD_analysis/evidence/smoke-20260804/illusion_1.0L/q_ctl"]},"schema_version":3,"scientific_decisions":["Original CCD uses literal full fields with no preliminary POD, row standardization, or whitening; centering is optional declared preprocessing.","Physical weighting is applied in W^(1/2) coordinates with computation dtype including U, P, and W before lossless casts; modes back-transform by W^(-1/2).","Lag identity is (block,timestamp); local block clocks must be strictly ordered, interleaved storage is supported, and integer delays traverse independent block sequences without wrapping.","A truncated retained space has the full leading value iff it intersects the full top eigenspace; a full top projector requires containment of that eigenspace.","Unequal/non-isotropic coefficient standardization generally changes the metric; exact degenerate POD covariance is isotropic and invariant under admissible unitary rotations.","Relative observable scaling changes the metric; one common complex scalar changes singular values only by its modulus.","No POD or CCD superiority claim is made.","Production original CCD reconstruction is explicitly a weighted field-snapshot projection, not observable prediction.","The reduced rectangular SVD retains at most min(LQ,M) modes; complete reconstruction is asserted only within the returned supported metric subspace.","The transparent phase convention fixes one phase per returned vector but does not identify individual modes inside degenerate singular blocks."],"test_results":[{"count":63,"covers":["mandatory live input reread on load","pre/post publish live validation","exact acquisition identities and role field equality","fully rehashed common shift with forged input_hashes rejection","missing/moved/changed input fail closed","explicit metadata-only unverified API","all prior tests"],"name":"live_acquisition_provenance_anchor","status":"passed"},{"count":20,"covers":["safe complex weighted dtype and exact W validation","block-local lag identity/order/interleaving/duplicates/Q1/odd/even","left/right singular relations and complex multiobservable scaling","complex full-rank weighted equivalence, coefficients and reconstruction","simple and degenerate truncated equality value/vector/projector cases","unequal-scale standardization and exact-degenerate isotropic invariance","explicit U/P centering","exact published Lyu N=10000 cycles, Q=128, 128 spatial points, noise amplitude 100","noiseless structure/order/ratio and multi-seed convergence statistics","published noisy individual mode ordering via sign/phase-invariant overlaps","literal P/U versus factorized accumulator equality for noninteger cycles and five chunk sizes","duplicate field pair, invalid delay kind/noninteger index, nonfinite U/P/W, and exact-only interpolation rejection"],"name":"original_ccd_derivation_contract","status":"final_independent_math_pass_20_focused_and_83_full_pass"},{"count":22,"covers":["public dataclass/config/result fit/decompose API and private reference separation","literal hand and random real/complex comparison against private reference","efficient diagonal and dense complex HPD weighting with W-orthonormal modes","direct rectangular cross-correlation SVD and chunk invariance","independent declared snapshot/observable centering and multiobservable normalization","exact timestamp/index lag construction with blocks, interleaving, cadence, duplicates and preserved U mapping","selected-rank reconstruction, residual monotonicity, projector orthogonality and supported full basis","singular equations, deterministic phase convention and degenerate projector semantics","field flatten roundtrip, strict invalid-input rejection and no POD/whitening/standardization API","smaller Lyu production output matching the literal private reference"],"name":"original_ccd_production","status":"final_independent_code_and_science_pass_after_remediation"},{"count":22,"covers":["dense original_ccd literal equivalence across chunk sizes 1,2,3,4,7","operator, singular values, subspaces, coefficients, weighted residuals","mask, flattening, quadrature weights, actions and clocks","provenance/schema/hash/no-clobber/reload failures","memory budget fail-closed and read-only real preflight"],"name":"real_ccd_preflight_tests","status":"passed_after_one_independent_review_remediation_cycle"}],"unique_next_entry":"user-decision-real-ccd-illusion-authorization","updated_at_utc":"2026-08-04T10:30:00Z"}},"implementation":{"artifact_schema":"ccd-karman-dynamic-role/v1 wrapping unchanged ccd-acquisition-artifact/v3 payloads","execution_order":["drl","constant_mean","target","zero"],"failure":"QUARANTINED state and immediate stop","fresh_child_per_role":true,"guards":{"conda":"pycuda_3_10","cuda_visible_devices":"exactly one token","ppo_device":"cpu inherited from active acquisition runtime"},"launch_delay_seconds":{"configurable":true,"default":120,"minimum":30},"lease":".runtime/ccd-karman-dynamic.lock exclusive O_EXCL; never steals unknown/stale lease","package":"src/CCD_analysis/karman_dynamic","phase_telemetry":"center sensor uy=sensors[:,3]","storage":"stable exact repository symlink to Optane campaign root; fresh no-clobber paths","validation":"wrapper and payload hash validation plus unchanged schema-v3 semantic reload"},"memory_update":"phase-domain DOWNGRADE decision saved as durable memory c29ef4dd-2b5a-4ee2-967e-320ab8c387e9","phase_domain_ccd":{"M":1299184,"N":10,"Q":1,"authoritative_artifact":"src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/drl-constant-phase-domain-ccd-v1","cfd_run":false,"circular_phase_offset_strength_ranges":[[0.03686067042927058,0.04427364714336846],[0.014714204296504796,0.017903672720069642],[0.003276551220145353,0.004146234741437647]],"claim_boundary":"exploratory circular phase co-variation only; phase offsets are not time-response lags, causality, mechanism, uncertainty, or observable prediction","commit":false,"cuda":false,"decision":"DOWNGRADE","numerical_rank":3,"primary_left_functions":[[-0.4583220232557895,0.827012093939793,0.3255640021194584],[0.7551389631852223,0.16915852898595662,0.6333644593366878],[0.4687281400537707,0.5361309434724195,-0.7020381344151515]],"primary_singular_values":[0.042040426069844525,0.015793916618985184,0.004144692782223726],"sensitivity":{"bin_and_origin":{"bins10_half_shift":{"leading_spectrum":[0.042145758048305906,0.01568263414654207,0.0041347443908958835],"left_function_absolute_cosines":[0.9999923741672698,0.9999814676956773,0.9999883569435609],"minimum_projector_cosine":0.9992017517945878,"projector_principal_cosines":[0.9999774221653056,0.9999053774513923,0.9992017517945878],"rank":3},"bins12":{"leading_spectrum":[0.04257601699311459,0.015923253696834123,0.004267269058409139],"left_function_absolute_cosines":[0.9999404578489239,0.999933225421032,0.999981858584874],"minimum_projector_cosine":0.9984900300457712,"projector_principal_cosines":[0.9999244461336624,0.9996396075352884,0.9984900300457712],"rank":3},"bins8":{"leading_spectrum":[0.042692869006954196,0.015136969269815262,0.003962017321380038],"left_function_absolute_cosines":[0.9998985681839145,0.9998665730455008,0.9999522870506812],"minimum_projector_cosine":0.9950375008200261,"projector_principal_cosines":[0.9990847144367746,0.9987079154920205,0.9950375008200261],"rank":3}},"harmonic_order":{"1":{"leading_spectrum":[0.0419665541344059,0.015500538804959673,7.103051907645594e-16],"left_function_absolute_cosines":[0.9999813077238623,0.9993781373886008],"minimum_projector_cosine":0.9909109684247569,"projector_principal_cosines":[0.9998197914146005,0.9909109684247569],"rank":2},"2":{"leading_spectrum":[0.041971882888983215,0.015753900173712976,0.003773743971892818],"left_function_absolute_cosines":[0.9999945506629131,0.9999855470879571,0.9999906623821551],"minimum_projector_cosine":0.9539332382496856,"projector_principal_cosines":[0.9999969612548216,0.9993229279284058,0.9539332382496856],"rank":3},"3":{"leading_spectrum":[0.04203942279108871,0.015793192308789807,0.0041402283997407615],"left_function_absolute_cosines":[0.9999999977750353,0.9999999850190134,0.99999998551243],"minimum_projector_cosine":0.9999509470741741,"projector_principal_cosines":[1.0000000000000269,0.9999996490040363,0.9999509470741741],"rank":3}}},"stability_gate":{"minimum_matched_left_function_absolute_cosine":0.9993781373886008,"minimum_projector_principal_cosine":0.9539332382496856,"rank_stable":false,"thresholds":{"left_function_absolute_cosine":0.8,"projector_cosine":0.9}},"status":"DOWNGRADE_COMPLETE_VERIFIED","verification":{"artifact_reload":"live parents and essential recomputation passed","focused_tests":"29 passed","full_tests":"177 passed in 54.64s","git_diff_check":"scoped passed","lints":"no errors"}},"phase_gate":{"amplitude_cv":0.0014891935085334647,"compact_artifact":"src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/drl-phase-compact-v1","complete_cycles":19,"constant_mean_physical_action":[-0.004870177712291479,-0.04144351929426193,0.04020744189620018],"dense_fields_deleted":false,"double_crossing_cycle_count":0,"low_amplitude_cycle_count":0,"period_cv":0.0002764637119787988,"period_mean_boundaries":18.07108881985732,"status":"PASS"},"phase_reduction":{"constant_mean":{"amplitude_cv":0.0036282907371154183,"complete_cycles":19,"passed":true,"period_cv":0.0002174294706184067},"deletion_authorized":false,"dense_fields_deleted":false,"drl":{"amplitude_cv":0.0014891935085334647,"complete_cycles":19,"passed":true,"period_cv":0.0002764637119787988},"target":{"amplitude_cv":0.00018344606644011053,"complete_cycles":19,"passed":true,"period_cv":8.217079190603127e-06},"zero":{"amplitude_cv":0.10138932703637261,"blockers":["period CV exceeds 0.05","amplitude CV exceeds 0.10"],"complete_cycles":11,"passed":false,"period_cv":0.08283078789842367}},"plan":"/home/frank14f/.cursor/plans/karman_dynamic_increment_a70b7955.plan.md","plan_modified":false,"recommended_exact_drl_smoke_command":"CONDA_DEFAULT_ENV=pycuda_3_10 CUDA_VISIBLE_DEVICES=0 PYTHONPATH=\"$PWD/src\" python -m CCD_analysis.karman_dynamic role --campaign-id karman-dynamic-v1 --role drl --output src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-smoke/drl --warmup-intervals 1 --collect-boundaries 2 --launch-delay-seconds 120 --smoke","schema_version":4,"scientific_contract":{"case":"karman_re100","claim_limits":["independent phase-conditioned means are not pointwise counterfactuals","closed-loop lag/phase structure is not causality or response time","old direct-dq and Q=1 results are immutable historical evidence, not the active estimand"],"code_reynolds":100,"decomposition":"q_D(phi)-q_C(phi)=(mean(q_D)-mean(q_C))+[(q_D(phi)-mean(q_D))-(q_C(phi)-mean(q_C))]","physical_re_D":50,"roles":["target","zero","drl","constant_mean"],"target_role":"cloaking-error reference only","unique_dynamic_comparison":"drl minus constant_mean derived from this campaign DRL retained effective_applied_action means","zero_role":"passive baseline only"},"smoke_semantics":{"accepted_command_changed":false,"artifact":"campaign.json acquisition_mode=smoke; production otherwise","cli":"--smoke forwarded to execute_role","schedule":"requires exactly warmup_intervals=1 and collect_boundaries=2","status":"BOUND_AND_PERSISTED"},"status":"KARMAN_DYNAMIC_FINAL_PUBLICATION_COMPLETE","supersedes":"Illusion next-entry and user-decision-real-ccd-illusion-authorization as active execution direction","temporal_lagged_ccd":{"M":1299184,"N":21,"Q":18,"authoritative_result":"src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/drl-temporal-negative-lag-ccd-v1","cadence_lattice_steps":800,"claim_boundary":"closed-loop temporal co-variation only; no causal or response-time claim","cycle_blocks":19,"field_estimand":"full-resolution mask-compressed q_DRL(t)-mean over all 360 retained q_DRL boundaries","lag_sign":"tau<0 means action precedes field","leading_left_lag_metrics":{"centroids_boundaries":[-7.969033358328657,-5.253600371158449,-9.001838828936586],"peak_lags_boundaries":[-6,0,-5]},"leading_singular_values":[0.002779967539429028,1.9148039234810842e-05,1.3248350320403833e-07],"memory":{"decision":"PASS","estimated_peak_ram_bytes":4555975920,"ram_budget_bytes":45000000000,"scratch_bytes":0},"numerical_rank":20,"observable_estimand":"exact same-boundary three-channel effective_applied_action fluctuation","primary_lags_boundaries":[-17,-16,-15,-14,-13,-12,-11,-10,-9,-8,-7,-6,-5,-4,-3,-2,-1,0],"sensitivity":{"common_support_N":21,"leading_relative_changes":[[-0.006715214835484911,0.019359853684949295,-0.05676239070776551],[-0.015839802077724563,0.04329979543236538,-0.060567147176236145]],"minimum_leading3_principal_cosine":0.9999987463372593,"native_support_N":[40,59],"native_support_warning":"spectrum and third leading direction are sample-support-sensitive","windows":[[-16,0],[-15,0]]},"status":"PRIMARY_COMPLETE_VERIFIED","verification":{"artifact_reload":"live role and phase hashes/provenance passed; publication performed full essential recomputation","commit":false,"cuda":false,"focused_tests":"26 passed","full_tests":"174 passed in 52.76s","git_diff_check":"scoped passed","lints":"no errors"}},"unique_next_entry":"none; accepted plan complete","updated_at_utc":"2026-08-06T12:05:00Z"} diff --git a/src/CCD_analysis/FINAL_RESULTS.md b/src/CCD_analysis/FINAL_RESULTS.md new file mode 100644 index 0000000..098e3fb --- /dev/null +++ b/src/CCD_analysis/FINAL_RESULTS.md @@ -0,0 +1,38 @@ +# CCD three-part reset: final result boundary + +## Direct-dq facts + +Two schema-v3, three-role, 450-control acquisitions completed with the corrected Legacy `q/U0` decoder and exact acquisition-relative timelines: + +- Kármán `karman_re100`: field interval 2000; retain exactly the 120 snapshots with relative lattice step `>120000` (122000–360000). The weighted vector-RMS of `e_target = q_ctl - q_target` is `0.15965716015602738`. Algebraic closure `e_target = dq_ctl - dq_tar` has maximum absolute residual `9.5367431640625e-07`. +- Illusion `illusion_1.0L`: field interval 1250; retain exactly the 144 snapshots with relative lattice step `>90000` (91250–270000). The weighted vector-RMS target error is `0.08031177071338107`. Maximum closure residual is `1.7881393432617188e-07`. + +Both results preserve full-resolution instantaneous role and difference fields, means, solver-mask intersection, exact x/D=35/40/45 profiles, mask-aware vorticity, and declared prefix/suffix convergence diagnostics. The 90/120 Kármán and 108/144 Illusion windows show decreasing mean-field deviations, but one trajectory cannot provide independent-realization uncertainty or by itself prove asymptotic convergence. + +These are same-time direct differences. They are not phase-conditioned results. `dq_ctl` and `dq_tar` share `-q_blk`; similarity between them is descriptive and is not causal mechanism evidence. The momentum-flux quantity is explicitly an incomplete proxy, not a complete momentum balance. + +Authoritative immutable results: + +- `evidence/direct-dq-karman-burn120000/` +- `evidence/direct-dq-illusion-authorized-burn90000/` + +A validated reload depends on the immutable live acquisition roots recorded by absolute path in each result; the result directories are not standalone portable evidence. + +## Original CCD capability + +`original_ccd/` implements the original full-field Lyu operator `A = P U†/(N sqrt(LQ))`, weighted modes, exact lag/block handling, coefficients, lag functions, and rank-selected reconstruction/residual. It does not use preliminary POD, row standardization, or whitening. The published-scale synthetic derivation and production-reference tests pass. + +The frozen real-case observable decision was executed for Kármán only. The authoritative Q=1, tau=0 result is `evidence/real-ccd-karman-q1-tau0-burn120000-v1/` (manifest SHA-256 `97d5cb300d642295bd3dffedd85d940fde06c8cc1c236fade390db78c3e3810e`). Its singular values are `0.14676751183519246`, `0.05920814023903697`, and `0.0063884442355860915`; squared cross-correlation strengths are `0.02154070253029336`, `0.003505603870565469`, and `0.00004081221975119316`. The three action means (front/upper/lower, native units) are `-0.004750394590640402`, `-0.0417400509895136`, and `0.03969304291531443`. The result has numerical rank 3, no degenerate singular blocks, modes shape `(1299184, 3)`, coefficients shape `(3, 120)`, and weighted relative residuals `0.7823805118116768`, `0.4698397615223296`, `0.45542496362972573` after complete blocks 1, 2, 3. Fresh-process live-provenance reload and essential identity recomputation passed. The singular values are cross-correlation strengths, not field energy, explained variance, canonical coefficients, causal effects, mechanism evidence, uncertainty, or a CCD-versus-POD comparison. Illusion CCD was not run and requires a separate user authorization decision. + + +## Karman figure subset + +The deterministic CPU artifact-derived package is `evidence/real-ccd-karman-figures-v1/`. It uses only the public verified real-CCD loader, retains the 1280 x 512 grid without downsampling, and publishes PNG/PDF figures plus quantitative Markdown/JSON interpretation. Selected reconstruction snapshots are acquisition-relative lattice steps 122000, 242000, and 360000; no full MxN reconstruction is persisted. The governing two-case figures review remains partial/in progress because Illusion was not run and awaits user authorization. The unique next entry remains that authorization decision. + +## Kármán dynamic-increment final publication + +The accepted-plan final stage is complete at `data/karman-dynamic/karman-dynamic-v1-production/publication-v2/`. Six artifact-only PNG/PDF figure pairs and concise `RESULTS.json`/`RESULTS.md` report the four-role mean performance decomposition, DRL-versus-constant 10-bin phase statistics and centered phase difference, temporal negative-lag CCD spectrum/modes/left lag functions/common-support sensitivity, and an explicit phase-domain **DOWNGRADE**. Zero appears only in mean statistics; no zero phase figure or claim exists because its phase gate failed. + +Mean target errors are zero `0.2177701`, constant mean `0.1680192`, and DRL `0.1599946`. Thus zero-to-constant reduces error by `0.0497509`; constant-to-DRL contributes a further `0.00802465`, about `13.9%` of the total zero-to-DRL mean-error reduction. Constant/DRL cycle-mean 10-bin phase target errors are `0.2126461`/`0.1821272`. Temporal leading strengths are `2.77997e-3`, `1.91480e-5`, and `1.32484e-7`; common-support leading-three principal cosines remain at least `0.999998746`, while native support is sample-support-sensitive. Phase-domain CCD remains DOWNGRADE because first-harmonic rank is 2 instead of primary rank 3. + +Strict code/science/claim review passed after one allowed remediation: `publication-v1` omitted the target bar despite the four-role requirement; immutable `publication-v2` fixes it. Dense fields were not deleted role-by-role: DRL is required for independent temporal recomputation, constant_mean and target for independent source-level mean/statistics recomputation, and zero has no compact field replacement after its failed phase gate. Focused Kármán tests: 26 passed; full active CCD suite: 178 passed; changed-file lints and scoped diff whitespace checks passed; old real-CCD/direct-dq loaders remained isolated from the publication process. diff --git a/src/CCD_analysis/Lyu23.md b/src/CCD_analysis/Lyu23.md deleted file mode 100644 index 9c96b00..0000000 --- a/src/CCD_analysis/Lyu23.md +++ /dev/null @@ -1,1000 +0,0 @@ -# Canonical correlation decomposition of numerical and experimental data for observable diagnosis - -B. Lyu $^{1}$ † - -$^{1}$ State Key Laboratory for Turbulence and Complex Systems, College of Engineering, Peking University, Beijing 100871, China - -(Received xx; revised xx; accepted xx) - -A flow decomposition method based on canonical correlation analysis is proposed in this paper to optimally dissect complex flows into mutually orthogonal modes that are ranked by their cross-correlation with an observable. It is particularly suitable for identifying the observable-correlated flow structures while effectively excluding those uncorrelated, even though they may be highly energetic. Therefore, this method is capable of extracting coherent flow features under low signal-to-noise ratios. A numerical validation is conducted and shows that the method can robustly identify the observable-correlated flow events even though the underlying signal is corrupted by random noise that is four orders of magnitude stronger. The temporal sampling frequency and duration of the observable determine the maximum and minimum frequencies to be resolved in the cross-correlation respectively, while those of the flow are to ensure convergence. These criteria are validated using synthetic examples. The decomposition method is subsequently used to analyse a turbulent channel flow, a subsonic turbulent jet and an unsteady vortex shedding from a cylinder, showing the effectiveness of observable-correlated structure identification and order reduction. This decomposition represents a data-driven method of effective order reduction for highly noisy numerical and experimental data and is suitable for identifying the source and descendant events of a given observable. It is hoped that this method will join the existing flow diagnosis tools, in particular for observable-related diagnosis and control. - -# 1. Introduction - -Many natural flows exhibit complex behaviour, such as the boundary layer formed over a sand dune or compressed air inside an aeroengine. This is particularly true at high Reynolds numbers, where most realistic engineering flows occur, because turbulence comes into play exhibiting a wide range of temporal and spatial scales. To understand, model, and possibly exert control on these flows, it is crucial to extract dominant structures and reduce the systems' degrees of freedom. - -Extensive research has been conducted to extract coherent features and decompose complex flows into a collection of simple modes. Well-established methods include the Proper Orthogonal Decomposition (POD), Dynamic Mode Decomposition (DMD), resolvent/input-output analysis and global stability analysis (Taira et al. 2017). Among these, POD and DMD fall into the category of data-driven approaches, while the resolvent/input-output analysis and global stability analysis are model-based. - -POD (Lumley 1967; Berkooz et al. 1993) is a particularly well-known data-driven method and represents a powerful tool for feature extraction and order reduction. Originating from Principal Component Analysis (PCA) in classical statistics, POD decomposes a complex flow into mutually orthogonal modes ranked by their fluctuation energy. If a flow is comprised of a few energetic coherent structures, POD effectively identifies them as leading-order modes. A linear combination of these leading-order modes then forms an optimal reduced-order representation of the total flow. POD may be used to extract the spatial or temporal structures (Lumley 1970; Sirovich 1987). These two structures are coupled, with the temporal structures representing the temporal variation of their corresponding spatial modes, and the spatial modes representing the spatial distribution of their corresponding temporal modes (Aubry 1991). This leads to the so-called Bi-orthogonal Decomposition (BOD). Recent years have also seen the increasingly widely-used Spectral Proper Orthogonal Decomposition (SPOD) in studying turbulent flows (Towne et al. 2018). In addition, to better capture the structures in transient and intermittent flows, conditional space-time POD (Schmidt and Schmid 2019) and multidimensional empirical mode decompositions (Souza et al. 2024) are proposed. These techniques are used to examine the acoustic bursts, the onset and evolution of the dynamic stall and intermittent vortex pairs, showing advantageous capability in resolving transient and intermittent events. It is worth noting that since POD relies on the underlying coherence within the flow to work, it is capable of identifying the flow structures that are dynamically nonlinear compared to linear model-based approaches. - -While POD aims to identify the coherent structures within a complex flow, DMD aims to extract temporal evolutionary information of the underlying dynamics captured in the data (Schmid 2010). The resulting representation is a dynamical system of fewer degrees of freedom. DMD starts by assuming a linear mapping between a sequence of the flow data, and the dynamics is extracted by examining the eigenvalues of a similarity matrix. For a linear system, this amounts to identifying the eigenmodes of the system. For nonlinear systems, DMD is connected with the modes of the so-called Koopman operator (Koopman 1931; Mezić 2013; Schmid 2022). Unlike POD, DMD modes capture the main “contributions” to the overall dynamics embedded in the data sequence. Recent years have seen numerous variants of DMD such as the extended DMD (Williams et al. 2015) and Residual DMD (Colbrook et al. 2023). More details on the recent development of DMD can be found in the recent review by Schmid (2022). - -As mentioned above, both POD and DMD are data-driven, while the resolvent analysis is based on the modal analysis of a linear operator. The resolvent analysis has an early origin in control theory and is based on the pseudospectrum of an operator (Trefethen et al. 1993; Taira et al. 2017), rather than the spectrum. For example, when the flow is decomposed into a base part and a fluctuation part, the Navier-Stokes equations can be rewritten and interpreted as a forced linear system, by which the evolution of the fluctuation part is governed. The nonlinear terms are collected on the right-hand side and interpreted as the forcing of the system. The resolvent modes are ranked by the energy gain between the response and forcing. Therefore, the resolvent analysis examines the gain properties of the linearized operator and has been successfully used to study turbulence from a linearized Navier-Stokes equation point of view (Farrell and Ioannou 1993; Mckeon 2010). Recent studies also show that the leading-order resolvent modes match the leading-order SPOD modes extracted from a numerically simulated high-speed jet (Schmidt et al. 2018). The input-output analysis (Jovanović 2021) is similar to the resolvent analysis in that a modal analysis is performed on a linearized operator. Input-output analysis differs from the conventional resolvent analysis in that a weight may be added to the operator to bias both the forcing and response towards interested domains or observables (Jeun et al. 2016). Therefore, input-output analysis may be regarded as a weighted resolvent analysis. - -In contrast to the resolvent analysis, model-driven global stability analysis (Theofilis 2011) examines the eigenvalue properties of an operator linearized around a base flow with multiple inhomogeneous spatial directions. In particular, it pays special attention to unstable modes, which would dominate the linear response of the system at large times. Note that through global stability analysis, the stable modes can also be obtained, which may play an important role in determining the transient dynamics of underlying flows. This is particularly true in fluid mechanics, where the linearized operators are often non-normal (Trefethen et al. 1993) and the transient growth can become crucial in determining the flow stability. In addition, an adjoint analysis of the operator may be performed to examine the receptivity problem, yielding modes that are similar to the optimal forcing modes in the resolvent analysis. - -POD and DMD, together with their variants, are common data-driven flow decomposition methods used in fluid mechanics. These provide important tools for probing the structures and dynamics of an underlying dynamical system. The ultimate goal of identifying the dominant structures or dynamics is, however, often to understand and possibly control some observables of the flow, such as to reduce the drag of a cylinder, minimise the unsteady force of a wing, or abate the noise emission from a jet. However, because POD modes are ranked by their fluctuation energy, the leading-order modes are not necessarily the most important structures as far as the observable is concerned, although they do carry the largest energy. For example, a large coherent structure effectively extracted from a turbulent subsonic jet using POD may be very inefficient at generating noise. In other words, the leading-order POD mode may not be the leading-order noise-generating flow structure. For example, it has been shown that a substantial number of near-field POD modes are required to reconstruct the acoustic field (Freund and Colonius 2009). Similarly, DMD extracts the dominant dynamics embedded within the flow without taking their connection with any observable into account. Consequently, the leading-order dynamic mode does not necessarily represent the flow events connected with the leading-order dynamics of the observables. - -That the energy rank may not be an appropriate measure, in particular for an observable-related diagnosis, is a well-recognised limitation of POD (Rowley 2005; Schmid 2010). One widely-used approach to overcome this difficulty is to use different norms to bias the decomposition towards interested observables or to use the extended POD (Maurel et al. 2001; Borée 2003). For example, Freund and Colonius (2009) performed the POD decomposition of a turbulent jet using various norms, including the near-field turbulent kinetic energy, near-field pressure, and far-field pressure. When the far-field pressure norm is used, the near-field flow quantities drop out in the correlation matrix and the resulting modes are effectively ranked only by the far-field pressure. Although the near-field flow can still be projected onto the far-field basis, the resulting near-field mode does not necessarily form a direct continuation of the far-field physics, particularly when the near- and far-field exhibit completely different dynamics or the far-field and near-field variables are characterised by pronounced phase delays. Note that the balanced POD proposed by Rowley (2005) is another similar technique to overcome the energy norm limitation of POD, which may be viewed as a special form of POD when the observability Gramian is used as the norm. - -On the other hand, the resolvent and input-output analyses decompose the flow to maximise the energy gain between the output and forcing based on the spectral theory of linear operators. Hence, the observable may be directly included in the choice of output. The resolvent and input-output analysis represent powerful tools to diagnose the flow structure and are capable of providing insightful understanding into a variety of turbulent flows (Mckeon 2010; Sharma and Mckeon 2013). In order to do so, a linearized operator describing the underlying system is often needed. In some cases, however, such an operator may not be readily known, while in others the linearized operator may not be an appropriate representation of the dynamical system, particularly in highly nonlinear systems. For example, an input-output analysis was performed on compressible subsonic and supersonic jets and found that a considerable number of modes were required to reconstruct the acoustic energy of subsonic jets (Jeun et al. 2016), which may be partly due to the limitation imposed by linearity. Such a limitation is also applicable to global stability analysis, where a linearized operator must be known in advance. - -In this paper, we aim to develop a data-driven flow decomposition method that is suitable for observable diagnosis based on flow and observable snapshots instead of linear operators. Instead of redefining the POD energy norm to bias towards the observable, the decomposition aims to introduce a rank based on a cross-correlation norm between the resulting modes and the observable, hence including both the flow and observable data in the correlation matrix. The decomposition method falls under the framework of canonical correlation analysis (CCA) (Hotelling 1936) in classical statistics. This paper is structured as follows: section 2 shows a mathematical formulation of the decomposition method. The physical significance of the resulting modes, the frequency and wavenumber resolutions, the effect of including multiple observables and the connection of the present decomposition to POD and the extended POD are discussed in detail sequentially. Section 3 validates the method by performing the decomposition on multiple synthetic flow fields. The effects of varying sampling frequency, duration and including multiple observables are also thoroughly validated. Section 4 applies this technique to both numerical and experimental data, demonstrating the potential use of such a method. The following section concludes the paper and lists some future work. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/75159b30fee58ee8c37eab1182a7d5ababa12d31b362597a80ac0b808f8a8b31.jpg) - - - -Figure 1: Schematic illustration of the two-dimensional flow snapshots sampled at time $t_{i}, i = 1, 2, 3, \ldots, N$ . Each snapshot contains flow data in both x and y directions, where x and y denotes the Cartesian coordinates of the flow domain. - - -# 2. The canonical correlation decomposition - -# 2.1. The decomposition procedure - -Assume that we have a sequence of snapshots $u_{i}$ obtained by sampling a flow field $u(\boldsymbol{x}, t)$ at time $t = t_{i}$ , where x represents the coordinates of the flow domain and i is an integer that takes the value of $1, 2, 3, \ldots, N$ . If the snapshots $u_{i}$ are sampled in time, $t_{i}$ increases sequentially as i increases, as shown in figure 1. If $u_{i}$ are, however, sampled in the ensemble space, each $t_{i}$ refers to the sampling time in its corresponding independent realisation and can therefore be independent of each other. In the most general case, $u_{i}$ can be sampled both in the time and ensemble space. Each snapshot of this sequence is obtained by discretizing the spatial domain on a mesh and represented by a column vector of length M. We write this snapshot sequence compactly in a matrix notation as - -$$ -\boldsymbol {U} = \left[ \boldsymbol {u} _ {1}, \boldsymbol {u} _ {2}, \boldsymbol {u} _ {3}, \dots , \boldsymbol {u} _ {N} \right]. \tag {2.1} -$$ - -For each snapshot $u_{i}$ , which is obtained by sampling a flow field at $t_{i}$ , assume that we can simultaneously sample an interested observable $p(t)$ of this flow at time $t_{i} + \tau_{j}$ , where j is an integer and takes the values of 1, 2, 3, ...Q with Q being a positive integer. We therefore obtain a sequence of the sampled observable $p_{i,j}$ , $j = 1, 2, 3...Q$ . Note that the sequence $p_{i,j}$ can be sampled at an earlier or later time of $t_{i}$ , depending on whether $\tau_{1}$ is a negative or positive value, respectively. This is important, and we will discuss its significance in the rest of this paper. - -For each integer i, we can define a column vector $p_{i}$ such that - -$$ -\boldsymbol {p} _ {i} = [ p _ {i, 1}, p _ {i, 2}, p _ {i, 3}, \dots p _ {i, Q} ] ^ {T}, \tag {2.2} -$$ - -where T denotes transpose. We then form a matrix P such that - -$$ -\boldsymbol {P} = [ \boldsymbol {p} _ {1}, \boldsymbol {p} _ {2}, \dots , \boldsymbol {p} _ {N - 1}, \boldsymbol {p} _ {N} ]. \tag {2.3} -$$ - -The key step is to construct a matrix A, representing the cross-correlation matrix between the flow and the observable, such that - -$$ -\boldsymbol {A} = \frac {1}{N \sqrt {Q}} \boldsymbol {P} \boldsymbol {U} ^ {\dagger}, \tag {2.4} -$$ - -where $\dagger$ denotes the Hermitian adjoint. The Hermitian adjoint here allows both P and U to be complex matrices. This is useful because both the observable and flow field can be just a Fourier component of the total fields (see the end of section 2.1 for more details). In the case where only real matrices are involved, the Hermitian adjoint $\dagger$ reduces to the simple transpose T. - -We then perform the standard Singular Value Decomposition (SVD) of matrix A, such that - -$$ -\boldsymbol {A} = \boldsymbol {R} \boldsymbol {\Sigma} \boldsymbol {V} ^ {\dagger}, \tag {2.5} -$$ - -where R and V are $Q \times Q$ and $M \times M$ unitary matrices respectively, while $\Sigma$ is a diagonal matrix of $Q \times M$ with the singular values $\sigma_{j}$ ( $j = 1, 2, 3 \ldots, \min(M, Q)$ ) as its diagonal elements. The column vectors of V represent the desired modes of the flow field $u_{i}$ , while those of R represent the normalised cross-correlation functions between the resulting modes and the observable. From SVD, it can be readily shown that these modes are mutually orthonormal and form a complete basis of $R^{M}$ . Therefore, the flow field $u_{i}$ can be conveniently decomposed as - -$$ -\boldsymbol {u} _ {i} = \sum_ {k = 1} ^ {N} a _ {k} (t _ {i}) \boldsymbol {v} _ {k}, \tag {2.6} -$$ - -where $v_{k}$ denotes the k-th column of V while $a_{k}(t_{i})$ denotes its corresponding expansion coefficient at time $t_{i}$ , or equivalently, - -$$ -u (\boldsymbol {x}, t) = \sum_ {k = 1} ^ {N} a _ {k} (t) \phi_ {k} (\boldsymbol {x}), \tag {2.7} -$$ - -where $\phi_{k}(\boldsymbol{x})$ denotes the basis function corresponding to $v_{k}$ , while $a_{k}(t)$ is the expansion coefficient of $u(\boldsymbol{x},t)$ using the basis $\phi_{k}(\boldsymbol{x})$ . As shown in section 2.2, these modes are ranked by their cross-correlation with the observable, and a significant order reduction may be expected if only a small number of modes are pronouncedly correlated with the observable. As will be shown in section 2.2, the decomposition method falls under the framework of CCA, therefore it will be referred to as the canonical correlation decomposition (CCD) in the rest of this paper. - -Note that, as mentioned above, both the observable and the flow can be just a Fourier component of the total fields. For example, the observable may be $\tilde{p}_{\omega}$ while the flow may be $\tilde{u}_{\omega}$ , where $\tilde{p}_{\omega}$ and $\tilde{u}_{\omega}$ represent the temporal Fourier components of the observable and flow at angular the frequency $\omega$ , respectively. In practice, a long flow snapshot sequence $u(\boldsymbol{x}, t_{k})$ ( $k = 1, 2, 3 \ldots$ ) obtained in experiments or simulations may be first partitioned into N segments; each segment may be regarded as a realisation in the ensemble space and then Fourier transformed in time and/or space to form the $u_{i}$ ( $i = 1, 2 \ldots N$ ) shown in (2.1). Similarly, a long observable sequence $p(t_{k} + \tau_{1})$ obtained in experiments or simulations may be partitioned into N segments; the ith segment is then Fourier transformed with respect to $t_{k}$ ( $\tau_{1}$ is a constant) to obtain $\tilde{p}_{\omega i}(\tau_{1})$ . In a similar manner, $\tilde{p}_{\omega i}(\tau_{2})$ , $\tilde{p}_{\omega i}(\tau_{3}), \ldots$ $\tilde{p}_{\omega i}(\tau_{Q})$ can be obtained, which are just $p_{i,2}$ , $p_{i,3}, \ldots p_{i,Q}$ shown in (2.2) ( $\tilde{p}_{\omega i}(\tau_{1})$ constitutes $p_{i,1}$ ). Note that when the observable and the flow are sampled at different frequencies, proper temporal alignment of them for each realisation must be ensured according to those described in section 2.1. Care must also be taken regarding the frequency resolutions of the flow and the observable when the Fourier transform is performed. CCD can then be performed according to (2.3) to (2.5), which may be regarded as a form of CCD decomposition in the spectral space. - -# 2.2. Physical significance of CCD modes - -The CCD represents an optimal decomposition that maximises the cross-correlation between the flow field u and the observable p. This can be shown mathematically as follows. Assuming the flow field is described by the function $u(\boldsymbol{x}, t)$ while the observable by $p(t + \tau)$ , where $\tau$ represents the time delay between flow and the observable. We form the cross-correlation $R(\tau, \boldsymbol{x})$ using - -$$ -R (\tau , \boldsymbol {x}) = \left\langle p ^ {*} (t + \tau) u (\boldsymbol {x}, t) \right\rangle , \tag {2.8} -$$ - -where $*$ represent the complex conjugate, while $\langle\cdot\rangle$ represents the temporal or ensemble average. In the latter case, the statistical processes represented by u and p are assumed to be stationary. For non-stationary processes, (2.8) explicitly depends on t, but the following derivation can still proceed. - -First, let us define an inner product in the Hilbert space defined on a domain $\Omega$ such that - -$$ -(f, g) = \int_ {\Omega} f (\boldsymbol {x}) g ^ {*} (\boldsymbol {x}) \mathrm{d} x ^ {n}, \tag {2.9} -$$ - -where $f(\boldsymbol{x})$ and $g(\boldsymbol{x})$ denote two functions within this space and n represents the dimension of $\Omega$ . A norm is therefore defined as $||f|| = (f, f)^{1/2}$ . Suppose we wish to find a function $\phi(\boldsymbol{x})$ of unit norm, such that the inner product between $R(\tau, \boldsymbol{x})$ and $\phi(\boldsymbol{x})$ , i.e. $(R, \phi)$ , obtains its maximum value in the $L_{2}$ norm. Mathematically, this is equivalent to - -$$ -\max _ {| | \phi | | = 1} \frac {1}{T} \int_ {\tau_ {0}} ^ {\tau_ {0} + T} | (R, \phi) | ^ {2} \mathrm{d} \tau , \tag {2.10} -$$ - -where $|\cdot|$ represents the complex modulus, and $\tau_{0}$ and T are two constants chosen such that the integration includes the entire interval where the integrand obtains non-negligible values. - -Physically, this amounts to finding the optimal function $\phi$ that most correlates with the observable. This is because the ensemble average in (2.8) commutes with the inner product in (2.10), i.e. - -$$ -(R, \phi) = \langle p ^ {*} (t + \tau) a _ {\phi} (t) \rangle , \tag {2.11} -$$ - -where $a_{\phi}(t)$ represents the expansion coefficient of the flow field u using the basis $\phi$ , i.e. - -$$ -a _ {\phi} (t) = (u, \phi). \tag {2.12} -$$ - -Clearly, we see from (2.11) and (2.12) that $(R,\phi)$ represents the cross-correlation function between the mode $\phi$ and the observable p. The $L_{2}$ norm of $(R,\phi)$ defined over an interval of length T is a natural measure of the correlation level between $\phi$ and p. We therefore define the correlation strength $C_{e}$ as the average of $|(R,\phi)|^{2}$ over the interval $[\tau_{0},\tau_{0}+T]$ , i.e. - -$$ -C _ {e} = \frac {1}{T} \int_ {\tau_ {0}} ^ {\tau_ {0} + T} | (R, \phi) | ^ {2} \mathrm{d} \tau . \tag {2.13} -$$ - -Evidently, if $\phi(\boldsymbol{x})$ maximises $C_{e}$ , it represents a flow structure that most correlates with the observable p. - -The function $\phi(\boldsymbol{x})$ that we seek can be obtained from an eigenvalue problem as follows. We know that $\phi(\boldsymbol{x})$ is a function of unit norm that yields a maximum $C_{e}$ , i.e. $\phi(x)$ satisfies - -$$ -\max _ {| | \phi | | = 1} \frac {1}{T} \int_ {\tau_ {0}} ^ {\tau_ {0} + T} | (R, \phi) | ^ {2} \mathrm{d} \tau . \tag {2.14} -$$ - -Classic calculus of variation shows that a necessary condition for (2.14) to hold is that $\phi$ is an eigenfunction of the correlation tensor, i.e. - -$$ -\int_ {\Omega} B (\boldsymbol {x}, \boldsymbol {x} ^ {\prime}) \phi (\boldsymbol {x} ^ {\prime}) \mathrm{d} x ^ {\prime n} = \lambda \phi (\boldsymbol {x}), \tag {2.15} -$$ - -where the correlation tensor is defined by - -$$ -B (\boldsymbol {x}, \boldsymbol {x} ^ {\prime}) = \frac {1}{T} \int_ {\tau_ {0}} ^ {\tau_ {0} + T} R (\tau , \boldsymbol {x}) R ^ {*} (\tau , \boldsymbol {x} ^ {\prime}) \mathrm{d} \tau , \tag {2.16} -$$ - -and the eigenvalue $\lambda$ corresponds to $C_e$ defined in (2.13) (Riesz and Nagy 1955). Clearly, the maximum $C_e$ is given by the largest eigenvalue. - -When the flow field and the observable are discretized, we can show that after multiplied by $\sqrt{Q}$ the matrix A defined in section 2.1 is identical to a discretized form of $R^{*}(\tau, \boldsymbol{x}')$ . The correlation tensor $B(\boldsymbol{x}, \boldsymbol{x}')$ then reduces to $A^{\dagger}A$ because - -$$ -B (\pmb {x}, \pmb {x} ^ {\prime}) = \frac {1}{T} \int_ {\tau_ {0}} ^ {\tau_ {0} + T} R (\tau , \pmb {x}) R ^ {*} (\tau , \pmb {x} ^ {\prime}) \mathrm{d} \tau \approx \frac {1}{Q} \sum_ {i = 1} ^ {Q} R (\tau_ {i}, \pmb {x}) R ^ {*} (\tau_ {i}, \pmb {x} ^ {\prime}) = \pmb {A} ^ {\dagger} \pmb {A}, (2. 1 7) -$$ - -where $\tau_{i}$ is the discretized values of $\tau$ . Equation 2.15 therefore reduces to a discretized eigenvalue problem of the matrix $A^{\dagger}A$ , i.e. - -$$ -\boldsymbol {A} ^ {\dagger} \boldsymbol {A} \boldsymbol {v} _ {k} = \lambda_ {k} \boldsymbol {v} _ {k}, \tag {2.18} -$$ - -where $v_{k}$ as defined in section 2.1 is the discretized form of the k-th eigenfunction $\phi(\boldsymbol{x})$ , while $\lambda_{k}$ is the k-th $\lambda$ in (2.15) subject to a discretization constant, whose exact value often carries no significance in practice. The eigenvalue problem of (2.18) is equivalent to the singular value decomposition shown in (2.5). Therefore, the column vectors of V are these optimal modes, while the corresponding column vectors of R are the normalised cross-correlation functions. In addition, the squares of the singular values $\sigma_{k}^{2}$ are precisely $\lambda_{k}$ , representing the correlation strength $C_{e}$ between the CCD modes and observable (subject to a discretization constant). In particular, when the components of p that correlate with their corresponding CCD modes of u are of equal energy, $\sigma_{k}^{2}$ also represent the observable-correlated energy of their corresponding CCD modes (subject to a constant), and the correlation ranking is identical to the ranking of the observable-correlated flow energy. In summary, instead of decomposing the flow field u based on its energy ranking using the classical POD, (2.5) yields a decomposition that is based on a cross-correlation ranking with an observable, or the observable-correlated energy ranking in the special case where the correlated components of p are of equal energy. - -Mathematically, the flow decomposition method can be shown to fall under the framework of CCA (Hotelling 1936) as follows. Given two column vectors $\boldsymbol{X} = (x_{1}, x_{2}, \ldots, x_{n})^{T}$ and $\boldsymbol{Y} = (y_{1}, y_{2}, \ldots, y_{m})^{T}$ of random variables with finite second moments, CCA seeks two vectors $\boldsymbol{a} (\boldsymbol{a} \in \mathbb{R}^{n})$ and $\boldsymbol{b} (\boldsymbol{b} \in \mathbb{R}^{m})$ such that the random variables $a^{T} X$ and $b^{T} Y$ yield the maximum correlation. The process may be continued in a subspace to yield a sequence of vector pairs. In the context of CCD, the flow field u may be regarded as the Y vector. However, the key part of the decomposition is to find a proper X vector. There are many ways X can be specified, such as the flow within a specific subdomain of interest. However, the essence and novelty of the present decomposition is to construct an X that consists of the observable sampled in a synchronised manner with the flow but at different time delays. Compared to POD or the extended POD, this time shift is an additional dimension used in CCD. As will be shown, the additional information embedded in this “hidden” shifted-time dimension is the key to yielding a more observable-targeting decomposition. More importantly, this permits independent sampling rates between the observable and the flow, which can be of great advantage. - -CCD possesses a number of key features that would be particularly useful for targeted flow diagnosis. First, the decomposition modes are not ranked by their energy, but by the correlation strength with the observable. Flow features that are not correlated with the observable can be effectively suppressed, while those correlated are promoted and ranked according to their correlation strength with the observable. This targets exclusively the observable and is, therefore, very useful in finding the sources or descendant structures of the observable. Second, as will be shown in section 3, the decomposition is robust even when the signal-to-noise ratio (SNR) is low. This is useful when only a small portion of the flow energy correlates with the observable, for example in the classical problem of aeroacoustic emission due to turbulence. Moreover, this robustness can be continuously improved when a longer time duration is used. This is therefore suitable for experimental diagnosis, where an arbitrarily long measurement may be readily performed. - -Third, as will be shown in section 3, the decomposition appears more capable of order or dimensionality reduction compared to POD. This is because CCD aims to decompose the flow only in the observable-correlated subspace, rather than the entire $\mathbb{R}^M$ . In fact, this fact may be used to estimate the convergence of the decomposition by examining how well observable can be reconstructed only using modes corresponding to non-zero singular values. Last but not least, the flexibility to use different sampling frequencies for the flow and the observable enables one to fully exploit the instrument's capabilities in experiments and numerical simulations. For example, it is well known that acoustic signals can often be sampled much faster using a microphone than the entire flow field using PIV. Similarly, in numerical simulations, the observable can also be sampled much faster than the flow due to limited storage requirements imposed by the observable at only a number of probe positions. Note that in general the sampling frequency of the observable is independent of that of the flow, provided $p_i$ properly aligns with $u_i$ as prescribed in section 2.1. In practice, if the sampling frequency of the observable is an integer multiple of that of the flow, it would be trivial to achieve such alignment. In other cases, clock-triggered synchronisation may be used to meet such a requirement in experiments. - -# 2.3. Frequency and wavenumber resolution and sampling delay - -In section 2.1 we mention that the matrix P is assumed to have Q rows and each adjacent row is shifted by time $\Delta\tau = \tau_{j+1} - \tau_j$ (assuming a constant sampling frequency). Moreover, p is sampled temporally behind u by a time $\tau_1$ (or ahead of u if $\tau_1$ is negative). In practice, the choice of Q, $\Delta\tau$ , and $\tau_1$ has significant physical implications. - -First, we show that $\Delta\tau$ and Q determine the maximum and minimum frequencies that can be resolved in the cross-correlation between the flow and the observable, respectively. To see this, we start by defining the correlation tensor $C(\tau', \tau)$ as - -$$ -C (\tau , \tau^ {\prime}) = \int_ {\Omega} R ^ {*} (\tau , \boldsymbol {x}) R (\tau^ {\prime}, \boldsymbol {x}) \mathrm{d} x ^ {n}. \tag {2.19} -$$ - -Similar to that shown in section 2.2, we can show that after discretization $C(\tau, \tau')$ reduces to $AA^{\dagger}$ subject to a scaled constant. The eigenvalue $\lambda$ defined in (2.15) can also be found by - -$$ -\frac {1}{T} \int_ {\tau_ {0}} ^ {\tau_ {0} + T} C (\tau , \tau^ {\prime}) \psi (\tau^ {\prime}) \mathrm{d} \tau^ {\prime} = \lambda \psi (\tau), \tag {2.20} -$$ - -where $\psi (\tau)$ corresponds to the column vectors of matrix $\pmb{R}$ defined in (2.5) in a discretized form. When the function $C(\tau ,\tau^{\prime})$ is of a homogeneous (stationary) form, i.e. - -$$ -C (\tau , \tau^ {\prime}) = C _ {0} (\tau - \tau^ {\prime}), \tag {2.21} -$$ - -equation (2.20) reduces to a Fourier expansion (Berkooz et al. 1993), i.e. - -$$ -\int_ {\tau_ {0}} ^ {\tau_ {0} + T} C _ {0} (\tau - \tau^ {\prime}) \mathrm{e} ^ {\mathrm{i} 2 \pi f \tau^ {\prime}} \mathrm{d} \tau^ {\prime} = \lambda T \mathrm{e} ^ {\mathrm{i} 2 \pi f \tau}, \tag {2.22} -$$ - -or equivalently, - -$$ -C (\tau , \tau^ {\prime}) = \sum_ {n} \lambda_ {n} T \mathrm{e} ^ {\mathrm{i} 2 \pi f _ {n} (\tau - \tau^ {\prime})}. \tag {2.23} -$$ - -Equation (2.23) indicates that the function $C(\tau, \tau')$ can be expanded into a Fourier series. When $C(\tau, \tau')$ is discretized, the well-known Nyquist's theorem demands that the sampling frequency $f_s^p \equiv 1 / \Delta \tau$ of the observable must be at least twice as large as the highest frequency to be resolved. Similarly, the total sampling duration $Q\Delta \tau$ determines the frequency resolution to be $1 / Q\Delta \tau$ . When the function $C(\tau, \tau')$ is not a homogeneous function, there are no general theorems, but we expect that the frequency requirement remains similar to the homogeneous case. In summary, $\Delta \tau$ determines the maximum frequency while $Q$ determines the frequency resolution similar to those in the Discrete Fourier Transform (DFT). - -Second, we note that the choice of $\tau_{1}$ depends on the physical time delay between p and u. In general, the observable may be temporally ahead of or behind the flow events depending on the causal relations between the two. For example, if p represents the upstream forcing imposed near the nozzle lip of a turbulent jet, then there must exist a finite time delay between the evolved downstream structure and p due to the finite propagation speed of jet instability waves. In this case, p is preferably sampled ahead of u in order to capture the physical correlation within a reasonably short sampling duration of p. A good estimation of $\tau_{1}$ would be around $-d/U_{c}$ , where d and $U_{c}$ represent the maximum distance between the flow and the observable and the convection velocity of the instability waves, respectively. Conversely, if p is temporally behind u then it must be sampled after u. For example, if the observable p represents the acoustic pressure at a distance r from the jet flow, a good estimate of $\tau_{1}$ would be around r/c, where c represents the speed of sound. In other more general flows, a good estimate of $\tau_{1}$ may be obtained by examining the cross-correlation function between the flow and the observable. $\tau_{1}$ should be chosen such that the correlation matrix A captures the entire correlation peaks. - -In addition to the sampling rate, sampling duration, and sampling delay of p, the temporal and spatial sampling of u also have physical implications. First, the spatial sampling rate of u has the conventional implication that it determines the maximum spatial wavenumber, whereas the length of the spatial sample determines the minimum wavenumber that can be resolved. This can be shown in a similar manner to those shown from $(2.19)$ to $(2.23)$ by considering the expansion of $B(\boldsymbol{x}, \boldsymbol{x}')$ . We omit a repetitive presentation here for brevity. - -Second, the temporal sampling rate of the flow $f_{s}^{u}$ (when $u_{i}$ is obtained via temporal sampling), however, has a different implication. The sampling rate here is not to determine the frequency limit, but mainly to ensure the convergence of the correlation between p and u. In particular, there is no need for the flow field and the observable to be sampled at the same frequency. This is an important advantage, because, as discussed in section 2.2, in experiments PIV can only be sampled at a much slower rate than that using a hot-wire or a microphone, whereas in numerical simulations sampling the flow field fast is impractical because of storage limit. However, such limitation does not exist for a number of interested observables. Therefore, the much higher sampling rate of the observable can be fully exploited by CCD in both experiments and numerical simulations. The fact that the sampling rates of the flow and observable are independent of each other is evident in the case that $u_{i}$ is obtained in the ensemble space. - -# 2.4. Inclusion of multiple observables - -In many applications, the appropriate observable is not necessarily limited by one. For example, to examine the dominant flow structures in a subsonic round jet that generates sound at $90^{\circ}$ to the jet centreline, the acoustic pressure at any azimuthal position is an appropriate choice due to the azimuthal statistical homogeneity. In such cases, upon defining a local coordinate system, each observable and the flow field in the local coordinates may be treated as an independent realisation. In such cases, using multiple observables is trivial by following section 2.1, i.e. allowing $u_{i}$ to be sampled both in the temporal and ensemble space. By doing so, the number of flow snapshots is increased by $N_{rl}$ fold, where $N_{rl}$ denotes the number of independent realisations. This would be very useful in improving the convergence of the resulting CCD modes. - -In cases where there is no apparent statistical homogeneity in the flow, multiple observables may still be included. For example, when a turbulent jet is forced in an upstream position (Crow and Champagne 1971), the introduced disturbance evolves downstream. One may wish to extract the coherent structures induced by the forcing using observable measurements downstream of the jet. In such cases, velocity fluctuations at any location within a reasonable distance from the forcing location may be used. However, each observable is likely to be heavily contaminated by turbulence. Using multiple observables are expected to improve the converge of the resulting modes. In such case, suppose that the matrix $P_{i}$ ( $i = 1, 2, 3 \ldots L$ ) can be formed using the i-th observable according to (2.3), then a straightforward way to include multiple observables - -is to form the total matrix P such that - -$$ -\boldsymbol {P} = \left[ \begin{array}{l} \boldsymbol {P} _ {1} \\ \boldsymbol {P} _ {2} \\ \dots \\ \boldsymbol {P} _ {L} \end{array} \right]. \tag {2.24} -$$ - -The normalisation constant Q in $(2.4)$ should be replaced by LQ. However, it is important to note that although P has L times as many rows as $P_{i}$ , this does not improve the temporal frequency resolution of the decomposition, which is still determined by $P_{i}$ . This is because, as illustrated in section 2.3, the temporal frequency resolution is determined by the duration of the time shift $Q\Delta\tau$ when $(2.8)$ is truncated and discretized; including more observables does not increase the length of this duration. Nevertheless, convergence of the resulting CCD modes may improve due to the effective inclusion of more data, particularly when highly noisy observables are used. For highly complicated flow data with a limited sampling duration, such as those obtained in numerical turbulent simulations, including multiple observables is expected to improve the convergence, i.e. reduce the uncertainty or noise of the resulting CCD modes. - -The choice of multiple observables, non matter in statistical homogeneous or inhomogeneous flows, must be made with care. As mentioned, the observables must be expected to resolve the same structures either due to statistical homogeneity or well-defined sources of the underlying problem. In the case where the multiple observables chosen are correlated with different events, including more observables would effectively seek an average between these flow structures, which may not be one's intention. For example, if one is interested in identifying the flow structures that are most correlated with the skin friction under a turbulent boundary layer, observables sampled at various streamwise stations are expected to resolve different structures. In such cases, using multiple observables may not be a worthwhile technique. - -# 2.5. Connection to POD and extended POD - -As shown in section 2.2, CCD is different from POD in that the decomposition is based on a cross-correlation rather than an energy norm. This difference is similar to that between CCA and its sister method PCA in classical statistics. Physically, CCA aims to find the “common parts” between two sets of variables, while PCA aims to find the main energetic structures. Mathematically, instead of decomposing the matrix $U^{\dagger}$ , a projection onto P is performed first in CCD. This shows that the decomposition takes into account the space spanned by P. Note this projection may result in a rank that is lower than that of the original flow; however, this is intended as one seeks to decompose $U^{\dagger}$ in the subspace correlated with the observable only. One could argue that this projection leads to a “lower-rank” behaviour by construction, as this would yield fewer singular values. However, the low-rank behaviour we discuss in the following sections is not characterised by fewer singular values, but rather characterised by a quick decay of singular values as the mode number increases and, perhaps more importantly, by a rapid reconstruction of the observable using fewer flow modes. - -As mentioned in section 1, the extended POD (Maurel et al. 2001; Borée 2003) is developed with a similar aim as the present decomposition, i.e. to better target the observable. One can show that the extended POD using a subdomain s is closely related to the degenerate case of CCD when no time shift is allowed between the observable and flow (using multiple observables in s). Mathematically, this implies Q = 1, $\tau_{1} = 0$ and the observable matrix P shown in (2.3) is a degenerate row vector of rank 1. In the special case where the subdomain of the extended POD only includes one observable point and only one mode results, the extended POD and degenerate CCD are identical subject to a normalisation constant. This can be shown as follows. - -Suppose that there exist L observables in the subdomain s. Since no time shift is allowed between the flow and observable, the matrix $P_{i}$ for each observable is a row vector. Hence, the assembled matrix P is a matrix of dimension $L \times N$ . Written in the matrix convention used in the present paper, the essential steps of the spatial extended POD start by decomposing P using POD or, equivalently, by SVD, i.e. - -$$ -\boldsymbol {P} ^ {\dagger} = \boldsymbol {R} _ {s} \boldsymbol {\Sigma} _ {s} \boldsymbol {V} _ {s} ^ {\dagger}, \tag {2.25} -$$ - -where both $R_{s}$ and $V_{s}$ are unitary matrices, the subscript s represents that this is a POD performed in the subdomain s. Note that it is the $P^{\dagger}$ that is decomposed. Right-multiplying (2.25) by $V_{s}$ , one obtains - -$$ -\boldsymbol {P} ^ {\dagger} \boldsymbol {V} _ {s} = \boldsymbol {R} _ {s} \boldsymbol {\Sigma} _ {s}. \tag {2.26} -$$ - -Taking the $k$ th column of both sides of (2.26) yields - -$$ -\boldsymbol {P} ^ {\dagger} \boldsymbol {V} _ {s, k} = \boldsymbol {R} _ {s, k} \sigma_ {s, k}, \tag {2.27} -$$ - -where $\sigma_{s,k}$ represents the kth diagonal element of $\Sigma_{s}$ . The right-hand side of (2.27) represents the temporal coefficient of the kth subdomain POD mode $V_{s,k}$ . The kth extended POD mode $V_{e,k}$ is obtained by projecting the flow U in the extended domain defined in (2.1) onto the kth temporal coefficient, followed by a normalisation, i.e. - -$$ -\boldsymbol {V} _ {e, k} = \frac {1}{\sigma_ {s , k} ^ {2}} \boldsymbol {U} \boldsymbol {P} ^ {\dagger} \boldsymbol {V} _ {s, k}. \tag {2.28} -$$ - -Following the procedure introduced in sections 2.1 and 2.4, the multiple-observable CCD yields - -$$ -\frac {1}{N \sqrt {L}} \boldsymbol {P} \boldsymbol {U} ^ {\dagger} = \boldsymbol {R} \boldsymbol {\Sigma} \boldsymbol {V} ^ {\dagger}, \tag {2.29} -$$ - -where R, $\Sigma$ and V are defined earlier in section 2.1. Left-multiplying (2.29) by $R^{\dagger}$ and then taking the Hermitian adjoint of both sides of the resulting equation yields, - -$$ -\frac {1}{N \sqrt {L}} \boldsymbol {U} \boldsymbol {P} ^ {\dagger} \boldsymbol {R} = \boldsymbol {V} \boldsymbol {\Sigma}. \tag {2.30} -$$ - -Taking the kth column of both sides of $(2.30)$ yields the kth multi-observable degenerate CCD mode - -$$ -\boldsymbol {V} _ {k} = \frac {1}{N \sigma_ {k} \sqrt {L}} \boldsymbol {U} \boldsymbol {P} ^ {\dagger} \boldsymbol {R} _ {k}. \tag {2.31} -$$ - -Comparing (2.28) and (2.31), one sees that the kth extended POD and degenerate CCD modes share much similarity. In particular, since both $V_{s}$ and R are unitary matrices of size $L \times L$ , $V_{s,k}$ and $R_{k}$ are of similar forms. This shows that both modes can be written as a projection of $UP^{\dagger}$ onto a unitary matrix of the same size. However, since $V_{s}$ is obtained by decomposing $P^{\dagger}$ while R by decomposing $PU^{\dagger}/N\sqrt{L}$ , in general, they are not the same. This represents the key difference between the two methods, i.e. one uses an energy-like rank in the subdomain only, while the other uses a correlation rank involving both the subdomain and full domain. It is also this difference that ensures the resulting CCD modes are orthogonal, while it is not necessarily so for the extended POD. - -However, in the special case where only one observable exists in the subdomains s and only one extended POD mode results, both $V_{e}$ and R reduce to 1. Clearly, in this case, the kth extended POD and degenerate CCD modes are identical, subject to a normalisation constant. This also suggests that a key difference between the two is that an extra dimension of time shift is allowed in CCD. It is in fact this difference that results in a more effective order-reduction, which will be discussed in the following sections. - -In summary, one can see that CCD is different from the extended POD in the following ways. First, CCD uses a norm involving both the subdomain and full domain, while the extended POD uses a norm defined in a subdomain space. Second, it is not the energy of the flow within a subdomain that is maximised, but the cross-correlation between the flow and the observable, which is the key difference from the extended POD. Last, the matrix P is formed by consecutively shifting the temporal delay between the flow and the observable. This is why although $A^{\dagger}A$ can be written as $U(P^{\dagger}P)U^{\dagger}$ , CCD is not weighted POD as $P^{\dagger}P$ is a non-diagonal matrix formed by time shifting the observable, instead of a diagonal weight independent of the flow variables. Note that P does not have to be within the flow field; instead, it can represent a variable outside the flow field, a Fourier component of the flow, a particular event in a complex flow, or an observable obtained by integrating the entire flow field. - -Apart from these differences, the connections between POD, extended POD and CCD can also be shown. For example, mathematically POD can be regarded as a special case of CCD when the observable is just an impulse exhibiting no spectral preferences. Specifically, if $p_{ij} = \delta_{i(N+1-i)}$ where $i = 1, 2, \ldots, N$ and $\delta_{ij}$ is the Kronecker delta function, we see that matrix A is a reversed $U^{\dagger}$ and CCD reduces to POD. Physically, this implies that p contains identical frequency components, and therefore exhibits no preferences in the spectral space. Therefore, U is decomposed into modes ranked purely by their energy. Similarly, mathematically CCD may reduce to the extended POD if P is a mathematically constructed simple diagonal weight matrix independent of any flow variables. The exact diagonal elements of course depend on the specific subdomain to be interrogated in the extended POD. - -# 3. Validation - -# 3.1. One-dimensional deterministic flow fields - -To validate that CCD can effectively extract flow events that correlate with an observable, even under very low SNR, we create an artificial one-dimensional unsteady flow field - -$$ -\begin{array}{l} u (x, t) = 2 \cos (t - x) + 1. 5 \cos (2 t) \cos (2 x) + \cos (3 t) \cos (3 x) \\ + 0. 5 \cos (4 t) \cos (4 x) + \cos (6 t) \cos (6 x) \exp (- 0. 1 (x - \pi) ^ {2}) + 1 0 0 r (t, x), \tag {3.1} \\ \end{array} -$$ - -where $r(t,x)$ represents a random noise field with a uniform probability distribution over $[-0.5,0.5]$ , while other terms represent given flow structures with different amplitudes. Note that the energy of the random noise field is deliberately chosen to be around $10^{4}$ times stronger than the defined flow structures. - -Suppose that p represents an observable of interest at a specific point of the flow field, for example, it may represent the skin friction fluctuations at one point on the bottom wall within a turbulent channel flow. It is known that some flow structures are the primary cause of the skin friction fluctuations while others have minimal effects on them. Therefore, as an illustration we suppose that p is generated by the flow events represented by the first, second, fourth and fifth terms in $(3.1)$ , but not by the third and last terms. For instance, p may be given by - -$$ -p (t) = \cos (t - \frac {\pi}{4}) + \sin (2 t - \frac {\pi}{3}) + \cos (4 t) + \cos (6 t - \frac {\pi}{1 2}). \tag {3.2} -$$ - -Note that the amplitudes of the terms shown in $(3.2)$ are chosen to be identical, although this is not at all necessary. In fact, they may be changed arbitrarily without affecting the validity of the decomposition, for instance, the amplitudes shown in $(3.1)$ can be used should one be interested. - -Suppose that the flow field u is sampled over $t \in [0, 2N\pi]$ at a sample frequency $f_{s}^{u} = 128/2\pi$ , where N is an integer representing the number of periodic cycles. Given the strong random noise in (3.1), N is chosen to be a large number (only necessary when strong noise is present). p, on the other hand, is sampled at the same sample frequency $f_{s}^{p} = f_{s}^{u}$ but for a slightly longer duration of $2(N + 1)\pi$ . According to section 2.1, by choosing $\tau_{1} = 0$ and Q = 128, we can construct a matrix P with 128 rows straightforwardly. Within each snapshot, the flow field is discretized on a mesh of 128 points uniformly distributed between $[0, 2\pi]$ . In this example, $\Delta\tau = 2\pi/128$ , therefore the maximum frequency that can be resolved is limited by around $64/2\pi$ . Similarly, Q = 128 implying that the frequency resolution is around $1/2\pi$ . - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/d9d99ac379847f175f6232b864e3bc539ae9171bf99fd22936981ae37e7d3c88.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/f35c492029a591bd7abab3e1a8320a1acadd4577449f4895b0ab5b9ed38817b5.jpg) - - - -Figure 2: Comparison of the spectra of CCD (a) and POD (b) when $N = 10^{4}$ , Q = 128 and $f_{s}^{p} = f_{s}^{u} = 128/2\pi$ . CCD is capable of effectively extracting the observable-correlated events leading to a low-rank spectrum, while POD results in a flat spectrum. - - -Following the procedures introduced in section 2.1, both matrix U and P can be easily constructed, where U is of a size of $128 \times 128N$ while P is of a size $128 \times 128N$ . Upon constructing the matrix A, the CCD can be carried out in a straightforward manner. The resulting CCD spectrum, i.e. the magnitude of the singular values against the mode number, is shown in figure 2(a). To facilitate a direct comparison, the POD spectrum is also shown in figure 2(b). - -Figure 2 shows the CCD spectrum with a desired low-rank behaviour. From figure 2(a) we see that the five modes that correlate with the observable can be robustly identified, even though the energy of the random noise is up to $10^{4}$ times stronger. Specifically, the first two identical singular values form a pair, revealing a flow event of travelling-wave nature, i.e. $\cos(t - x)$ . The first mode of the pair corresponds to $\sin(t + \phi)\sin(x + \phi)$ while the other to $\cos(t + \phi)\cos(x + \phi)$ ( $\phi$ is an arbitrary phase delay), as demonstrated in figure 3(a). The third, fourth, and fifth singular values correspond to the flow events described by the second, fifth, and fourth terms in (3.1), respectively. These can be confirmed by examining the corresponding mode vectors shown in figure 3(b-d). Most importantly, the $\cos(3x)$ mode, which does not correlate with the observable, is robustly removed in the CCD spectrum. This shows that CCD can effectively remove those uncorrelated flow events while only keeping those correlated, and therefore works well for an observable-targeted feature extraction and order reduction. - -The sixth to the ninth singular values $(\sigma_{j}^{2})$ shown in figure 2(a), which are two orders of magnitude weaker than the first few modes, are artefacts introduced by the strong random noise. Note, however, that these unphysical modes can be further suppressed robustly if the flow field is sampled for a longer duration (larger N). All other values of $\sigma_{j}^{2}$ are below $10^{-24}$ and therefore not shown within the given range. As discussed in section 2.2, the singular values represent the correlation strengths between corresponding CCD modes and the observable. In this illustrative case, the observable is comprised of four modes of equal amplitude, as shown in (3.2), therefore the singular values in figure 2(a) are precisely the observable-correlated fluctuation energy (subject to a fixed constant), as evidenced in figure 3 (for example $\sigma_{1}^{2}:\sigma_{2}^{2}:\sigma_{3}^{2}=2^{2}:2^{2}:1.5^{2}$ ). - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/f9d4d3688ccfda05ec3b750e581d1b55461d83c70a23f13a277578fc45232eb3.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/09200b6a65af1b7c860c05cc93d12cf46f5519325a13aac274dff76240343715.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/b1274b88dc1871df86a5616ecc6333da8d2b868230695d475db0bb4fcf975e05.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/258c99e1ad45bc9fcf905fef10804234fc32d070203d12f4e0b75d193e1bff73.jpg) - - - -Figure 3: Extracted CCD modes when $N = 10^{4}$ , Q = 128 and $f_{s}^{p} = f_{s}^{u} = 128/2\pi$ . They are most correlated with the observable p (corresponding to the first, second, fifth and fourth terms in (3.2), respectively). - - -On the other hand, figure 2(b) shows that due to the strong random noise the POD spectrum is completely corrupted and shown as a flat line. The low-rank behaviour embedded within the data therefore cannot be identified. This is expected, because POD modes are ranked by their corresponding fluctuation energy. The random noise present in the flow field is up to $10^{4}$ times stronger than the observable-correlated events, and therefore completely dominates the POD spectrum. More importantly, even though POD may be able to extract the coherent structures when weaker noise is present, it cannot separate the observable-correlated flow structures from those uncorrelated in the same way as CCD does, since no information of $p$ is used. For example, the second term of (3.1) would stay in the POD spectrum and also exhibit as a dominant mode. - -Having validated the decomposition, one can straightforwardly demonstrate the effects of varying the sampling frequency, duration, time shift and including multiple observables. The results agree well with the arguments discussed in sections 2.3 and 2.4. For conciseness, however, we do not include them in this section, but rather have it shown in Appendix A. - -# 3.2. One-dimensional statistical flow fields - -The example shown in figures 2 and 3 illustrates the capability of CCD in extracting flow events from highly noisy data. The temporal signals given in (3.1) are deterministic; we can show in a similar manner that CCD can also effectively extract the observable-correlated flow events when the temporal variation is statistical, such as those exhibited in many turbulent flows. To show this, we construct an artificial one-dimensional flow field - -$$ -u (x, t) = 3 s _ {1} (t) \cos x + 2 s _ {2} (t) \cos 3 x + s _ {3} (t) \cos 6 x \exp (- 0. 1 (x - \pi) ^ {2}) + 1 0 r (t, x), \tag {3.3} -$$ - -where $s_{i}(t)$ (i = 1, 2, 3) represent three statistical processes. The $s_{i}(t)$ series are generated by a random number generator with different seeds in MATLAB and then filtered using three different 6th-order Butterworth filters. More specifically, $s_{1}(t)$ is filtered using a bandpass filter with lower and upper cut-off frequencies of $0.2f_{s}$ and $0.4f_{s}$ , respectively. The $s_{2}(t)$ and $s_{3}(t)$ series are filtered using low-pass filters with cut-off frequencies of $0.2f_{s}$ and $0.15f_{s}$ , respectively. For illustrative purposes, we also add a random noise field that is two orders of magnitude more energetic than $s_{3}(t)$ . Suppose that the observable p is generated by the flow events represented by the second and third terms in (3.3), but not by the first, i.e. - -$$ -p (t) = s _ {2} (t - \frac {\pi}{3}) + s _ {3} (t) + 2 \left[ s _ {3} (t) ^ {2} - \overline {{s _ {3} (t) ^ {2}}} \right] + 3 \left[ s _ {3} (t) ^ {3} - \overline {{s _ {3} (t) ^ {3}}} \right] + r (t). \qquad (3. 4) -$$ - -Note that because the observable may be non-linearly related to the flow dynamics, we also add in (3.4) two nonlinear terms of $s_3(t)$ , as shown by the two bracket terms. Similarly, the observable may be also subject to noise contamination. A statistical random noise $r(t)$ , with a uniform distribution over $[-0.5, 0.5]$ , is therefore also added. The flow field is again sampled at $f_s^u = 128 / 2\pi$ on a uniform spatial mesh of 128 points over the time interval $[0, 2N\pi]$ , while $p$ is sampled over $[0, 2(N + 1)\pi]$ using the same frequency $f_s^p = 128 / 2\pi$ . - -Routine use of the decomposition yields the CCD spectrum and the first two modes, as shown in figures 4(a) and 4(b), respectively. Clearly, the leading-order mode corresponds to the second term in (3.3), while the second-order mode the third. This can be clearly seen from figure 4(b). It is worth noting that the observable also contains the square and cube of $s_3(t)$ , but this does not appear to affect the identification of the second mode. Indeed, CCD works by maximizing the correlation between the flow field and the observables, but in general it does not limit the observable being a linear function of the flow field. Additionally, the first term of (3.3), due to it being uncorrelated with $p$ , is effectively removed in the CCD spectrum. Other higher-order modes are more than two orders of magnitude lower than the first two. Again, as $N$ increases, these unphysical modes can be further suppressed, while the physical modes resolved more accurately. Note that in this illustrative example, the observable p is also corrupted by the random noise, but CCD continues to work robustly. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/03108d0c82d261a4cb2cbcd8380144ae5c5fe14e0bfead9f68c1bce1bbb30360.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/cc91f66662bd8489047a732a67289db47468d885f224eaa3351740b57b16ae86.jpg) - - - -Figure 4: The CCD spectrum (a) and the first and second modes (b). N takes the value of $10^{4}$ but $10^{3}$ may also be used at the expense of convergence. - - -# 4. Applications to numerical and experimental data - -Having validated the method, in this section CCD is used to decompose numerical and experimental data in order to demonstrate its potential use in fluid mechanics. Three flows are used, including a turbulent channel flow, a subsonic jet and a wake flow past a cylinder. Where possible, POD results are also included for comparison. In all cases, the simple $L_{2}$ norm of the flow $u_{i}$ is used in POD. - -# 4.1. Turbulent channel flow - -As an illustrative example, we first apply CCD to a Direct Numerical Simulation (DNS) database of turbulent channel flows. The database was obtained from a turbulent channel flow using the code developed by Lee and Moser (2015). The computational domain is of $4\pi H \times 2H \times 2\pi H$ in the streamwise $(x)$ , wall-normal $(y)$ and spanwise $(z)$ directions, respectively, where H denotes the half-height of the channel. The domain is discretized using 192, 128 and 192 points in x, y and z directions, respectively. The friction Reynolds number $Re_{\tau}$ defined as $\rho u_{\tau}H/\mu$ , where $\rho$ , $\mu$ and $u_{\tau}$ denote the fluid density, dynamic viscosity and friction velocity at the wall respectively, is around 180. The time step is fixed at $0.01H/U_{b}$ , where $U_{b}$ is the bulk flow velocity. The flow is sampled every 100 time steps, resulting in a sampling frequency of $f_{s}^{u} = U_{b}/H$ . In total, 1687 snapshots of the flow field are recorded. - -In turbulent channel flows, skin friction represents a significant operational cost in applications such as long-range oil transport (Kim 2011). The control of turbulent skin friction is therefore of particular interest and has been studied extensively in the literature (Gad-el Hak 2007). To understand the physical mechanism concerning its generation and suppression, it is crucial to extract the turbulent flow structures that determine the skin friction. CCD is therefore suitable for such a diagnosis. As mentioned in section 2, without the data storage limit, the observable is allowed to be sampled at a much higher frequency than the flow field. In this example, the sampling frequency $f_{s}^{p} = 10f_{s}^{u}$ , resulting in an interval of $\Delta\tau = 0.1H/U_{b}$ and 16870 samples for the skin friction. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/55f0684256957b3b9f104ebd3d7e303e68f8cab3d6ecb15bacc7ca3ae231abed.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/bc88a20258ee3a01741790eccc7dac177e4799bea7e6b3cba953fa9df6fa6e21.jpg) - - - -Figure 5: The spectra of (a) CCD and POD; The CCD spectrum exhibits a much steeper decay as mode number n increases, indicating a more effective order reduction. - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/b9b11fc53d3a151dd7cdb4d517d89357530d251cc44def4400b002a38cefbaba.jpg) - - - -Figure 6: Reconstruction of the dimensionless wall friction coefficient using the first 6 CCD (a) and POD (b) modes, respectively. - - -Considering the statistical homogeneity of the flow, we use the skin friction sampled at $x = 2\pi H$ and $z = \pi H$ on the lower wall (y = -H) as the observable and choose the streamwise velocity as the flow variable in order to extract the coherent structures. Considering the short temporal correlation scale, we choose Q = 100 and $\tau_{1} = -50\Delta\tau$ . Using the procedures described in section 2, we perform CCD and obtain the resulting singular values and CCD modes. The singular values are shown in figure 5(a). Also shown is the spectrum from POD in figure 5(b), where the streamwise velocity is decomposed. Comparing the two we see that the CCD spectrum is markedly different from that of POD. In particular, the CCD spectrum exhibits a much quicker decay. For example, higher-order modes ( $\geqslant$ 5) are one order of magnitude lower, whereas the POD spectrum is rather flat. This signals a quicker reconstruction of the skin friction using CCD modes. Indeed, using the first 6 modes recovers more than 80% of the total skin friction at the observer point, as shown in figure 6(a). The high-frequency deviation may be further reduced if the observable is allowed to be sampled faster. In contrast, the first 6 POD modes only recover less than 5% energy, as shown in figure 6(b). - - -B. Lyu - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/865b6a2a8053c9fd40cf26474029139a6d7700d1b997182ef4a11856bdbb8704.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/bbbcb6b7099211517e4dc62668f088b5681ad9a76e8939cb70b6df3a3075a2ce.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/c8d69272b6b20fe9fd0c5e998dbaad05f4a934f818ebd147a29ba299da6b981d.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/3765fe6ce327f660150ff8d16fe72570f3dc43ba6f68c434e5b40ae928b8ad30.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/b1963d6769f79ff07fa864527a14c703e2ed10e8d79157056a288fa73015dfc4.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/cf82d01dca62fcd8a0abfcb8ab08ec8cbd65d4f9f734a991072395c125cb6d23.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/fc1497e2afc7993ff3384bd9456d0494f9833275398ac335f2a6e6d682a9684b.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/a07cc2eb31bc18262172a5dc9a7cf33ea065c59858e98e8f75c55f78c7b10cbf.jpg) - - - -Figure 7: Front views of the first four CCD (a,c,e,g) and POD (b, d, f, h) modes. The z coordinate is fixed at $\pi H$ for (a, c, e, g). The spanwise widths of the streaks in (a,c,e,g) are around 0.2H. - - -The resulting CCD modes are shown in figure 7. We see that the CCD modes take the form of streamwise streaks slightly above the bottom wall, in accordance with current understanding. More importantly, figure 7 also shows that they are spatially localized around the observer point. This is particularly true in the spanwise direction with a streak width of less than 0.2H. Moreover, higher-order modes have increasingly short spatial and temporal scales. To the best knowledge of the authors, such a quantitative and unambiguous characterization of these structures specifically targeting the skin fluctuation in the middle of the wall has not been reported in the literature. In contrast, although the POD modes take the form of streaks, they are not localized around the observer point, but stretched in the streamwise direction and scattered in the spanwise direction instead. Moreover, the first few modes do not exhibit a clear decrease of either spatial or temporal scales, signalling a slower reconstruction of the skin friction. - -Although CCD focuses on examining the flow structures that contribute to the skin friction at one individual point, it in fact does not lose generality. This is because the flow is homogeneous in the streamwise direction; the structures that generate the skin friction at other locations on the wall remain identical (subject to a shift in space). However, by focusing on the observable at a specific point, one expects to obtain a more effective order reduction since our interest is more focused. The fact that the flow is homogeneous can also be exploited to improve the convergence of the resulting flow. Instead of using the observable at one point, one can use multiple points along different spanwise or streamwise locations. They can be treated as independent realisations, with which the resulting mode indeed converges better. However, since the structures remain similar to those in figure 7, we omit showing their contours repetitively. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/f1e07351702f169b80320f5082cf7cde6dde1e8aebaa08d8dd15fa3d0bc004fd.jpg) - - - -Figure 8: The spectrum of extended POD using (a) a subdomain of size $4\pi H \times 2\pi H$ and (b) a subdomain consisting of the observable point only; Reconstructed skin friction fluctuations using (c) a subdomain of size $4\pi H \times 2\pi H$ (using the first 6 extended POD modes) and (d) a subdomain of only one point (only 1 mode results). - - -Note that part of the reason why the skin friction reconstruction using POD is slow is due to its use of energy within the entire domain as the norm. Since the extended POD can be used to target more towards the observable, it is interesting to compare it with CCD in detail. To show this, we first perform the extended POD using skin friction on the wall. The resulting singular values and reconstruction of the skin friction using the first 6 extended POD modes are shown in figures 8(a) and 8(c), respectively. The resulting spectrum of singular values exhibits a similar slow decay to that shown in POD. This is consistent with a similar reconstruction of the skin friction, as shown in figure 8(c), where a limited time range from 650 to 1150 is shown for clarity. However, comparing to figure 6(b), the skin friction reconstruction appears slightly improved when the wall shear stress is used as the subdomain in the extended POD. - -One is, therefore, interested in seeing how much the reconstruction can improve by using increasingly small subdomains centring around the observable. In the ultimate case, the subdomain can be chosen to consist of the observable point only. We choose to perform extended POD using such a special subdomain. Note that this is identical to the degenerate CCD where no time shift is included between the observable and the flow. We expect the resulting mode to better target the observable, which is indeed the case, as shown in figure 8(d). The extended POD modes can capture an overall trend in the skin friction variation. However, it is important to note that since there is only one mode available, as can be seen in its spectrum shown in figure 8(b), this is the best reconstruction one can achieve using the extended POD. - -On the other hand, since the decomposition is also a degenerate case of CCD, this represents the worst reconstruction one would obtain using CCD. Indeed, by including the dimension of time shifts, P would have a rank of more than 1, and the reconstruction using CCD improves considerably, as shown in figure 6(a). Note that the reconstruction further improves as one includes more CCD modes, the family of which forms a complete orthonormal set. In addition, figure 8(d) shows that only an overall trend of the skin friction is captured in the reconstruction, and the deviation occurs mainly in the high-frequency regime. This is expected, since this degenerate CCD corresponds to a sampling interval $\Delta\tau = \infty$ for the observable, hence a failure to resolve high-frequency components. - -In summary, using only one point where the observable is located in the extended POD better targets the observable, but the resulting one mode limits the capability of separating multiple flow structures that possibly coexist within the flow. To do that, a sufficiently large region is preferred, compromising the observable specificity. This appears a trade-off between targeting a local observable and separating multiple flow structures. CCD does not have this limitation, and this relaxation is enabled by exploiting the “hidden” time-shift dimension. This reflects a key difference between CCD and the extended POD. More importantly, this also adds the flexibility of fully exploiting a different (possibly much higher) sampling frequency. - -Figures 5 and 7 show that CCD works well in extracting the coherent structures that are most correlated with the given observable. This is further evidenced by a quick reconstruction of the skin friction using the first few CCD modes. Note again that in this example the observable is sampled at a much higher frequency than the flow. This flexibility, as mentioned in section 2.5, plays an important role in the successful feature extraction and order reduction. - -# 4.2. Turbulent subsonic round jets - -In this section, we apply CCD to a numerical dataset of a turbulent subsonic round jet. Using the pressure fluctuations as the observable we can examine the flow structures that are most correlated with them. Directly resolving far-field pressure fluctuation is rarely possible in numerical turbulence simulations, hence in this example we examine the near-field pressure fluctuation instead. This is expected to suffice for the purposes of demonstrating the potential use of CCD. The near-field dynamics of turbulent jets is expected to connect with their mixing and acoustic characteristics, and is therefore studied extensively in the literature. Order reduction techniques are widely used. This includes POD or the extended POD with a variety of norms (Freund and Colonius 2009; Sinha et al. 2014; Schmidt and Schmid 2019), the resolvent/input-output analysis (Jeun et al. 2016; Pickering et al. 2021; Bugeat et al. 2024) and other source identification methods that we do not aim to show exhaustively. Moreover, the near-field pressure fluctuations are crucial in determining installed jet noise (Lyu et al. 2017; Lyu and Dowling 2019), therefore its modelling and control have practical uses. - -The numerical data is extracted from an earlier work (Lyu et al. 2017), where an LES simulation of a subsonic round jet was performed. Only a slice of data on one azimuthal plane is used, but it should be sufficient for illustration purposes. The jet Mach number is $M_{j} = 0.5$ while the nozzle diameter D is 2 inches. The computational domain is axisymmetric, with the streamwise coordinate x extending from 0 to 20D and lateral coordinate r extending to 4D. The computational domain is discretized using 512 and 97 points in the x and r directions, respectively. - -We choose the near-field pressure fluctuation at x/D = 10 and r/D = 4 as the observable. At this close distance, the observable is likely to include both acoustic and hydrodynamic pressure fluctuations. In addition, we choose the pressure field as the flow variable to be decomposed. The same is used in a reference POD decomposition. The flow is sampled at a frequency of $f_{s}^{u} = 4U_{j}/D$ for a duration of $200D/U_{j}$ , where $U_{j}$ is the jet exit velocity. The near-field pressure p is sampled at the same frequency but for a longer duration of $280D/U_{j}$ . This results in a Q value of 320. Due to the short distance between the flow field and the near-field pressure fluctuations, we choose the time delay $\tau_{1}$ to be 0. With the procedure described in section 2, the CCD spectrum is shown in figure 9. Also shown is the POD spectrum to facilitate a direct comparison. Only the first 50 singular values are shown. Compared to POD, the CCD spectrum exhibits a more rapid decay as the mode number n increases. In particular, at small mode numbers the CCD spectrum shows a clear low-rank behaviour. The first two modes are almost one order of magnitude stronger than higher-order modes. This is in direct contrast to the POD spectrum, where the low-rank behaviour is not pronounced. - - -B. Lyu - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/ed2416630caf8a7e633240660706c8c2ad2e36cbf4cf8f3664ab743eeb40cf65.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/94a35f6dbc30c508188e1c78404cbb7a4910b8f5645e0af80ab78ff5ebecdf7d.jpg) - - - -Figure 9: The CCD (a) and POD (b) spectra of the unsteady pressure field on a x - r plane. The CCD spectrum shows a clear low-rank behaviour compared to POD. - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/d57282eccc17d2a7b74668be55a1b7f86842ab13ee4ae7bd7713ec82255ebb0d.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/036d397805145727f48ebb633d3fce4c3129f2524771dae27ba396b1f98ffd58.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/ced6f482a7ba46d53a669ae5161a52ee018c56531d8bfc9927040c574c46d8b6.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/f761b2a0c7920391eca6a623ed35c3eaacec42da3d1440aa963937f957af81f4.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/6b83a3ed648ef0aa2e737b3dd91f4fa6d0a2896d96ce7c16e9fea22017ca34f8.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/b740daf0a2daf061f4534a66918c53e44a7f8aeac9a5e829d61d310b4a8577b5.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/f3320ef81e4f61036125a3eb030a3340bbd69460b79610d299632a2750b3d189.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/6c3a17a7d8e35014acc0c1134de9f0e1b1018259f04ed633719934041ffdf95d.jpg) - - - -Figure 10: The first 4 CCD modes (a,c,e,g) and PSD spectra (b,d,f,h) of their corresponding temporal coefficients, where the blue dashed line in (b) represents the spectrum of the observable with its magnitude scaled for an easier comparison; mode 1 (a-b), mode 2 (c-d), mode 3 (e-f), mode 4 (g-h). As the mode number increases the CCD modes are characterised by increasingly short spatial scale and high frequency components. - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/7a72190498f60f37d6494d3a3e264f9d7387ba5a002145bdbb3fd1e4fb2ce03f.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/c221a5bcbe34f4d0f29f6420c226ba099a584b986859c4c5f47862f01c463732.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/8bec60aae520fae10155c2ace2f740d71f39445b17c9b9579d50430061c49e4f.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/2411eaac70dabfed7f0b19c75eedb3384dc989d6e04e0e51a7703a9c40ac0719.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/247f44018ba1c3b65f2334d08e9473fb617ae4a2a2584038c9ac959a29ca26b4.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/d8d9afc59f4e684a99aebb5f9425361596ead54fa8872e0d85da07d913db547c.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/88bbc97f19feb01f3c9a2ad364a03a1b57702c845339de2db15dae422dcd8f44.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/57ea1777cce4c3a019a732d1be9ac8f5d6c3e1e7baf1cefab566d3bca99c76b0.jpg) - - - -Figure 11: The first 4 POD modes (a,c,e,g) and PSD spectra (b,d,f,h) of their corresponding temporal coefficients, where the blue dashed line in (b) represents the spectrum of the observable with its magnitude scaled for an easier comparison; mode 1 (a-b), mode 2 (c-d), mode 3 (e-f), mode 4 (g-h). As the mode number increases the POD modes have larger spatial scales with more low-frequency components. - - -This can be understood from figures 10 and 11, where the first 4 CCD and POD modes $\phi_{k}(\boldsymbol{x})$ and the corresponding Power Spectral Densities (PSDs) of their temporal expansion coefficients $a_{k}(t)$ are shown, respectively. Clearly, the first two CCD modes are large flow structures exhibiting relatively low-frequency behaviour, whereas the leading-order POD modes have much shorter scales with a well-known dominant frequency at around St = 0.3, where St is the Strouhal number defined using $U_{j}$ and D. The PSD spectrum of the observable is also included with its magnitude scaled for an easier comparison. Since the observer is located at x/D = 10 and r/D = 4, the pressure fluctuations inevitably include the signatures of the downstream large coherent structures. The similar first two singular values shown in figure 9 and similar mode shapes shown in figure 10 indicate a convection behaviour of this large structure. CCD decomposition can take this into consideration and yield an observable-relevant low-frequency fluctuation mode. The leading-order POD modes, on the other hand, are ranked only by the fluctuation energy and, therefore, are not as relevant as the CCD modes. - -Note that the singular value represents a measure of the cross-correlation in the $L_{2}$ norm and, therefore, is in general not equal to the energy of the CCD modes contained in the flow, nor is it equal to the energy of the corresponding correlated component in the observable. Nevertheless, since the decomposition targets more at the observable, we expect that it can reconstruct the pressure fluctuations at the observable position using much fewer modes. This is indeed the case, as shown in figure 12, where only the first two CCD and POD modes are included to calculate the reconstructed pressure fluctuations at the observable position, respectively. As can be seen from figure 12(a), using two CCD modes can yield a good reconstruction, which is in contrast to POD shown in figure 12(b). Note that in this application we use the pressure field as the flow u, whereas in general a combined velocity and pressure field may be used. We can show that a similar result may also be obtained when a combination of velocity and pressure fluctuations is used in CCD. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/685524a0e5480890de753d757ae531177ee58edca6910bd11b3d94cc9b2ade0c.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/11302fcaa990840f7041e3ef018434e6c9429b914919cb51afa1b7ddb1d6e049.jpg) - - - -Figure 12: The reconstructed pressure fluctuations at the observable location using the first 2 CCD (a) and POD modes (b), respectively. - - -At large mode numbers, the CCD spectrum shows a steeper decay, and higher-order modes tend to have increasingly short scales together with higher frequencies, as shown in figures 9 and 10, respectively. Note that the singular values represent the correlation strength between the CCD modes and the observable, therefore the decay of the singular values is determined by both the energy of the flow and the observable and the coherence decay between them. Therefore, the steeper CCD spectrum suggests that although the pressure fluctuations consist of energetic structures of various scales, they may not be equivalently important in contributing to the observable, therefore the coherence between the two may decrease rapidly. On the other hand, the POD spectrum decays much more slowly, and as the mode number increases the POD mode starts to capture more downstream large structures with more low-frequency content, as shown in figure 11. That the POD spectrum decays more slowly is attributed to the fact that the POD spectrum is determined solely by the energy of flow and, therefore, does not depend on its coherence with the observable. - -Figure 10 shows that each CCD mode corresponds to a unique temporal variation. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/7e9e4ba9cb744e735a908a00db57c49110b20db75285ab7279fb10f02d6cfaf9.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/6155e91309e045c5be247e3aaf27416a9c829c8a60b927a595b5cab862f3690b.jpg) - - - -Figure 13: The mean (a) and instantaneous (b) streamwise velocity distributions of the immediately downstream wake over a cylinder flow (Renn et al. 2023). The black diamond, red circle and white square are located at x/D = 1.5 but y/D = 6.1, y/D = 5.7, and y/D = 4.9, respectively. - - -Unlike the Fourier analysis, each of these temporal variations is spectrally broadband. In essence, CCD decomposition works as a special spectral transform of the flow based on its correlation with the observable. Note, however, that the sampling frequency and duration are limited in this simulation, and further analysis using longer samples is needed for better statistical convergence. In addition, due to current data availability, we only decompose the near-field pressure, and it would be interesting to apply this technique to extract acoustically dominant flow features in future studies. Nevertheless, it suffices for the purpose of demonstrating the potential application of CCD. - -# 4.3. Unsteady wake flows over cylinders - -In this example, we apply CCD to the experimental data of an unsteady wake flow behind a cylinder. The experiment was performed in a water tunnel using the two-dimensional time-resolved Particle Image Velocimetry (PIV) technique. The cylinder had a diameter of $D = 9.53 \mathrm{~mm}$ while the Reynolds number was fixed at 650. The interrogation window was a rectangle immediately behind a cylinder in the wake and measured $13D \times 9D$ in the streamwise ( $x$ ) and cross-stream ( $y$ ) directions, respectively. Details of the experimental setup can be found in Renn et al. (2023). The velocity field was sampled at a frequency of around $50 \mathrm{~Hz}$ on a mesh of $N_x = 133$ and $N_y = 89$ , and in total $N = 8250$ snapshots were obtained. The mean and instantaneous streamwise velocity fields are shown in figure 13 for reference. As shown in figure 13, the mean flow exhibits the expected symmetry across the wake, while the instantaneous velocity field shows a clear vortex shedding behaviour behind the cylinder. The vortex shedding occurring when the Reynolds number exceeds a critical number is one iconic feature of the flow over cylinders. Given its wide applications such as wind blowing over chimneys and high-rise buildings, its control has attracted significant attention in the fluid mechanics community (Choi et al. 2008). Many techniques exist, including both passive and active controls. Earlier studies show that using the feedback signal recorded in the wake, vortex shedding can be successfully suppressed or even eliminated at low Reynolds numbers (Williams and Zhao 1989; Roussopoulos 1993; Park et al. 1994). In designing a closed-loop active control system such as the one in Park et al. (1994), one primary interest is to identify the optimal location to place the feedback sensor. Ideally, the observable, such as the cross-stream velocity, at the feedback sensor location should maintain a strong correlation with the vortex structures shed from the cylinder. CCD may be used to give an initial assessment of the correlation between the sensed signal and the vortex structures. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/1e1e17728ddad124f470cc584db2d0c76cf0b1231ed9da2713c82436aedb7efc.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/c024dffea9e961154e7f4b43ddfdf7a0b8f95d0148bb740e5be0d18c7112692c.jpg) - - - -Figure 14: The CCD (a) and POD (b) spectra of the streamwise velocity fluctuations on an x-y plane with the cross-stream velocity fluctuation at y/D = 5.7 as the observable. - - -As an illustration, we choose the observable to be the cross-wake velocity (Park et al. 1994) in the initial shear layer behind the cylinder, for example at x/D = 1.5 and y/D = 5.7 as shown by the red circular dot in figure 13(a). Using this observable, we may decompose the streamwise velocity field using CCD. Again, the streamwise velocity is correspondingly used in POD. The time shift $\tau_{1}$ is chosen to be $-4Q\Delta\tau/5$ while Q is chosen to be N/3. Figures 14(a) and (b) show the CCD and POD spectra, respectively. Clearly, both CCD and POD capture the dominant vortex shedding behaviour, and the two nearly identical singular values reflect a convecting behaviour of the shed vortices. Figures 15 and 16 show the corresponding first three CCD and POD modes and their corresponding PSDs, respectively. Clearly the first two vortex shedding modes from both CCD and POD are virtually identical, which can be seen from both the mode shape and their corresponding PSD spectra. The CCD spectrum shows a slightly smaller singular value for the third mode, which is somewhat more symmetric, whereas the similar mode resulting from POD obtains a similar singular value compared to the leading-order mode. This suggests that although this mode carries one of the largest energy, it is slightly less correlated with the cross-stream velocity fluctuation at x/D = 1.5 and y/D = 5.7. - -If we keep x/D = 1.5 but move the observable position further away from the shear layer, for example, to y/D = 4.9 and y/D = 6.1 as shown by the white square and black diamond symbols respectively in figure 13, these three modes can still be identified using CCD, but their relative singular values changed significantly, as shown in figure 17. This implies that these modes correlate differently to different observables. Evidently, the first and second modes in figure 17(a) represent the vortex shedding modes. Their mode shapes are similar to those shown in figure 15(a) and (b), so we omit a repetitive presentation. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/f7bbbafa2f5d037c742b95e57b95063ea8544e4141145c01b79935f8576aec65.jpg) - - - -Figure 15: The first 3 CCD modes (a,c,e) and PSD spectra (b,d,f) of their corresponding temporal coefficients; mode 1 (a-b), mode 2 (c-d) and mode 3 (e-f). - - -However, it is important to note that these singular values are much larger compared to those shown in figure 14(a), suggesting they are more strongly correlated with the observable. More importantly, the third singular value drops rapidly, almost one order of magnitude weaker than the leading-order mode. From the feedback control point of view, this would be a good candidate for placing the feedback sensor owing to its higher correlation with our interested flow events and simultaneously a higher SNR. Figure 17(b) shows that the singular values corresponding to the vortex shedding modes are slightly lower than those shown in figure 14(a) with an even stronger leading-order non-shedding mode. Consequently, this would be a position to be avoided for placing the feedback sensor. This may be why the wake centreline was used to place the feedback sensors in Park et al. (1994). - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/4e131b3f4172e290869bc6016058fde48507d4a60d8b65427485e11baaf1f530.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/87d501398c09037834baf561db4b80a2662de7b7843cae3d2209904fc91a6a26.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/26e9f993057fd4755b21ccfe2773934221e1ae5e0e4f7650055c46c25dab733b.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/bac251285e48233274d6c0fc6e5cbeb2076ac83519056848054165a5cb9f3af7.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/825b65945aa9e4958c4b07409d471cccf8729f26d8d600301514f90c69845c1d.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/2d508ae0d4f1abc16dd0666d7a42d95eda390d8134965cd509d7318d69cf097f.jpg) - - - -Figure 16: The first 3 POD modes (a,c,e) and PSD spectra (b,d,f) of their corresponding temporal variation coefficients; mode 1 (a-b), mode 2 (c-d) and mode 3 (e-f). - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/6f2fecafe70c5574de69b22829965a75b748cd4d373bd72d2e4e6abf3dbd5473.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/c47f038310a2071c4a49a9f662ea1ced7e79a0997f56d497029ab36e3cd47f80.jpg) - - - -Figure 17: The CCD spectra of the streamwise velocity fluctuations when the observable is located at $y / D = 4.9$ (a) and $y / D = 6.1$ (b), respectively. - - -# 5. Conclusion - -A data-driven method referred to as CCD is proposed in this paper to decompose complex flows into modes ranked by their correlation strength with an observable. The method is based on the canonical correlation analysis in classical statistics. The method is validated for both deterministic and statistical flow events. First, the results show that CCD can effectively extract the observable-correlated flow features while suppressing those uncorrelated in both cases. CCD, therefore, results in more low-rank spectra compared to POD. Second, CCD can effectively extract those observable-correlated flow structures even under low SNRs. Third, numerical validation shows that the sampling frequency and duration of the observable determine the frequency limit and resolution while that of the flow are to ensure the convergence of the cross-correlation. Longer sampling of the flow and including multiple observables can improve the convergence of the resulting CCD modes. Therefore, CCD is particularly suitable for experimental data because long samples can be more conveniently obtained. Lastly, as no linearity is assumed, CCD is capable of extracting nonlinear flow events similar to POD, provided a non-negligible correlation exists between the flow and the observable. - -As an illustrative example, the method is first used to analyse a turbulent channel flow obtained using DNS. The flow structures that are most correlated with the skin friction at the point in the middle of the bottom wall are extracted. It is shown that CCD yields a spectrum of singular values that decays rapidly as the mode number increases compared to POD. The first 6 CCD modes effectively recover more than 80% of the skin friction fluctuations. The extended POD using only one observable point can better target the observable, and is found to be equivalent to the degenerate case of CCD when no time shift between the flow and observable is used. The CCD modes take the form of streamwise streaks slightly above the wall. More importantly, the streamwise and spanwise extent of these streaks are unambiguously determined. As the mode number increases, CCD modes have increasingly short spatial and temporal scales. - -In a subsequent example, CCD is used to decompose the unsteady pressure field of a turbulent subsonic jet using a near-field pressure fluctuation as the observable. Results show that CCD results in a steeper spectrum compared to POD. In particular, the CCD spectrum exhibits a clear low-rank behaviour and the corresponding modes correspond to the large coherent flow structures that convect downstream. The first two CCD modes recover 80% of the energy of the near-field pressure fluctuations. The method is subsequently applied to analyse the unsteady vortex shedding behind a cylinder. It shows that similar modes to POD can be robustly identified, but their strengths depend crucially on the observable and its locations, suggesting that these modes correlate differently with observables at different locations. This diagnosis would be useful for determining the optimal location for placing the feedback sensor in a closed-loop control of the vortex shedding behind a cylinder. - -Note that the examples shown in the paper are only for illustrations. They suffice for the purpose of demonstrating the potential uses of CCD, but further improvements are needed for a more in-depth analysis. For example, we can see that both the sampling frequency and sampling duration in the jet example are rather limited; therefore, a faster sampling of the observable, together with a longer sampling duration, is needed for more accurate diagnosis. In addition, a possible far-field noise diagnosis using pressure or the Lighthill stress tensors as the flow variables may be conducted. These, together with the application of CCD in the spectral space such as that shown in section 2.1, form some of our future work. - -# Acknowledgements - -The author wishes to thank Prof. Ann Dowling and Prof. Tim Colonius for the stimulating discussions on including multiple observables and the low-rank properties of CCD modes. The author would like to gratefully thank Prof. Jie Yao for sharing the DNS data of channel flows. The author is very grateful to Dr. Cong Wang and his collaborators for agreeing to use their PIV data on cylinder wakes. The author wishes to thank Dr. I. Naqavi who performed the LES simulations in our last collaborative publication (Lyu et al. 2017), from which part of the data is extracted and reused. The author also wishes to gratefully acknowledge the National Natural Science Foundation of China (NSFC) under the grant number 12472263. - -Declaration of interests. The authors report no conflict of interest. - -# Appendix A. The effects of sampling frequency, duration and multiple observables - -In this appendix, we aim to demonstrate the effects of sampling frequency, duration and multiple observables using the one-dimensional synthetic example introduced in section 3.1. We first demonstrate the effects of varying the sampling frequency of the observable $f_{s}^{p}$ . In figure 3, a large sampling frequency of $128 / 2\pi$ is used. This is a sufficiently large number considering one only needs to resolve the approximately maximum frequency of $6/2\pi$ required by the fifth term in (3.1). According to Nyquist's theorem, a minimum sampling frequency of $12/2\pi$ is needed. To demonstrate the validity of this requirement, we first perform CCD using $f_{s}^{p} = 12/2\pi$ , resulting in extracted modes virtually the same as those in figure 3. Subsequently, we use an under-resolved sampling frequency of $10/2\pi$ while are other parameters remain unchanged. The resulting modes are shown in figure 18. Clearly, although the low-frequency modes shown in figure 18(a) and (b) can still be correctly captured, the high-frequency modes shown in figure 18(c) and (d) start to differ from their correct forms. This is because the under-sampling causes aliasing effects in the decomposition, leading to incorrect modes 4 and 5. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/8caa0e061600aba4bfc66fef0b215ecb9cd5e035b8b68936d44066475f7a17d6.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/29af093c51a7cb9ae90c3d7d1a632a023f0277ea93bc2089593633b6ad228e5f.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/8d956b65ab4cd42412075bf0cca3e20aed43639cb59e41fb4a9c5cea2dfe8e92.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/5222f11258621410683f089893b4205cd6273a29d957276156075a37beb08546.jpg) - - - -Figure 18: Extracted CCD modes when the observable is sampled using an under-resolved frequency $f_{s}^{p}=10/2\pi$ while $N=10^{4}$ , Q=128 and $f_{s}^{u}=128/2\pi$ . With this low sampling rate, modes 4 and 5 cannot be captured accurately. - - -We then demonstrate the effects of varying the sampling duration of the observable. In figure 3, Q is taken as 128 as this is the minimum value to use in order to resolve the flow structures given in (3.1), resulting in a frequency resolution of $f_{s}/Q = 1/2\pi$ . If a smaller Q such as 64 is used, one would expect the failure of resolving the low-frequency structures. This is precisely the case, as shown in figure 19, where Q = 64 while all other parameters remain the same. As shown in figure 19(a), the mode at the lowest frequency of $1/2\pi$ is extracted incorrectly. In addition, the next mode at the frequency of $1/\pi$ seems resolved incorrectly as well (see figure 19(b)). In fact, the modes shown in figure 19(a) and (b) appear to have somehow mixed. This is expected, since the frequency resolution is only $1/\pi$ , but the two modes differ from each other only by $1/2\pi$ . On the other hand, modes shown in figure 19(c) and (d) are characterised by frequencies of $3/\pi$ and $2/\pi$ respectively, and therefore have been resolved correctly. Figure 19 clearly shows that the number of shifted rows Q determines the frequency resolution in a similar manner to that in DFT. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/b542af658dcb7e3fbfb053b05b57cc726ddd96a330af4f418020a4f450ec1a7a.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/231fd69522c6ea1fa65d7ae079cccc6e77630c3de88daae4f89209cbc84811da.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/85990d650d16f62ec1f11045c6f18a9c2c159b557dff447031c662cb2e787a09.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/21762de206ccaf2b373f53d78c12e6f23f77a7dac6067dc9deb0ad24e583594b.jpg) - - - -Figure 19: Extracted CCD modes when the observable is sampled for a short duration, i.e. Q = 64, while $N = 10^{4}$ and $f_{s}^{p} = f_{s}^{u} = 128/2\pi$ . With this short sampling duration, modes 1 and 2 cannot be resolved correctly. - - -We are now in a position to illustrate the effects of sampling frequency and duration of the flow u. Figure 3 shows that the extracted modes are subject to small random noise. This noise decreases rapidly as N increases. As mentioned in section 2.3, this is because the sampling rate and duration of the flow are to ensure the convergence of the correlation tensor (they do not affect the frequency limit and resolution). In figures 2 and 3, N is taken to be 10000. This is a large number because we deliberately chose an SNR that is as low as $10^{-4}$ . A small N can also be used at the expense of augmented noise in the resolved modes. For example, figure 20 shows the extracted CCD modes when N = 1000 (a-b) and N = 100 (c-d), respectively. Only modes 1, 2 and 4 are shown for brevity. Clearly, we see that using a duration of N = 1000 results in CCD modes that converge less well but are still unambiguously identified. When N reduces to 100, the resolved CCD modes are further corrupted by the random noise, nevertheless, the structures of the modes can still be recognized. Note that N here denotes the number of cycles of the sampled flow. At the lowest frequency of 100 Hz widely used in the fluid mechanics literature, N = 100 yields a sampling duration of 1 s. In the experiments, a record of 100 s can be easily managed. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/2b63b306e6f06875a6f646c5b9f2df292aba101e31f07d7b414f8a0f96454439.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/3a1b364cfb2f8137241ba3187610264e9f3a3c7d90403567919081740506b26c.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/b7bec8203f82af7afc047aaaf00c0621b96d2a666cc518a3b18cf753a55a0b11.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/87919f87582ba19f15ba4cb43d057b185d0673b5d0f3e1960d31179c9e0d65ba.jpg) - - - -Figure 20: The extracted CCD modes when N = 1000 (a-b) and N = 100 (c-d) while other parameters remain the same as those in figure 3. Convergence increasingly deteriorates as N decreases. - - -Figure 20 shows that the CCD modes are corrupted significantly by random noise at an SNR $< 10^{-4}$ when $N = 100$ . However, when the random noise is only two orders of magnitude more energetic, $N = 100$ yields sufficiently well-resolved CCD modes. In general, we find that to obtain the same level of convergence, $N$ scales roughly as 1/SNR. Conversely, if longer samples are readily available, CCD can robustly extract the observable-correlated flow events at the same level of convergence at an even lower SNR. Therefore, CCD is especially suitable for analysing data acquired in experiments, where the data may be recorded for as long as desired. - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/ff840f5d3e2bb2d3c8effb1047afa59ad6eb1f74606f1d6d935828cf74750804.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/08b39d3e2e764d3e4e8c3fd9caa05e394c22a5b863b70c947269fb8ac42e31ed.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/d542ba6ff43a49f4d8358f0c61266c713fd2557114f0a23456f26b0db34b1dab.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/b6a310d176adbf9e7e038abe006423d0d710f1ef243f4733d8c626caaf2a16e9.jpg) - - - -Figure 21: The extracted CCD modes using a low flow sampling frequency of $f_{s}^{u} = 16/2\pi$ while $N = 8 \times 10^{4}$ , Q = 128, $f_{s}^{p} = 128/2\pi$ . Negligible change occurs compared to figure 3 because the same number of flow snapshots are used for the temporal average. - - -It is worth noting that although the extracted modes in figure 20 are subject to stronger noise contamination due to the insufficient convergence level, they do not suffer from aliasing effects due to under sampling. The fact that the sampling frequency and duration of the flow do not affect the frequency limit and resolution can be even more clearly demonstrated in figure 21. Figure 21 shows the extracted modes when the flow is only sampled at $16 / 2\pi$ . However, to exclude the effects of insufficient convergence, the total number of flow snapshots is kept the same. Comparing figures 3 and 21 reveals that the extracted modes are virtually identical, demonstrating the independence of the frequency resolution on the temporal duration and frequency of sampling. - -We are now in a position to demonstrate the effects of including multiple observables. The flow field u takes the same form of $(3.1)$ . However, more observables need to be defined. To ensure that the observables are not only similar to $(3.2)$ but also exhibit variations, we construct up to ten observables $p_{i}(t)$ ( $i = 1, 2, 3 \ldots 10$ ) such that - -$$ -\begin{array}{l} p _ {i} (t) = \left(1 + 0. 2 r _ {1 i}\right) \cos \left(t - \frac {\pi}{4}\right) + \left(1 + 0. 2 r _ {2 i}\right) \sin \left(2 t - \frac {\pi}{3}\right) + \left(1 + 0. 2 r _ {3 i})\right) \cos (4 t) \tag {A1} \\ + (1 + 0. 2 r _ {4 i}) \cos (6 t - \frac {\pi}{1 2}) + 1 0 0 r _ {i} (t), \\ \end{array} -$$ - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/40244f4a1338849948b2a9e818a3f6883f8b412db8f10921fc1ef33c754ae8f1.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/424e568b81c5f4645fa4445be64905ffd44aa417a66d7d119493976b7e75698d.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/48f6568c702256ae79fe87eb84810a68c1fe7a054bab7925234e0258ff1d64f3.jpg) - - -![image](https://cdn-mineru.openxlab.org.cn/result/2026-06-07/b70089ee-fa1a-4e57-9e24-8f0805a1690f/b95bfd3f8d3670f7bbf72c281e1ad202ff0e9891921273c1272eb0fa3ff03ba4.jpg) - - - -Figure 22: The extracted CCD modes 1 and 4 when 1 (a-b) and 10 (c-d) observables are included. Including multiple observables improves the convergences of the resulting CCD modes. - - -where $r_{ji}$ (j = 1, 2, 3, 4) are random numbers between [-0.5, 0.5] and $r_{i}(t)$ a random function with a uniform distribution over [-0.5, 0.5]. Note that a very strong random noise (SNR $\sim 10^{-4}$ ) is also added in each observable in order to demonstrate the validity of the decomposition with strongly contaminated observables. - -Following the procedure listed in section 2.4, multiple observables can be included straightforwardly to form the matrix P. When identical parameters to those in figure 3 are used, the extracted modes using either 1 or 10 observables are shown in figures 22(a,b) and 22(c,d), respectively. Only modes 1, 2 and 4 are shown for brevity. Clearly, when the observables are also strongly contaminated, the extracted modes converge less satisfactorily, as shown in figure 22(a-b). However, by including 10 observables, the quality of the extracted modes improves significantly. Figure 22 shows that including more observables can indeed improve the convergence of the resulting CCD modes, particularly when the observables are corrupted by noise. - - - -N. Aubry. On the hidden beauty of the proper orthogonal decomposition. Theoretical and Computational Fluid Dynamics, 2:339-352, 1991. - - - - - -G. Berkooz, P. Holmes, and J. L. Lumley. The proper orthogonal decomposition in the analysis of turbulent flow. Annual Review of Fluid Mechanics, 25:539–575, 1993. - - - - - -J. Borée. Extended proper orthogonal decomposition: a tool to analyse correlated events in turbulent flows. Experiments in Fluids, 35:188–192, 2003. - - - - - -B. Bugeat, U. Karban, A. Agarwal, L. Lesshafft, and P. Jordan. Acoustic resolvent analysis of turbulent jets. Theoretical and Computational Fluid Dynamics, 38:687–706, 2024. - - - - - -H. Choi, W. Jeon, and J. Kim. Control of flow over a bluff body. Annual Review of Fluid Mechanics, 40:113–139, 2008. - - - - - -M. J. Colbrook, L. J. Ayton, and M. Szöke. Residual dynamic mode decomposition: robust and verified Koopmanism. Journal of Fluid Mechanics, 955(A21), 2023. - - - - - -S. C. Crow and F. H. Champagne. Orderly structure in jet turbulence. Journal of Fluid Mechanics, 48:547–591, 1971. - - - - - -B. Farrell and J. Ioannou. Stochastic forcing of the linearized Navvier-Stokes equations. Physics of Fluids, 5(11):2600-2609, 1993. - - - - - -J. B. Freund and T. Colonius. Turbulence and sound-field POD analysis of a turbulent jet. International Journal of Aeroacoustics, 8(4):337–354, 2009. - - - - - -Mohamed Gad-el Hak. Flow control: passive, active, and reactive flow management. Cambridge University Press, 2007. - - - - - -H. Hotelling. Relations between two sets of variates. Biometrika, 28:321-377, 1936. - - - - - -J. Jeun, J. W. Nichols, and M. R. Jovanović. Input-output analysis of high-speed axisymmetric isothermal jet noise. Physics of Fluids, 28(4):047101, 2016. - - - - - -M. R. Jovanović. From bypass transition to flow control and data-driven turbulence modeling: an input-output viewpoint. Annual Review of Mechanics, 53:311–45, 2021. - - - - - -John Kim. Physics and control of wall turbulence for drag reduction. Philosophical Transactions of the Royal Society of London A: Mathematical, Physical and Engineering Sciences, 369 (1940):1396–1411, 2011. - - - - - -B. O. Koopman. Hamiltonian systems and transformations in Hilbert space. PNAS, 17:315-318, 1931. - - - - - -M. Lee and R. Moser. Direct numerical simulation of turbulent channel flow up to Re 5200. Journal of Fluid Mechanics, 774:395–415, 2015. - - - - - -J. L. Lumley. The structure of inhomogeneous turbulence. Atmospheric Turbulence and Wave Propagation, pages 166–178, 1967. ed. AM Yaglom, VI Tatarsky, Nauka, Moscow. - - - - - -J. L. Lumley. Stochastic tools in turbulence. Academic Press, 1970. - - - - - -B. Lyu and A. Dowling. Modelling installed jet noise due to the scattering of jet instability waves by swept wings. Journal of Fluid Mechanics, 870:760–783, 2019. - - - - - -B. Lyu, A. Dowling, and I. Naqavi. Prediction of installed jet noise. Journal of Fluid Mechanics, 811:234–268, 2017. - - - - - -S. Maurel, J. Borée, and J. L. Lumley. Extended proper orthogonal decomposition: application to jet/vortex interaction. Flow, turbulence and combustion, 67:125–136, 2001. - - - - - -B. J. Mckeon. A critical-layer framework for turbulent pipe flow. Journal of Fluid Mechanics, 658:336–382, 2010. - - - - - -I. Mezić. Analysis of fluid flows via spectral properties of the Koopman operator. Annual Review of Fluid Mechanics, 45:357–378, 2013. - - - - - -D. S. Park, D. M. Ladd, and E. W. Hendricks. Feedback control of von Karman vortex shedding behind a circular cylinder at low Reynolds numbers. Physics of Fluids, 6(2390), 1994. - - - - - -E. Pickering, A. Towne, P. Jordan, and T. Colonius. Resolvent-based modelling of turbulent jet noise. The Journal of the Acoustical Society of America, 150:2421–2433, 2021. - - - - - -P. I. Renn, C. Wang, S. Lale, Z. Li, A. Anandkumar, and M. Gharib. Forecasting subcritical cylinder wakes with Fourier Neural Operators. ArXiv, 2301:08290, 2023. - - - - - -F. Riesz and B. Nagy. Functional analysis. Ungar, New York, 1955. - - - - - -K. Roussopoulos. Feedback control of vortex shedding at low Reynolds numbers. Journal of Fluid Mechanics, 248:267–296, 1993. - - - - - -C. W. Rowley. Model reduction for fluids using balanced proper orthogonal decomposition. International Journal of Bifurcation and Chaos, 15(3):997-1013, 2005. - - - - - -P. Schmid. Dynamic mode decomposition of numerical and experimental data. Journal of Fluid Mechanics, 656:5-28, 2010. - - - - - -P. Schmid. Dynamic mode decomposition and its variants. Annual Review of Fluid Mechanics, 54:225-254, 2022. - - - - - -O. T. Schmidt and P. J. Schmid. A conditional space-time pod formalism for intermittent and rare events/ example of acoustic bursts in turbulent jets. Journal of Fluid Mechanics, 867:R2, 2019. - - - - - -O. T. Schmidt, A. Towne, G. Rigas, T. Colonius, and G. A. Brés. Spectral analysis of jet turbulence. Journal of Fluid Mechanics, 855:953–982, 2018. - - - - - -A. S. Sharma and B. J. Mckeon. On coherent structure in wall turbulence. Journal of Fluid Mechanics, 728:196–238, 2013. - - - - - -A. Sinha, D. Rodriguez, A. B. Guillaume, and T. Colonius. Wavepacket models for supersonic jet noise. Journal of Fluid Mechanics, 742:71–95, 2014. - - - - - -L. Sirovich. Turbulence and the dynamics of coherent structures, Parts I-III. Quarterly of Applied Mathematics, 45(3):561–590, 1987. - - - - - -L. F.de Souza, R. F. Miotto, and W. R. Wolf. Analysis of transient and intermittent flows using a multidimensional empirical mode decomposition. Theoretical and Computational Fluid Dynamics, 38:291-311, 2024. - - - - - -K. Taira, S. Brunton, S. T. M. Dawson, C. W. Rowley, T. Colonius, B. J. Mckeon, O. T. Schmidt, S. Gordeyev, V. Theofilis, and L. S. Ukeiley. Modal analysis of fluid flows: an overview. AIAA Journal, 55:4013–4041, 2017. - - - - - -V. Theofilis. Global linear instability. Annual Review of Fluid Mechanics, 43(1):319–352, 2011. - - - - - -A. Towne, O. T. Schmidt, and T. Colonius. Spectral proper orthogonal decomposition and its relationship to dynamic mode decomposition and resolvent analysis. Journal of Fluid Mechanics, 847:821–867, 2018. - - - - - -L. N. Trefethen, A. E. Trefethen, S. C. Redyy, and T. A. Driscoll. Hydrodynamic stability without eigenvalues. Science, 261(5121):578–584, 1993. - - - - - -J. E. Williams and B. C. Zhao. The active control of vortex shedding. Journal of Fluids and Structures, 3:115–122, 1989. - - - - - -M. O. Williams, I. G. Kevrekidis, and C. W. Rowley. A data-driven approximation of the Koopman operator: extending dynamic mode decomposition. Journal of Nonlinear Science, 25:1307–1346, 2015. - diff --git a/src/CCD_analysis/PIPELINE.md b/src/CCD_analysis/PIPELINE.md deleted file mode 100644 index faeccc9..0000000 --- a/src/CCD_analysis/PIPELINE.md +++ /dev/null @@ -1,164 +0,0 @@ -# CCD Analysis Pipeline - -> Correction-field CCD analysis for fluidic pinball DRL control. -> Core question: does `dq_ctl` (what the controller adds) match `dq_tar` (what the target requires)? - -## Quick Start - -```bash -cd src/CCD_analysis - -# Panorama comparison figure (primary output) -conda run -n pycuda_3_10 python3 correction_analysis/compare_dqctl_scenes.py - -# CCD quantitative decomposition -conda run -n pycuda_3_10 python3 correction_analysis/decompose_corrections.py - -# Single-scene zone diagnostics -conda run -n pycuda_3_10 python3 correction_analysis/diagnose_corrections.py -``` - -## Pipeline Architecture - -``` -[Data Collection] [Phase Alignment] [Correction Fields] [Analysis] -scripts/collect_*.py → detect_period.py → compute_correction_ → compare_dqctl_scenes.py - (GPU, device 2) replay_fields.py fields.py decompose_corrections.py - (CPU/GPU) (CPU) diagnose_corrections.py - (CPU) -``` - -## Dual-clock field sampling - -Legacy DRL 的控制时钟与 DDF 保存时钟必须独立：模型仍只在固定控制周期边界读取完整 observation 和预测动作，DDF 可以在周期内任意 lattice step 读取。通用 API、OID 复用方式、输出时间轴合同和 GPU A/B 结果见 [`DUAL_CLOCK_SAMPLING.md`](DUAL_CLOCK_SAMPLING.md)。 - -## Key Conventions - -### Geometry (unified 2026-06-25) -- **All scenes**: pinball center ≈ 613 px, sensors at 800 px (40×L0) -- Collected at source — no post-processing translation needed - -### Correction Fields -- **`dq_blk = q_blk − q_in`**: pinball blockage (passive) -- **`dq_ctl = q_ctl − q_blk`**: control correction (active) — **primary analysis object** -- **`dq_tar = q_tar − q_blk`**: target correction (theoretical) -- Core question: **O(dq_ctl, dq_tar)** — how well does control match theory? - -### Observation Normalization -- **Force-first**: `obs = [forces/force_norm, sensors/sens_norm]` -- Each scene computes its own `force_norm_fact`, `sens_deviation`, `sens_norm_fact` during collection -- Same norm values MUST be used during inference - -### Reynolds Number -- Code Re uses reference length 2D = 40: `Re = U0×40/ν` -- Physical Re_D uses D = 20: `Re_D = Re/2` -- Default: Re=100 → Re_D=50, nu=0.004 - -### Inlet -- Parabolic velocity profile (not uniform) -- Top/bottom walls: no-slip bounce-back -- U0 = 0.01 (centerline, lattice units) - -### Vortex Bug History (2026-06-29) -Three bugs in `collect_vortex.py` caused incorrect Lamb data: -1. **Cylinder order**: add order is front→TOP(+y)→BOTTOM(−y); reversed caused wrong bias mapping -2. **Obs swap**: used `[sensors/force_norm, forces/sens_norm]` instead of force-first `[forces/force_norm, sensors/sens_norm]` -3. **Missing fade-in/out**: 25-step transition from steady-cloak bias to PPO action required - -See `collect_vortex.py` header and `ccd_knowledge.md` §12 for full details. - -## Results Index - -All figures in `results/figures/`; CCD JSON in `results/ccd/`. - -### Panorama (main deliverable) - -| # | Figure | Content | -|---|--------|---------| -| 01 | `01_panorama_all_scenes.png` | 7 scenes × 4 quantities (ux_mean, uy_mean, RMS, vorticity) | -| 02 | `02_cloak_comparison.png` | 4 cloak scenes (steady, karman, vortex_lamb, vortex_taylor) | -| 03 | `03_illusion_comparison.png` | 3 illusion scenes (0.75L, 1.0L, 1.5L) | - -### Per-Scene dq_ctl vs dq_tar - -| # | Figure | Scene | -|---|--------|-------| -| 04 | `04_steady_cloak_cancel.png` | Steady cloak cancellation test | -| 05 | `05_illusion_075L_ctl_vs_tar.png` | Illusion 0.75L | -| 06 | `06_illusion_10L_ctl_vs_tar.png` | Illusion 1.0L | -| 07 | `07_illusion_15L_ctl_vs_tar.png` | Illusion 1.5L | -| 08 | `08_karman_ctl_vs_tar.png` | Karman cloak re100 | -| 09 | `09_vortex_lamb_ctl_vs_tar.png` | Vortex Lamb | -| 10 | `10_vortex_taylor_ctl_vs_tar.png` | Vortex Taylor | - -### Vortex Diagnosis - -| # | Figure | Content | -|---|--------|---------| -| 11 | `11_vortex_lamb_diagnosis.png` | Lamb sensor + action comparison | -| 12 | `12_vortex_taylor_diagnosis.png` | Taylor sensor + action comparison | -| 13 | `13_vortex_lamb_vorticity.png` | Lamb vorticity field evolution | -| 14 | `14_vortex_taylor_vorticity.png` | Taylor vorticity field evolution | -| 15 | `15_vortex_lamb_target_vorticity.png` | Lamb target (no pinball) reference | -| 16 | `16_vortex_taylor_target_vorticity.png` | Taylor target (no pinball) reference | - -### Key Numerical Results - -**Correction-field CCD (2026-06-28, unified geometry)** - -| Metric | 0.75L | 1.0L | 1.5L | -|--------|:-----:|:----:|:----:| -| O(dq_ctl, dq_tar) mode1 (r=6) | 0.383 | **0.926** | **0.922** | -| O(dq_ctl, dq_tar) mode1 (r=10) | 0.320 | 0.684 | 0.661 | -| Force-CCD m80 (r=6) | 2 | 2 | 1 | -| Action sigma1 (r=6) | 1.49 | 1.17 | **0.20** | - -**Cloak dq_ctl RMS (cropped x=[300,1100])** - -| Scene | RMS | Type | -|-------|:---:|------| -| steady_cloak | 0.196 | Steady, open-loop | -| karman_re100 | 0.397 | Periodic, PPO closed-loop | -| vortex_lamb | 0.146 | Transient, PPO closed-loop | -| vortex_taylor | 0.188 | Transient, PPO closed-loop | - -**Key findings:** -- Cloak mechanism is **independent of upstream condition** (steady/vortex street/transient vortex all share the same dq_ctl structure) -- Illusion 1.0L achieves near-perfect overlap (O=0.926) via "cloak physics + target frequency modulation" -- Illusion 1.5L uses a fundamentally different mechanism (high-freq modulation, action sigma1=0.20 vs 1.17-1.49) -- 0.75L overlap dropped from 0.564→0.383 after fixing geometry alignment — old number was inflated - -### CCD Quantitative (JSON) - -| File | Content | -|------|---------| -| `results/ccd/correction_ccd_results.json` | Force/Action-CCD per scene (r=6,8,10), O(dq_ctl,dq_tar) per mode | -| `results/ccd/zone_metrics.json` | Per-zone KE and enstrophy | - -## Full Documentation - -| File | Content | -|------|---------| -| `PIPELINE.md` | This file — pipeline overview, conventions, results index | -| `README.md` | Quick start | -| `ccd_knowledge.md` | Complete knowledge base (theory, methodology, detailed results, bug history) | -| `Lyu23.md` | CCD method paper (Lyu 2023) | - -## Adding New Training Results - -When new DRL models are trained (e.g. on new CelerisLab solver): - -1. **Collect fields**: Run the appropriate `scripts/collect_*.py` with the new model path -2. **Phase alignment**: `detect_period.py` → `replay_fields.py` (for periodic scenes) -3. **Correction fields**: Already handled by `compute_correction_fields.py` — just ensure the scene name is registered in `configs.py` `_SCENE_MAP` -4. **Regenerate figures**: `compare_dqctl_scenes.py` and `diagnose_corrections.py` -5. **Regenerate CCD**: `decompose_corrections.py` -6. **Symlink**: Add new figures to `results/figures/` - -The `configs.py` `SCENES` dict is the single source of truth — add new scenes there and all analysis scripts automatically pick them up. - -## Environment - -- **Conda**: `pycuda_3_10` -- **GPU**: Device 2 (check with `nvidia-smi` before collection) -- **CPU-only steps**: POD, CCD, analysis scripts diff --git a/src/CCD_analysis/README.md b/src/CCD_analysis/README.md index f434163..804f525 100644 --- a/src/CCD_analysis/README.md +++ b/src/CCD_analysis/README.md @@ -1,57 +1,114 @@ -# CCD_analysis: Correction-Field CCD Pipeline +# CCD analysis: Kármán dynamic-increment campaign -Analyzes DRL-controlled fluidic pinball using **correction-field decomposition** + **Canonical Correlation Decomposition (CCD/Lyu23)**. Core question: does `dq_ctl` (what the controller adds) match `dq_tar` (what the target requires)? +This is the active authority. It supersedes the Illusion-next-entry checkpoint. The unique scientific question is the Legacy `karman_re100` (code Re100, physical `Re_D=50`) increment of time-varying DRL over constant control fixed to the same fresh DRL run's retained, three-channel `effective_applied_action` mean. `zero` is only the passive baseline; `target` only defines cloaking error. Neither is the dynamic CCD subtraction. -## Start Here +The frozen identity is `q_D(phi)-q_C(phi) = (mean(q_D)-mean(q_C)) + [(q_D(phi)-mean(q_D))-(q_C(phi)-mean(q_C))]`. The first term is a policy-induced statistical mean change; the second is phase-coherent unsteady change. Independent phase-conditioned trajectories are not pointwise counterfactuals, response measurements, or causal effects. Historical acquisition/direct-dq/Q=1 results below remain immutable evidence, but are superseded as the next execution direction. -**→ [`PIPELINE.md`](PIPELINE.md)** — pipeline overview, results index, conventions, new training integration guide. +`karman_dynamic/` is the campaign-specific schema and launcher. Roles are `target`, `zero`, `drl`, `constant_mean`; execution is DRL first, then constant_mean, target, zero. It wraps the unchanged active schema-v3 acquisition payload and reuses its exact clocks, q/U0 decoder, solver mask, requested/effective actions, and fresh role runtime. Campaign telemetry explicitly stores center-sensor `uy=sensors[:,3]`. Constant mean is hash-bound to the fresh DRL wrapper and retained effective actions, without hand entry or symmetry forcing. -**→ [`DUAL_CLOCK_SAMPLING.md`](DUAL_CLOCK_SAMPLING.md)** — 在固定 DRL 控制周期内按独立时钟读取/保存 DDF；CCD 与 OID 共用。 +CFD execution requires `CONDA_DEFAULT_ENV=pycuda_3_10`, exactly one `CUDA_VISIBLE_DEVICES` token, CPU PPO inference, an exclusive campaign lease, a stable exact Optane symlink, fresh no-clobber roots, semantic reload after each child, and at least 30 s between starts (default 120 s). Failure quarantines the campaign and stops later roles. Do not run real CFD from `pinball_math`. -## Quick Commands +Current status: all four production roles were fresh-loaded. Immutable phase publications exist for every role at sibling `ROLE-phase-compact-v1` paths. DRL, constant_mean, and target pass; zero fails closed (period CV `0.0828308 > 0.05`, amplitude CV `0.101389 > 0.10`) and therefore publishes metrics only, with no compact fields. DRL–constant phase differences remain authorized because those two independent gates pass; zero phase-target metrics are unavailable. The immutable four-role result is `data/karman-dynamic/karman-dynamic-v1-production/dynamic-increment-v2`. Mean target errors are zero `0.2177701`, constant_mean `0.1680192`, and DRL `0.1599946`; corresponding target-error reductions are `0.0497509` and `0.00802465`. Dense `payload/fields.npz` remains intact for all roles and deletion is not yet authorized because zero has no sufficient compact replacement. The earlier `dynamic-increment-v1` is immutable partial-stage evidence superseded by v2. + + +The authoritative primary temporal result is `data/karman-dynamic/karman-dynamic-v1-production/drl-temporal-negative-lag-ccd-v1`. It uses full-resolution mask-compressed `q_DRL(t)-mean(q_DRL)` and exact same-boundary three-channel effective-action fluctuations at 800-step cadence, with complete phase cycles as non-crossing blocks. The predeclared grid is `tau/800 = -17,...,0` (`Q=18`, about one measured shedding period), giving `N=21` complete columns and `M=1,299,184`. Leading cross-correlation strengths are `2.77997e-3`, `1.91480e-5`, and `1.32484e-7`; mode-1 left-lag energy peaks at `tau/800=-6` and has centroid `-7.969`, while modes 2/3 peak at `0/-5`. On fixed common support, dropping the oldest one/two lags changes the first three strengths by at most `1.58%`, `4.33%`, and `6.06%`, with leading-three weighted-subspace principal cosines at least `0.9999987`. Native endpoint support grows to `N=40/59` and substantially changes strengths and the third leading subspace direction, so those native-window comparisons are sample-support-sensitive and are not interpreted as timing evidence. This is closed-loop temporal co-variation only, with no causal or response-time claim. + +The final deterministic artifact-only publication is `data/karman-dynamic/karman-dynamic-v1-production/publication-v2`. It contains six concise PNG/PDF figure pairs plus hash-bound JSON/Markdown, generated after fresh live-provenance reloads and essential recomputation. `publication-v1` is immutable superseded pre-review evidence; final review found that its first panel omitted the target reference, and the single remediation added the explicit fourth role in v2. No figure reports zero phase results. + +Dense-field deletion decision: **RETAIN ALL FOUR ROLE SOURCES**. DRL dense fields remain required to independently recompute the temporal CCD and its provenance; constant_mean and target dense fields remain required to independently recompute their mean/statistical products from source; zero failed the phase gate and has no compact field artifact, so its dense source is the only source-level basis for its accepted mean/statistics. The compact products suffice for currently published downstream figures, but not for every source-level downstream recomputation/provenance contract; therefore deletion is not independently justified. + +The exploratory phase-domain CCD is complete at `data/karman-dynamic/karman-dynamic-v1-production/drl-constant-phase-domain-ccd-v1` and is **DOWNGRADED**, not retained as a stable three-direction result. It uses only the separately centered phase-coherent difference `Δq′_phase(φ)=(q_DRL(φ)-mean(q_DRL))-(q_constant_mean(φ)-mean(q_constant_mean))` and the DRL phase-conditioned effective-action fluctuation in the literal weighted Lyu `Q=1` operator. The 10-bin strengths are `0.0420404`, `0.0157939`, and `0.00414469` (rank 3). The 8/12-bin and half-bin-origin projectors are stable (minimum leading-three cosine `0.995038`), and left functions are stable (minimum absolute cosine `0.999867` across those bin tests), but first-harmonic truncation has rank 2 rather than 3; therefore rank stability fails. Circular offsets change strengths but are phase offsets only, never time-response lags. The immutable artifact is published because its scientific contract and downgrade boundary are explicit; it supports only low-order exploratory circular co-variation. + +Recommended smoke plan command (prints fresh child command; no CFD): ```bash -cd src/CCD_analysis - -# Panorama comparison (main output) -conda run -n pycuda_3_10 python3 correction_analysis/compare_dqctl_scenes.py - -# CCD quantitative analysis -conda run -n pycuda_3_10 python3 correction_analysis/decompose_corrections.py - -# Zone diagnostics -conda run -n pycuda_3_10 python3 correction_analysis/diagnose_corrections.py +PYTHONPATH="$PWD/src" conda run -n pinball_math python -m CCD_analysis.karman_dynamic orchestrate --campaign-id karman-dynamic-v1 --root src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-smoke --warmup-intervals 1 --collect-boundaries 2 --launch-delay-seconds 120 --smoke ``` -## Directory +Recommended exact DRL smoke execution command (real CUDA CFD; run only after Optane mapping/lease preflight): -``` -CCD_analysis/ - README.md # This file - PIPELINE.md # Primary entry — overview, results, conventions - ccd_knowledge.md # Full knowledge base (theory, methods, bug history) - Lyu23.md # CCD method paper - configs.py # Scene registry (single source of truth) - results/ # Canonical outputs - figures/ # Symlinks to all core figures (numbered) - ccd/ # Final CCD JSON results - correction_analysis/ # All analysis scripts - scripts/ # GPU data collection + phase alignment - utils/ # Core algorithms (POD, CCD, field loading) - data/ # Raw data + generated figures - ccd/ # Round 5 frozen baseline (do not modify) +```bash +CONDA_DEFAULT_ENV=pycuda_3_10 CUDA_VISIBLE_DEVICES=0 PYTHONPATH="$PWD/src" python -m CCD_analysis.karman_dynamic role --campaign-id karman-dynamic-v1 --role drl --output src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-smoke/drl --warmup-intervals 1 --collect-boundaries 2 --launch-delay-seconds 120 --smoke ``` -## Key Documentation +## Superseded historical authority (immutable evidence) -| File | Content | -|------|---------| -| `PIPELINE.md` | **Primary entry** — pipeline, convention, results index | -| `ccd_knowledge.md` | Complete knowledge base (509 lines) | -| `results/figures/` | All core figures with numerical prefix ordering | +This is the active authoritative surface. Historical material under `archive/2026-08-03-pre-three-part-reset/payload/` is immutable, non-authoritative, and never imported by active code. -## Environment +## Acquisition status +`acquisition/` now defines CPU-import-safe contract/artifact schema v3 for exactly `karman_re100` and `illusion_1.0L`, each with roles `q_target`, `q_blk`, and `q_ctl`. It includes frozen geometry, signed front/upper/lower action identities, exact absolute lattice clocks, same-step telemetry, solver-derived masks, full-grid coordinates/velocity fields, controller/history identity, no-clobber publication, and fresh-process sequential orchestration. + +Kármán's code label Re100 uses the historical `2D` reference and is physically `Re_D=50`. Its front/rear/sensor x locations are 30/31.3/40 D. Illusion uses the strict +11D deployment geometry: front/rear/sensors/target x = 30/31.3/41/31 D, rear y = ±0.75 D, sensors y = +2/0/-2 D. This Illusion deployment differs from training geometry; replay/history smoke is mandatory before production. + +Velocity fields are the Legacy solver's nondimensional velocity `q/U0`, decoded exactly on solver-flagged fluid cells as `ux=(f1+f5+f8-f3-f6-f7)/u0` and `uy=(f2+f5+f6-f4-f7-f8)/u0`. This is not momentum divided by density. Nonfluid cells are exact zero and their unused populations may be ignored. The decoder schema and formula hash are mandatory artifact identities. + +All six artifacts under `evidence/smoke-20260804/` were produced with the incorrect density-normalized decoder. They are withdrawn, invalid under the current schema, and retained only as immutable negative evidence; they must not be overwritten. Those v2 artifacts are also structurally obsolete because they lack complete control-boundary lineage. The corrected schema-v3 smoke and the two 450-control three-role production acquisitions have completed. Schema v3 persists the exact pre-action `(150,12)` FIFO, every interval-average `(control_count,12)` boundary observation independently of field cadence, every policy source/input observation and source hash, zero-origin policy harmonic phase indices, and complete requested-action control histories. Current production roots are `evidence/production-20260804-q-over-u0-v3-karman-450-fi2000/` and `evidence/production-20260804-q-over-u0-v3-illusion-authorized-450-fi1250/`. + +`q_target` is a desired reference generated with different bodies. It is not a same-checkpoint counterfactual. Equal absolute time establishes same-time sampling only; `phase_reference` is evidence for later phase validation and does not itself prove same phase. + +### Commands + +CPU tests (no CFD): + +```bash +PYTHONPATH="$PWD/src" conda run -n pinball_math python -m pytest src/CCD_analysis/tests/test_acquisition.py -q ``` -conda run -n pycuda_3_10 + +Inspect the sequential fresh-process command plan without running CFD: + +```bash +PYTHONPATH="$PWD/src" conda run -n pinball_math python -m CCD_analysis.acquisition orchestrate --case karman_re100 --output /fresh/path --control-count 450 --field-interval 2000 ``` + +CFD entry points require `CONDA_DEFAULT_ENV=pycuda_3_10` and fail otherwise. Do not use `conda run` if it does not propagate that variable correctly. The first independent gate failed on its no-op runner/EMA lifecycle, and the second re-review failed on exact initialization and controller-reference semantics. Both sets are now remediated. Initialization is explicitly `stabilize → full current/temp/lifecycle checkpoint → zero-action normalization trajectory → exact restore → FIFO warmup`, with zero post-stabilization EMA restored before warmup. Karman's first PPO input is exactly zero. Illusion PPO consumes only the frozen training normalization (`9ec5dd…`) and two target-force harmonics (`f13566…`) from the frozen Illusion training-reference files; newly measured +11D eight-channel harmonics are phase evidence only, never controller input. The deployment/reference mismatch still requires the strict +11D compatibility replay/history smoke. The third review then failed only because solver, rollout, and policy-phase clocks were conflated. They are now explicit: solver-absolute lattice/control lineage comes from the public solver accessor; acquisition-relative lattice/control starts at zero; policy harmonic phase independently starts at zero. `lattice_steps`/`sample_ids` are solver-absolute, while `acquisition_relative_lattice_steps`, rollout-relative `control_indices`, and solver-absolute control indices are persisted separately. A final static review found that snapshots called the boundary-only clock API during an active split. Legacy now has a distinct `active_step_clock_state()` valid only after at least one completed split step; snapshots use it, while initialization and post-interval checks retain boundary-only `solver_clock_state()`. The corrected q/U0 decoder, complete lineage contract, and explicit orchestration schedule passed fresh CUDA smoke before the two production acquisitions. Kármán used 450 controls with field interval 2000; Illusion used 450 controls with field interval 1250. These runs establish successful acquisition and exact artifact lineage, not physical-phase equality or a mechanism claim. + +## Direct-dq status + +`direct_dq/` is the completed CPU-only strict same-time analysis core. It accepts three explicit completed active acquisition artifact directories, revalidates each manifest plus file/config/state-array hashes, and requires exact case/role/schema, finite float32 `(time,x,y)` fields, exact float32 coordinates, a common grid, and exact full acquisition-relative timelines before selection. Solver-absolute origins may differ by role when each lineage is internally valid. A required exclusive `--start-after-relative-step` burn-in bound and optional inclusive `--end-at-relative-step` select samples by exact integer physical-step inequalities only; no boundary lookup, nearest match, or index trimming is allowed. A supplied end must be the terminal selected sample, and the interval must be nonempty. There is no trimming, nearest-time/station substitution, phase guessing, crop, translation, or coordinate-generated mask. + +Only selected role columns enter the analysis. Results preserve the original common timeline, exact selection bounds, selected original indices, selected steps, and selected count. The transparent analysis domain is the intersection of the three preserved solver-derived fluid masks. Outputs include instantaneous and full-resolution time-mean `e_target`, `dq_ctl`, and `dq_tar`; all three role means; exact-station streamwise profiles; weighted vector RMS target error; signed target-relative streamwise deficit; an explicitly incomplete momentum-flux proxy; positive-deficit wake area/centroid/width; mask-aware nonuniform-coordinate vorticity; and declared nested prefix/suffix convergence diagnostics. `dq_ctl` and `dq_tar` share `-q_blk`, so agreement is not mechanism evidence. Phase-conditioned output fails closed because cross-role physical-phase equality has not been independently proven. Prefix/suffix windows are not independent-realization uncertainty. + +Run on completed artifacts in `pinball_math` (repeat station/window options as needed; station tokens are preserved and canonically converted to exact float32 grid values without nearest/tolerance matching; the final window must equal the full sample count): + +```bash +PYTHONPATH="$PWD/src" conda run -n pinball_math python -m CCD_analysis.direct_dq \ + --case karman_re100 --q-target /path/q_target --q-blk /path/q_blk --q-ctl /path/q_ctl \ + --output /fresh/result --start-after-relative-step 120000 \ + --station-x-D 35 --station-x-D 40 --station-x-D 45 \ + --window-size 30 --window-size 60 --window-size 90 --window-size 120 +``` + +For the completed Illusion acquisition, use burn step `90000`, stations `35,40,45`, and windows `36,72,108,144`. The canonical production grids contain each station exactly once as float32. Convergence windows are validated against the selected count. + +Results are immutable/no-clobber atomic directories containing `arrays.npz`, `summary.json`, `config.json`, `input_hashes.json`, and a hash manifest. Provenance-validated `load_result()` always rereads all three acquisition directories at their recorded absolute paths, revalidates their complete semantics and hashes, and requires their masks/grid/full timeline to equal the persisted result and each selected instantaneous role array to equal the exact selected live input columns. Therefore results are not portable by themselves: moving, deleting, or changing an acquisition directory makes provenance validation fail closed. `load_result_metadata_unverified()` is explicitly metadata/internal-science-only and cannot support a provenance claim; publication and CLI never use it as success validation. The completed real results are `evidence/direct-dq-karman-burn120000/` (120 retained fields) and `evidence/direct-dq-illusion-authorized-burn90000/` (144 retained fields). Their summaries report weighted vector-RMS target errors of 0.1596571602 and 0.08031177071338107, respectively. These are direct same-time estimands after the declared burn-in selections; they are not phase-conditioned, independent-realization uncertainty, or causal mechanism evidence. Tests additionally use synthetic artifacts written through the active acquisition writer. + +## Original CCD production status + +`original_ccd/ORIGINAL_CCD_MATH.md` is the active mathematical contract for the original full-field Lyu CCD. The public CPU package now implements literal full-field `U`, lag-stacked `P`, `A = P U†/(N sqrt(LQ))`, direct rectangular SVD, physical diagonal or dense complex-HPD weighting, separately declared `U`/`P` centering, exact timestamp/block lag construction, weighted modes and physical-amplitude coefficients, and selected-mode reconstruction/residuals. There is no POD pre-reduction, whitening, row standardization, nearest-time matching, or implicit centering. + +`build_lagged_observables()` preserves exact admitted field-column indices, and `fit()` applies that mapping to `U`. Reconstruction is the weighted projection of field snapshots onto selected CCD modes, not observable prediction. Exactly degenerate singular blocks identify subspaces/projectors rather than unique individual modes; the optional deterministic phase convention does not resolve that non-uniqueness. See `original_ccd/README.md` for API usage. + +Production tests compare against the private literal derivation reference for hand, random real/complex, diagonal/dense weighted, centered, multiobservable, and exact-lag cases; verify singular equations, chunk invariance, weighted orthogonality, full supported-basis reconstruction, truncation residual/projector behavior, validation failures, and a smaller Lyu equations (3.1)-(3.2) production case. The exact published-scale stochastic derivation test remains in the full suite and is not redundantly rerun in a second production test. The production API tests run no CFD. The real Kármán CCD result is reported below; Illusion CCD was not run. No observable prediction, causal result, or CCD-versus-POD claim is included. + +## Real-case CCD contract status + +`original_ccd/REAL_CASE_CCD_CONTRACT.md` freezes the first real-data estimand for both cases: centered full-resolution `dq_ctl` on the authoritative solver-mask intersection; centered q_ctl `effective_applied_action` front/upper/lower channels at each exact field time in native units; and the literal weighted original CCD with `Q=1`, `tau=0`. It fixes component-major mask flattening, coordinate quadrature, mean-action algebra, prohibited preprocessing, provenance validation, OOM-safe passes, immutable result requirements, and claim limits. The active `real_ccd/` package implements mandatory direct-dq/live-acquisition provenance validation, exact q_ctl field-time effective actions, component-major mask compression, explicit two-sided centering and coordinate weighting, a three-pass 3-by-M thin-SVD decomposition, streamed coefficients/block residuals, conservative fail-closed RAM/scratch accounting, an immutable verified result loader, and a CPU-only `preflight`/`run` CLI with explicit direct-dq root, chunk size, safe host admission budget, fresh output, and no-clobber publication. + +The authoritative Kármán Q=1, tau=0 decomposition is complete at `evidence/real-ccd-karman-q1-tau0-burn120000-v1/`, using the immutable `evidence/direct-dq-karman-burn120000/` input, chunk size 8, 120 samples, and 1,299,184 spatial degrees of freedom. Singular values (cross-correlation strengths) are `[0.14676751183519246, 0.05920814023903697, 0.0063884442355860915]`; their squares are `[0.02154070253029336, 0.003505603870565469, 0.00004081221975119316]`. The numerical rank is 3 with no degenerate blocks, and weighted relative residuals after complete blocks 1/2/3 are `[0.7823805118116768, 0.4698397615223296, 0.45542496362972573]`. These values are not field energy, explained variance, or canonical coefficients. The fresh-process loader passed live provenance and essential-identity recomputation. Illusion CCD was not run and is not authorized by this completion; the unique next entry is a user decision on `real-ccd-illusion` authorization. + +The Kármán-only figure subset is complete at `evidence/real-ccd-karman-figures-v1/`: full-resolution PNG/PDF spectrum, authoritative mean and three physical modes, zero-lag action left vectors, exact-step coefficients, complete-block residuals, and on-demand rank reconstructions at relative steps 122000/242000/360000. The two-case figures todo remains `in_progress`: Illusion figures await the same user authorization decision as Illusion CCD, which remains the unique next entry. + +Run CPU tests (no CFD): + +```bash +PYTHONPATH="$PWD/src" conda run -n pinball_math python -m pytest src/CCD_analysis/tests -q +``` + +## Other active parts + +- `original_ccd/` — production original full-field CCD API and its frozen mathematical contract. +- `tests/` — active CPU tests only. +- `evidence/` — machine-readable review evidence. + +`EXECUTION_CHECKPOINT.json` is the authoritative resumable execution state. diff --git a/src/CCD_analysis/__init__.py b/src/CCD_analysis/__init__.py new file mode 100644 index 0000000..be2d73e --- /dev/null +++ b/src/CCD_analysis/__init__.py @@ -0,0 +1,5 @@ +"""Active three-part CCD analysis surface. + +Historical implementations are isolated under ``archive/`` and are not part of +this package's import graph. +""" diff --git a/src/CCD_analysis/acquisition/__init__.py b/src/CCD_analysis/acquisition/__init__.py new file mode 100644 index 0000000..f44e987 --- /dev/null +++ b/src/CCD_analysis/acquisition/__init__.py @@ -0,0 +1,7 @@ +"""CPU-safe exact-time acquisition contract package.""" +from .contracts import CASES, ROLES, case_snapshot +from .dual_clock import DualClockCollector, ExactTelemetry, field_steps, solver_fluid_mask +from .runtime import preflight, role_spec, run_role_acquisition +from .solver_state import copy_ping_pong_ddf, d2q9_q_over_u0_xy +from .validation import validate_acquisition_semantics +__all__=["CASES","ROLES","case_snapshot","DualClockCollector","ExactTelemetry","field_steps","solver_fluid_mask","preflight","role_spec","run_role_acquisition","copy_ping_pong_ddf","d2q9_q_over_u0_xy","validate_acquisition_semantics"] diff --git a/src/CCD_analysis/acquisition/__main__.py b/src/CCD_analysis/acquisition/__main__.py new file mode 100644 index 0000000..13be635 --- /dev/null +++ b/src/CCD_analysis/acquisition/__main__.py @@ -0,0 +1,2 @@ +from .cli import main +raise SystemExit(main()) diff --git a/src/CCD_analysis/acquisition/artifacts.py b/src/CCD_analysis/acquisition/artifacts.py new file mode 100644 index 0000000..d6f1dd8 --- /dev/null +++ b/src/CCD_analysis/acquisition/artifacts.py @@ -0,0 +1,99 @@ +"""Immutable, fsync-backed acquisition artifact transactions.""" +from __future__ import annotations +import ctypes +import errno +from hashlib import sha256 +import json, os, shutil, uuid +from pathlib import Path +from typing import Mapping, Any +import numpy as np +from .contracts import ARTIFACT_SCHEMA_ID, canonical_json +from .validation import array_sha256, validate_acquisition_semantics + + +def file_sha256(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def rename_noreplace(source: Path, destination: Path) -> None: + """Atomically publish a directory without replacing any existing inode.""" + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if renameat2 is None: + raise RuntimeError("atomic no-replace publication unavailable: libc renameat2 is missing") + renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + renameat2.restype = ctypes.c_int + result = renameat2(-100, os.fsencode(source), -100, os.fsencode(destination), 1) + if result == 0: + return + code = ctypes.get_errno() + if code in (errno.EEXIST, errno.ENOTEMPTY): + raise FileExistsError(destination) + if code in (errno.ENOSYS, errno.EINVAL, errno.ENOTSUP): + raise RuntimeError("atomic no-replace publication unavailable; refusing unsafe fallback") from OSError(code, os.strerror(code)) + raise OSError(code, os.strerror(code), destination) + + +def _hash_array(value: np.ndarray) -> str: + return sha256(np.ascontiguousarray(value).tobytes()).hexdigest() + + +def _fsync_file(path: Path) -> None: + with path.open("rb") as stream: + os.fsync(stream.fileno()) + + +def _validate_sha(value: np.ndarray, key: str) -> None: + if value.ndim != 0 or value.dtype.kind not in "SU": raise ValueError(f"{key} must be a scalar string") + text = str(value.item()) + if len(text) != 64: raise ValueError(f"{key} must be SHA256") + int(text, 16) + + +class ArtifactTransaction: + def __init__(self, destination: str | Path): + self.destination = Path(destination) + self.partial = self.destination.with_name(f".{self.destination.name}.partial.{os.getpid()}.{uuid.uuid4().hex}") + self.active = False + + def __enter__(self): + if self.destination.exists(): raise FileExistsError(self.destination) + self.destination.parent.mkdir(parents=True, exist_ok=True) + self.partial.mkdir(); self.active = True + return self + + def write(self, *, arrays: Mapping[str, Any], config: dict, state: Mapping[str, Any]): + if not self.active: raise RuntimeError("transaction inactive") + data, state_arrays = validate_acquisition_semantics(arrays=arrays, config=config, state=state) + np.savez_compressed(self.partial/"fields.npz", **data) + (self.partial/"config.json").write_bytes(canonical_json(config)) + np.savez_compressed(self.partial/"controller_state.npz", **state_arrays) + for path in self.partial.iterdir(): + if path.is_file(): _fsync_file(path) + files = {path.name:file_sha256(path) for path in sorted(self.partial.iterdir()) if path.is_file()} + manifest = {"schema_id":ARTIFACT_SCHEMA_ID,"complete":True,"files":files,"state_array_sha256":{key:array_sha256(value) for key,value in state_arrays.items()},"config_sha256":sha256(canonical_json(config)).hexdigest(),"field_count":data["ux"].shape[0]} + validate_acquisition_semantics(arrays=data, config=config, state=state_arrays, manifest=manifest) + (self.partial/"manifest.json").write_bytes(canonical_json(manifest)); _fsync_file(self.partial/"manifest.json") + return manifest + + def publish(self): + manifest = json.loads((self.partial/"manifest.json").read_text()) + if not manifest.get("complete") or manifest.get("schema_id") != ARTIFACT_SCHEMA_ID: raise RuntimeError("partial artifact is not complete") + for name, digest in manifest["files"].items(): + if file_sha256(self.partial/name) != digest: raise RuntimeError("artifact hash validation failed") + with np.load(self.partial/"fields.npz", allow_pickle=False) as fields, np.load(self.partial/"controller_state.npz", allow_pickle=False) as state: + config=json.loads((self.partial/"config.json").read_text()) + validate_acquisition_semantics(arrays={key:fields[key] for key in fields.files},config=config,state={key:state[key] for key in state.files},manifest=manifest) + rename_noreplace(self.partial, self.destination) + directory_fd = os.open(self.destination.parent, os.O_RDONLY) + try: os.fsync(directory_fd) + finally: os.close(directory_fd) + self.active = False + return self.destination + + def __exit__(self, typ, value, tb): + if self.active: shutil.rmtree(self.partial, ignore_errors=True); self.active=False diff --git a/src/CCD_analysis/acquisition/cli.py b/src/CCD_analysis/acquisition/cli.py new file mode 100644 index 0000000..981fd2e --- /dev/null +++ b/src/CCD_analysis/acquisition/cli.py @@ -0,0 +1,58 @@ +"""Acquisition CLI and fresh-process sequential role orchestration.""" +from __future__ import annotations +import argparse, os, subprocess, sys +from pathlib import Path +from .contracts import CASES, ROLES, case_snapshot +from .compatibility import publish_certificate, validate_certificate +from .runtime import preflight, require_cfd_environment, run_role_acquisition + +def _validate_schedule(case:str,control_count:int,field_interval:int)->tuple[int,...]: + if case not in CASES: raise ValueError("unsupported case") + if type(control_count) is not int or control_count < 1: + raise ValueError("control_count must be a positive integer") + if type(field_interval) is not int or field_interval < 1: + raise ValueError("field_interval must be a positive integer") + horizon = CASES[case].sample_interval * control_count + if horizon % field_interval: + raise ValueError("horizon must be exactly divisible by field_interval") + schedule = tuple(range(field_interval, horizon + 1, field_interval)) + if not schedule or schedule[-1] != horizon: + raise ValueError("field schedule must be nonempty and include the terminal step") + return schedule + + +def role_command(case:str,role:str,output:Path,*,control_count:int,field_interval:int)->list[str]: + _validate_schedule(case,control_count,field_interval) + if role not in ROLES: raise ValueError("unsupported role") + return [sys.executable,"-m","CCD_analysis.acquisition","role","--case",case,"--role",role,"--output",str(output),"--control-count",str(control_count),"--field-interval",str(field_interval)] + + +def orchestrate(case:str,output:Path,*,control_count:int,field_interval:int,run:bool=False,compatibility_certificate:Path|None=None)->list[list[str]]: + _validate_schedule(case,control_count,field_interval) + if output.exists(): raise FileExistsError(output) + if CASES[case].history_smoke_required: + if compatibility_certificate is None: raise ValueError("Illusion production requires --compatibility-certificate") + validate_certificate(compatibility_certificate) + commands=[role_command(case,r,output/r,control_count=control_count,field_interval=field_interval) for r in ROLES] + if run: + require_cfd_environment(); env=dict(os.environ) + for command in commands: subprocess.run(command,check=True,env=env) + return commands +def main(argv:list[str]|None=None)->int: + p=argparse.ArgumentParser(); sub=p.add_subparsers(dest="command",required=True) + role=sub.add_parser("role"); role.add_argument("--case",choices=CASES,required=True); role.add_argument("--role",choices=ROLES,required=True); role.add_argument("--output",type=Path,required=True); role.add_argument("--control-count",type=int,required=True); role.add_argument("--field-interval",type=int,required=True) + orch=sub.add_parser("orchestrate"); orch.add_argument("--case",choices=CASES,required=True); orch.add_argument("--output",type=Path,required=True); orch.add_argument("--control-count",type=int,required=True); orch.add_argument("--field-interval",type=int,required=True); orch.add_argument("--compatibility-certificate",type=Path); orch.add_argument("--execute",action="store_true") + cert=sub.add_parser("certify-illusion-pilot"); cert.add_argument("--pilot",type=Path,required=True); cert.add_argument("--output",type=Path,required=True) + check=sub.add_parser("preflight"); check.add_argument("--case",choices=CASES) + args=p.parse_args(argv) + if args.command=="preflight": + import json + report=preflight(args.case); print(json.dumps(report,indent=2)); return 0 if report["ready_for_independent_pre_cfd_gate"] else 2 + if args.command=="certify-illusion-pilot": + print(publish_certificate(args.pilot,args.output)); return 0 + if args.command=="orchestrate": + for command in orchestrate(args.case,args.output,control_count=args.control_count,field_interval=args.field_interval,run=args.execute,compatibility_certificate=args.compatibility_certificate): print(" ".join(command)) + return 0 + require_cfd_environment(); case_snapshot(args.case,args.role) + run_role_acquisition(case=args.case,role=args.role,output=args.output,control_count=args.control_count,field_interval=args.field_interval) + return 0 diff --git a/src/CCD_analysis/acquisition/compatibility.py b/src/CCD_analysis/acquisition/compatibility.py new file mode 100644 index 0000000..85e5d9b --- /dev/null +++ b/src/CCD_analysis/acquisition/compatibility.py @@ -0,0 +1,72 @@ +"""Fail-closed Illusion +11D policy compatibility certificates.""" +from __future__ import annotations +from hashlib import sha256 +import json +from pathlib import Path +from typing import Any +import numpy as np +from .artifacts import file_sha256, rename_noreplace +from .contracts import canonical_json +from CCD_analysis.direct_dq.io import load_acquisition_artifact + +COMPATIBILITY_SCHEMA_ID = "ccd-illusion-policy-compatibility/v1" +CONTRACT = { + "schema_id": COMPATIBILITY_SCHEMA_ID, + "case_id": "illusion_1.0L", + "role": "q_ctl", + "required_control_count": 150, + "required_field_interval": 600, + "required_policy_input_width": 14, + "normalized_action_absolute_limit": 1.0, + "physical_action_absolute_limit": 0.1, + "field_q_over_u0_absolute_limit": 5.0, + "raw_observation_absolute_limit": 5.0, + "policy_input_absolute_limit": 1.0, + "acceptance": "all active acquisition semantics pass; exact 150-control FIFO/policy lineage; all listed finite declared bounds pass", +} +CONTRACT_SHA256 = sha256(canonical_json(CONTRACT)).hexdigest() + +def evaluate(pilot: str | Path) -> dict[str, Any]: + root = Path(pilot).resolve() + artifact = load_acquisition_artifact(root, expected_case="illusion_1.0L", expected_role="q_ctl") + with np.load(root / "controller_state.npz", allow_pickle=False) as z: + state = {key: z[key] for key in z.files} + arrays = artifact.fields + checks = { + "control_count_exact": int(artifact.config["clock_domains"]["acquisition_relative_control_final"]) == 150, + "field_interval_exact": int(artifact.config["acquisition"]["field_interval"]) == 600, + "initial_fifo_shape_exact": state["initial_fifo_history"].shape == (150, 12), + "boundary_history_shape_exact": state["boundary_observation_history"].shape == (150, 12), + "policy_input_shape_exact": state["policy_input_observation_history"].shape == (150, 14), + "policy_source_hash_count_exact": state["policy_source_observation_sha256"].shape == (150,), + "normalized_action_bound": float(np.max(np.abs(state["requested_normalized_action_history"]))) <= 1.0, + "physical_action_bound": float(np.max(np.abs(state["requested_physical_action_history"]))) <= 0.1, + "field_bound": max(float(np.max(np.abs(arrays["ux"]))), float(np.max(np.abs(arrays["uy"])))) <= 5.0, + "raw_observation_bound": max(float(np.max(np.abs(state["initial_fifo_history"]))), float(np.max(np.abs(state["boundary_observation_history"])))) <= 5.0, + "policy_input_bound": float(np.max(np.abs(state["policy_input_observation_history"]))) <= 1.0, + } + metrics = { + "normalized_action_max_abs": float(np.max(np.abs(state["requested_normalized_action_history"]))), + "physical_action_max_abs": float(np.max(np.abs(state["requested_physical_action_history"]))), + "field_q_over_u0_max_abs": max(float(np.max(np.abs(arrays["ux"]))), float(np.max(np.abs(arrays["uy"])))), + "raw_observation_max_abs": max(float(np.max(np.abs(state["initial_fifo_history"]))), float(np.max(np.abs(state["boundary_observation_history"])))), + "policy_input_max_abs": float(np.max(np.abs(state["policy_input_observation_history"]))), + } + passed = all(checks.values()) + return {"schema_id": COMPATIBILITY_SCHEMA_ID, "status": "PASS" if passed else "FAIL", "production_authorized": passed, "contract": CONTRACT, "contract_sha256": CONTRACT_SHA256, "pilot_path": str(root), "pilot_manifest_sha256": file_sha256(root / "manifest.json"), "checks": checks, "metrics": metrics, "claim_boundary": "runtime compatibility only; not accuracy, stability, physical-phase, or mechanism validation"} + +def publish_certificate(pilot: str | Path, output: str | Path) -> Path: + destination = Path(output) + report = evaluate(pilot) + if report["status"] != "PASS": raise RuntimeError("Illusion compatibility contract failed") + partial = destination.with_name(f".{destination.name}.partial") + if destination.exists() or partial.exists(): raise FileExistsError(destination) + partial.write_bytes(canonical_json(report)) + rename_noreplace(partial, destination) + return destination + +def validate_certificate(path: str | Path) -> dict[str, Any]: + certificate = Path(path).resolve(); report = json.loads(certificate.read_bytes()) + if certificate.read_bytes() != canonical_json(report) or report != evaluate(report.get("pilot_path", "")) or report.get("status") != "PASS" or report.get("production_authorized") is not True: + raise ValueError("Illusion compatibility certificate is invalid or stale") + return report diff --git a/src/CCD_analysis/acquisition/contracts.py b/src/CCD_analysis/acquisition/contracts.py new file mode 100644 index 0000000..5fc70a4 --- /dev/null +++ b/src/CCD_analysis/acquisition/contracts.py @@ -0,0 +1,195 @@ +"""Frozen CPU-only acquisition contracts for the active CCD cases.""" +from __future__ import annotations +from dataclasses import asdict, dataclass +from hashlib import sha256 +import json +import numpy as np +from pathlib import Path +from typing import Literal + +Role = Literal["q_target", "q_blk", "q_ctl"] +ROLES: tuple[Role, ...] = ("q_target", "q_blk", "q_ctl") +SCHEMA_ID = "ccd-acquisition-contract/v3" +ARTIFACT_SCHEMA_ID = "ccd-acquisition-artifact/v3" +ACTION_IDENTITIES = ("front_ccw_positive", "upper_ccw_positive", "lower_ccw_positive") +BODY_ORDER = ("sensor_upper", "sensor_center", "sensor_lower", "front", "upper", "lower") +COORDINATE_FRAME_SCHEMA_ID = "ccd-lattice-coordinate-frame/v1" +COORDINATE_REFERENCE_LENGTH_LATTICE = 20.0 +COORDINATE_AXIS_KEYS = {"count", "origin_lattice", "spacing_lattice"} +COORDINATE_FRAME_KEYS = {"schema_id", "dtype", "axis_order", "reference_length_lattice", "x", "y"} +VELOCITY_DECODER_SCHEMA_ID = "legacy-d2q9-q-over-u0/v1" +VELOCITY_DECODER_FORMULA = ( + "ux=(f1+f5+f8-f3-f6-f7)/u0; " + "uy=(f2+f5+f6-f4-f7-f8)/u0; fluid cells only; nonfluid=0" +) +VELOCITY_DECODER_FORMULA_SHA256 = sha256(VELOCITY_DECODER_FORMULA.encode()).hexdigest() + +@dataclass(frozen=True) +class Geometry: + front_x_D: float + rear_x_D: float + sensor_x_D: float + target_x_D: float | None + rear_y_D: tuple[float, float] = (0.75, -0.75) + sensor_y_D: tuple[float, float, float] = (2.0, 0.0, -2.0) + body_order: tuple[str, ...] = BODY_ORDER + action_identities: tuple[str, ...] = ACTION_IDENTITIES + +@dataclass(frozen=True) +class CaseContract: + case_id: str + code_reynolds: int + physical_re_D: float + sample_interval: int + u0: float + viscosity: float + geometry: Geometry + target_kind: str + q_target_caveat: str + deployment_warning: str | None + history_smoke_required: bool + +KARMAN = CaseContract( + "karman_re100", 100, 50.0, 800, 0.01, 0.004, + Geometry(30.0, 31.3, 40.0, None), "disturbance-cylinder wake", + "q_target has different bodies and is a reference trajectory, not a same-checkpoint counterfactual.", + None, False, +) +ILLUSION = CaseContract( + "illusion_1.0L", 100, 50.0, 600, 0.01, 0.004, + Geometry(30.0, 31.3, 41.0, 31.0), "single 1.0D target-cylinder wake", + "q_target has different bodies and is a reference trajectory, not a same-checkpoint counterfactual.", + "Strict +11D deployment geometry differs from policy training geometry; replay/history smoke is mandatory before production.", + True, +) +CASES = {c.case_id: c for c in (KARMAN, ILLUSION)} +ROLE_SEMANTICS = { + "q_target": "desired reference flow with case-specific target bodies", + "q_blk": "passive pinball flow with exactly zero requested actuation", + "q_ctl": "controlled pinball flow driven by the frozen policy contract", +} + +def canonical_json(value: object) -> bytes: + return (json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n").encode() + +def canonical_coordinate_axis(count: int, *, origin_lattice: float, spacing_lattice: float, reference_length_lattice: float) -> np.ndarray: + """Generate the exact persisted float32 lattice coordinate sequence.""" + if type(count) is not int or count < 1: + raise ValueError("coordinate count must be a positive integer") + origin = np.float32(origin_lattice); spacing = np.float32(spacing_lattice); reference = np.float32(reference_length_lattice) + if not np.isfinite(origin) or not np.isfinite(spacing) or not np.isfinite(reference) or spacing <= 0 or reference <= 0: + raise ValueError("coordinate frame must be finite with positive spacing/reference length") + values = (origin + np.arange(count, dtype=np.float32) * spacing) / reference + if not np.isfinite(values).all(): raise ValueError("generated coordinate sequence must be finite") + return values + + +def canonical_coordinate_frame(nx: int, ny: int, *, x_origin_lattice: float = 0.0, y_origin_lattice: float | None = None, spacing_lattice: float = 1.0, reference_length_lattice: float = COORDINATE_REFERENCE_LENGTH_LATTICE) -> dict: + if y_origin_lattice is None: y_origin_lattice = -(ny - 1) / 2 + frame = {"schema_id": COORDINATE_FRAME_SCHEMA_ID, "dtype": "float32", "axis_order": "time,x,y", "reference_length_lattice": float(reference_length_lattice), "x": {"count": nx, "origin_lattice": float(x_origin_lattice), "spacing_lattice": float(spacing_lattice)}, "y": {"count": ny, "origin_lattice": float(y_origin_lattice), "spacing_lattice": float(spacing_lattice)}} + for axis in ("x", "y"): + spec = frame[axis]; canonical_coordinate_axis(spec["count"], origin_lattice=spec["origin_lattice"], spacing_lattice=spec["spacing_lattice"], reference_length_lattice=frame["reference_length_lattice"]) + return frame + + +def validate_coordinate_arrays(x_D: np.ndarray, y_D: np.ndarray, frame: object) -> tuple[np.ndarray, np.ndarray]: + if not isinstance(frame, dict) or set(frame) != COORDINATE_FRAME_KEYS or frame.get("schema_id") != COORDINATE_FRAME_SCHEMA_ID or frame.get("dtype") != "float32" or frame.get("axis_order") != "time,x,y": raise ValueError("coordinate frame schema is not exact") + reference = frame["reference_length_lattice"] + if type(reference) not in (int, float) or not np.isfinite(reference) or reference <= 0: raise ValueError("coordinate reference length is invalid") + actual = {"x": np.asarray(x_D), "y": np.asarray(y_D)} + for axis in ("x", "y"): + spec = frame[axis] + if not isinstance(spec, dict) or set(spec) != COORDINATE_AXIS_KEYS or type(spec["count"]) is not int or spec["count"] < 1 or any(type(spec[key]) not in (int, float) or not np.isfinite(spec[key]) for key in ("origin_lattice", "spacing_lattice")) or spec["spacing_lattice"] <= 0: raise ValueError(f"{axis} coordinate declaration is invalid") + value = actual[axis]; expected = canonical_coordinate_axis(spec["count"], origin_lattice=spec["origin_lattice"], spacing_lattice=spec["spacing_lattice"], reference_length_lattice=reference) + if value.dtype != np.float32 or value.ndim != 1 or not np.isfinite(value).all() or not np.array_equal(value, expected): raise ValueError(f"{axis}_D must exactly equal the declared canonical float32 lattice sequence") + return actual["x"], actual["y"] + + +def case_snapshot(case_id: str, role: Role) -> dict: + if case_id not in CASES: raise ValueError(f"case must be one of {tuple(CASES)}") + if role not in ROLES: raise ValueError(f"role must be one of {ROLES}") + case=CASES[case_id] + value={"schema_id":SCHEMA_ID,"case":asdict(case),"role":role,"role_semantics":ROLE_SEMANTICS[role], + "timeline_semantics":"absolute lattice steps; equal time does not establish equal physical phase", + "phase_reference_semantics":"sampled evidence only; phase equality requires separate validation", + "action_semantics":{"requested_normalized":"policy output", "requested_physical":"full solver body command", + "effective_applied":"solver EMA command at the completed lattice step"}, + "velocity_decoder":{"schema_id":VELOCITY_DECODER_SCHEMA_ID,"quantity":"nondimensional velocity q/U0", + "formula":VELOCITY_DECODER_FORMULA,"formula_sha256":VELOCITY_DECODER_FORMULA_SHA256}} + value["geometry_sha256"]=sha256(canonical_json(asdict(case.geometry))).hexdigest() + value["config_sha256"]=sha256(canonical_json(value)).hexdigest() + return value + + +# Dependency-safe frozen runtime authority. This module imports no solver/GPU code. +REPO_ROOT = Path(__file__).resolve().parents[3] +ACTION_FORMULA = "physical[-3:]=(normalized*8+bias)*u0; legacy EMA a_next=0.9*a_prev+0.1*target" +ACTION_FORMULA_SHA256 = sha256(ACTION_FORMULA.encode()).hexdigest() +MODEL_BINDINGS = { + "karman_re100": ("models/old/d1a3o12_re100.zip", "148240c1dcb8b11d8e5a0a9e991f6874c4e799c9744707bf0fb32a8efeb0468d"), + "illusion_1.0L": ("models/250525/d1a3o14_250525_imit_1L_2U_600S.zip", "570fbdb08fba1170cbab53de9ede372a9536e0bb8419dda0f0f2db20fd20583d"), +} +CONFIG_BINDINGS = { + "configs/legacy_configs/config_cuda.json": "77140dc4b4983a783e601c145761c4d0a7d9b79e821d3925a8a10a59055c8565", + "configs/legacy_configs/config_flowfield.json": "a0428733b94a5dc4b01a75cd55a935c51cddea49feb61cb85e59e2b04a865cce", +} +ILLUSION_TRAINING_BINDINGS = { + "normalization_path": "src/SR_analysis/data/illusion/illusion_1L/norm.json", + "normalization_sha256": "9ec5ddbe68fb441cdb660c17b51bf84ac348ca87d565081a21cb092a5245e4fc", + "harmonics_path": "src/SR_analysis/data/illusion/illusion_1L/target_harmonics.json", + "harmonics_sha256": "f135660ee3533c81d7dd175a4530f83e728ba52ec9485738ec0ad953e4a6378d", +} + +@dataclass(frozen=True) +class ObjectSpec: + kind: str + identity: str + center_D: tuple[float, float] + radius_D: float + +@dataclass(frozen=True) +class RoleSpec: + case_id: str + role: str + objects: tuple[ObjectSpec, ...] + control_interval: int + physical_action_width: int + observation_slices: dict[str, tuple[int, int] | None] + controller: str + fifo_len: int + harmonic_channels: int + model_path: str | None + + +def role_spec(case_id: str, role: str) -> RoleSpec: + if case_id not in CASES or role not in ROLES: raise ValueError("unsupported case/role") + case=CASES[case_id]; g=case.geometry + sensors=tuple(ObjectSpec("sensor",name,(g.sensor_x_D,y),.25) for name,y in zip(("sensor_upper","sensor_center","sensor_lower"),g.sensor_y_D)) + pinball=(ObjectSpec("cylinder","front",(g.front_x_D,0.),.5),ObjectSpec("cylinder","upper",(g.rear_x_D,g.rear_y_D[0]),.5),ObjectSpec("cylinder","lower",(g.rear_x_D,g.rear_y_D[1]),.5)) + if case_id=="karman_re100": + objects=(ObjectSpec("cylinder","disturbance",(10.,0.),1.),)+sensors+(() if role=="q_target" else pinball) + slices={"disturbance_force":(0,2),"sensors":(2,8),"pinball_forces":None if role=="q_target" else (8,14)}; harmonics=0 + else: + target=(ObjectSpec("cylinder","target",(float(g.target_x_D),0.),.5),) + objects=target+sensors if role=="q_target" else sensors+pinball + slices={"disturbance_force":(0,2),"sensors":(2,8),"pinball_forces":None} if role=="q_target" else {"disturbance_force":None,"sensors":(0,6),"pinball_forces":(6,12)} + harmonics=8 if role=="q_target" else (2 if role=="q_ctl" else 0) + model_path=str(REPO_ROOT/MODEL_BINDINGS[case_id][0]) if role=="q_ctl" else None + return RoleSpec(case_id,role,objects,case.sample_interval,len(objects),slices,"ppo_history" if role=="q_ctl" else "zero",150,harmonics,model_path) + + +def role_spec_identity(spec: RoleSpec) -> str: + return sha256(canonical_json(asdict(spec))).hexdigest() + + +def expected_source_bindings(case_id: str, role: str) -> dict[str, str]: + if case_id not in CASES or role not in ROLES: raise ValueError("unsupported case/role") + result=dict(CONFIG_BINDINGS) + if role=="q_ctl": result[MODEL_BINDINGS[case_id][0]]=MODEL_BINDINGS[case_id][1] + return result + + +def expected_controller_identity(case_id: str, role: str) -> dict[str, object]: + if case_id=="illusion_1.0L" and role=="q_ctl": return dict(ILLUSION_TRAINING_BINDINGS) + if role=="q_ctl": return {"kind":"generated_karman_normalization"} + return {"kind":"not_policy_consumed","measured_phase_evidence":case_id=="illusion_1.0L" and role=="q_target"} diff --git a/src/CCD_analysis/acquisition/dual_clock.py b/src/CCD_analysis/acquisition/dual_clock.py new file mode 100644 index 0000000..f420108 --- /dev/null +++ b/src/CCD_analysis/acquisition/dual_clock.py @@ -0,0 +1,95 @@ +"""CPU-safe exact-time dual-clock collection.""" +from __future__ import annotations +from collections.abc import Callable, Iterable, Mapping +from typing import Any +import numpy as np + +TELEMETRY_KEYS=("sample_ids","acquisition_relative_lattice_steps","solver_absolute_control_indices","requested_normalized_action","requested_physical_action","effective_applied_action","disturbance_force","pinball_forces","sensors","phase_reference") + +def field_steps(total_steps:int, interval:int, *, absolute_origin:int=0)->tuple[int,...]: + if type(total_steps) is not int or total_steps<1 or type(interval) is not int or interval<1 or type(absolute_origin) is not int or absolute_origin<0: raise ValueError("timeline values must be positive integers and origin non-negative") + return tuple(absolute_origin+i for i in range(interval,total_steps+1,interval)) + +def velocity_from_public_accessor(flow:Any)->tuple[np.ndarray,np.ndarray]: + accessor=getattr(flow,"current_step_velocity_field",None) + if not callable(accessor): raise RuntimeError("solver lacks safe public current_step_velocity_field accessor") + ux,uy=(np.asarray(v) for v in accessor()) + if ux.dtype!=np.float32 or uy.dtype!=np.float32 or ux.shape!=uy.shape or ux.ndim!=2 or not np.isfinite(ux).all() or not np.isfinite(uy).all(): raise ValueError("velocity accessor must return finite float32 matching 2D fields") + return ux.copy(),uy.copy() + +def solver_fluid_mask(flow:Any)->np.ndarray: + accessor=getattr(flow,"completed_flags_xy",None) + if not callable(accessor): raise RuntimeError("solver-derived mask unavailable; coordinate fallback is forbidden") + flags=np.asarray(accessor()) + if flags.dtype!=np.uint8 or flags.ndim!=2: raise ValueError("solver flags must be uint8 2D") + mask=(flags & np.uint8(1)) != 0 + if not mask.any(): raise ValueError("solver fluid mask is empty") + return mask.copy() + +class ExactTelemetry: + def __init__(self, *, role:str, observation_slices:Mapping[str,tuple[int,int]|None], phase_probe:Callable[[Any],Any], physical_width:int): + if role not in {"q_target","q_blk","q_ctl"}: raise ValueError("invalid role") + if type(physical_width) is not int or physical_width < 1: raise ValueError("physical_width must be positive") + if set(observation_slices) != {"disturbance_force","pinball_forces","sensors"}: raise ValueError("observation slices must be exact") + for bounds in observation_slices.values(): + if bounds is not None and (not isinstance(bounds,tuple) or len(bounds)!=2 or any(type(v) is not int for v in bounds) or bounds[0]<0 or bounds[1]<=bounds[0]): raise ValueError("invalid observation slice") + self.role,self.slices,self.phase_probe,self.physical_width=role,dict(observation_slices),phase_probe,physical_width + self.normalized=np.zeros(3,np.float32); self.physical=np.zeros(physical_width,np.float32) + def set_requested(self, normalized:Any, physical:Any)->None: + n,p=np.asarray(normalized),np.asarray(physical) + if n.dtype!=np.float32 or n.shape!=(3,) or p.dtype!=np.float32 or p.shape!=(self.physical_width,) or not np.isfinite(n).all() or not np.isfinite(p).all(): raise ValueError("requested actions have wrong exact dtype/shape/finiteness") + if np.any(n < -1) or np.any(n > 1): raise ValueError("normalized action is outside [-1,1]") + if self.role!="q_ctl" and (np.any(n) or np.any(p)): raise ValueError("reference/passive requested action must be exactly zero") + self.normalized,self.physical=n.copy(),p.copy() + def __call__(self, flow:Any, solver_absolute_step:int, acquisition_relative_step:int, solver_absolute_control_index:int)->dict[str,np.ndarray]: + obs_access=getattr(flow,"current_step_observation",None); action_access=getattr(flow,"current_effective_action",None) + if not callable(obs_access) or not callable(action_access): raise RuntimeError("same-step telemetry accessors unavailable") + raw=np.asarray(obs_access()); out={} + if raw.dtype != np.float32 or raw.ndim != 1 or not np.isfinite(raw).all(): raise ValueError("raw observation must be finite float32 rank-1") + for key,bounds in self.slices.items(): + width = 6 if key != "disturbance_force" else 2 + out[key]=np.zeros(width,np.float32) if bounds is None else raw[slice(*bounds)].copy() + if out[key].dtype != np.float32 or out[key].shape != (width,) or not np.isfinite(out[key]).all(): raise ValueError(f"{key} has invalid width/dtype/finiteness") + phase=np.asarray(self.phase_probe(flow)); effective=np.asarray(action_access()) + if phase.dtype!=np.float32 or phase.shape!=(1,) or effective.dtype!=np.float32 or effective.shape!=(self.physical_width,) or not np.isfinite(phase).all() or not np.isfinite(effective).all(): raise ValueError("phase/effective accessor contract violated") + out.update(sample_ids=np.asarray(solver_absolute_step,np.int64),acquisition_relative_lattice_steps=np.asarray(acquisition_relative_step,np.int64),solver_absolute_control_indices=np.asarray(solver_absolute_control_index,np.int64),requested_normalized_action=self.normalized.copy(),requested_physical_action=self.physical.copy(),effective_applied_action=effective.copy(),phase_reference=phase.copy()) + if set(out)!=set(TELEMETRY_KEYS): raise ValueError("telemetry schema mismatch") + return out + +class DualClockCollector: + def __init__(self, flow:Any, *, control_interval:int, control_count:int, absolute_origin:int, field_steps_absolute:Iterable[int], telemetry:Callable[[Any,int],Mapping[str,Any]], velocity_reader:Callable[[Any],tuple[np.ndarray,np.ndarray]]=velocity_from_public_accessor): + self.flow=flow; self.control_interval=control_interval; self.control_count=control_count; self.origin=absolute_origin; self.steps=tuple(field_steps_absolute); self.telemetry=telemetry; self.velocity_reader=velocity_reader + horizon=absolute_origin+control_interval*control_count + if control_interval<1 or control_count<1 or tuple(sorted(set(self.steps)))!=self.steps or any(s<=absolute_origin or s>horizon for s in self.steps): raise ValueError("invalid absolute field timeline") + clocks=flow.solver_clock_state() + if clocks["solver_absolute_lattice_clock"]!=absolute_origin: raise ValueError("collector origin contradicts solver absolute lattice clock") + self.solver_control_origin=clocks["solver_absolute_control_clock"] + self.next_control=0; self.next_field=0; self.ux=[]; self.uy=[]; self.saved_steps=[]; self.snapshots={k:[] for k in TELEMETRY_KEYS} + def run_interval(self,index:int,target:np.ndarray)->np.ndarray: + if index!=self.next_control: raise ValueError("control intervals must be sequential") + start=self.origin+index*self.control_interval; stop=start+self.control_interval; due=[] + while self.next_fieldstart: due.append(self.steps[self.next_field]) + self.next_field+=1 + self.flow.begin_control_interval(self.control_interval,target); current=start + for boundary in due if due and due[-1]==stop else [*due,stop]: + if boundary>current: self.flow.run_control_segment(boundary-current); current=boundary + if boundary in due: + clocks=self.flow.active_step_clock_state() + expected_control=self.solver_control_origin+index + if clocks["solver_absolute_lattice_clock"]!=boundary or clocks["solver_absolute_control_clock"]!=expected_control: raise RuntimeError("solver clock lineage mismatch at sampled step") + ux,uy=self.velocity_reader(self.flow); snap=self.telemetry(self.flow,boundary,boundary-self.origin,expected_control) + self.ux.append(ux); self.uy.append(uy); self.saved_steps.append(boundary) + for key in TELEMETRY_KEYS:self.snapshots[key].append(np.asarray(snap[key]).copy()) + obs=np.asarray(self.flow.end_control_interval()).copy() + clocks=self.flow.solver_clock_state() + if clocks!={"solver_absolute_lattice_clock":stop,"solver_absolute_control_clock":self.solver_control_origin+index+1}: raise RuntimeError("solver clocks did not advance exactly one control interval") + self.next_control+=1; return obs + def arrays(self)->dict[str,np.ndarray]: + if self.next_control!=self.control_count or tuple(self.saved_steps)!=self.steps: raise RuntimeError("rollout is incomplete") + steps=np.asarray(self.saved_steps,np.int64); rel=steps-self.origin + out={"ux":np.asarray(self.ux,np.float32),"uy":np.asarray(self.uy,np.float32),"lattice_steps":steps, + "control_indices":(rel-1)//self.control_interval,"control_offsets":(rel-1)%self.control_interval+1} + out.update({k:np.stack(v) for k,v in self.snapshots.items()}) + if not np.array_equal(out["sample_ids"],steps) or not np.array_equal(out["acquisition_relative_lattice_steps"],rel): raise RuntimeError("telemetry clock domains are not exact-time aligned") + return out diff --git a/src/CCD_analysis/acquisition/runtime.py b/src/CCD_analysis/acquisition/runtime.py new file mode 100644 index 0000000..12e0c4d --- /dev/null +++ b/src/CCD_analysis/acquisition/runtime.py @@ -0,0 +1,514 @@ +"""Role-specific CFD runtime boundary; importing this module is CPU/CUDA safe.""" +from __future__ import annotations +from dataclasses import asdict +from hashlib import sha256 +import ast +import json +import zipfile +import os +from pathlib import Path +from typing import Any, Protocol +import numpy as np +from .contracts import (ACTION_FORMULA, ACTION_FORMULA_SHA256, CASES, CONFIG_BINDINGS, ILLUSION_TRAINING_BINDINGS, MODEL_BINDINGS, ROLES, CaseContract, ObjectSpec, RoleSpec, canonical_coordinate_axis, canonical_coordinate_frame, canonical_json, case_snapshot, expected_controller_identity, expected_source_bindings, role_spec, role_spec_identity, VELOCITY_DECODER_FORMULA, VELOCITY_DECODER_FORMULA_SHA256, VELOCITY_DECODER_SCHEMA_ID) +from .artifacts import ArtifactTransaction, file_sha256 +from .dual_clock import DualClockCollector, ExactTelemetry, field_steps, solver_fluid_mask + +REQUIRED_CFD_ENV = "pycuda_3_10" +POLICY_DEVICE = "cpu" +REPO_ROOT = Path(__file__).resolve().parents[3] +LEGACY_CONFIG_DIR = REPO_ROOT / "configs" / "legacy_configs" +MODEL_FILES = {case: REPO_ROOT/path for case,(path,_) in MODEL_BINDINGS.items()} +EXPECTED_SHA256 = {**CONFIG_BINDINGS, **{path:digest for path,digest in MODEL_BINDINGS.values()}} +ILLUSION_TRAINING_NORM = REPO_ROOT / ILLUSION_TRAINING_BINDINGS["normalization_path"] +ILLUSION_TRAINING_HARMONICS = REPO_ROOT / ILLUSION_TRAINING_BINDINGS["harmonics_path"] +ILLUSION_TRAINING_NORM_SHA256 = ILLUSION_TRAINING_BINDINGS["normalization_sha256"] +ILLUSION_TRAINING_HARMONICS_SHA256 = ILLUSION_TRAINING_BINDINGS["harmonics_sha256"] +REQUIRED_SOLVER_APIS = ( + "begin_control_interval", "run_control_segment", "current_step_observation", + "current_effective_action", "current_step_velocity_field", "current_step_ddf_checkpoint", + "completed_flags_xy", "active_step_clock_state", "solver_clock_state", "full_state_checkpoint", "restore_full_state", "end_control_interval", +) + + +def require_cfd_environment() -> None: + actual = os.environ.get("CONDA_DEFAULT_ENV") + if actual != REQUIRED_CFD_ENV: + raise RuntimeError(f"CFD acquisition requires CONDA_DEFAULT_ENV={REQUIRED_CFD_ENV}; got {actual!r}") + + +class FlowProtocol(Protocol): + def begin_control_interval(self, total_steps: int, action_target: np.ndarray) -> None: ... + def run_control_segment(self, num_steps: int) -> None: ... + def current_step_observation(self) -> np.ndarray: ... + def current_effective_action(self) -> np.ndarray: ... + def current_step_velocity_field(self) -> tuple[np.ndarray, np.ndarray]: ... + def current_step_ddf_checkpoint(self) -> dict[str, Any]: ... + def full_state_checkpoint(self) -> dict[str, Any]: ... + def restore_full_state(self, checkpoint: dict[str, Any]) -> None: ... + def completed_flags_xy(self) -> np.ndarray: ... + def active_step_clock_state(self) -> dict[str, int]: ... + def solver_clock_state(self) -> dict[str, int]: ... + def end_control_interval(self) -> np.ndarray: ... + + + +def _load_legacy_configs() -> tuple[Any, Any]: + from LegacyCelerisLab import utils + cuda_cfg = utils.load_cuda_config(str(LEGACY_CONFIG_DIR / "config_cuda.json")) + field_cfg = utils.load_flow_field_config(str(LEGACY_CONFIG_DIR / "config_flowfield.json")) + return cuda_cfg, field_cfg + + +def _load_policy(path: str, *, policy_device: str, s_dim: int): + """Load PPO for CPU inference only; CUDA belongs exclusively to the CFD solver.""" + if policy_device != POLICY_DEVICE: + raise ValueError(f"policy_device must be {POLICY_DEVICE!r}; got {policy_device!r}") + import torch + from torch.nn import Module + from stable_baselines3 import PPO + import gymnasium as gym + from gymnasium import spaces + class Sin(Module): + def forward(self, x): + return torch.sin(x) + class Dummy(gym.Env): + observation_space = spaces.Box(-1, 1, shape=(s_dim,), dtype=np.float32) + action_space = spaces.Box(-1, 1, shape=(3,), dtype=np.float32) + def reset(self, *, seed=None, options=None): + return np.zeros(s_dim, np.float32), {} + def step(self, action): + return np.zeros(s_dim, np.float32), 0.0, False, False, {} + return PPO.load(path, env=Dummy(), device=POLICY_DEVICE) + + +class RoleRuntime: + """Constructed one-role adapter; CFD advancement remains an explicit caller action.""" + def __init__(self, flow: FlowProtocol, spec: RoleSpec, case: CaseContract, policy: Any = None, *, cfd_device: int = 0, policy_device: str = POLICY_DEVICE): + if type(cfd_device) is not int or cfd_device < 0: raise ValueError("cfd_device must be a nonnegative logical device") + if policy_device != POLICY_DEVICE: raise ValueError(f"policy_device must be {POLICY_DEVICE!r}") + self.flow, self.spec, self.case, self.policy = flow, spec, case, policy + self.cfd_device, self.policy_device = cfd_device, policy_device + self.fifo: list[np.ndarray] = [] + self.normalization: dict[str, np.ndarray | float] | None = None + self.harmonics: list[dict] | None = None + self.solver_absolute_control_clock = 0 + self.solver_absolute_lattice_clock = 0 + self.acquisition_relative_control_index = 0 + self.acquisition_relative_lattice_clock = 0 + self.policy_harmonic_phase_index = 0 + self.requested_normalized = np.zeros(3, np.float32) + self.requested_physical = np.zeros(spec.physical_action_width, np.float32) + self.initial_fifo_history: np.ndarray | None = None + self.boundary_observation_history: list[np.ndarray] = [] + self.policy_source_observation_history: list[np.ndarray] = [] + self.policy_input_observation_history: list[np.ndarray] = [] + self.policy_harmonic_phase_indices: list[int] = [] + self.requested_normalized_action_history: list[np.ndarray] = [] + self.requested_physical_action_history: list[np.ndarray] = [] + + def set_controller_state(self, *, fifo_history: np.ndarray, normalization: dict[str, Any], + harmonics: list[dict] | None = None, + solver_absolute_control_clock: int = 0, + solver_absolute_lattice_clock: int = 0, + policy_harmonic_phase_index: int = 0) -> None: + fifo = np.asarray(fifo_history) + if fifo.dtype != np.float32 or fifo.shape != (self.spec.fifo_len, 12) or not np.isfinite(fifo).all(): + raise ValueError("controller FIFO must be finite float32 (150,12)") + norm = validate_normalization(normalization) + checked_harmonics = validate_harmonics(harmonics, self.spec.harmonic_channels) + clocks=(solver_absolute_control_clock,solver_absolute_lattice_clock,policy_harmonic_phase_index) + if any(type(value) is not int or value<0 for value in clocks): raise ValueError("clock domains must be nonnegative integers") + self.fifo = [row.copy() for row in fifo] + self.initial_fifo_history = fifo.copy() + self.boundary_observation_history = [] + self.policy_source_observation_history = [] + self.policy_input_observation_history = [] + self.policy_harmonic_phase_indices = [] + self.requested_normalized_action_history = [] + self.requested_physical_action_history = [] + self.normalization = norm + self.harmonics = checked_harmonics + self.solver_absolute_control_clock=solver_absolute_control_clock; self.solver_absolute_lattice_clock=solver_absolute_lattice_clock + self.acquisition_relative_control_index=0; self.acquisition_relative_lattice_clock=0 + self.policy_harmonic_phase_index=policy_harmonic_phase_index + + def requested_action(self) -> tuple[np.ndarray, np.ndarray]: + source = np.asarray(self.fifo[-1], np.float32).copy() + phase_index = self.policy_harmonic_phase_index + if self.spec.role != "q_ctl": + self.requested_normalized.fill(0); self.requested_physical.fill(0) + obs = np.zeros(12 if self.spec.case_id == "karman_re100" else 14, np.float32) + self.policy_source_observation_history.append(np.zeros(12, np.float32)) + self.policy_input_observation_history.append(obs) + self.policy_harmonic_phase_indices.append(phase_index) + self.requested_normalized_action_history.append(self.requested_normalized.copy()) + self.requested_physical_action_history.append(self.requested_physical.copy()) + return self.requested_normalized.copy(), self.requested_physical.copy() + if self.policy is None or self.normalization is None or len(self.fifo) != self.spec.fifo_len: + raise RuntimeError("controlled role requires loaded policy and complete history/normalization") + raw = self.fifo[-1] + if self.spec.case_id=="karman_re100" and self.policy_harmonic_phase_index==0: + obs=np.zeros(12,np.float32) + else: + force = raw[6:12] / self.normalization["force_norm_fact"] + sensor = (raw[:6] - self.normalization["sens_deviation"]) / self.normalization["sens_norm_fact"] + values = [*force, *sensor] + if self.spec.case_id == "illusion_1.0L": + if self.harmonics is None: raise RuntimeError("Illusion harmonics are unavailable") + target = reconstruct_harmonics(self.policy_harmonic_phase_index, self.harmonics)[:2] + values.extend((target / self.normalization["force_norm_fact"]).tolist()) + obs = np.clip(np.asarray(values, np.float32), -1, 1) + self.last_policy_observation=obs.copy() + self.policy_source_observation_history.append(source) + self.policy_input_observation_history.append(obs.copy()) + self.policy_harmonic_phase_indices.append(phase_index) + predicted = np.asarray(self.policy.predict(obs, deterministic=True)[0]) + if predicted.dtype != np.float32 or predicted.shape not in {(3,), (1,3)} or not np.isfinite(predicted).all(): raise ValueError("policy output must be finite float32 width 3") + normalized = predicted.reshape(3) + if np.any(normalized < -1) or np.any(normalized > 1): raise ValueError("policy output is outside [-1,1]") + bias = np.asarray((0., -4., 4.) if self.spec.case_id == "karman_re100" else (0., -2., 2.), np.float32) + physical = np.zeros(self.spec.physical_action_width, np.float32) + physical[-3:] = (normalized * 8 + bias) * self.case.u0 + self.requested_normalized, self.requested_physical = normalized, physical + self.requested_normalized_action_history.append(normalized.copy()) + self.requested_physical_action_history.append(physical.copy()) + return normalized.copy(), physical.copy() + + def controller_state_identity(self) -> dict[str, Any]: + if not self.fifo or self.normalization is None: + raise RuntimeError("controller state is incomplete") + checkpoint = self.flow.current_step_ddf_checkpoint() + if self.initial_fifo_history is None: + raise RuntimeError("initial controller FIFO was not retained") + boundaries = np.asarray(self.boundary_observation_history, np.float32) + sources = np.asarray(self.policy_source_observation_history, np.float32) + policy_inputs = np.asarray(self.policy_input_observation_history, np.float32) + phases = np.asarray(self.policy_harmonic_phase_indices, np.int64) + normalized_history = np.asarray(self.requested_normalized_action_history, np.float32) + physical_history = np.asarray(self.requested_physical_action_history, np.float32) + source_hashes = np.asarray([sha256(np.ascontiguousarray(row).tobytes()).hexdigest() for row in sources]) + norm_hash = sha256(canonical_json({k: np.asarray(v).tolist() for k, v in self.normalization.items()})).hexdigest() + harmonics_hash = sha256(canonical_json(self.harmonics or [])).hexdigest() + if self.spec.case_id=="illusion_1.0L" and self.spec.role=="q_ctl": + norm_hash=ILLUSION_TRAINING_NORM_SHA256; harmonics_hash=ILLUSION_TRAINING_HARMONICS_SHA256 + model_hash = sha256(Path(self.spec.model_path).read_bytes()).hexdigest() if self.spec.model_path else sha256(b"zero-controller").hexdigest() + return { + "current_ddf": checkpoint["current_ddf"], "temp_ddf": checkpoint["temp_ddf"], + "current_raw_observation": np.asarray(self.flow.current_step_observation(), np.float32), + "fifo_history": np.asarray(self.fifo, np.float32), + "initial_fifo_history": self.initial_fifo_history.copy(), + "boundary_observation_history": boundaries, + "policy_source_observation_history": sources, + "policy_source_observation_sha256": source_hashes, + "policy_input_observation_history": policy_inputs, + "policy_harmonic_phase_indices": phases, + "requested_normalized_action_history": normalized_history, + "requested_physical_action_history": physical_history, + "persisted_effective_ema_action": np.asarray(self.flow.current_effective_action(), np.float32), + "policy_harmonic_phase_index": np.asarray(self.policy_harmonic_phase_index,np.int64), + "solver_absolute_control_clock": np.asarray(self.solver_absolute_control_clock,np.int64), + "solver_absolute_lattice_clock": np.asarray(self.solver_absolute_lattice_clock,np.int64), + "acquisition_relative_control_index": np.asarray(self.acquisition_relative_control_index,np.int64), + "acquisition_relative_lattice_clock": np.asarray(self.acquisition_relative_lattice_clock,np.int64), + "normalization_hash": np.asarray(norm_hash), + "harmonics_hash": np.asarray(harmonics_hash), + "model_hash": np.asarray(model_hash), + "cuda_config_hash": np.asarray(EXPECTED_SHA256["configs/legacy_configs/config_cuda.json"]), + "flow_config_hash": np.asarray(EXPECTED_SHA256["configs/legacy_configs/config_flowfield.json"]), + "config_hash": np.asarray(role_spec_identity(self.spec)), + "geometry_hash": np.asarray(sha256(canonical_json(asdict(self.case.geometry))).hexdigest()), + "action_formula_hash": np.asarray(ACTION_FORMULA_SHA256), + "velocity_decoder_formula_hash": np.asarray(VELOCITY_DECODER_FORMULA_SHA256), + } + + +def validate_normalization(value: dict[str, Any]) -> dict[str, np.ndarray]: + if set(value) != {"force_norm_fact","sens_deviation","sens_norm_fact"}: raise ValueError("normalization keys are not exact") + force=np.asarray(value["force_norm_fact"]); deviation=np.asarray(value["sens_deviation"]); scale=np.asarray(value["sens_norm_fact"]) + if force.dtype != np.float32 or force.ndim != 0 or not np.isfinite(force) or force <= 0: raise ValueError("force normalization must be positive finite float32 scalar") + if deviation.dtype != np.float32 or deviation.shape != (6,) or not np.isfinite(deviation).all(): raise ValueError("sensor deviation invalid") + if scale.dtype != np.float32 or scale.shape != (6,) or not np.isfinite(scale).all() or np.any(scale <= 0): raise ValueError("sensor scales must be positive finite float32") + return {"force_norm_fact":force.copy(),"sens_deviation":deviation.copy(),"sens_norm_fact":scale.copy()} + + +def validate_harmonics(value: list[dict] | None, channels: int) -> list[dict] | None: + if channels == 0: + if value not in (None, []): raise ValueError("harmonics forbidden for this role") + return None + if not isinstance(value,list) or len(value)!=channels: raise ValueError(f"harmonic reference requires exactly {channels} channels") + checked=[] + for harmonic in value: + if set(harmonic)!={"dc","amps","freqs","phases"}: raise ValueError("harmonic keys invalid") + arrays=[np.asarray(harmonic[key],np.float64) for key in ("amps","freqs","phases")] + if not arrays[0].ndim==arrays[1].ndim==arrays[2].ndim==1 or not arrays[0].shape==arrays[1].shape==arrays[2].shape or not all(np.isfinite(a).all() for a in arrays) or not np.isfinite(float(harmonic["dc"])): raise ValueError("harmonic arrays invalid") + checked.append({"dc":float(harmonic["dc"]),"amps":arrays[0].tolist(),"freqs":arrays[1].tolist(),"phases":arrays[2].tolist()}) + return checked + + +def analyze_harmonics(states: np.ndarray, count: int = 5) -> list[dict]: + states=np.asarray(states) + if states.dtype!=np.float32 or states.shape!=(150,8) or not np.isfinite(states).all(): raise ValueError("target harmonic source must be float32 (150,8)") + result=[] + for channel in range(8): + transform=np.fft.rfft(states[:,channel]); frequencies=np.fft.rfftfreq(150); amplitudes=2*np.abs(transform)/150 + indices=np.argsort(amplitudes[1:])[::-1][:count]+1 + result.append({"dc":float(transform[0].real/150),"amps":amplitudes[indices].tolist(),"freqs":frequencies[indices].tolist(),"phases":np.angle(transform)[indices].tolist()}) + return result + + +def verify_policy_spaces(policy: Any, observation_width: int) -> None: + obs=getattr(policy,"observation_space",None); action=getattr(policy,"action_space",None) + if getattr(obs,"shape",None)!=(observation_width,) or getattr(action,"shape",None)!=(3,): raise ValueError("loaded policy spaces mismatch") + if np.dtype(getattr(obs,"dtype",None))!=np.float32 or np.dtype(getattr(action,"dtype",None))!=np.float32: raise ValueError("loaded policy space dtype mismatch") + if not np.allclose(obs.low,-1) or not np.allclose(obs.high,1) or not np.allclose(action.low,-1) or not np.allclose(action.high,1): raise ValueError("loaded policy bounds mismatch") + device = getattr(policy, "device", None) + if device is None or str(device) != POLICY_DEVICE: raise ValueError("loaded policy device must be CPU") + + +def verify_solver_objects(flow: Any, spec: RoleSpec) -> list[dict[str,Any]]: + table=getattr(flow,"objects",None) + if not isinstance(table,dict) or list(table.keys()) != list(range(len(spec.objects))): raise ValueError("solver object IDs are not exact contiguous insertion order") + center_y=(int(flow.FIELD_SHAPE[1])-1)/2; diameter=20.0; persisted=[] + for index,obj in enumerate(spec.objects): + actual=table[index]; center=(obj.center_D[0]*diameter,center_y+obj.center_D[1]*diameter,0.0); radius=obj.radius_D*diameter + if actual.get("type")!=obj.kind or tuple(actual.get("center",()))!=center or float(actual.get("radius",-1))!=radius: raise ValueError(f"solver object {index} contradicts role spec") + persisted.append({"id":index,"identity":obj.identity,"kind":obj.kind,"center_lattice":center,"radius_lattice":radius}) + if np.asarray(getattr(flow,"action",None)).shape != (spec.physical_action_width,): raise ValueError("solver action width contradicts role spec") + return persisted + +def reconstruct_harmonics(t: int, harmonics: list[dict]) -> np.ndarray: + result = np.zeros(len(harmonics), np.float32) + for i, harmonic in enumerate(harmonics): + value = float(harmonic["dc"]) + for amp, freq, phase in zip(harmonic["amps"], harmonic["freqs"], harmonic["phases"]): + value += float(amp) * np.cos(2 * np.pi * float(freq) * t + float(phase)) + result[i] = value + return result + + +def build_role_runtime(*, case: str, role: str, output: Path | None = None, + device_id: int = 0, policy_device: str = POLICY_DEVICE, flow_factory: Any = None, + config_loader: Any = None, policy_loader: Any = None) -> RoleRuntime: + """Build exactly one role in one process; does not advance CFD or write output.""" + require_cfd_environment() + if policy_device != POLICY_DEVICE: raise ValueError(f"policy_device must be {POLICY_DEVICE!r}; CUDA policy inference is forbidden") + spec = role_spec(case, role) + using_default_config = config_loader is None + config_loader = config_loader or _load_legacy_configs + if using_default_config: + for relative in ("configs/legacy_configs/config_cuda.json","configs/legacy_configs/config_flowfield.json"): + if file_sha256(REPO_ROOT/relative) != EXPECTED_SHA256[relative]: raise RuntimeError(f"bound source digest mismatch: {relative}") + if flow_factory is None: + from LegacyCelerisLab import FlowField + flow_factory = FlowField + cuda_cfg, field_cfg = config_loader() + field_cfg = field_cfg._replace(viscosity=CASES[case].viscosity, velocity=CASES[case].u0) + flow = flow_factory(field_cfg, cuda_cfg, device_id=device_id) + center_y = (int(flow.FIELD_SHAPE[1]) - 1) / 2 + diameter = 20.0 + for obj in spec.objects: + center = (obj.center_D[0] * diameter, center_y + obj.center_D[1] * diameter, 0.0) + if obj.kind == "sensor": flow.add_sensor(center, obj.radius_D * diameter) + else: flow.add_cylinder(center, obj.radius_D * diameter) + policy = None + if role == "q_ctl": + model_relative=str(Path(spec.model_path).relative_to(REPO_ROOT)) + if file_sha256(Path(spec.model_path)) != EXPECTED_SHA256[model_relative]: raise RuntimeError("bound policy model digest mismatch") + loader = policy_loader or _load_policy + policy = loader(spec.model_path, policy_device=policy_device, s_dim=12 if case == "karman_re100" else 14) + verify_policy_spaces(policy, 12 if case == "karman_re100" else 14) + if hasattr(policy, "set_random_seed"): policy.set_random_seed(0 if case == "karman_re100" else 19) + verify_solver_objects(flow, spec) + return RoleRuntime(flow, spec, CASES[case], policy, cfd_device=device_id, policy_device=policy_device) + + + +def _canonical_history_observation(raw: np.ndarray, spec: RoleSpec) -> np.ndarray: + raw=np.asarray(raw) + if raw.dtype!=np.float32 or raw.ndim!=1 or not np.isfinite(raw).all(): raise ValueError("boundary observation invalid") + result=np.zeros(12,np.float32) + sensors=spec.observation_slices["sensors"] + forces=spec.observation_slices["pinball_forces"] + if sensors is not None: result[:6]=raw[slice(*sensors)] + if forces is not None: result[6:12]=raw[slice(*forces)] + return result + + +def _normalization_from_history(history: np.ndarray) -> dict[str,np.ndarray]: + history=np.asarray(history,np.float32) + force=float(6*np.max(np.abs(history[:,6:12]))) + deviation=np.mean(history[:,:6],axis=0,dtype=np.float32) + scales=5*np.max(np.abs(history[:,:6]-deviation),axis=0) + # Fail closed: a zero observed scale cannot define policy normalization. + return validate_normalization({"force_norm_fact":np.asarray(force,np.float32),"sens_deviation":deviation.astype(np.float32),"sens_norm_fact":scales.astype(np.float32)}) + + + +def load_illusion_training_reference(*, repo_root:Path=REPO_ROOT) -> tuple[dict[str,np.ndarray],list[dict],dict[str,str]]: + norm_path=repo_root/ILLUSION_TRAINING_NORM.relative_to(REPO_ROOT); harmonics_path=repo_root/ILLUSION_TRAINING_HARMONICS.relative_to(REPO_ROOT) + if file_sha256(norm_path)!=ILLUSION_TRAINING_NORM_SHA256 or file_sha256(harmonics_path)!=ILLUSION_TRAINING_HARMONICS_SHA256: raise RuntimeError("frozen Illusion training reference digest mismatch") + norm_doc=json.loads(norm_path.read_text()); normalization=validate_normalization({key:np.asarray(norm_doc[key],np.float32) for key in ("force_norm_fact","sens_deviation","sens_norm_fact")}) + harmonics_doc=json.loads(harmonics_path.read_text()); harmonics=validate_harmonics(harmonics_doc.get("harmonics",harmonics_doc) if isinstance(harmonics_doc,dict) else harmonics_doc,2) + identity={"normalization_path":str(norm_path),"normalization_sha256":ILLUSION_TRAINING_NORM_SHA256,"harmonics_path":str(harmonics_path),"harmonics_sha256":ILLUSION_TRAINING_HARMONICS_SHA256} + return normalization,harmonics,identity + +def initialize_role(runtime: RoleRuntime, *, target_harmonics: list[dict] | None = None, + stabilization_steps: int | None = None) -> None: + """Stabilize; checkpoint; measure norm; exact restore; then warm FIFO.""" + flow,spec,case=runtime.flow,runtime.spec,runtime.case + stabilize=int(4*int(flow.FIELD_SHAPE[0])/case.u0) if stabilization_steps is None else stabilization_steps + if type(stabilize) is not int or stabilize<1: raise ValueError("stabilization_steps must be positive") + zero=np.zeros(spec.physical_action_width,np.float32); flow.run(stabilize,zero) + if np.any(np.asarray(flow.current_effective_action())): raise RuntimeError("post-stabilization EMA must be exactly zero") + checkpoint=flow.full_state_checkpoint() + norm_history=[]; measured_target=[] + for _ in range(spec.fifo_len): + flow.run(spec.control_interval,zero); raw=np.asarray(flow.obs) + norm_history.append(_canonical_history_observation(raw,spec)) + if spec.case_id=="illusion_1.0L" and spec.role=="q_target": measured_target.append(raw.copy()) + measured_harmonics=analyze_harmonics(np.asarray(measured_target,np.float32)) if measured_target else None + flow.restore_full_state(checkpoint) + if np.any(np.asarray(flow.current_effective_action())) or np.any(np.asarray(flow.action)): raise RuntimeError("restored post-stabilization EMA/action must be exactly zero") + if spec.role=="q_ctl" and spec.case_id=="illusion_1.0L": + normalization,harmonics,identity=load_illusion_training_reference(); runtime.controller_reference_identity=identity + elif spec.role=="q_ctl": + normalization=_normalization_from_history(np.asarray(norm_history,np.float32)); harmonics=None; runtime.controller_reference_identity={"kind":"generated_karman_normalization"} + else: + normalization={"force_norm_fact":np.asarray(1,np.float32),"sens_deviation":np.zeros(6,np.float32),"sens_norm_fact":np.ones(6,np.float32)} + harmonics=measured_harmonics if spec.role=="q_target" else None + runtime.controller_reference_identity={"kind":"not_policy_consumed","measured_phase_evidence":bool(measured_harmonics)} + warm=np.zeros(spec.physical_action_width,np.float32) + if spec.role=="q_ctl": warm[-3:]=np.asarray((0,-4,4) if spec.case_id=="karman_re100" else (0,-1,1),np.float32)*case.u0 + history=[] + for _ in range(spec.fifo_len): + flow.run(spec.control_interval,warm); history.append(_canonical_history_observation(np.asarray(flow.obs),spec)) + runtime.set_controller_state(fifo_history=np.asarray(history,np.float32),normalization=normalization,harmonics=harmonics,solver_absolute_control_clock=flow.solver_clock_state()["solver_absolute_control_clock"],solver_absolute_lattice_clock=flow.solver_clock_state()["solver_absolute_lattice_clock"],policy_harmonic_phase_index=0) + runtime.measured_target_harmonics=measured_harmonics + runtime.initial_policy_observation = np.zeros(12,np.float32) if spec.case_id=="karman_re100" and spec.role=="q_ctl" else None + + +def _phase_probe(runtime: RoleRuntime): + spec=runtime.spec; flow=runtime.flow + center_y=int((int(flow.FIELD_SHAPE[1])-1)/2) + x=int(round(runtime.case.geometry.sensor_x_D*20.0)) + def probe(_:Any)->np.ndarray: + velocity=np.asarray(flow.current_step_velocity_probe((x,center_y))) + if velocity.dtype!=np.float32 or velocity.shape!=(2,) or not np.isfinite(velocity).all(): raise ValueError("phase velocity probe invalid") + return velocity[:1].copy() + return probe + + +def _runtime_config(runtime:RoleRuntime, *, solver_lattice_origin:int, solver_control_origin:int, field_interval:int, objects:list[dict], coordinate_frame:dict) -> dict: + config=case_snapshot(runtime.spec.case_id,runtime.spec.role) + config["runtime"]={"role_spec":asdict(runtime.spec),"role_spec_sha256":role_spec_identity(runtime.spec),"physical_action_width":runtime.spec.physical_action_width,"solver_objects":objects,"coordinate_frame":coordinate_frame,"policy_device":runtime.policy_device,"cfd_device":runtime.cfd_device,"action_formula":ACTION_FORMULA,"action_formula_sha256":ACTION_FORMULA_SHA256,"velocity_decoder":{"schema_id":VELOCITY_DECODER_SCHEMA_ID,"quantity":"nondimensional velocity q/U0","u0":float(runtime.case.u0),"formula":VELOCITY_DECODER_FORMULA,"formula_sha256":VELOCITY_DECODER_FORMULA_SHA256}} + normalization={key:np.asarray(value).tolist() for key,value in runtime.normalization.items()} + config["controller_sources"]={"normalization":normalization,"normalization_content_sha256":sha256(canonical_json(normalization)).hexdigest(),"controller_harmonics":runtime.harmonics or [],"controller_harmonics_content_sha256":sha256(canonical_json(runtime.harmonics or [])).hexdigest(),"identity":getattr(runtime,"controller_reference_identity",expected_controller_identity(runtime.spec.case_id,runtime.spec.role)),"measured_plus11_phase_harmonics":getattr(runtime,"measured_target_harmonics",None),"compatibility":"Illusion PPO always consumes frozen training reference; +11D measured target harmonics are phase evidence only and require replay/history compatibility smoke"} + config["source_sha256"]=expected_source_bindings(runtime.spec.case_id,runtime.spec.role) + config["clock_domains"]={"solver_absolute_lattice_origin":solver_lattice_origin,"solver_absolute_control_origin":solver_control_origin,"solver_absolute_lattice_final":runtime.solver_absolute_lattice_clock,"solver_absolute_control_final":runtime.solver_absolute_control_clock,"acquisition_relative_lattice_final":runtime.acquisition_relative_lattice_clock,"acquisition_relative_control_final":runtime.acquisition_relative_control_index,"policy_harmonic_phase_final":runtime.policy_harmonic_phase_index} + config["acquisition"]={"field_interval":field_interval,"checkpoint_lifecycle":"after final completed control boundary; raw observation is final lattice step, public obs is interval average; DDF current is completed and temp is previous/work","control_history":"complete boundary-average lineage independent of field cadence","policy_input_contract":"reconstruct_from_prior_boundary_history" if runtime.spec.role=="q_ctl" else "not_applicable_explicit_zero"} + return config + + +def run_role_acquisition(*, case:str, role:str, output:Path, control_count:int, + field_interval:int, runtime:RoleRuntime|None=None, + runtime_builder:Any=build_role_runtime, + initializer:Any=initialize_role, + target_harmonics:list[dict]|None=None, + stabilization_steps:int|None=None) -> Path: + """Execute, validate and atomically publish one complete role acquisition.""" + if output.exists(): raise FileExistsError(output) + if type(control_count) is not int or control_count<1 or type(field_interval) is not int or field_interval<1: raise ValueError("counts/intervals must be positive") + horizon=CASES[case].sample_interval*control_count + if horizon%field_interval!=0: raise ValueError("horizon must be divisible by field_interval; terminal field is required") + schedule=field_steps(horizon,field_interval) + if not schedule or schedule[-1]!=horizon: raise ValueError("field schedule must be nonempty and include terminal boundary") + runtime=runtime or runtime_builder(case=case,role=role,output=output) + if runtime.spec.case_id!=case or runtime.spec.role!=role: raise ValueError("runtime role mismatch") + objects=verify_solver_objects(runtime.flow,runtime.spec) + initializer(runtime,target_harmonics=target_harmonics,stabilization_steps=stabilization_steps) + live=runtime.flow.solver_clock_state() + if live!={"solver_absolute_lattice_clock":runtime.solver_absolute_lattice_clock,"solver_absolute_control_clock":runtime.solver_absolute_control_clock}: raise RuntimeError("runtime clocks contradict live solver after warmup") + telemetry=ExactTelemetry(role=role,observation_slices=runtime.spec.observation_slices,phase_probe=_phase_probe(runtime),physical_width=runtime.spec.physical_action_width) + horizon=runtime.spec.control_interval*control_count + collector=DualClockCollector(runtime.flow,control_interval=runtime.spec.control_interval,control_count=control_count,absolute_origin=runtime.solver_absolute_lattice_clock,field_steps_absolute=field_steps(horizon,field_interval,absolute_origin=runtime.solver_absolute_lattice_clock),telemetry=telemetry) + solver_lattice_origin=runtime.solver_absolute_lattice_clock; solver_control_origin=runtime.solver_absolute_control_clock + for index in range(control_count): + normalized,physical=runtime.requested_action(); telemetry.set_requested(normalized,physical) + boundary=np.asarray(collector.run_interval(index,physical)) + canonical_boundary=_canonical_history_observation(boundary,runtime.spec) + runtime.boundary_observation_history.append(canonical_boundary.copy()) + runtime.fifo.pop(0); runtime.fifo.append(canonical_boundary) + runtime.acquisition_relative_control_index+=1; runtime.acquisition_relative_lattice_clock+=runtime.spec.control_interval; runtime.policy_harmonic_phase_index+=1 + runtime.solver_absolute_control_clock+=1; runtime.solver_absolute_lattice_clock+=runtime.spec.control_interval + if runtime.flow.solver_clock_state()!={"solver_absolute_lattice_clock":runtime.solver_absolute_lattice_clock,"solver_absolute_control_clock":runtime.solver_absolute_control_clock}: raise RuntimeError("runtime and solver clocks diverged after interval") + arrays=collector.arrays(); flags=solver_fluid_mask(runtime.flow) + nx,ny=flags.shape + coordinate_frame=canonical_coordinate_frame(nx,ny) + reference=coordinate_frame["reference_length_lattice"] + arrays.update(x_D=canonical_coordinate_axis(nx,origin_lattice=coordinate_frame["x"]["origin_lattice"],spacing_lattice=coordinate_frame["x"]["spacing_lattice"],reference_length_lattice=reference),y_D=canonical_coordinate_axis(ny,origin_lattice=coordinate_frame["y"]["origin_lattice"],spacing_lattice=coordinate_frame["y"]["spacing_lattice"],reference_length_lattice=reference),fluid_mask=flags) + config=_runtime_config(runtime,solver_lattice_origin=solver_lattice_origin,solver_control_origin=solver_control_origin,field_interval=field_interval,objects=objects,coordinate_frame=coordinate_frame) + state=runtime.controller_state_identity() + state["config_hash"]=np.asarray(sha256(canonical_json(config)).hexdigest()) + with ArtifactTransaction(output) as transaction: + transaction.write(arrays=arrays,config=config,state=state) + return transaction.publish() + +def _solver_methods_from_source(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == "FlowField": + return {child.name for child in node.body if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef))} + return set() + + +def preflight(case: str | None = None, *, repo_root: Path = REPO_ROOT) -> dict[str, Any]: + """CPU-only, fail-closed readiness inspection. It never imports CUDA or PPO.""" + cases = tuple(CASES) if case is None else (case,) + if any(item not in CASES for item in cases): raise ValueError("unsupported case") + checks: list[dict[str, Any]] = [] + def record(name: str, ok: bool, detail: Any): checks.append({"name": name, "ok": bool(ok), "detail": detail}) + for relative in ("configs/legacy_configs/config_cuda.json","configs/legacy_configs/config_flowfield.json"): + path=repo_root/relative; actual=file_sha256(path) if path.is_file() else None + record(relative,actual==EXPECTED_SHA256[relative],{"path":str(path),"expected_sha256":EXPECTED_SHA256[relative],"actual_sha256":actual}) + for path,expected,name in ((repo_root/ILLUSION_TRAINING_NORM.relative_to(REPO_ROOT),ILLUSION_TRAINING_NORM_SHA256,"illusion.training_norm"),(repo_root/ILLUSION_TRAINING_HARMONICS.relative_to(REPO_ROOT),ILLUSION_TRAINING_HARMONICS_SHA256,"illusion.training_harmonics")): + actual=file_sha256(path) if path.is_file() else None; record(name,actual==expected,{"path":str(path),"expected_sha256":expected,"actual_sha256":actual}) + methods = _solver_methods_from_source(repo_root / "LegacyCelerisLab/driver.py") + missing = sorted(set(REQUIRED_SOLVER_APIS) - methods) + record("solver_read_only_apis", not missing, {"required": REQUIRED_SOLVER_APIS, "missing": missing}) + for case_id in cases: + model = repo_root / MODEL_FILES[case_id].relative_to(REPO_ROOT); relative=str(MODEL_FILES[case_id].relative_to(REPO_ROOT)) + actual=file_sha256(model) if model.is_file() else None + record(f"{case_id}.model",actual==EXPECTED_SHA256[relative],{"path":str(model),"expected_sha256":EXPECTED_SHA256[relative],"actual_sha256":actual}) + try: + with zipfile.ZipFile(model) as archive: metadata=json.loads(archive.read("data")) + observation_shape=tuple(metadata["observation_space"]["_shape"]); action_shape=tuple(metadata["action_space"]["_shape"]) + spaces_ok=observation_shape==((12,) if case_id=="karman_re100" else (14,)) and action_shape==(3,) and metadata["observation_space"]["dtype"]==metadata["action_space"]["dtype"]=="float32" + except Exception as error: + spaces_ok=False; observation_shape=action_shape=(); metadata_error=repr(error) + record(f"{case_id}.policy_archive_spaces",spaces_ok,{"observation_shape":observation_shape,"action_shape":action_shape,"error":locals().get("metadata_error")}) + specs = [role_spec(case_id, role) for role in ROLES] + record(f"{case_id}.role_specs", all(s.objects and s.physical_action_width == len(s.objects) for s in specs), + [{"role": s.role, "objects": [o.identity for o in s.objects], "identity": role_spec_identity(s)} for s in specs]) + contract = CASES[case_id] + record(f"{case_id}.history", specs[2].fifo_len == 150, {"fifo_len": specs[2].fifo_len}) + record(f"{case_id}.harmonics", case_id != "illusion_1.0L" or (specs[0].harmonic_channels == 8 and specs[2].harmonic_channels == 2), + {"measured_q_target_channels": specs[0].harmonic_channels, "frozen_q_ctl_target_force_channels": specs[2].harmonic_channels}) + geometry = asdict(contract.geometry) + expected = ({"front_x_D": 30.0, "rear_x_D": 31.3, "sensor_x_D": 40.0, "target_x_D": None} + if case_id == "karman_re100" else + {"front_x_D": 30.0, "rear_x_D": 31.3, "sensor_x_D": 41.0, "target_x_D": 31.0}) + geometry_ok = all(geometry[key] == value for key, value in expected.items()) + geometry_ok = geometry_ok and tuple(geometry["rear_y_D"]) == (.75, -.75) and tuple(geometry["sensor_y_D"]) == (2., 0., -2.) + record(f"{case_id}.geometry", geometry_ok, geometry) + record("initialization_contract", all(token in Path(__file__).read_text() for token in ("full_state_checkpoint()","restore_full_state(checkpoint)","post-stabilization EMA must be exactly zero")),{"sequence":"stabilize -> full checkpoint -> zero normalization trajectory -> exact restore -> FIFO warmup"}) + record("clock_domain_contract",all(token in Path(__file__).read_text() for token in ("solver_absolute_lattice_clock","solver_absolute_control_clock","acquisition_relative_lattice_clock","acquisition_relative_control_index","policy_harmonic_phase_index")),{"solver":"absolute lifecycle lineage","acquisition":"relative rollout zero-origin","policy_phase":"independent zero-origin"}) + record("policy_device_contract", POLICY_DEVICE=="cpu" and "device=POLICY_DEVICE" in Path(__file__).read_text() and "device=f\"cuda:" not in Path(__file__).read_text(), {"policy_device":POLICY_DEVICE,"cfd_device":"separate logical device selected at runtime","cuda_initialized":False}) + record("policy_initial_state_contract", "self.policy_harmonic_phase_index==0" in Path(__file__).read_text() and "reconstruct_harmonics(self.policy_harmonic_phase_index" in Path(__file__).read_text(),{"karman_first":"exact zero 12-vector","illusion_first":"warmup boundary normalization + frozen training harmonic phase zero"}) + record("runner_reachable",callable(run_role_acquisition) and "transaction.publish()" in Path(__file__).read_text(),{"callable":True,"publishes_artifact":True}) + record("action_formula",ACTION_FORMULA_SHA256==sha256(ACTION_FORMULA.encode()).hexdigest(),{"formula":ACTION_FORMULA,"sha256":ACTION_FORMULA_SHA256}) + record("velocity_decoder",VELOCITY_DECODER_FORMULA_SHA256==sha256(VELOCITY_DECODER_FORMULA.encode()).hexdigest(),{"schema_id":VELOCITY_DECODER_SCHEMA_ID,"quantity":"q/U0","formula":VELOCITY_DECODER_FORMULA,"sha256":VELOCITY_DECODER_FORMULA_SHA256}) + ok = all(check["ok"] for check in checks) + blockers = [check["name"] for check in checks if not check["ok"]] + return {"schema_id": "ccd-acquisition-preflight/v1", "ready_for_independent_pre_cfd_gate": ok, + "cfd_executed": False, "runtime_success_claimed": False, "checks": checks, "blockers": blockers, + "production_blockers": ["Illusion +11D replay/history smoke not run"] if "illusion_1.0L" in cases else []} diff --git a/src/CCD_analysis/acquisition/solver_state.py b/src/CCD_analysis/acquisition/solver_state.py new file mode 100644 index 0000000..4497f4f --- /dev/null +++ b/src/CCD_analysis/acquisition/solver_state.py @@ -0,0 +1,91 @@ +"""CPU-testable solver extraction and checkpoint-copy primitives.""" +from __future__ import annotations +from hashlib import sha256 +from collections.abc import Callable +import numpy as np + + +LEGACY_FLUID_BIT = np.uint8(0b00000001) + + +def _fluid_mask_xy(mask_or_flags: np.ndarray, nx: int, ny: int) -> np.ndarray: + """Validate an exact mask or decode Legacy's FLUID bit from solver flags.""" + source = np.asarray(mask_or_flags) + if source.shape != (nx, ny): + raise ValueError("solver fluid mask/flags must match the exact (nx, ny) grid") + if source.dtype == np.bool_: + mask = source + elif source.dtype == np.uint8: + # Legacy driver.py and kernels/macros.h both define FLUID as bit 0. + mask = (source & LEGACY_FLUID_BIT) != 0 + else: + raise ValueError("solver fluid mask must be bool or Legacy flags must be uint8") + if not mask.any(): + raise ValueError("solver fluid mask is empty") + return np.ascontiguousarray(mask, dtype=np.bool_) + + +def d2q9_q_over_u0_xy(flat_ddf: np.ndarray, nx: int, ny: int, + fluid_mask_or_flags: np.ndarray, u0: float + ) -> tuple[np.ndarray, np.ndarray]: + """Decode Legacy nondimensional velocity ``q/U0`` on fluid cells. + + This is the exact archived Legacy formula: D2Q9 momentum components are + divided by the configured reference velocity ``u0``. No density division + is performed. Nonfluid populations are ignored and export exact zero. + """ + if type(nx) is not int or type(ny) is not int or nx < 1 or ny < 1: + raise ValueError("nx and ny must be positive integers") + if type(u0) not in (int, float) or not np.isfinite(u0) or u0 <= 0: + raise ValueError("u0 must be positive and finite") + flat = np.asarray(flat_ddf) + if flat.dtype != np.float32 or flat.ndim != 1 or flat.size != 9 * nx * ny: + raise ValueError("D2Q9 storage must be flat float32 with size 9*nx*ny") + mask = _fluid_mask_xy(fluid_mask_or_flags, nx, ny) + populations = flat.reshape(9, ny, nx).transpose(2, 1, 0) + fluid = populations[mask] + if not np.isfinite(fluid).all(): + raise ValueError("D2Q9 fluid populations contain non-finite values") + ux = np.zeros((nx, ny), dtype=np.float32) + uy = np.zeros((nx, ny), dtype=np.float32) + ux[mask] = ((fluid[:, 1] + fluid[:, 5] + fluid[:, 8] + - fluid[:, 3] - fluid[:, 6] - fluid[:, 7]) / np.float32(u0)).astype(np.float32) + uy[mask] = ((fluid[:, 2] + fluid[:, 5] + fluid[:, 6] + - fluid[:, 4] - fluid[:, 7] - fluid[:, 8]) / np.float32(u0)).astype(np.float32) + if not np.isfinite(ux[mask]).all() or not np.isfinite(uy[mask]).all(): + raise ValueError("decoded fluid velocity is non-finite") + return ux, uy + + +def copy_ping_pong_ddf(copy_current: Callable[[np.ndarray], None], + copy_temp: Callable[[np.ndarray], None], size: int + ) -> dict[str, np.ndarray | str]: + """Copy current(completed) then temp(previous/work) without changing either.""" + if type(size) is not int or size < 1: + raise ValueError("checkpoint size must be a positive integer") + current = np.empty(size, dtype=np.float32) + temp = np.empty(size, dtype=np.float32) + copy_current(current) + copy_temp(temp) + if not np.isfinite(current).all() or not np.isfinite(temp).all(): + raise ValueError("checkpoint DDF buffers must be finite") + return { + "current_ddf": current, + "temp_ddf": temp, + "current_sha256": sha256(current.tobytes()).hexdigest(), + "temp_sha256": sha256(temp.tobytes()).hexdigest(), + } + + +def ema_step(previous: np.ndarray, target: np.ndarray, steps: int, weight: float = 0.1) -> np.ndarray: + """Pure reference for Legacy's per-lattice-step action EMA.""" + old = np.asarray(previous) + goal = np.asarray(target) + if old.dtype != np.float32 or goal.dtype != np.float32 or old.shape != goal.shape: + raise ValueError("EMA arrays must have matching float32 shape") + if type(steps) is not int or steps < 0 or not np.isfinite(weight) or not 0 < weight <= 1: + raise ValueError("invalid EMA step contract") + result = old.copy() + for _ in range(steps): + result = ((1.0 - weight) * result + weight * goal).astype(np.float32) + return result diff --git a/src/CCD_analysis/acquisition/validation.py b/src/CCD_analysis/acquisition/validation.py new file mode 100644 index 0000000..d37c473 --- /dev/null +++ b/src/CCD_analysis/acquisition/validation.py @@ -0,0 +1,251 @@ +"""Authoritative semantic validator for active acquisition artifacts.""" +from __future__ import annotations +from dataclasses import asdict +from hashlib import sha256 +from pathlib import Path +from typing import Any, Mapping +import json +import numpy as np +from .contracts import (ACTION_FORMULA, ACTION_FORMULA_SHA256, ARTIFACT_SCHEMA_ID, CASES, CONFIG_BINDINGS, ILLUSION_TRAINING_BINDINGS, MODEL_BINDINGS, ROLES, SCHEMA_ID, canonical_json, case_snapshot, expected_controller_identity, expected_source_bindings, role_spec, role_spec_identity, validate_coordinate_arrays, VELOCITY_DECODER_FORMULA, VELOCITY_DECODER_FORMULA_SHA256, VELOCITY_DECODER_SCHEMA_ID) +from .dual_clock import TELEMETRY_KEYS + + +def _reconstruct_harmonics(t: int, harmonics: list[dict]) -> np.ndarray: + result = np.zeros(len(harmonics), np.float32) + for index, harmonic in enumerate(harmonics): + value = float(harmonic["dc"]) + for amp, freq, phase in zip(harmonic["amps"], harmonic["freqs"], harmonic["phases"]): + value += float(amp) * np.cos(2 * np.pi * float(freq) * t + float(phase)) + result[index] = value + return result + +FIELD_KEYS = {"ux", "uy", "x_D", "y_D", "fluid_mask", "lattice_steps", "control_indices", "control_offsets", *TELEMETRY_KEYS} +STATE_KEYS = {"current_ddf", "temp_ddf", "current_raw_observation", "fifo_history", "initial_fifo_history", "boundary_observation_history", "policy_source_observation_history", "policy_source_observation_sha256", "policy_input_observation_history", "policy_harmonic_phase_indices", "requested_normalized_action_history", "requested_physical_action_history", "persisted_effective_ema_action", "policy_harmonic_phase_index", "solver_absolute_control_clock", "solver_absolute_lattice_clock", "acquisition_relative_control_index", "acquisition_relative_lattice_clock", "normalization_hash", "harmonics_hash", "model_hash", "cuda_config_hash", "flow_config_hash", "config_hash", "geometry_hash", "action_formula_hash", "velocity_decoder_formula_hash"} +CONFIG_KEYS = {"schema_id", "case", "role", "role_semantics", "timeline_semantics", "phase_reference_semantics", "action_semantics", "velocity_decoder", "geometry_sha256", "config_sha256", "runtime", "controller_sources", "source_sha256", "clock_domains", "acquisition"} +LEGACY_ZERO_ROLE_RUNTIME_KEYS = {"role_spec", "role_spec_sha256", "physical_action_width", "solver_objects", "coordinate_frame", "action_formula", "action_formula_sha256", "velocity_decoder"} +RUNTIME_KEYS = LEGACY_ZERO_ROLE_RUNTIME_KEYS | {"policy_device", "cfd_device"} +ROLE_SPEC_KEYS = {"case_id", "role", "objects", "control_interval", "physical_action_width", "observation_slices", "controller", "fifo_len", "harmonic_channels", "model_path"} +CLOCK_KEYS = {"solver_absolute_lattice_origin", "solver_absolute_control_origin", "solver_absolute_lattice_final", "solver_absolute_control_final", "acquisition_relative_lattice_final", "acquisition_relative_control_final", "policy_harmonic_phase_final"} +CONTROLLER_KEYS = {"normalization", "normalization_content_sha256", "controller_harmonics", "controller_harmonics_content_sha256", "identity", "measured_plus11_phase_harmonics", "compatibility"} +ACQUISITION_KEYS = {"field_interval", "checkpoint_lifecycle", "control_history", "policy_input_contract"} +MANIFEST_KEYS = {"schema_id", "complete", "files", "state_array_sha256", "config_sha256", "field_count"} + + +def array_sha256(value: np.ndarray) -> str: + return sha256(np.ascontiguousarray(value).tobytes()).hexdigest() + + +def require_sha256(value: Any, label: str) -> str: + if not isinstance(value, str) or len(value) != 64: + raise ValueError(f"{label} must be SHA256") + try: + int(value, 16) + except ValueError as exc: + raise ValueError(f"{label} must be SHA256") from exc + return value + + +def _state_sha(value: np.ndarray, label: str) -> str: + if value.ndim != 0 or value.dtype.kind not in "SU": + raise ValueError(f"{label} must be a scalar string") + return require_sha256(str(value.item()), label) + + +def _finite_json(value: Any, label: str) -> None: + try: + canonical_json(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{label} must be finite canonical-JSON compatible") from exc + + +def validate_acquisition_semantics(*, arrays: Mapping[str, Any], config: Mapping[str, Any], state: Mapping[str, Any], manifest: Mapping[str, Any] | None = None, expected_case: str | None = None, expected_role: str | None = None) -> tuple[dict[str, np.ndarray], dict[str, np.ndarray]]: + """Validate content semantics, not merely internal artifact hashes.""" + if not isinstance(config, Mapping) or set(config) != CONFIG_KEYS: + raise ValueError("acquisition config top-level schema is not exact") + case_id, role = config.get("case", {}).get("case_id"), config.get("role") + if case_id not in CASES or role not in ROLES or (expected_case is not None and case_id != expected_case) or (expected_role is not None and role != expected_role): + raise ValueError("artifact exact case/role contract mismatch") + config_json = json.loads(canonical_json(config)) + frozen = json.loads(canonical_json(case_snapshot(case_id, role))) + for key in ("schema_id", "case", "role", "role_semantics", "timeline_semantics", "phase_reference_semantics", "action_semantics", "velocity_decoder", "geometry_sha256", "config_sha256"): + if config_json.get(key) != frozen[key]: + raise ValueError(f"artifact exact case/role contract mismatch: {key}") + if config["schema_id"] != SCHEMA_ID: + raise ValueError("acquisition contract schema mismatch") + + runtime, clocks, acquisition, controller = config["runtime"], config["clock_domains"], config["acquisition"], config["controller_sources"] + runtime_keys = set(runtime) if isinstance(runtime, Mapping) else set() + allowed_runtime_schema = runtime_keys == RUNTIME_KEYS or (role in {"q_target", "q_blk"} and runtime_keys == LEGACY_ZERO_ROLE_RUNTIME_KEYS) + if not allowed_runtime_schema or not isinstance(runtime["role_spec"], Mapping) or set(runtime["role_spec"]) != ROLE_SPEC_KEYS: + raise ValueError("runtime/role-spec config schema is not exact") + spec = runtime["role_spec"] + authoritative_spec = json.loads(canonical_json(asdict(role_spec(case_id, role)))) + if json.loads(canonical_json(spec)) != authoritative_spec or runtime["role_spec_sha256"] != role_spec_identity(role_spec(case_id, role)): + raise ValueError("runtime role spec differs from frozen authoritative role_spec") + width = runtime["physical_action_width"] + objects, persisted_objects = spec["objects"], runtime["solver_objects"] + if type(width) is not int or width < 1 or not isinstance(objects, (list, tuple)) or len(objects) != width or not isinstance(persisted_objects, list) or len(persisted_objects) != width: + raise ValueError("runtime physical action/object width is invalid") + inferred_center_y = None + for index, (declared, persisted) in enumerate(zip(objects, persisted_objects)): + if not isinstance(declared, Mapping) or set(declared) != {"kind", "identity", "center_D", "radius_D"} or not isinstance(persisted, Mapping) or set(persisted) != {"id", "identity", "kind", "center_lattice", "radius_lattice"}: raise ValueError("solver object schema is not exact") + center_D, center_lattice = declared["center_D"], persisted["center_lattice"] + if not isinstance(center_D, (list, tuple)) or len(center_D) != 2 or not isinstance(center_lattice, (list, tuple)) or len(center_lattice) != 3 or not all(type(v) in (int, float) and np.isfinite(v) for v in (*center_D, *center_lattice)): raise ValueError("solver object coordinates are invalid") + candidate_center_y = float(center_lattice[1]) - float(center_D[1]) * 20.0 + inferred_center_y = candidate_center_y if inferred_center_y is None else inferred_center_y + if persisted["id"] != index or persisted["identity"] != declared["identity"] or persisted["kind"] != declared["kind"] or float(center_lattice[0]) != float(center_D[0]) * 20.0 or float(center_lattice[2]) != 0.0 or candidate_center_y != inferred_center_y or float(persisted["radius_lattice"]) != float(declared["radius_D"]) * 20.0: raise ValueError("solver object semantics contradict role spec") + if runtime_keys == RUNTIME_KEYS and (runtime["policy_device"] != "cpu" or type(runtime["cfd_device"]) is not int or runtime["cfd_device"] < 0): + raise ValueError("runtime compute-device roles are invalid") + if runtime["action_formula"] != ACTION_FORMULA or runtime["action_formula_sha256"] != ACTION_FORMULA_SHA256: + raise ValueError("runtime action formula differs from frozen authority") + decoder = runtime["velocity_decoder"] + if (not isinstance(decoder, Mapping) or set(decoder) != {"schema_id", "quantity", "u0", "formula", "formula_sha256"} + or decoder["schema_id"] != VELOCITY_DECODER_SCHEMA_ID + or decoder["quantity"] != "nondimensional velocity q/U0" + or type(decoder["u0"]) not in (int, float) or not np.isfinite(decoder["u0"]) or decoder["u0"] <= 0 + or float(decoder["u0"]) != float(config["case"]["u0"]) + or decoder["formula"] != VELOCITY_DECODER_FORMULA + or decoder["formula_sha256"] != VELOCITY_DECODER_FORMULA_SHA256): + raise ValueError("runtime velocity decoder differs from frozen q/U0 authority") + if not isinstance(clocks, Mapping) or set(clocks) != CLOCK_KEYS or any(type(clocks[key]) is not int or clocks[key] < 0 for key in CLOCK_KEYS): + raise ValueError("clock-domain config schema is not exact") + if not isinstance(acquisition, Mapping) or set(acquisition) != ACQUISITION_KEYS or type(acquisition["field_interval"]) is not int or acquisition["field_interval"] < 1 or not isinstance(acquisition["checkpoint_lifecycle"], str) or acquisition["control_history"] != "complete boundary-average lineage independent of field cadence" or acquisition["policy_input_contract"] != ("reconstruct_from_prior_boundary_history" if role == "q_ctl" else "not_applicable_explicit_zero"): + + raise ValueError("acquisition config schema is not exact") + if not isinstance(controller, Mapping) or set(controller) != CONTROLLER_KEYS: + raise ValueError("controller-source config schema is not exact") + normalization = controller["normalization"] + if not isinstance(normalization, Mapping) or set(normalization) != {"force_norm_fact", "sens_deviation", "sens_norm_fact"}: + raise ValueError("controller normalization schema is not exact") + force = np.asarray(normalization["force_norm_fact"]); deviation = np.asarray(normalization["sens_deviation"]); scale = np.asarray(normalization["sens_norm_fact"]) + if force.ndim != 0 or not np.isfinite(force) or float(force) <= 0 or deviation.shape != (6,) or scale.shape != (6,) or not np.isfinite(deviation).all() or not np.isfinite(scale).all() or np.any(scale <= 0): + raise ValueError("controller normalization semantics are invalid") + if sha256(canonical_json(normalization)).hexdigest() != require_sha256(controller["normalization_content_sha256"], "normalization_content_sha256"): + raise ValueError("normalization content hash mismatch") + harmonics = controller["controller_harmonics"] + if not isinstance(harmonics, list) or sha256(canonical_json(harmonics)).hexdigest() != require_sha256(controller["controller_harmonics_content_sha256"], "controller_harmonics_content_sha256"): + raise ValueError("controller harmonics identity mismatch") + if not isinstance(controller["identity"], Mapping) or not isinstance(controller["compatibility"], str): + raise ValueError("controller source identity/compatibility is invalid") + expected_identity = expected_controller_identity(case_id, role) + if case_id == "illusion_1.0L" and role == "q_ctl": + actual_identity = dict(controller["identity"]); actual_identity["normalization_path"] = str(Path(actual_identity.get("normalization_path", "")).resolve().relative_to(Path(__file__).resolve().parents[3])) if Path(actual_identity.get("normalization_path", "")).is_absolute() else actual_identity.get("normalization_path") + actual_identity["harmonics_path"] = str(Path(actual_identity.get("harmonics_path", "")).resolve().relative_to(Path(__file__).resolve().parents[3])) if Path(actual_identity.get("harmonics_path", "")).is_absolute() else actual_identity.get("harmonics_path") + if actual_identity != expected_identity: raise ValueError("Illusion training reference paths/hashes differ from frozen authority") + elif controller["identity"] != expected_identity: + raise ValueError("controller source identity differs from frozen authority") + _finite_json(controller["measured_plus11_phase_harmonics"], "measured phase harmonics") + if config["source_sha256"] != expected_source_bindings(case_id, role): + raise ValueError("source paths/hashes differ from frozen case/role authority") + + if set(arrays) != FIELD_KEYS: + raise ValueError("artifact arrays do not match unified schema") + data = {key: np.asarray(value) for key, value in arrays.items()} + ux, uy, x, y, mask = data["ux"], data["uy"], data["x_D"], data["y_D"], data["fluid_mask"] + if ux.dtype != np.float32 or uy.dtype != np.float32 or ux.ndim != 3 or ux.shape != uy.shape or not np.isfinite(ux).all() or not np.isfinite(uy).all(): + raise ValueError("fields must be finite matching float32 (time,x,y)") + count, nx, ny = ux.shape + if count < 1: raise ValueError("field time axis must be nonempty") + validate_coordinate_arrays(x, y, runtime["coordinate_frame"]) + if x.shape != (nx,) or y.shape != (ny,): raise ValueError("declared coordinates must match field axes") + if mask.dtype != np.bool_ or mask.shape != (nx, ny) or not mask.any(): + raise ValueError("saved solver fluid mask must match nonempty (x,y) grid") + if np.any(ux[:, ~mask] != np.float32(0)) or np.any(uy[:, ~mask] != np.float32(0)): + raise ValueError("solid-cell velocities must be exact float32 zero") + integer_keys = ("lattice_steps", "control_indices", "control_offsets", "sample_ids", "acquisition_relative_lattice_steps", "solver_absolute_control_indices") + for key in integer_keys: + if data[key].dtype != np.int64 or data[key].shape != (count,): raise ValueError(f"{key} must be int64 length time") + steps, relative = data["lattice_steps"], data["acquisition_relative_lattice_steps"] + interval, origin, control_origin = config["case"]["sample_interval"], clocks["solver_absolute_lattice_origin"], clocks["solver_absolute_control_origin"] + expected_controls = (relative - 1) // interval + if np.any(np.diff(steps) <= 0) or np.any(relative <= 0) or not np.array_equal(data["sample_ids"], steps) or not np.array_equal(relative, steps - origin) or not np.array_equal(data["control_indices"], expected_controls) or not np.array_equal(data["control_offsets"], (relative - 1) % interval + 1) or not np.array_equal(data["solver_absolute_control_indices"], control_origin + expected_controls): + raise ValueError("solver/acquisition timeline relations are invalid") + if relative[-1] != clocks["acquisition_relative_lattice_final"] or steps[-1] != clocks["solver_absolute_lattice_final"] or clocks["solver_absolute_control_final"] - control_origin != clocks["acquisition_relative_control_final"] or clocks["policy_harmonic_phase_final"] != clocks["acquisition_relative_control_final"]: + raise ValueError("final clock domains contradict timeline/control lineage") + shapes = {"requested_normalized_action": (count, 3), "requested_physical_action": (count, width), "effective_applied_action": (count, width), "disturbance_force": (count, 2), "pinball_forces": (count, 6), "sensors": (count, 6), "phase_reference": (count, 1)} + for key, shape in shapes.items(): + value = data[key] + if value.dtype != np.float32 or value.shape != shape or not np.isfinite(value).all(): raise ValueError(f"{key} has invalid dtype/shape/finiteness") + normalized, physical, effective = data["requested_normalized_action"], data["requested_physical_action"], data["effective_applied_action"] + if np.any(normalized < -1) or np.any(normalized > 1): raise ValueError("normalized actions outside bounds") + if role != "q_ctl": + if np.any(normalized) or np.any(physical) or np.any(effective): raise ValueError("q_target/q_blk requested and effective actions must be exactly zero") + else: + bias = np.asarray((0., -4., 4.) if case_id == "karman_re100" else (0., -2., 2.), np.float32) + u0 = np.float32(config["case"]["u0"]); expected_physical = np.zeros_like(physical); expected_physical[:, -3:] = (normalized * np.float32(8) + bias) * u0 + lower = (np.asarray([-8., -8., -8.], np.float32) + bias) * u0; upper = (np.asarray([8., 8., 8.], np.float32) + bias) * u0 + if not np.array_equal(physical, expected_physical): raise ValueError("q_ctl requested physical actions contradict frozen formula") + if np.any(physical[:, :-3]) or np.any(effective[:, :-3]) or np.any(effective[:, -3:] < lower) or np.any(effective[:, -3:] > upper): raise ValueError("q_ctl effective EMA actions violate actuated-channel semantics/bounds") + + if set(state) != STATE_KEYS: + raise ValueError("controller state identity inventory is not exact") + state_data = {key: np.asarray(value) for key, value in state.items()} + ddf_size = 9 * nx * ny + for key in ("current_ddf", "temp_ddf"): + if state_data[key].dtype != np.float32 or state_data[key].shape != (ddf_size,) or not np.isfinite(state_data[key]).all(): raise ValueError(f"{key} has invalid D2Q9 storage") + if state_data["current_raw_observation"].dtype != np.float32 or state_data["current_raw_observation"].ndim != 1 or state_data["current_raw_observation"].size < 1 or not np.isfinite(state_data["current_raw_observation"]).all(): raise ValueError("raw observation invalid") + fifo = state_data["fifo_history"] + initial_fifo = state_data["initial_fifo_history"] + boundaries = state_data["boundary_observation_history"] + sources = state_data["policy_source_observation_history"] + source_hashes = state_data["policy_source_observation_sha256"] + policy_inputs = state_data["policy_input_observation_history"] + phase_indices = state_data["policy_harmonic_phase_indices"] + normalized_controls = state_data["requested_normalized_action_history"] + physical_controls = state_data["requested_physical_action_history"] + control_count = clocks["acquisition_relative_control_final"] + s_dim = 12 if case_id == "karman_re100" else 14 + for label, value, shape in (("FIFO/history", fifo, (150, 12)), ("initial FIFO/history", initial_fifo, (150, 12)), ("boundary observation history", boundaries, (control_count, 12)), ("policy source observation history", sources, (control_count, 12)), ("policy input observation history", policy_inputs, (control_count, s_dim)), ("requested normalized control history", normalized_controls, (control_count, 3)), ("requested physical control history", physical_controls, (control_count, width))): + if value.dtype != np.float32 or value.shape != shape or not np.isfinite(value).all(): raise ValueError(f"{label} has invalid dtype/shape/finiteness") + if phase_indices.dtype != np.int64 or phase_indices.shape != (control_count,) or not np.array_equal(phase_indices, np.arange(control_count, dtype=np.int64)): raise ValueError("policy harmonic phase indices must be exact zero-origin control lineage") + if source_hashes.dtype.kind not in "SU" or source_hashes.shape != (control_count,) or any(str(value) != array_sha256(sources[index]) for index, value in enumerate(source_hashes.tolist())): raise ValueError("policy source observation hashes mismatch") + expected_fifo = np.concatenate((initial_fifo, boundaries), axis=0)[-150:] + if not np.array_equal(fifo, expected_fifo): raise ValueError("terminal FIFO must equal rolling append of initial FIFO and all boundaries") + expected_sources = np.zeros_like(sources) + if role == "q_ctl": + expected_sources[0] = initial_fifo[-1] + if control_count > 1: expected_sources[1:] = boundaries[:-1] + if not np.array_equal(sources, expected_sources): raise ValueError("policy source observations must use the appropriate prior FIFO/boundary history or explicit not-applicable zeros") + expected_inputs = np.zeros_like(policy_inputs) + if role == "q_ctl": + for index, raw in enumerate(expected_sources): + if not (case_id == "karman_re100" and index == 0): + force_values = raw[6:12] / np.float32(force) + sensor_values = (raw[:6] - deviation.astype(np.float32)) / scale.astype(np.float32) + values = [*force_values, *sensor_values] + if case_id == "illusion_1.0L": + target = _reconstruct_harmonics(index, harmonics)[:2] / np.float32(force) + values.extend(target.tolist()) + expected_inputs[index] = np.clip(np.asarray(values, np.float32), -1, 1) + if not np.array_equal(policy_inputs, expected_inputs): raise ValueError("policy inputs do not reconstruct exactly from prior history, normalization, harmonics, and initial semantics") + if not np.array_equal(normalized, normalized_controls[data["control_indices"]]) or not np.array_equal(physical, physical_controls[data["control_indices"]]): + raise ValueError("field-time requested actions contradict complete control action histories") + expected_control_physical = np.zeros_like(physical_controls) + if role == "q_ctl": + bias = np.asarray((0., -4., 4.) if case_id == "karman_re100" else (0., -2., 2.), np.float32) + expected_control_physical[:, -3:] = (normalized_controls * np.float32(8) + bias) * np.float32(config["case"]["u0"]) + if not np.array_equal(physical_controls, expected_control_physical): raise ValueError("control action histories contradict frozen role/action formula") + if state_data["persisted_effective_ema_action"].dtype != np.float32 or state_data["persisted_effective_ema_action"].shape != (width,) or not np.isfinite(state_data["persisted_effective_ema_action"]).all() or not np.array_equal(state_data["persisted_effective_ema_action"], effective[-1]): raise ValueError("persisted EMA action must equal terminal sampled effective action") + clock_pairs = (("solver_absolute_lattice_clock", "solver_absolute_lattice_final"), ("solver_absolute_control_clock", "solver_absolute_control_final"), ("acquisition_relative_lattice_clock", "acquisition_relative_lattice_final"), ("acquisition_relative_control_index", "acquisition_relative_control_final"), ("policy_harmonic_phase_index", "policy_harmonic_phase_final")) + for state_key, config_key in clock_pairs: + value = state_data[state_key] + if value.dtype != np.int64 or value.ndim != 0 or int(value) != clocks[config_key]: raise ValueError(f"{state_key} does not exactly match final clock domain") + state_hash_values = {key: _state_sha(state_data[key], key) for key in STATE_KEYS if key.endswith("_hash")} + if state_hash_values["geometry_hash"] != config["geometry_sha256"] or state_hash_values["action_formula_hash"] != runtime["action_formula_sha256"] or state_hash_values["velocity_decoder_formula_hash"] != runtime["velocity_decoder"]["formula_sha256"] or state_hash_values["config_hash"] != sha256(canonical_json(config)).hexdigest(): + raise ValueError("state semantic identity hashes contradict config") + identity_values = {value for key, value in controller["identity"].items() if key.endswith("_sha256") and isinstance(value, str)} + if state_hash_values["normalization_hash"] not in {controller["normalization_content_sha256"], *identity_values} or state_hash_values["harmonics_hash"] not in {controller["controller_harmonics_content_sha256"], *identity_values}: + raise ValueError("state controller-source hashes contradict config") + if state_hash_values["cuda_config_hash"] != CONFIG_BINDINGS["configs/legacy_configs/config_cuda.json"] or state_hash_values["flow_config_hash"] != CONFIG_BINDINGS["configs/legacy_configs/config_flowfield.json"]: + raise ValueError("state solver config hashes differ from frozen authority") + expected_model_hash = MODEL_BINDINGS[case_id][1] if role == "q_ctl" else sha256(b"zero-controller").hexdigest() + if state_hash_values["model_hash"] != expected_model_hash: raise ValueError("state model hash differs from frozen authority") + if case_id == "illusion_1.0L" and role == "q_ctl" and (state_hash_values["normalization_hash"] != ILLUSION_TRAINING_BINDINGS["normalization_sha256"] or state_hash_values["harmonics_hash"] != ILLUSION_TRAINING_BINDINGS["harmonics_sha256"]): raise ValueError("state Illusion training reference hashes differ from frozen authority") + + if manifest is not None: + if not isinstance(manifest, Mapping) or set(manifest) != MANIFEST_KEYS or manifest["schema_id"] != ARTIFACT_SCHEMA_ID or manifest["complete"] is not True or manifest["field_count"] != count: + raise ValueError("artifact manifest semantic schema is not exact") + if set(manifest["state_array_sha256"]) != STATE_KEYS or any(array_sha256(state_data[key]) != manifest["state_array_sha256"][key] for key in STATE_KEYS): + raise ValueError("manifest state array hash inventory is invalid") + if manifest["config_sha256"] != sha256(canonical_json(config)).hexdigest(): raise ValueError("manifest config hash mismatch") + return data, state_data diff --git a/src/CCD_analysis/ccd/run_ccd.py b/src/CCD_analysis/ccd/run_ccd.py deleted file mode 100644 index 35452f3..0000000 --- a/src/CCD_analysis/ccd/run_ccd.py +++ /dev/null @@ -1,346 +0,0 @@ -"""CCD analysis pipeline: POD + force/action CCD. - -New data format (fields_aligned.npz + phase_plan.json). -Target-only POD basis. Per-force observable (primary=SigmaFy). -Short Q_delay=6 for force/action. 1.5L flagged as special_mechanism. - -Usage: - conda run -n pycuda_3_10 python ccd/run_ccd.py - -Requires fields_aligned.npz and phase_plan.json in data/ directories. -""" -from __future__ import annotations - -import json -import os -import sys -import time - -import numpy as np - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR, SCENES, NX, NY -from CCD_analysis.utils.resampling import ( - compute_pod, cumulative_energy, e95_index, - compute_reduced_ccd, - load_aligned_fields, make_force_obs, - build_field_matrix, project_into_basis, -) - -# -- Protocol constants --------------------------------------------------- -R_CANDIDATES = [6, 8, 10] -CCD_Q = 6 # short, near-synchronous window for force/action -DIAMETERS_MAIN = [0.75, 1.0] -DIAMETERS_ALL = [0.75, 1.0, 1.5] -CV_T_RELAXED = 0.12 - - -# --------------------------------------------------------------------------- -# Preflight check (built-in) -# --------------------------------------------------------------------------- - -def preflight(scene_name: str) -> dict: - """Load and verify one scene's data. Returns meta or raises.""" - cfg = SCENES[scene_name] - scene_id = cfg["scene_id"] - data_dir = os.path.join(DATA_DIR, scene_id, scene_name) - - # Check fields_aligned.npz - fa_path = os.path.join(data_dir, "fields_aligned.npz") - if not os.path.isfile(fa_path): - raise FileNotFoundError(f"{fa_path} not found") - - fd = np.load(fa_path) - ux = fd["ux"] - print(f" {scene_name}: fields_aligned ux shape={ux.shape} " - f"(expect ({cfg.get('n_cycles', 4) * cfg.get('n_pts', 24)}, {NX}, {NY}))", - flush=True) - fd.close() - - # Check phase_plan.json - plan_path = os.path.join(DATA_DIR, "resampled", scene_name, "phase_plan.json") - if not os.path.isfile(plan_path): - raise FileNotFoundError(f"{plan_path} not found") - - import json - with open(plan_path) as f: - plan = json.load(f) - - n_total = plan["n_cycles"] * plan["n_pts"] - if n_total != ux.shape[0]: - print(f" WARNING: phase_plan has {n_total} snapshots but fields has {ux.shape[0]}", - flush=True) - - gate = plan["gate"] - cv_t = plan["CV_T"] - print(f" gate={gate}, CV_T={cv_t:.4f}, " - f"N_raw={plan['N_raw_per_cycle']:.1f}, rho={plan['rho_interp']:.2f}", - flush=True) - - if gate not in ("strict", "relaxed") and cv_t is not None and cv_t > CV_T_RELAXED: - print(f" WARNING: gate='{gate}' — does not pass relaxed gate (CV_T <= {CV_T_RELAXED})", - flush=True) - - # Check telemetry - tele_found = False - for p in [os.path.join(data_dir, "controlled.npz"), os.path.join(data_dir, "sensors.npz")]: - if os.path.isfile(p): - td = np.load(p) - if "forces" in td: - print(f" forces: {td['forces'].shape}", flush=True) - if "actions" in td: - print(f" actions: {td['actions'].shape}", flush=True) - td.close() - tele_found = True - break - if not tele_found: - raise FileNotFoundError(f"No telemetry found in {data_dir}") - - return { - "gate": gate, - "CV_T": cv_t, - "n_snapshots": ux.shape[0], - "N_raw_per_cycle": plan.get("N_raw_per_cycle"), - } - - -def compute_modal_overlap(W_dict: dict, diam: float, r: int, - obs_label: str = "force") -> list: - """Compute pairwise modal overlaps for a given diameter and r.""" - keys = [k for k in W_dict - if f"{diam}L_" in k and f"_{obs_label}_r{r}" in k] - overlaps = [] - for i, ka in enumerate(keys): - for kb in keys[i + 1:]: - Wa, Wb = W_dict[ka], W_dict[kb] - n = min(Wa.shape[1], Wb.shape[1], 5) - for k in range(n): - ov = float(abs( - Wa[:, k] / (np.linalg.norm(Wa[:, k]) + 1e-12) @ - Wb[:, k] / (np.linalg.norm(Wb[:, k]) + 1e-12) - )) - overlaps.append({ - "case_a": ka.split(f"_{obs_label}_r{r}")[0], - "case_b": kb.split(f"_{obs_label}_r{r}")[0], - "mode": k + 1, - "O": ov, - }) - return overlaps - - -# --------------------------------------------------------------------------- -# Main pipeline -# --------------------------------------------------------------------------- - -def main(): - print("=" * 60, flush=True) - print("CCD Pipeline (Round 5 — fields_aligned, target-only basis)", flush=True) - print("=" * 60, flush=True) - - out_dir = os.path.join(DATA_DIR, "ccd") - os.makedirs(out_dir, exist_ok=True) - all_results = {} - W_dict = {} # for modal overlap - - # -- Preflight -- - print("\n--- Preflight check ---", flush=True) - all_scenes = ["pinball"] - for diam in DIAMETERS_ALL: - all_scenes.append(f"target_cylinder_{diam}L") - all_scenes.append(f"illusion_{diam}L") - - preflight_ok = {} - for sn in all_scenes: - try: - meta = preflight(sn) - preflight_ok[sn] = meta - print(f" OK", flush=True) - except (FileNotFoundError, AssertionError, KeyError) as e: - print(f" FAILED: {e}", flush=True) - preflight_ok[sn] = None - - # -- Load all data -- - print("\n--- Loading data ---", flush=True) - data_cache = {} - for sn in all_scenes: - if preflight_ok.get(sn) is None: - continue - t0 = time.time() - try: - d = load_aligned_fields(sn) - data_cache[sn] = d - print(f" {sn}: loaded ({len(d['ux'])} snapshots, " - f"{time.time() - t0:.1f}s)", flush=True) - except (FileNotFoundError, AssertionError, KeyError) as e: - print(f" {sn}: FAILED — {e}", flush=True) - - # -- Per-diameter CCD -- - print("\n--- CCD per diameter ---", flush=True) - - for diam in DIAMETERS_ALL: - tgt_name = f"target_cylinder_{diam}L" - ill_name = f"illusion_{diam}L" - - tgt_data = data_cache.get(tgt_name) - ill_data = data_cache.get(ill_name) - pin_data = data_cache.get("pinball") - - if tgt_data is None: - print(f"\n SKIP {diam}L: missing target data", flush=True) - continue - - print(f"\n{'=' * 60}", flush=True) - print(f"Diameter {diam}L", flush=True) - print(f"{'=' * 60}", flush=True) - - is_special = (diam not in DIAMETERS_MAIN) - if is_special: - print(f" Note: {diam}L flagged as special-mechanism case", flush=True) - - # -- Build target-only POD basis -- - Q_tgt = build_field_matrix(tgt_data["ux"], tgt_data["uy"]) - mean_f, modes, sv, coeffs = compute_pod(Q_tgt) - energy = cumulative_energy(sv) - e95 = e95_index(energy) - print(f" Target-only POD: E95={e95}", flush=True) - for i in range(min(8, len(sv))): - print(f" mode {i + 1}: energy={energy[i]:.4f}", flush=True) - - # -- Project illusion and pinball into target basis -- - proj_cache = {tgt_name: coeffs} # already in target basis - - if ill_data is not None: - proj_cache[ill_name] = project_into_basis( - ill_data["ux"], ill_data["uy"], modes, mean_f) - - if pin_data is not None: - proj_cache["pinball"] = project_into_basis( - pin_data["ux"], pin_data["uy"], modes, mean_f) - - # -- CCD for each r and each case -- - for r in R_CANDIDATES: - print(f"\n r={r}:", flush=True) - modes_r = modes[:, :r] - - for name in [tgt_name, ill_name, "pinball"]: - d = data_cache.get(name) - if d is None: - continue - if name not in proj_cache: - continue - - a_r = proj_cache[name][:r, :] - N = a_r.shape[1] - - # --- Force-CCD (primary: SigmaFy) --- - frc = d.get("forces") - if frc is not None: - for f_mode, f_label in [("fy", "force_fy"), - ("fx", "force_fx"), - ("joint", "force_joint")]: - y_f = make_force_obs(frc, name, mode=f_mode) - y_f = y_f[:, :N] - W, sig, Rmat, z, No, Nv = compute_reduced_ccd( - a_r[:, :N], y_f, Q_delay=CCD_Q) - en = cumulative_energy(sig) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 0 - - key = f"{diam}L_{name}_{f_label}_r{r}" - W_dict[key] = W - all_results[key] = { - "diam": diam, "case": name, - "obs": f_label, "r": r, - "m80": m80, "N": Nv, - "sigma_top3": [float(sig[i]) - for i in range(min(3, len(sig)))], - "special_mechanism": is_special, - } - if f_mode == "fy": - print(f" {key}: m80={m80}, " - f"sigma1={float(sig[0]):.4f}", flush=True) - - # --- Action-CCD (illusion only) --- - act = d.get("actions") - if act is not None: - y_a = act.T # (3, N) - W, sig, Rmat, z, No, Nv = compute_reduced_ccd( - a_r[:, :N], y_a[:, :N], Q_delay=CCD_Q) - en = cumulative_energy(sig) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 0 - - key = f"{diam}L_{name}_action_r{r}" - W_dict[key] = W - all_results[key] = { - "diam": diam, "case": name, - "obs": "action", "r": r, - "m80": m80, "N": Nv, - "sigma_top3": [float(sig[i]) - for i in range(min(3, len(sig)))], - "special_mechanism": is_special, - } - print(f" {key}: m80={m80}, " - f"sigma1={float(sig[0]):.4f}", flush=True) - - # -- Modal overlaps (r=6, force_fy primary) -- - print(f"\n Modal overlap (r=6, force_fy):", flush=True) - ov_list = compute_modal_overlap(W_dict, diam, 6, "force_fy") - for ov in ov_list: - print(f" O({ov['case_a']}, {ov['case_b']}) " - f"mode{ov['mode']} = {ov['O']:.4f}", flush=True) - - # -- Reconstruction quality (POD basis check) -- - # Project target fields back onto its own POD basis and check residual - q_rec = modes[:, :r] @ coeffs[:r, :] + mean_f[:, None] - res = Q_tgt.astype(np.float64) - q_rec - r2 = 1.0 - np.sum(res ** 2) / np.sum(Q_tgt.astype(np.float64) ** 2) - print(f" Target self-reconstruction R2 (r={r}): {r2:.4f}", flush=True) - - # -- Cross-diameter comparison (0.75L illusion in 1.0L basis) -- - print("\n--- Cross-diameter: 0.75L -> 1.0L basis ---", flush=True) - d10_cache = data_cache.get("target_cylinder_1.0L") - d075_i = data_cache.get("illusion_0.75L") - if d10_cache is not None and d075_i is not None: - Q_10 = build_field_matrix(d10_cache["ux"], d10_cache["uy"]) - mf_10 = np.mean(Q_10, axis=1) - U_10, _, _ = np.linalg.svd(Q_10 - mf_10[:, None], full_matrices=False) - modes_10_6 = U_10[:, :6] - - # Project 0.75L illusion - a_075 = project_into_basis(d075_i["ux"], d075_i["uy"], - modes_10_6, mf_10)[:6, :] - frc_075 = d075_i.get("forces") - if frc_075 is not None: - y_f = make_force_obs(frc_075, "illusion_0.75L", mode="fy") - W_cross, _, _, _, _, _ = compute_reduced_ccd(a_075, y_f, Q_delay=CCD_Q) - - # Compare with 1.0L illusion in its own basis - d10_i = data_cache.get("illusion_1.0L") - if d10_i is not None: - a_10 = project_into_basis(d10_i["ux"], d10_i["uy"], - modes_10_6, mf_10)[:6, :] - frc_10 = d10_i.get("forces") - if frc_10 is not None: - y_f10 = make_force_obs(frc_10, "illusion_1.0L", mode="fy") - W_10, _, _, _, _, _ = compute_reduced_ccd(a_10, y_f10, Q_delay=CCD_Q) - n = min(W_cross.shape[1], W_10.shape[1], 5) - for k in range(n): - ov = float(abs( - W_cross[:, k] / (np.linalg.norm(W_cross[:, k]) + 1e-12) @ - W_10[:, k] / (np.linalg.norm(W_10[:, k]) + 1e-12) - )) - print(f" Cross-diam O(0.75L->1.0L) mode{k + 1} = {ov:.4f}", - flush=True) - - # -- Save -- - with open(os.path.join(out_dir, "ccd_results.json"), "w") as f: - json.dump(all_results, f, indent=2) - print(f"\nSaved to {out_dir}/ccd_results.json", flush=True) - print(f"Total entries: {len(all_results)}", flush=True) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/ccd/validate.py b/src/CCD_analysis/ccd/validate.py deleted file mode 100644 index ea6c661..0000000 --- a/src/CCD_analysis/ccd/validate.py +++ /dev/null @@ -1,270 +0,0 @@ -"""LOCO and blocked-split validation for CCD (Round 5). - -Reuses shared data loader from resampling.py. -Target-only POD basis. Q_delay=6 for force/action. - -0.75L and 1.0L only. 1.5L excluded from validation. - -Usage: - conda run -n pycuda_3_10 python ccd/validate.py -""" -from __future__ import annotations - -import json -import os -import sys -import time - -import numpy as np - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR -from CCD_analysis.utils.resampling import ( - compute_reduced_ccd, cumulative_energy, - load_aligned_fields, make_force_obs, - build_field_matrix, project_into_basis, -) - -R_LIST = [6, 8, 10] -CCD_Q = 6 -N_CYCLES = 4 -N_PTS = 24 -DIAMETERS = [0.75, 1.0] - - -def r2(y_true: np.ndarray, y_pred: np.ndarray) -> float: - """Coefficient of determination.""" - ss_r = np.sum((y_true - y_pred) ** 2) - ss_t = np.sum((y_true - np.mean(y_true)) ** 2) - return float(1.0 - ss_r / (ss_t + 1e-12)) - - -def reconstruct_observable(W, sigma, R, a_test, y_train): - """Reconstruct observable from CCD modes. - - Returns dict with 'mode1' and 'm80' reconstructions. - """ - am = np.mean(a_test, axis=1, keepdims=True) - as_ = np.std(a_test, axis=1, keepdims=True) + 1e-12 - a_test_z = (a_test - am) / as_ - z_test = W.T @ a_test_z - - ym = np.mean(y_train, axis=1, keepdims=True) - ys = np.std(y_train, axis=1, keepdims=True) + 1e-12 - half = CCD_Q // 2 - m_obs = y_train.shape[0] - - en = cumulative_energy(sigma) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 1 - - results = {} - - # Mode-1 - if R.shape[1] >= 1: - pz_1 = R[:, :1] * sigma[:1] @ z_test[:1, :] - yp_1 = pz_1[half * m_obs:(half + 1) * m_obs, :] * ys + ym - results["mode1"] = yp_1 - else: - results["mode1"] = np.zeros_like(y_train[:, :a_test.shape[1]]) - - # M80 - n_rm = min(m80, R.shape[1]) - if n_rm >= 1: - pz_m = R[:, :n_rm] * sigma[:n_rm] @ z_test[:n_rm, :] - yp_m = pz_m[half * m_obs:(half + 1) * m_obs, :] * ys + ym - results["m80"] = yp_m - else: - results["m80"] = np.zeros_like(y_train[:, :a_test.shape[1]]) - - return results - - -def run_single_diameter(diam: float, scene_data: dict) -> dict: - """Run validation for one diameter. Returns results dict.""" - tgt_key = f"target_cylinder_{diam}L" - ill_key = f"illusion_{diam}L" - - tgt_d = scene_data[tgt_key] - ill_d = scene_data[ill_key] - unc_d = scene_data["pinball"] - - # Pre-build target-only field matrices - tgt_f = build_field_matrix(tgt_d["ux"], tgt_d["uy"]) - ill_f = build_field_matrix(ill_d["ux"], ill_d["uy"]) - unc_f = build_field_matrix(unc_d["ux"], unc_d["uy"]) - - diam_results = {} - pod_cache = {} - - # Pre-compute target-only POD for each fold - for fold in range(N_CYCLES): - test_cyc = fold - train_cyc = [c for c in range(N_CYCLES) if c != test_cyc] - train_idx = sorted([c * N_PTS + p for c in train_cyc for p in range(N_PTS)]) - for r in R_LIST: - Q_ref = tgt_f[:, train_idx] - mf = np.mean(Q_ref, axis=1) - U, _, _ = np.linalg.svd(Q_ref - mf[:, None], full_matrices=False) - pod_cache[("loco", fold, r)] = (mf, U[:, :r]) - - # Blocked split - train_idx_full = list(range(0, 2 * N_PTS)) - for r in R_LIST: - Q_ref = tgt_f[:, train_idx_full] - mf = np.mean(Q_ref, axis=1) - U, _, _ = np.linalg.svd(Q_ref - mf[:, None], full_matrices=False) - pod_cache[("blocked", 0, r)] = (mf, U[:, :r]) - - # -- LOCO -- - print("\n--- LOCO (4-fold) ---", flush=True) - loco_results = {} - for r in R_LIST: - for obs in ["force_fy", "force_fx", "action"]: - fold_r2_m1, fold_r2_m80 = [], [] - for fold in range(N_CYCLES): - test_cyc = fold - train_cyc = [c for c in range(N_CYCLES) if c != test_cyc] - train_idx = sorted([c * N_PTS + p for c in train_cyc for p in range(N_PTS)]) - test_idx = sorted([c * N_PTS + p for c in [test_cyc] for p in range(N_PTS)]) - - mf, modes_r = pod_cache[("loco", fold, r)] - - for name, d, fld in [ - (tgt_key, tgt_d, tgt_f), (ill_key, ill_d, ill_f), ("pinball", unc_d, unc_f) - ]: - if obs == "action" and "illusion" not in name: - continue - if d.get("forces") is None and "force" in obs: - continue - - a_train = modes_r.T @ (fld[:, train_idx] - mf[:, None]) - a_test = modes_r.T @ (fld[:, test_idx] - mf[:, None]) - - if "force" in obs: - f_mode = obs.split("_")[1] # "fy" or "fx" - y_train = make_force_obs(d["forces"][train_idx], name, mode=f_mode) - y_test = make_force_obs(d["forces"][test_idx], name, mode=f_mode) - else: - y_train = d["actions"][train_idx, :].T - y_test = d["actions"][test_idx, :].T - - W, sigma, Rmat, _, _, _ = compute_reduced_ccd(a_train, y_train, Q_delay=CCD_Q) - recon = reconstruct_observable(W, sigma, Rmat, a_test, y_train) - - ch_m1 = [r2(y_test[c], recon["mode1"][c]) for c in range(y_test.shape[0])] - ch_m80 = [r2(y_test[c], recon["m80"][c]) for c in range(y_test.shape[0])] - fold_r2_m1.append(float(np.mean(ch_m1))) - fold_r2_m80.append(float(np.mean(ch_m80))) - - if fold_r2_m1: - key = f"LOCO_{obs}_r{r}" - loco_results[key] = { - "mode1": { - "mean": float(np.mean(fold_r2_m1)), - "std": float(np.std(fold_r2_m1)), - }, - "m80": { - "mean": float(np.mean(fold_r2_m80)), - "std": float(np.std(fold_r2_m80)), - }, - } - print(f" {key}: R2_m1={loco_results[key]['mode1']['mean']:.4f}+-" - f"{loco_results[key]['mode1']['std']:.4f} " - f"R2_m80={loco_results[key]['m80']['mean']:.4f}+-" - f"{loco_results[key]['m80']['std']:.4f}", flush=True) - - # -- Blocked split -- - print("\n--- Blocked Split (train=0-47, test=48-95) ---", flush=True) - blocked = {} - test_idx_full = list(range(2 * N_PTS, 4 * N_PTS)) - for r in R_LIST: - for obs in ["force_fy", "force_fx", "action"]: - mf, modes_r = pod_cache[("blocked", 0, r)] - per_case_m1, per_case_m80 = {}, {} - - for name, d, fld in [ - (tgt_key, tgt_d, tgt_f), (ill_key, ill_d, ill_f), ("pinball", unc_d, unc_f) - ]: - if obs == "action" and "illusion" not in name: - continue - if d.get("forces") is None and "force" in obs: - continue - - a_train = modes_r.T @ (fld[:, train_idx_full] - mf[:, None]) - a_test = modes_r.T @ (fld[:, test_idx_full] - mf[:, None]) - - if "force" in obs: - f_mode = obs.split("_")[1] - y_train = make_force_obs(d["forces"][train_idx_full], name, mode=f_mode) - y_test = make_force_obs(d["forces"][test_idx_full], name, mode=f_mode) - else: - y_train = d["actions"][train_idx_full, :].T - y_test = d["actions"][test_idx_full, :].T - - W, sigma, Rmat, _, _, _ = compute_reduced_ccd(a_train, y_train, Q_delay=CCD_Q) - recon = reconstruct_observable(W, sigma, Rmat, a_test, y_train) - - ch_m1 = [r2(y_test[c], recon["mode1"][c]) for c in range(y_test.shape[0])] - ch_m80 = [r2(y_test[c], recon["m80"][c]) for c in range(y_test.shape[0])] - per_case_m1[name] = float(np.mean(ch_m1)) - per_case_m80[name] = float(np.mean(ch_m80)) - - key = f"blocked_{obs}_r{r}" - blocked[key] = { - "mode1": { - "mean": float(np.mean(list(per_case_m1.values()))), - "per_case": per_case_m1, - }, - "m80": { - "mean": float(np.mean(list(per_case_m80.values()))), - "per_case": per_case_m80, - }, - } - print(f" {key}: R2_m1={blocked[key]['mode1']['mean']:.4f} " - f"R2_m80={blocked[key]['m80']['mean']:.4f}", flush=True) - - diam_results["LOCO"] = loco_results - diam_results["blocked_split"] = blocked - return diam_results - - -def run(): - print("=" * 60, flush=True) - print("CCD Validation (Round 5)", flush=True) - print("=" * 60, flush=True) - - t_start = time.time() - all_results = {} - - for diam in DIAMETERS: - tgt_key = f"target_cylinder_{diam}L" - ill_key = f"illusion_{diam}L" - print(f"\n{'=' * 60}", flush=True) - print(f"Diameter {diam}L", flush=True) - print(f"{'=' * 60}", flush=True) - - t0 = time.time() - scene_data = { - tgt_key: load_aligned_fields(tgt_key), - ill_key: load_aligned_fields(ill_key), - "pinball": load_aligned_fields("pinball"), - } - print(f" Data loaded in {time.time() - t0:.0f}s", flush=True) - - t1 = time.time() - all_results[f"{diam}L"] = run_single_diameter(diam, scene_data) - print(f" Analysis done in {time.time() - t1:.0f}s", flush=True) - - out_dir = os.path.join(DATA_DIR, "ccd") - with open(os.path.join(out_dir, "validation_results.json"), "w") as f: - json.dump(all_results, f, indent=2) - - print(f"\nTotal: {time.time() - t_start:.0f}s", flush=True) - print(f"Saved to {out_dir}/validation_results.json", flush=True) - - -if __name__ == "__main__": - run() diff --git a/src/CCD_analysis/ccd_knowledge.md b/src/CCD_analysis/ccd_knowledge.md deleted file mode 100644 index aafbaf1..0000000 --- a/src/CCD_analysis/ccd_knowledge.md +++ /dev/null @@ -1,509 +0,0 @@ -# CCD Analysis Knowledge Base — 论文知识库 (2026-06-25) - -> 统一文档。整合了 Round 5-6 的完整分析结果、correction-field 框架、Vortex/Cloak/Illusion 对比、几何对齐方案、以及所有操作流程。写论文时以本文档为准。 - ---- - -## 1. 项目概览 - -### 1.1 物理系统 - -**Fluidic Pinball**: 三个等间距圆柱（直径 D=20 格点，中心间距 0.75D）在二维通道内呈倒三角排列（间距 30°）。每个圆柱可独立旋转。DRL 控制器（PPO）每 SAMPLE_INTERVAL 步输出三个转速动作 [-1,1]^3，映射到物理表面速度。 - -**场景分类**: - -```mermaid -flowchart LR - subgraph Cloak [Cloak：涡街伪装] - Steady[均匀来流 → 下游恢复均匀] - Karman[涡街来流 → 下游维持涡街] - Vortex[瞬态涡事件 → 涡不受畸变] - end - subgraph Illusion [Illusion：流场欺骗] - I075[0.75L: 伪装成更小圆柱] - I10[1.0L: 伪装成等大圆柱] - I15[1.5L: 伪装成更大圆柱] - end -``` - -**控制机制**（经 SR 和 correction-field 双重验证）: -- 后两圆柱（Top/Bottom）: 恒速反向旋转（cloak 约 ±0.313），补偿 pinball 后速度亏损 -- 前端圆柱（Front）: 动态调节，控制升力（$\alpha_F = \Delta a_F/\Delta t - 14.952\mu C_{l,\text{tot}}$） - -### 1.2 核心方法：Correction-field 框架 - -**基本思想**: 不直接分析受控流场 `q_ctl`，而是分析**控制施加的修正**： - -| 符号 | 含义 | 计算方式 | -|------|------|----------| -| `q_in` | 均匀来流（干净通道） | 空通道采集 | -| `q_blk` | pinball 无控制 | 原始 pinball 涡街 | -| `q_ctl` | pinball + DRL 控制 | PPO 推理采集 | -| `q_tar` | 目标流场 | 目标圆柱/涡街参考 | -| `dq_blk = q_blk - q_in` | pinball 阻塞场 | pinball 对来流的畸变 | -| `dq_ctl = q_ctl - q_blk` | **控制修正场** | 控制器在 pinball 基础上加的改变 | -| `dq_tar = q_tar - q_blk` | **目标修正场** | 理论需达到的修正 | - -**核心问题**: `dq_ctl` 是否等于 `dq_tar`？即控制做的"修正"是否就是理论上需要的"修正"？ - -### 1.3 关键发现 - -1. **Cloak 机制统一**：Steady/Karman/Vortex 三种场景的 dq_ctl 高度一致——补偿速度亏损（正 ux）+ 偶极子效应。控制策略相同，不论上游来流条件。 - -2. **Illusion 1.0L 与 Cloak 共享机制**：dq_ctl 结构定性一致。Illusion 本质是通过"Cloak 机制"让下游看起来像目标圆柱。 - -3. **Illusion 0.75L 效率降低**：O(dqctl,dqtar)=0.564，控制效率低于 1.0L - -4. **Illusion 1.5L 特殊机制**：O(dqctl,dqtar)=0.667，action sigma1=0.28（其他 1.13-1.39），强相位漂移，修正集中在近体区 - -5. **Force/Signature 分离**：力相关结构和传感器相关结构在瞬时 tau=0 时分离（O=0.01-0.55），对流延迟后共享（O=0.72-0.81） - ---- - -## 2. 方法细节 - -### 2.1 物理参数 - -| 参数 | 值 | 说明 | -|------|-----|------| -| NX, NY | 1280, 512 | LBM 格点数 | -| L0 | 20 | 基准长度单位（1 圆柱直径） | -| U0 | 0.01 | 入口中心流速（抛物线分布） | -| nu | 0.004 | 默认运动粘度 | -| Re_code | 100 | 参考长度 2×D_cyl → Re_D=50 | -| D_cyl | L0=20 | 单一圆柱直径 | -| CENTER_Y | 255.5 | 通道中心 y 坐标 | - -**Reynolds 数约定**： -- `re_code` 使用参考长度 D_REF = 2*D = 40（匹配模型文件命名） -- `Re_D = re_code / 2` 是真实物理雷诺数（使用单圆柱直径） -- 默认场景 `re_code=100` → `Re_D=50` - -### 2.2 场景几何位置（2026-06-25 统一更新） - -**所有场景已统一几何**：采集端直接在 `configs.py` 中设置统一坐标，无需后处理平移。 - -| 组 | 场景 | pinball 中心 | sensor x | 来源 configs | -|----|------|-------------|--------|-------------| -| **所有场景** | pinball, steady, karman, vortex, illusion_*, target_cylinder_* | **613 px** | **800 px** | configs.py UNIFIED 注释 | - -- pinball: front x=30×L0=600, rear x=31.3×L0=626 → 中心 ≈ 613 px -- target cylinder: x=30.65×L0=613 px -- sensors: x=40×L0=800 px - -### 2.3 几何对齐方案（已废弃，仅保留备用） - -统一几何后不再需要后处理平移。`utils/field_translate.py` 保留作为可选工具，但不参与默认 pipeline。 - -
-旧方案（参考） - -```python -SHIFT_ILLUSION_TO_CLOAK = +220 px # 旧 illusion → cloak -SHIFT_CLOAK_TO_ILLUSION = -220 px -``` -
- ---- - -## 3. 数据采集 - -### 3.1 GPU 采集脚本 - -| 脚本 | 功能 | 输出目录 | -|------|------|----------| -| `scripts/collect_target_cylinder.py` | 目标圆柱（单圆柱涡街） | `data/target_cylinder/{diam}L/` | -| `scripts/collect_pinball.py` | pinball 无控制基线 | `data/pinball/pinball/` | -| `scripts/collect_empty_channel.py` | 空通道 | `data/target_channel/target_channel/` | -| `scripts/collect_illusion.py` | Illusion PPO 推理 | `data/illusion/{scene}/` | -| `scripts/collect_karman.py` | Karman cloak PPO 推理 | `data/karman/karman_re100/` | -| `scripts/collect_karman_q_in.py` | 涡街无 pinball（Karman 参考） | `data/karman_target/karman_q_in/` | -| `scripts/collect_karman_q_blk.py` | 涡街+pinball 无控制（Karman 参考） | `data/karman_blocked/karman_q_blk/` | -| `scripts/collect_steady_cloak.py` | 稳态 cloak（开环恒速） | `data/steady_cloak/steady_cloak/` | -| `scripts/collect_vortex.py` | Vortex cloak（Taylor/Lamb） | `data/vortex_{type}/` | - -### 3.2 采集流程（以 Karman 为例） - -``` -Phase 1: Target recording - FlowField → add_sensor(40*L0) ×3 → stabilize → add_vortex/cylinder - → run(SI, zero) × F来O_LEN → save target.npz - -Phase 2: Add pinball + Norm - restore → add_cylinder(pinball) ×3 → stabilize - → run(SI, zero) × FIFO_LEN → compute norm - → run(SI, bias_action) × FIFO_LEN → save ddf+fifo checkpoint - -Phase 3: Controlled PPO inference - restore + warmup → for step in range(n_steps): - model.predict(obs) → action → run(SI, action_arr) → save telemetry + field -``` - -### 3.3 Norm 计算 - -``` -force_norm_fact = 6 × max|forces| -sens_deviation = mean(sensors, axis=0) -sens_norm_fact[i] = 5 × max|sensors[:,i] - sens_deviation[i]| -obs = clip[forces/force_norm, (sens - deviation)/sens_norm], to [-1, 1] -``` - -**注意**：Norm 是在 scene-specific 采集时计算的，不同场景的 norm 值不同。推理时必须使用对应场景的 norm。 - -### 3.4 Phase Alignment 流程 - -``` -detect_period.py: - sensors[:, 3] → FFT → dominant frequency + period - zero-crossing detection → cycle boundaries (CV_T) - select best 4-cycle window → map 4×24=96 uniform phase points - → save phase_plan.json - -replay_fields.py: - load phase_plan + ddf_checkpoint + actions - replay PPO → at each phase_plan step_index → save velocity field - → save fields_aligned.npz + replay_verify.json -``` - -### 3.5 数据状态 - -| 场景 | scene_id | 帧数 | 格式 | -|------|----------|------|------| -| pinball | pinball | 96 | fields_aligned.npz | -| target_cylinder_{0.75,1.0,1.5}L | target_cylinder | 96 | fields_aligned.npz | -| illusion_{0.75,1.0,1.5}L | illusion | 96 | fields_aligned.npz | -| karman_re100 | karman | 72 (3周期) | fields_aligned.npz | -| karman_q_in | karman_target | 96 | fields_aligned.npz | -| karman_q_blk | karman_blocked | 96 | fields_aligned.npz | -| steady_cloak | steady_cloak | 500 | fields.npz (旧格式) | -| target_channel | target_channel | 100 | fields.npz (旧格式) | -| vortex_{lamb/taylor}/target/uncontrolled | 各自 scene_id | 150 | fields.npz (瞬态) | - ---- - -## 4. CCD 方法 - -### 4.1 算法原理（Lyu23） - -CCD（Canonical Correlation Decomposition）通过 CCA 在流场和可观测量之间找到相关性最大的方向： - -```python -# Reduced CCD in POD coefficient space -def compute_reduced_ccd(pod_coeffs, observable, Q_delay=6): - # 1. Construct lagged observable matrix P (with symmetric delay window) - # 2. Standardize P and A (POD coefficients) - # 3. Cross-correlation matrix: C = P @ A^T / (N * sqrt(Q)) - # 4. SVD(C) → R, sigma, W - # W = CCD directions in POD space - # sigma = correlation strength (sorted descending) - # z = CCD temporal coefficients -``` - -**物理意义**： -- sigma[0] = 第一 CCD 模式的相关系数 -- m80 = 需要多少模式来捕获 80% 的总相关性（compactness 指标） -- O_k = 两个 case 之间第 k 模的重叠（内积绝对值） - -### 4.2 三条分析线 - -| 线 | Observable | 问题 | 物理意义 | -|----|-----------|------|----------| -| **Force** (主) | SigmaFy = sum(Fy_i) | 哪些修正结构决定升力？ | 力相关流场结构 | -| **Force** (次) | SigmaFx | 哪些修正结构决定阻力？ | 不可靠（R2~0.4） | -| **Action** | [omega1, omega2, omega3] | 控制器直接调制哪些结构？ | 动作相关流场结构 | -| **Signature** | e(t+tau) = s_ctl(t+tau) - s_tar(t+tau) | 哪些结构决定未来传感器误差？ | 传感器相关流场结构 | - -### 4.3 验证方法 - -**LOCO (Leave-One-Cycle-Out)**: -- 4 个涡街周期，留 1 做测试，3 做训练 -- 训练 CCD → 预测可观测量 → 计算 R2 -- 通过阈值：R2_m80 > 0.4 - -| Observable | 0.75L R2_m80 | 1.0L R2_m80 | 结论 | -|-----------|-------------|-------------|------| -| force_fy | 0.65 ± 0.08 | 0.64 ± 0.02 | PASS | -| force_fx | 0.38 ± 0.23 | 0.43 ± 0.11 | WARNING | -| signature tau=0 | 0.50 ± 0.09 | 0.49 ± 0.04 | PASS | -| signature tau=tau_c | 0.51 ± 0.09 | 0.53 ± 0.03 | PASS | - -### 4.4 Zone-Restricted CCD - -统一几何后所有场景共用一套 zone 定义（定义见 `diagnose_corrections.py` 的 `define_zones_karman()`）： - -| 区域 | x 范围 (像素) | 包含 | -|------|-------------|------| -| near_body | 580-720 | pinball 周围 | -| body_wake | 720-850 | 近尾流 | -| sensor_zone | 780-850 | 传感器区域 | - ---- - -## 5. 结果 - -### 5.1 Correction-field CCD 主表（2026-06-28 更新，统一几何后重跑） - -| 指标 | 0.75L | 1.0L | 1.5L | -|------|-------|------|------| -| **O(dqctl, dqtar) mode1 (r=6)** | **0.383** | **0.926** | **0.922** | -| **O(dqctl, dqtar) mode1 (r=10)** | **0.320** | **0.684** | **0.661** | -| force_fy m80 (r=6) | 2 | 2 | **1** | -| action sigma1 (r=6) | 1.49 | 1.17 | **0.20** | -| Phase drift | low | low | **high** | -| 1.5L special: rank sensitivity | — | — | O drops 0.922→0.661 (r=6→10) | - -**关键变化**（vs 6月15日旧版，Illusion 旧几何 pinball x≈393px）： -- 0.75L O 从 0.564 → **0.383**（-32%）：旧几何的空间错位虚高了 overlap。统一几何后揭示真实匹配度远低于预期。 -- 1.0L O 从 0.913 → **0.926**（+1.4%）：基本不变，1.0L 的控制修正与目标一致。 -- 1.5L O(r=6)=**0.922**（首次获得）：dominant mode 匹配度高，但更高阶 mode 快速发散（r=10 时降至 0.661），反映多尺度控制策略。 -- 1.5L action sigma1=**0.20**（远低于 0.75L 的 1.49 和 1.0L 的 1.17）：确认高频调制机制下动作与流场结构的映射极其分散。 - -以下 force-sig overlap 和 zone 数据来自 6 月 15 日旧版（待用统一几何和新 zone 定义重跑）： - -### 5.2 三区域 Force-Signature Overlap - -**0.75L** — sensor_zone 在 tau=0 时近乎正交： - -| Zone | O(force, sig) tau=0 | O(force, sig) tau=tau_c | -|------|--------------------|------------------------| -| near_body | 0.262 | 0.827 | -| body_wake | 0.269 | **0.917** | -| sensor_zone | **0.010** | 0.722 | - -**1.0L** — 更均匀，没有近零区域： - -| Zone | O(force, sig) tau=0 | O(force, sig) tau=tau_c | -|------|--------------------|------------------------| -| near_body | 0.596 | 0.596 | -| body_wake | 0.509 | 0.483 | -| sensor_zone | 0.594 | **0.730** | - -### 5.3 Cloak 全景对比（Steady / Karman / Vortex dq_ctl） - -所有 cloak 场景的 dq_ctl 展现了**一致的物理机制**： -1. **速度亏损补偿**：pinball 后方正 ux（红色），控制加速尾流 -2. **偶极子效应**：圆柱附近旋转产生的偶极子模式 -3. **涡量结构**：三种场景涡量分布高度相似 - -量化指标： - -| 场景 | dq_ctl RMS (crop) | 性质 | -|------|------------------|------| -| steady_cloak | 0.196 | 稳态，开环 | -| karman_re100 | 0.397 | 周期，PPO 闭环 | -| vortex_lamb | 0.164 | 瞬态，PPO 闭环（fade-in/out + swapped norm） | -| vortex_taylor | 0.203 | 瞬态，PPO 闭环（fade-in/out + swapped norm） | - -**结论**：不论上游条件如何（稳态/周期涡街/瞬态涡），控制策略的基本物理机制一致——"通过后两圆柱旋转补偿 pinball 阻塞引起的速度亏损，前端圆柱调节升力"。这与 SR 分析的结论完全吻合。 - -### 5.4 Steady Cloak 定量化 - -| 指标 | 值 | -|------|-----| -| 全局波动抑制 | ≈0% | -| 残留/阻塞比 | 81% | -| 传感器区残留 | 13% | - -**结论**：开环恒速旋转几乎无法抑制波动。需要闭环 DRL 控制。 - -### 5.5 Action-CCD Mode 1 - -Action-CCD 找出了控制器直接调制的主要结构。对 Cloak 场景，action-CCD mode 1 ≈ dq_ctl mean field，确认了"控制调制的结构 = correction-field 的主成分"的直觉。 - ---- - -## 6. Correction-field 诊断图 - -### 6.1 图例说明 - -所有位于 `data/figures/` 下（无 colorbar，干净布局，裁剪到 x=300-1100）： - -| 图 | 内容 | -|----|------| -| `corr_comparison_all_scenes.png` | 7 场景全景（4 cloak + 3 illusion），4 行 × 7 列 | -| `corr_cloak_comparison_dqctl.png` | Cloak 四场景 dq_ctl 对比 | -| `corr_illusion_comparison_dqctl.png` | Illusion 三直径 dq_ctl 对比 | -| `corr_{scene}_ctl_vs_tar.png` | 单场景 dq_ctl vs dq_tar 对比（2×2） | -| `steady_cloak_cancel_test.png` | Steady cloak 抵消检验 | - -### 6.2 全景对比图的读法 - -每行 = 一个物理量（ux_mean / uy_mean / RMS / vorticity） -每列 = 一个场景 -颜色统一（同一行在所有列间共享 vmax） - -**核心观察**： -- cloak 四列间 dq_ctl 高度一致 → 控制策略不依赖上游条件 -- illusion_1.0L 与 cloak 定性一致 → 1.0L illusion=cloak 机制 -- illusion_0.75L 偏弱但定性相似 → 控制效率降低 -- illusion_1.5L 结构突变 → 完全不同的策略 - ---- - -## 7. 操作流程 - -### 7.1 环境 - -```bash -conda run -n pycuda_3_10 -# Python 3.10+, numpy, matplotlib, LegacyCelerisLab, stable-baselines3 -# CUDA device 2 (采集用) -``` - -### 7.2 快速重新生成所有图 - -```bash -# 1. Correction-field pipeline（CPU） -conda run -n pycuda_3_10 python3 correction_analysis/diagnose_corrections.py - -# 2. 对比图（CPU） -conda run -n pycuda_3_10 python3 correction_analysis/compare_dqctl_scenes.py -``` - -### 7.3 从零开始完整流程 - -```bash -# Step 1: GPU 数据采集（每次 > ~4min） -# （已有数据则可跳过） - -# Step 2: Phase alignment（CPU） -python3 scripts/detect_period.py --scene {scene_name} - -# Step 3: Field replay（GPU） -conda run -n pycuda_3_10 python3 scripts/replay_fields.py --scene {scene_name} --device 2 - -# Step 4: Correction-field 分析（CPU） -conda run -n pycuda_3_10 python3 correction_analysis/compute_correction_fields.py -conda run -n pycuda_3_10 python3 correction_analysis/diagnose_corrections.py - -# Step 5: 对比图（CPU） -conda run -n pycuda_3_10 python3 correction_analysis/compare_dqctl_scenes.py -``` - -### 7.4 添加新场景 - -1. 在 `configs.py` 中添加场景注册 -2. 在 `compute_correction_fields.py` 的 `_SCENE_MAP` 中添加映射 -3. 在 `_resolve_source()` 中添加加载器分发 -4. 编写 GPU 采集脚本（参考 `collect_vortex.py`） -5. 在 `diagnose_corrections.py` 的 `SCENE_TYPES` 中添加场景 -6. 运行采集 → detect_period → replay_fields → diagnose_corrections - ---- - -## 8. 代码结构 - -``` -src/CCD_analysis/ - ccd_knowledge.md ← 本文档（唯一知识库） - configs.py ← 场景元数据（统一几何） - README.md ← 快速入口 - Lyu23.md ← CCD 方法文献 - ccd/ ← Round 5 冻结基线（勿改） - utils/ - resampling.py ← POD, CCD, 场加载 - field_translate.py ← 场平移（备用，不参与默认 pipeline） - load_vortex_fields.py ← 瞬态 vortex 场加载 - cfd_interface.py ← LegacyCelerisLab 封装 (GPU) - scripts/ - detect_period.py ← 周期检测 → phase_plan.json - replay_fields.py ← 场回放 → fields_aligned.npz - collect_*.py ← GPU 数据采集 - correction_analysis/ - compute_correction_fields.py ← correction-field 计算 - diagnose_corrections.py ← 诊断图生成 - compare_dqctl_scenes.py ← 多场景对比图 - decompose_corrections.py ← CCD 定量分解（POD + force/action CCD） - run_signature_line.py ← signature CCD - run_15L_correction.py ← 1.5L 专项分析 - run_zone_ccd.py ← zone-restricted CCD - run_steady_metrics.py ← steady cloak 定量度量 - visualize_action_ccd.py ← action-CCD mode1 可视化 - process_legacy_steady.py ← 旧格式加载 - data/ - {scene_id}/{scene_name}/ ← 各场景数据 - resampled/ ← phase_plan.json - ccd/ ← JSON 结果文件 - figures/ ← PNG 诊断图 - old_data/ ← 归档（旧报告、旧脚本、旧 resampled 数据） -``` - ---- - -## 9. 硬规则 - -1. **Illusion 只用 o14 模型**（S_DIM=14，`_2U` 系列）。不使用 o12 模型。 -2. **`_2U` 表示 S_DIM=14**（2 个额外的目标力通道），不是 2× 速度。U0 始终是 0.01。 -3. **action_bias ≠ preset_action**。bias 是 DRL action scaling（illusions=[0,-2,2]），preset 是 FIFO warmup 用的动作数组。 -4. **默认物理参数**：u0=0.01, nu=0.004（所有模型一致）。 -5. **主分析对象 = dq_ctl**（不是 q_ctl）。 -6. **Steady cloak 不要使用 phase-based CCD**——它是稳态问题。 -7. **Karman 的物理问题不同**（incoming-street preservation vs target generation），不要硬套 illusion 模板。 -8. **所有场景已在采集时统一几何**（pinball 中心 613 px，sensor x=800 px）；`field_translate.py` 仅作备用工具，不参与默认 pipeline。 -9. **旧数据格式**（`fields.npz`）：(N, NX, NY)；**新对齐格式**（`fields_aligned.npz`）：(N, NX, NY)；**加载后统一转 (N, NY, NX)**。 - ---- - -## 10. 不能写进论文的结论（内部参考，勿引用） - -以下陈述看似合理但没有被充分证据支持，不得写进正式论文： -- "Illusion 已证明使用完全不同于 target 的物理机制。" -- "低 overlap 已足以证明 force 通道与 target 正交。" -- "CCD 已经直接识别了壁面涡量生成机制。" -- "far wake 模态就是瞬时力的主载体。" - ---- - -## 11. 未完成工作（2026-06-28 更新） - -| 方向 | 状态 | 说明 | -|------|------|------| -| Karman cloak CCD 分析 | 数据已齐，分析延后 | 问题定义不同（distortion compensation） | -| Vortex 数据采集 Bug 修复 | 已完成 | 修复 cylinder order swap（BOTTOM=id4, TOP=id5）+ FIFO warmup 导致涡量消失；重采集 Taylor/Lamb | -| SR-CCD-OID 映射 | 草稿 | 归档 `data/old_data/sr_ccd_oid_mapping.md`，需最终校正 | -| 统一几何后 CCD 重跑 | 已完成 | `correction_ccd_results.json` 已更新（含 1.5L） | -| Vortex 对比图更新 | 已完成 | `compare_dqctl_scenes.py` 已用修正数据重跑 | -| 项目目录清理 + 文档更新 | 已完成 | 移除废弃目录、SI=200 错误数据、旧诊断图 | - ---- - -## 12. Vortex 采集 Bug 排查经验（2026-06-29） - -`collect_vortex.py` 中发现了四个独立 bug，按发现顺序： - -**Bug 1 — 圆柱顺序对调** -- 训练 env `legacy_env_vortex.py` 添加顺序：front(id3) → TOP(+y, id4) → BOTTOM(-y, id5) -- action 映射：`temp[3]=front, temp[4]=TOP(bias=-4), temp[5]=BOTTOM(bias=+4)` -- 脚本先后把 TOP/BOTTOM 添加顺序写反 → 后两圆柱旋转方向全错 -- 症状：Lamb front 剧烈振荡 (std=0.28)，dipole 对称性完全破坏 - -**Bug 2 — 涡量消失** -- FIFO warmup 在涡加入后跑 150×800=120000 步 → 涡从 x=15 漂出域外 (1280 lu) -- 症状：涡量场只有 pinball 尾流，完全看不到 vortex 结构 - -**Bug 3（核心）— 观测值归一化顺序错误** -- 训练 env 产出：`obs = [forces/force_norm, sensors/sens_norm]`（force 先） -- 脚本将 channel 和 norm 互换：`[sensors/force_norm, forces/sens_norm]` -- 模型收到不匹配分布的反馈 -- 症状：Lamb cross-corr 仅 0.73, Taylor 仅 0.50；后圆柱同向转而非反向 - -**Bug 4 — fade-in/out 缺失** -- `uni_test.ipynb` 有 25 步渐入 + 25 步渐出到 steady-cloak bias -- 脚本直接给全量 PPO action，无过渡 - -**最终修正方案**： -1. Cylinder order 匹配训练 env -2. Bias 使用 uni_test 值：[-5, +5]（FIFO），[-5.1, +5.1]（fade target） -3. Obs 归一化：`forces_norm = obs[6:12]/force_norm`，`sens_norm = (obs[0:6]-sens_dev)/sens_norm`，`hstack([forces_norm, sens_norm])` -4. 25-step fade-in / 25-step fade-out - -**修正后验证**： - -| 场景 | sim | cross-corr | active front mean | dq_ctl RMS | -|------|:---:|:----------:|:-----------------:|:----------:| -| vortex_lamb | 0.946 | 0.974 | 0.007 | 0.146 | -| vortex_taylor | 0.923 | 0.953 | -0.030 | 0.188 | - -**其他脚本审计**：`collect_karman.py` 和 `collect_illusion.py` 使用 `build_observation()` 函数，该函数正确实现了 force-first 归一化，无需修改。`collect_vortex.py` 是唯一手动构建 obs 的脚本。 - -**代码注释**：`collect_vortex.py` 的文件头 docstring 和关键行号均有 BUG HISTORY 和 BUG-FIX 标记。 diff --git a/src/CCD_analysis/configs.py b/src/CCD_analysis/configs.py deleted file mode 100644 index 79be148..0000000 --- a/src/CCD_analysis/configs.py +++ /dev/null @@ -1,291 +0,0 @@ -"""Unified scene configuration for CCD_analysis. - -All scene metadata in one place. Each scene dict contains all parameters -needed for data collection, resampling, POD, and CCD. - -CRITICAL: Illusion models use ONLY 2U series (d1a3o14_*). -1U series (d1a3o12_*) are NOT to be used. -All models use u0=0.01, nu=0.004. - -Re convention: - - "re_code" uses reference length 2*D (matching model file naming). - - Re_D = re_code / 2 is the true physical Reynolds number. -""" -from __future__ import annotations - -import os -from typing import Any, Dict, List, Optional - -# -- Root paths --------------------------------------------------------------- -_PROJ = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -MODEL_DIR = os.path.join(_PROJ, "..", "models") -LEGACY_CFG_DIR = os.path.join(os.path.dirname(__file__), "configs") -DATA_DIR = os.path.join(os.path.dirname(__file__), "data") - -# -- Physics constants ------------------------------------------------------- -U0 = 0.01 # standard inlet center velocity (all models use this) -# NOTE: "2U" in model name means S_DIM=14 (2 extra target force obs), NOT u0 scaling -D_CYL = 20.0 -D_REF = 40.0 -L0 = 20.0 -NX = 1280 -NY = 512 -CENTER_Y = (NY - 1) / 2.0 -FIFO_LEN = 150 -# CONV_LEN is per-scene. Illusion=36, Karman/Steady=30. -# Set locally in collection scripts, not as a global here. - - -def nu_from_re(re_code: float, u0: float = U0) -> float: - return u0 * D_REF / re_code - - -# -- Scene definitions ------------------------------------------------------- -SCENES: Dict[str, Any] = {} - -# -- Pure Pinball (uncontrolled baseline) ------------------------------------ -SCENES["pinball"] = { - "scene_id": "pinball", - "re_code": 100, - "has_disturbance": False, - "sample_interval": 800, - "source": "open_loop", - "model_name": None, - "n_objects_env": 6, - "obs_slice": (0, 12), - "sensor_x": 40.0, - "pinball_front_x": 30.0, - "pinball_rear_x": 31.3, - "target_type": "periodic", - "s_dim": 12, - "u0": U0, - "nu": nu_from_re(100), -} - -# -- Steady Cloak (open-loop constant rotation) ------------------------------ -SCENES["steady_cloak"] = { - "scene_id": "steady_cloak", - "re_code": 100, - "has_disturbance": False, - "sample_interval": 800, - "source": "open_loop", - "model_name": None, - "n_objects_env": 6, - "obs_slice": (0, 12), - "sensor_x": 40.0, - "pinball_front_x": 30.0, - "pinball_rear_x": 31.3, - "target_type": "steady", - "s_dim": 12, - "u0": U0, - "nu": nu_from_re(100), - "omega_front": 0.0, - "omega_rear_scale": 5.1, -} - -# -- Karman Cloak re100 (PPO, cloak validation only) ------------------------- -SCENES["karman_re100"] = { - "scene_id": "karman", - "re_code": 100, - "has_disturbance": True, - "sample_interval": 800, - "action_scale": 8.0, - "action_bias": (0.0, -4.0, 4.0), - "source": "PPO_inference", - "model_name": "d1a3o12_re100", - "model_subdir": "old", - "n_objects_env": 7, - "obs_slice": (2, 14), - "sensor_x": 40.0, - "pinball_front_x": 30.0, - "pinball_rear_x": 31.3, - "target_type": "periodic", - "s_dim": 12, - "u0": U0, - "nu": nu_from_re(100), -} - -# -- Karman q_in (target/incoming vortex street, no pinball) ------------------ -SCENES["karman_q_in"] = { - "scene_id": "karman_target", - "re_code": 100, - "has_disturbance": True, - "sample_interval": 800, - "source": "open_loop", - "n_objects_env": 4, - "obs_slice": (0, 8), - "sensor_x": 40.0, - "pinball_front_x": None, - "pinball_rear_x": None, - "target_type": "periodic", - "s_dim": None, - "u0": U0, - "nu": 0.004, -} - -# -- Karman q_blk (pinball in vortex street, zero control) ------------------- -SCENES["karman_q_blk"] = { - "scene_id": "karman_blocked", - "re_code": 100, - "has_disturbance": True, - "sample_interval": 800, - "source": "open_loop", - "n_objects_env": 7, - "obs_slice": (2, 14), - "sensor_x": 40.0, - "pinball_front_x": 30.0, - "pinball_rear_x": 31.3, - "target_type": "periodic", - "s_dim": None, - "u0": U0, - "nu": 0.004, -} - -# -- Illusion scenes (S_DIM=14) ----------------------------------------------- -# All use u0=0.01, SAMPLE_INTERVAL per diameter, nu=0.004 confirmed -# Sweep results: 0.004=0.962, 0.008=0.957, 0.002=0.882 -# "2U" in model name = S_DIM=14 (2 extra target force dimensions), NOT 2x velocity -_ILLUSION_2U = [ - ("illusion_0.75L", "d1a3o14_250525_imit_075L_2U_400S", 0.75, 400), - ("illusion_1.0L", "d1a3o14_250525_imit_1L_2U_600S", 1.0, 600), - ("illusion_1.5L", "d1a3o14_250525_imit_15L_2U", 1.5, 800), -] -for key, mn, diam, si in _ILLUSION_2U: - SCENES[key] = { - "scene_id": "illusion", - "target_diameter": diam, - "re_code": 100, # u0=0.01, nu=0.004 - "has_disturbance": False, - "sample_interval": si, - "conv_len": 36, # Illusion uses 36 (see legacy_env_imit.py) - "action_scale": 8.0, - "action_bias": (0.0, -2.0, 2.0), - "source": "PPO_inference", - "model_name": mn, - "model_subdir": "250525", - "n_objects_env": 6, - "obs_slice": (0, 12), - "sensor_x": 40.0, # UNIFIED: was 30.0 - "pinball_front_x": 30.0, # UNIFIED: was 19.0 - "pinball_rear_x": 31.3, # UNIFIED: was 20.3 - "target_type": "periodic", - "s_dim": 14, # CRITICAL: all 2U are 14-dim - "u0": U0, # 0.01 (NOT 0.02) - "nu": 0.004, # confirmed correct via sweep: 0.004=0.962, 0.008=0.957, 0.002=0.882 - } - -# -- Target cylinders (per-diameter, for signature-CCD reference) ------------ -# Each illusion diameter needs its own target cylinder data -# SAMPLE_INTERVAL per diameter, matching the corresponding illusion scene -for diam, si in [(0.75, 400), (1.0, 600), (1.5, 800)]: - key = f"target_cylinder_{diam}L" - SCENES[key] = { - "scene_id": "target_cylinder", - "target_diameter": diam, - "re_code": 100, # u0=0.01, nu=0.004 - "has_disturbance": False, - "sample_interval": si, # per-diameter, matching corresponding illusion - "conv_len": 36, # matching corresponding illusion - "source": "open_loop", - "model_name": None, - "n_objects_env": 4, # 1 cylinder + 3 sensors - "obs_slice": (0, 8), # cylinder force(2) + sensor(6) - "sensor_x": 40.0, # UNIFIED: was 30.0 - "cylinder_x": 30.65, # UNIFIED: was 20.0 (pinball center) - "target_type": "periodic", - "s_dim": None, - "u0": U0, # 0.01, matching illusion - "nu": 0.004, # confirmed correct via sweep: 0.004=0.962, 0.008=0.957, 0.002=0.882 - } - -# -- Target Channel (empty channel, for steady metrics) ---------------------- -SCENES["target_channel"] = { - "scene_id": "target_channel", - "re_code": 100, - "has_disturbance": False, - "sample_interval": 800, - "source": "open_loop", - "model_name": None, - "n_objects_env": 3, - "obs_slice": (0, 6), - "sensor_x": 40.0, - "pinball_front_x": None, - "pinball_rear_x": None, - "target_type": "steady", - "s_dim": 6, - "u0": U0, - "nu": nu_from_re(100), -} - -# -- Vortex cloak scenes (transient, Taylor monopole & Lamb dipole) --------- -# Taylor: monopole vortex, strength=0.03*U0, r=2*L0=40 -# Lamb: dipole vortex, strength=0.5*U0, r=2*L0=40 -# Both: MAX_STEPS=150, action_scale=4, action_bias=(0,-4,4), s_dim=12 -# Geometry: pinball at (30, 31.3)xL0, sensors at 40*xL0 -# Vortex at 10*xL0 (target) or 15*xL0 (pinball phase) -_VORTEX_SCENES = [ - ("vortex_lamb", "vortex_lamb", "lamb", 0.50), - ("vortex_taylor", "vortex_taylor", "taylor", 0.03), - ("vortex_uncontrolled_lamb", None, "lamb", 0.50), - ("vortex_uncontrolled_taylor", None, "taylor", 0.03), - ("vortex_target_lamb", None, "lamb", 0.50), - ("vortex_target_taylor", None, "taylor", 0.03), -] -for key, mn, vtype, vstrength in _VORTEX_SCENES: - is_controlled = mn is not None - SCENES[key] = { - "scene_id": key, - "re_code": 100, - "has_disturbance": False, - "sample_interval": 800, - "vortex_type": vtype, - "vortex_strength": vstrength, - "conv_len": 30, - "action_scale": 4.0 if is_controlled else None, - "action_bias": (0.0, -4.0, 4.0) if is_controlled else None, - "source": "PPO_inference" if is_controlled else "open_loop", - "model_name": mn, - "model_subdir": "old", - "n_objects_env": 6, - "obs_slice": (0, 12), - "sensor_x": 40.0, - "pinball_front_x": 30.0, - "pinball_rear_x": 31.3, - "target_type": "transient", - "max_steps": 150, - "s_dim": 12 if is_controlled else None, - "u0": U0, - "nu": nu_from_re(100), - } - - -# -- Utility helpers --------------------------------------------------------- - -def get_scene(name: str) -> dict: - if name not in SCENES: - raise KeyError(f"Unknown scene: {name}. Available: {list(SCENES.keys())}") - return dict(SCENES[name]) - - -def get_scene_list(scene_id: Optional[str] = None) -> List[str]: - if scene_id is None: - return list(SCENES.keys()) - return [k for k, v in SCENES.items() if v["scene_id"] == scene_id] - - -def model_path_for_scene(scene_name: str) -> Optional[str]: - cfg = get_scene(scene_name) - mn = cfg.get("model_name") - if mn is None: - return None - subdir = cfg.get("model_subdir", "old") - p = os.path.join(MODEL_DIR, subdir, f"{mn}.zip") - return p if os.path.isfile(p) else None - - -def data_dir_for_scene(scene_name: str) -> str: - cfg = get_scene(scene_name) - scene_id = cfg["scene_id"] - d = os.path.join(DATA_DIR, scene_id, scene_name) - os.makedirs(d, exist_ok=True) - return d diff --git a/src/CCD_analysis/configs/config_cuda.json b/src/CCD_analysis/configs/config_cuda.json deleted file mode 100644 index 3d4c10b..0000000 --- a/src/CCD_analysis/configs/config_cuda.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "multi_gpu": false, - "gpu_connection": "NVLink", - "required_cuda_capability": "7.0", - "threads_per_block": 128, - "X_1U": 128, - "Y_1U": 32, - "Z_1U": 1 -} \ No newline at end of file diff --git a/src/CCD_analysis/configs/config_flowfield.json b/src/CCD_analysis/configs/config_flowfield.json deleted file mode 100644 index f0ed50b..0000000 --- a/src/CCD_analysis/configs/config_flowfield.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "data_type": "FP32", - "dimensionality": 2, - "lattice": 9, - "field_dim_in_U": [10, 16, 1], - "viscosity": 0.004, - "velocity": 0.01, - "boundary_conditions": { - "x": ["parabolic", "outflow"], - "y": ["noslip", "noslip"], - "z": ["none", "none"] - } -} \ No newline at end of file diff --git a/src/CCD_analysis/correction_analysis/__init__.py b/src/CCD_analysis/correction_analysis/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/CCD_analysis/correction_analysis/compare_dqctl_scenes.py b/src/CCD_analysis/correction_analysis/compare_dqctl_scenes.py deleted file mode 100644 index f420a65..0000000 --- a/src/CCD_analysis/correction_analysis/compare_dqctl_scenes.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Generate comparison figures across all cloak & illusion scenarios. - -All dq_ctl fields use unified geometry (pinball center at ~613px, sensors at ~800px), -set during GPU collection (configs.py UNIFIED coordinates). -Figures zoom into the region around the pinball/cylinder (x=300-1100) to exclude -boundary artifacts. - -1. All-scenes panorama: steady_cloak, karman_re100, vortex_lamb, vortex_taylor, - illusion_0.75L, illusion_1.0L, illusion_1.5L -2. Illusion-only comparison: 0.75L, 1.0L, 1.5L - -Usage: - conda run -n pycuda_3_10 python correction_analysis/compare_dqctl_scenes.py -""" -from __future__ import annotations - -import os -import sys - -import numpy as np -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR, NX, NY -from CCD_analysis.correction_analysis.compute_correction_fields import ( - compute_correction, -) - -FIG_DIR = os.path.join(DATA_DIR, "figures") -os.makedirs(FIG_DIR, exist_ok=True) - -# Display crop region (pixels) — around pinball at x~613 -CROP_X0, CROP_X1 = 300, 1100 - -# Scene groups -CLOAK_SCENES = ["steady_cloak", "karman_re100", "vortex_lamb", "vortex_taylor"] -ILLUSION_SCENES = ["illusion_0.75L", "illusion_1.0L", "illusion_1.5L"] -ALL_SCENES = CLOAK_SCENES + ILLUSION_SCENES - -SCENE_LABELS = { - "steady_cloak": "Steady Cloak", - "karman_re100": "Karman Cloak", - "vortex_lamb": "Vortex Lamb", - "vortex_taylor": "Vortex Taylor", - "illusion_0.75L": "Illusion 0.75L", - "illusion_1.0L": "Illusion 1.0L", - "illusion_1.5L": "Illusion 1.5L", -} - -FIELD_METRICS = [ - ("ux_mean", r"mean $u_x$", "RdBu_r", True), - ("uy_mean", r"mean $u_y$", "RdBu_r", True), - ("rms", "RMS", "viridis", False), - ("vorticity", r"$\omega_z$", "RdBu_r", True), -] - - -def compute_metrics(st: str) -> dict | None: - """Load dq_ctl for a scene and compute metrics.""" - try: - corr = compute_correction(st) - dq = corr.get("dq_ctl") - if dq is None: - return None - ux, uy = dq["ux"], dq["uy"] - return { - "ux_mean": np.mean(ux, axis=0), - "uy_mean": np.mean(uy, axis=0), - "rms": np.sqrt(np.std(ux, axis=0)**2 + np.std(uy, axis=0)**2), - "vorticity": np.gradient(np.mean(uy, axis=0), axis=1) - - np.gradient(np.mean(ux, axis=0), axis=0), - } - except Exception as e: - print(f" SKIP {st}: {e}") - return None - - -def crop_field(f: np.ndarray) -> np.ndarray: - """Crop to display region (NY, NX_cropped).""" - return f[:, CROP_X0:CROP_X1] - - -def plot_comparison(scene_list: str | list, name: str): - """Generate a grid of dq_ctl metrics for selected scenes.""" - if isinstance(scene_list, str): - scene_list = [scene_list] - - # Load all fields - fields = {} - for st in scene_list: - print(f" Loading {st}...", flush=True) - m = compute_metrics(st) - if m is not None: - fields[st] = m - - n_scenes = len(fields) - if n_scenes == 0: - print(" No valid fields, skipping") - return - - scene_names = list(fields.keys()) - n_rows = len(FIELD_METRICS) - - # Compute global vmax per metric from CROPPED fields - metric_vmax = {} - for mkey, _, _, _ in FIELD_METRICS: - all_vals = np.concatenate( - [abs(crop_field(fields[s][mkey])).ravel() for s in fields]) - vmax = float(np.percentile(all_vals[np.isfinite(all_vals)], 99.5)) - metric_vmax[mkey] = max(vmax, 1e-12) - - fig, axes = plt.subplots(n_rows, n_scenes, - figsize=(3.5 * n_scenes, 3.0 * n_rows)) - if n_rows == 1: - axes = [axes] - if n_scenes == 1: - axes = [[a] for a in axes] - - nx_crop = CROP_X1 - CROP_X0 - extent = (CROP_X0, CROP_X1, 0, NY - 1) - - for row, (mkey, mlabel, cmap, symmetric) in enumerate(FIELD_METRICS): - for col, sn in enumerate(scene_names): - ax = axes[row][col] - f = crop_field(fields[sn][mkey]) - vmax = metric_vmax[mkey] - - kwargs = {"cmap": cmap, "origin": "lower", - "aspect": "equal", "extent": extent} - if symmetric: - kwargs["vmin"] = -vmax - kwargs["vmax"] = vmax - else: - kwargs["vmin"] = 0 - kwargs["vmax"] = vmax - - ax.imshow(f, **kwargs) - ax.tick_params(left=False, right=False, labelleft=False, - bottom=False, top=False, labelbottom=False) - - if row == 0: - ax.set_title(SCENE_LABELS.get(sn, sn), fontsize=10) - if col == 0: - ax.set_ylabel(mlabel, fontsize=10) - - plt.suptitle(f"dq_ctl: [{', '.join(SCENE_LABELS.get(s,s) for s in scene_names)}]", - fontsize=12, y=1.01) - plt.tight_layout() - path = os.path.join(FIG_DIR, name) - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - print(f" Saved: {path}", flush=True) - - # Stats - print(f"\n --- RMS (cropped region) ---") - for sn in scene_names: - rms_crop = crop_field(fields[sn]["rms"]) - rms_val = float(np.sqrt(np.mean(rms_crop**2))) - print(f" {sn:22s}: RMS={rms_val:.6f}") - - -def main(): - print("=" * 60) - print("Comparison: dq_ctl across all cloak & illusion scenes") - print("=" * 60) - - # 1. All 7 scenes panorama - print("\n--- All 7 scenes panorama ---") - plot_comparison(ALL_SCENES, "corr_comparison_all_scenes.png") - - # 2. Illusion-only (3 diameters) - print("\n--- Illusion-only comparison ---") - plot_comparison(ILLUSION_SCENES, "corr_illusion_comparison_dqctl.png") - - # 3. Cloak-only (4 scenes) for reference - print("\n--- Cloak-only comparison ---") - plot_comparison(CLOAK_SCENES, "corr_cloak_comparison_dqctl.png") - - print("\nDone!") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/correction_analysis/compute_correction_fields.py b/src/CCD_analysis/correction_analysis/compute_correction_fields.py deleted file mode 100644 index 53d8280..0000000 --- a/src/CCD_analysis/correction_analysis/compute_correction_fields.py +++ /dev/null @@ -1,434 +0,0 @@ -"""Build q_in, q_blk, q_ctl, q_tar field references and compute Delta corrections. - -Each scene type maps to different data sources: - -| Scene type | q_in | q_blk | q_ctl | q_tar | -|-----------------|-------------------|---------|---------------------|--------------------------| -| illusion_0.75L | target_channel* | pinball | illusion_0.75L | target_cylinder_0.75L | -| illusion_1.0L | target_channel* | pinball | illusion_1.0L | target_cylinder_1.0L | -| illusion_1.5L | target_channel* | pinball | illusion_1.5L | target_cylinder_1.5L | -| steady_cloak | target_channel* | pinball | steady_cloak* | None (target=q_in) | -| karman_re100 | karman_q_in | karman_q_blk | karman_re100 | karman_q_in | - -(*) loaded via load_legacy_steady(). Vortex scenes use load_vortex_fields(). - All scenes now use UNIFIED geometry (pinball center at 613px, sensors at 800px). -""" -from __future__ import annotations - -import os -import sys -import warnings -from typing import Any, Optional - -import numpy as np - -from CCD_analysis.configs import DATA_DIR, NX, NY -from CCD_analysis.utils.resampling import ( - load_aligned_fields, - build_field_matrix as _build_field_matrix, -) - -from CCD_analysis.correction_analysis.process_legacy_steady import load_legacy_steady -from CCD_analysis.utils.load_vortex_fields import load_vortex_fields - - -# --------------------------------------------------------------------------- -# Source mapping -# --------------------------------------------------------------------------- - -def _resolve_source(name: str) -> Optional[dict]: - """Load a named data source, dispatching to the correct loader. - - Parameters - ---------- - name : str - Scene or special name: - - 'target_channel', 'steady_cloak' -> load_legacy_steady - - 'vortex_*' -> load_vortex_fields (transient) - - 'karman_re100' -> _load_karman_re100 (handles 72 vs 96 mismatch) - - all others -> load_aligned_fields - - Returns - ------- - dict or None - """ - _LEGACY_SCENES = {"target_channel", "steady_cloak"} - _VORTEX_SCENES = { - "vortex_lamb", "vortex_taylor", - "vortex_uncontrolled_lamb", "vortex_uncontrolled_taylor", - "vortex_target_lamb", "vortex_target_taylor", - } - - if name in _LEGACY_SCENES: - return load_legacy_steady(name) - elif name in _VORTEX_SCENES: - return load_vortex_fields(name) - elif name == "karman_re100": - return _load_karman_re100() - else: - return load_aligned_fields(name) - - -# --------------------------------------------------------------------------- -# Karman re100 special loader (handles 72 vs 96 frame mismatch) -# --------------------------------------------------------------------------- - -def _load_karman_re100() -> dict: - """Load karman_re100 aligned fields, handling the 72 vs 96 frame mismatch. - - karman_re100's fields_aligned.npz has 72 snapshots (3 cycles x 24 pts) - but the phase_plan.json lists 96 step_indices (4 cycles x 24 pts). - This loader truncates step_indices to match. - """ - import json as _json - - scene_name = "karman_re100" - from CCD_analysis.configs import SCENES as _SCENES - - cfg = _SCENES[scene_name] - scene_id = cfg["scene_id"] - data_dir = os.path.join(DATA_DIR, scene_id, scene_name) - - # Load fields_aligned.npz - fa_path = os.path.join(data_dir, "fields_aligned.npz") - fd = np.load(fa_path) - ux_raw = fd["ux"] - uy_raw = fd["uy"] - N = ux_raw.shape[0] # 72 - fd.close() - - # Transpose (N, NX, NY) -> (N, NY, NX) - ux = np.ascontiguousarray(ux_raw.transpose(0, 2, 1)) - uy = np.ascontiguousarray(uy_raw.transpose(0, 2, 1)) - - # Load phase_plan, truncate to N - plan_path = os.path.join(DATA_DIR, "resampled", scene_name, "phase_plan.json") - with open(plan_path) as f: - plan = _json.load(f) - step_indices = list(plan["step_indices"][:N]) - - # Load telemetry - tele_path = os.path.join(data_dir, "controlled.npz") - td = np.load(tele_path) - - result = { - "ux": ux, - "uy": uy, - "forces": td["forces"][step_indices] if "forces" in td else None, - "actions": td["actions"][step_indices] if "actions" in td else None, - "sensors": td["sensors"][step_indices] if "sensors" in td else None, - "meta": { - "scene": scene_name, - "scene_id": scene_id, - "gate": plan.get("gate", "unknown"), - "CV_T": plan.get("CV_T"), - "f_dom": plan.get("f_dom"), - "N_raw_per_cycle": plan.get("N_raw_per_cycle"), - "rho_interp": plan.get("rho_interp"), - "sample_interval": cfg.get("sample_interval"), - "note": "step_indices truncated from 96 to 72 (3 cycles, not 4)", - }, - "step_indices": step_indices, - } - td.close() - return result - - -# --------------------------------------------------------------------------- -# Correction computation -# --------------------------------------------------------------------------- - -# Scene map: (scene_type -> (q_in_source, q_blk_source, q_ctl_source, q_tar_source)) -_SCENE_MAP = { - "illusion_0.75L": ("target_channel", "pinball", "illusion_0.75L", "target_cylinder_0.75L"), - "illusion_1.0L": ("target_channel", "pinball", "illusion_1.0L", "target_cylinder_1.0L"), - "illusion_1.5L": ("target_channel", "pinball", "illusion_1.5L", "target_cylinder_1.5L"), - "steady_cloak": ("target_channel", "pinball", "steady_cloak", None), - "karman_re100": ("karman_q_in", "karman_q_blk", "karman_re100", "karman_q_in"), - "vortex_lamb": ("target_channel", "vortex_uncontrolled_lamb", "vortex_lamb", "vortex_target_lamb"), - "vortex_taylor": ("target_channel", "vortex_uncontrolled_taylor", "vortex_taylor", "vortex_target_taylor"), -} - - -def get_diameter(scene_type: str) -> Optional[float]: - """Extract target diameter from scene type string (e.g. 'illusion_1.0L' -> 1.0).""" - if "illusion" in scene_type or "target_cylinder" in scene_type: - try: - return float(scene_type.split("_")[-1].replace("L", "")) - except (ValueError, IndexError): - return None - return None - - -def subtract_fields(q_a: dict, q_b: dict) -> Optional[dict]: - """Compute q_a - q_b field difference. - - Both must have the same N. Returns dict with: - ux, uy : (N, NY, NX) -- field difference - forces : from q_a (reference) - sensors : from q_a (reference) - actions : from q_a (reference) - meta : combined - step_indices : from q_a - - Parameters - ---------- - q_a : dict -- reference (minuend) - q_b : dict -- subtrahend - - Returns - ------- - dict or None if either input is None - """ - if q_a is None or q_b is None: - return None - - N_a = q_a["ux"].shape[0] - N_b = q_b["ux"].shape[0] - if N_a != N_b: - raise ValueError( - f"Frame count mismatch: q_a has {N_a} frames, q_b has {N_b}. " - "Use common_length() to align." - ) - - return { - "ux": q_a["ux"] - q_b["ux"], - "uy": q_a["uy"] - q_b["uy"], - "forces": q_a.get("forces"), - "sensors": q_a.get("sensors"), - "actions": q_a.get("actions"), - "meta": {**q_a.get("meta", {}), "delta_from": q_b.get("meta", {}).get("scene", "unknown")}, - "step_indices": q_a.get("step_indices"), - } - - -def dict_to_field_matrix(q: dict) -> np.ndarray: - """Wrapper: build_field_matrix(q['ux'], q['uy']) with error checking. - - Parameters - ---------- - q : dict -- must contain 'ux' and 'uy' with shape (N, NY, NX). - - Returns - ------- - Q : (2 * NX * NY, N) ndarray -- snapshot matrix for POD. - """ - if q is None: - raise ValueError("Cannot build field matrix from None") - ux = q["ux"] - uy = q["uy"] - if ux.ndim != 3 or ux.shape[-2:] != (NY, NX): - raise ValueError( - f"Expected field shape (N, {NY}, {NX}), got {ux.shape}" - ) - return _build_field_matrix(ux, uy) - - -def compute_correction(scene_type: str) -> dict: - """Load q_in, q_blk, q_ctl, q_tar and compute Delta fields. - - Parameters - ---------- - scene_type : str -- one of the keys in _SCENE_MAP. - - Returns - ------- - dict with: - scene_type : str - diam : float or None - q_in, q_blk, q_ctl, q_tar : dict or None -- raw loaded data - dq_blk, dq_ctl, dq_tar : dict or None -- field differences - dq_tar_minus_blk : dict or None - N : int -- aligned frame count (min across all loaded sources) - meta : combined metadata dict - """ - if scene_type not in _SCENE_MAP: - raise KeyError( - f"Unknown scene_type: {scene_type}. " - f"Available: {list(_SCENE_MAP.keys())}" - ) - - q_in_name, q_blk_name, q_ctl_name, q_tar_name = _SCENE_MAP[scene_type] - diam = get_diameter(scene_type) - - print(f"\n{'=' * 60}") - print(f"Computing correction fields for: {scene_type}") - print(f" q_in = {q_in_name}, q_blk = {q_blk_name}, " - f"q_ctl = {q_ctl_name}, q_tar = {q_tar_name}") - print(f"{'=' * 60}") - - # -- Load all sources -- - q_in = _resolve_source(q_in_name) if q_in_name else None - q_blk = _resolve_source(q_blk_name) if q_blk_name else None - q_ctl = _resolve_source(q_ctl_name) if q_ctl_name else None - q_tar = _resolve_source(q_tar_name) if q_tar_name else None - - # -- Determine aligned N -- - all_N = [] - for label, q in [("q_in", q_in), ("q_blk", q_blk), ("q_ctl", q_ctl), ("q_tar", q_tar)]: - if q is not None: - n = q["ux"].shape[0] - all_N.append(n) - print(f" {label}: {n} frames, shape={q['ux'].shape}") - else: - print(f" {label}: None") - - N = min(all_N) if all_N else 0 - - # -- Compute Delta fields -- - # dq_blk = q_ctl - q_blk (controller adds beyond pinball) - # dq_tar = q_tar - q_blk (target cylinder wake beyond pinball) - # dq_ctl = q_ctl - q_in (ctl perturbation from inflow) - # dq_tar_in = q_tar - q_in (target perturbation from inflow) - - dq_blk = _safe_subtract(q_blk, q_in, "dq_blk = q_blk - q_in (pinball blockage)") - dq_ctl = _safe_subtract(q_ctl, q_blk, "dq_ctl = q_ctl - q_blk (control correction)") - dq_tar = _safe_subtract(q_tar, q_blk, "dq_tar = q_tar - q_blk (target correction)") - - result: dict[str, Any] = { - "scene_type": scene_type, - "diam": diam, - "q_in": q_in, - "q_blk": q_blk, - "q_ctl": q_ctl, - "q_tar": q_tar, - "dq_blk": dq_blk, # q_blk - q_in - "dq_ctl": dq_ctl, # q_ctl - q_blk - "dq_tar": dq_tar, # q_tar - q_blk - "N": N, - "meta": { - "scene_type": scene_type, - "q_in": q_in_name, - "q_blk": q_blk_name, - "q_ctl": q_ctl_name, - "q_tar": q_tar_name, - "N_aligned": N, - }, - } - - return result - - -def _safe_subtract(q_a: Optional[dict], q_b: Optional[dict], - label: str) -> Optional[dict]: - """Subtract fields with optional trimming and None safety.""" - if q_a is None or q_b is None: - print(f" {label}: skipped (None input)") - return None - - N_a = q_a["ux"].shape[0] - N_b = q_b["ux"].shape[0] - - if N_a != N_b: - N_min = min(N_a, N_b) - print(f" {label}: N mismatch ({N_a} vs {N_b}), " - f"trimming to min N={N_min}") - q_a_trim = _trim_to(q_a, N_min) - q_b_trim = _trim_to(q_b, N_min) - else: - q_a_trim = q_a - q_b_trim = q_b - - dq = subtract_fields(q_a_trim, q_b_trim) - if dq is not None: - _print_field_summary(f" {label}", dq["ux"], dq["uy"]) - return dq - - -def _trim_to(q: dict, N: int) -> dict: - """Trim first N frames from field dict.""" - return { - "ux": q["ux"][:N], - "uy": q["uy"][:N], - "forces": q.get("forces")[:N] if q.get("forces") is not None else None, - "sensors": q.get("sensors")[:N] if q.get("sensors") is not None else None, - "actions": q.get("actions")[:N] if q.get("actions") is not None else None, - "step_indices": q.get("step_indices")[:N] if q.get("step_indices") is not None else None, - "meta": q.get("meta", {}), - } - - -def _print_field_summary(label: str, ux: np.ndarray, uy: np.ndarray) -> None: - """Print one-line field statistics.""" - ux_mean = ux.mean() - uy_mean = uy.mean() - ux_rms = ux.std() - uy_rms = uy.std() - mag_mean = np.sqrt(ux_mean**2 + uy_mean**2) - print(f" {label}:") - print(f" shape = {ux.shape}") - print(f" ux_mean = {ux_mean:.6f} uy_mean = {uy_mean:.6f}") - print(f" ux_rms = {ux_rms:.6f} uy_rms = {uy_rms:.6f}") - print(f" |q|_mean = {mag_mean:.6f}") - - -# --------------------------------------------------------------------------- -# Main (test / verification) -# --------------------------------------------------------------------------- - -if __name__ == "__main__": - print("=" * 60) - print("Testing compute_correction_fields.py") - print("=" * 60) - - # -- 1. Load target_channel + pinball, compute dq_blk -- - print("\n--- 1. Loading target_channel (legacy) + pinball (aligned) ---") - target_channel = load_legacy_steady("target_channel") - pinball = load_aligned_fields("pinball") - - _print_field_summary("target_channel (mean)", target_channel["ux"], target_channel["uy"]) - _print_field_summary("pinball (mean)", pinball["ux"], pinball["uy"]) - - # Verify shapes - print(f"\n target_channel: N={target_channel['ux'].shape[0]}, " - f"shape={target_channel['ux'].shape}") - print(f" pinball: N={pinball['ux'].shape[0]}, " - f"shape={pinball['ux'].shape}") - print(f" sensors (target): {target_channel['sensors'].shape}") - print(f" sensors (pinball): {pinball['sensors'].shape}") - print(f" forces (pinball): {pinball['forces'].shape}") - - # -- 2. Compute dq_blk = pinball - target_channel -- - # Trim to smaller N - dq_blk_test = _safe_subtract(pinball, target_channel, "pinball - target_channel") - if dq_blk_test is not None: - print(f"\n dq_blk shape: {dq_blk_test['ux'].shape}") - print(f" dq_blk ux_mean (mean of difference): {dq_blk_test['ux'].mean():.6f}") - - # -- 3. Test full compute_correction for illusion_1.0L -- - print("\n--- 3. Full correction field pipeline: illusion_1.0L ---") - result = compute_correction("illusion_1.0L") - - print(f"\n--- Result summary for {result['scene_type']} ---") - print(f" diam = {result['diam']}") - print(f" N = {result['N']}") - print(f" q_in = {result['q_in']['ux'].shape if result['q_in'] else None}") - print(f" q_blk = {result['q_blk']['ux'].shape if result['q_blk'] else None}") - print(f" q_ctl = {result['q_ctl']['ux'].shape if result['q_ctl'] else None}") - print(f" q_tar = {result['q_tar']['ux'].shape if result['q_tar'] else None}") - - for key in ["dq_blk", "dq_ctl", "dq_tar", "dq_tar_minus_blk"]: - dq = result.get(key) - if dq is not None: - print(f" {key}: mean(ux)={dq['ux'].mean():.6f}, " - f"mean(uy)={dq['uy'].mean():.6f}") - else: - print(f" {key}: None") - - # -- 4. Verify dict_to_field_matrix -- - print("\n--- 4. Testing dict_to_field_matrix ---") - Q = dict_to_field_matrix(pinball) - print(f" pinball snapshot matrix: {Q.shape} " - f"(expect ({2 * NX * NY}, 96))") - assert Q.shape == (2 * NX * NY, 96), f"Unexpected shape: {Q.shape}" - print(f" Q range: [{Q.min():.6f}, {Q.max():.6f}]") - - # -- 5. Test steady_cloak -- - print("\n--- 5. Testing steady_cloak correction ---") - result_sc = compute_correction("steady_cloak") - print(f" steady_cloak N = {result_sc['N']}") - if result_sc.get("dq_blk") is not None: - print(f" dq_blk (cloak-pinball) ux_mean: " - f"{result_sc['dq_blk']['ux'].mean():.6f}") - - print("\nAll tests passed.") diff --git a/src/CCD_analysis/correction_analysis/decompose_corrections.py b/src/CCD_analysis/correction_analysis/decompose_corrections.py deleted file mode 100644 index b906e26..0000000 --- a/src/CCD_analysis/correction_analysis/decompose_corrections.py +++ /dev/null @@ -1,223 +0,0 @@ -"""Minimal correction-field CCD: POD + force/action CCD on dq_ctl. - -Simplified version — processes only illusion_0.75L and illusion_1.0L. -No LOCO validation (separate step). Outputs CCD results and overlaps. - -Usage: - conda run -n pycuda_3_10 python correction_analysis/decompose_corrections.py -""" -from __future__ import annotations - -import json -import os -import sys -import time - -import numpy as np - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR -from CCD_analysis.utils.resampling import ( - compute_pod, cumulative_energy, e95_index, - compute_reduced_ccd, make_force_obs, -) -from CCD_analysis.correction_analysis.compute_correction_fields import ( - compute_correction, dict_to_field_matrix, -) - -R_CANDIDATES = [6, 8, 10] -CCD_Q = 6 -SCENE_TYPES = ["illusion_0.75L", "illusion_1.0L", "illusion_1.5L", "steady_cloak"] -DIAMETERS_MAIN = [0.75, 1.0] -DIAMETER_SPECIAL = 1.5 # flagged as special_mechanism (high-freq modulation) - - -def compute_modal_overlap(W_dict, scene_label, r, obs_label="force_fy"): - keys = [k for k in W_dict - if scene_label in k and f"_{obs_label}_r{r}" in k] - overlaps = [] - for i, ka in enumerate(keys): - for kb in keys[i + 1:]: - Wa, Wb = W_dict[ka], W_dict[kb] - n = min(Wa.shape[1], Wb.shape[1], 5) - for k in range(n): - ov = float(abs( - Wa[:, k] / (np.linalg.norm(Wa[:, k]) + 1e-12) @ - Wb[:, k] / (np.linalg.norm(Wb[:, k]) + 1e-12) - )) - overlaps.append({ - "case_a": ka.split(f"_{obs_label}_r{r}")[0], - "case_b": kb.split(f"_{obs_label}_r{r}")[0], - "mode": k + 1, - "O": ov, - }) - return overlaps - - -def _scene_to_target_name(scene_type): - if "illusion" in scene_type: - parts = scene_type.split("_") - if len(parts) >= 2: - return f"target_cylinder_{parts[1]}" - return None - - -def main(): - print("=" * 60, flush=True) - print("Correction-field CCD (Phase 3) — dq_ctl", flush=True) - print("=" * 60, flush=True) - - out_dir = os.path.join(DATA_DIR, "ccd") - os.makedirs(out_dir, exist_ok=True) - all_results = {} - W_dict = {} - - # Load correction fields - print("\n--- Loading correction fields ---", flush=True) - cache = {} - for st in SCENE_TYPES: - t0 = time.time() - try: - corr = compute_correction(st) - cache[st] = corr - dq = corr["dq_ctl"] - if dq is not None: - print(f" {st}: dq_ctl {dq['ux'].shape[0]} frames, " - f"forces={'✓' if dq['forces'] is not None else '✗'}, " - f"actions={'✓' if dq['actions'] is not None else '✗'}, " - f"{time.time()-t0:.1f}s", flush=True) - except Exception as e: - print(f" {st}: FAILED — {e}", flush=True) - - # CCD on dq_ctl - for st in SCENE_TYPES: - corr = cache.get(st) - if corr is None: - continue - dq_ctl = corr["dq_ctl"] - dq_tar = corr["dq_tar"] - if dq_ctl is None: - continue - - diam = corr.get("diam") - is_special = (diam is not None and diam >= DIAMETER_SPECIAL) - flag = " [SPECIAL MECHANISM — high-freq modulation]" if is_special else "" - print(f"\n--- {st} (diam={diam}){flag} ---", flush=True) - - Q_ctl = dict_to_field_matrix(dq_ctl) - N = Q_ctl.shape[1] - print(f" dq_ctl: shape={Q_ctl.shape}", flush=True) - - # POD: target-only (dq_tar) or direct (dq_ctl) - if dq_tar is not None: - Q_tar = dict_to_field_matrix(dq_tar) - print(f" dq_tar: shape={Q_tar.shape}", flush=True) - mf_tar, modes_tar, sv_tar, coeffs_tar = compute_pod(Q_tar) - # Project dq_ctl into target basis - dc = dq_ctl - q_proj = np.column_stack([ - np.concatenate([dc["ux"][s].ravel(), dc["uy"][s].ravel()]) - for s in range(N) - ]) - a_ctl = modes_tar.T @ (q_proj - mf_tar[:, None]).astype(np.float64) - a_tar = coeffs_tar - print(f" POD: target-only basis (E95={e95_index(cumulative_energy(sv_tar))})", flush=True) - else: - mf, modes, sv, coeffs = compute_pod(Q_ctl) - a_ctl = coeffs - a_tar = None - print(f" POD: direct dq_ctl (E95={e95_index(cumulative_energy(sv))})", flush=True) - - for r in R_CANDIDATES: - a_r = a_ctl[:r, :] - Nv = a_r.shape[1] - print(f"\n r={r}: N={Nv}", flush=True) - - # Force-CCD - frc = dq_ctl.get("forces") - if frc is not None: - for fmode, flabel in [("fy","force_fy"), ("fx","force_fx")]: - y = make_force_obs(frc, st, mode=fmode)[:, :Nv] - W, sig, _, _, _, _ = compute_reduced_ccd(a_r, y, Q_delay=CCD_Q) - en = cumulative_energy(sig) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 0 - key = f"{st}_dqctl_{flabel}_r{r}" - W_dict[key] = W - all_results[key] = { - "scene": st, "diam": diam, "obs": flabel, "r": r, - "m80": m80, "N": sig.size, - "sigma_top3": [float(sig[i]) for i in range(min(3,len(sig)))], - "special_mechanism": is_special, - } - if fmode == "fy": - print(f" {key}: m80={m80} s1={float(sig[0]):.4f}", flush=True) - - # Action-CCD (illusion only) - act = dq_ctl.get("actions") - if act is not None: - y_a = act.T[:, :Nv] - W, sig, _, _, _, _ = compute_reduced_ccd(a_r, y_a, Q_delay=CCD_Q) - en = cumulative_energy(sig) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 0 - key = f"{st}_dqctl_action_r{r}" - W_dict[key] = W - all_results[key] = { - "scene": st, "diam": diam, "obs": "action", "r": r, - "m80": m80, "N": sig.size, - "sigma_top3": [float(sig[i]) for i in range(min(3,len(sig)))], - "special_mechanism": is_special, - } - print(f" {key}: m80={m80} s1={float(sig[0]):.4f}", flush=True) - - # Target force-CCD reference (if available) - if dq_tar is not None and a_tar is not None: - a_tr = a_tar[:r, :Nv] - frc_t = dq_tar.get("forces") - if frc_t is not None: - tname = _scene_to_target_name(st) or f"{st}_tar" - y_t = make_force_obs(frc_t[:Nv], tname, mode="fy") - Wt, sig_t, _, _, _, _ = compute_reduced_ccd(a_tr, y_t, Q_delay=CCD_Q) - kt = f"{st}_dqtar_force_fy_r{r}" - W_dict[kt] = Wt - all_results[kt] = { - "scene": st, "diam": diam, "obs": "force_fy_tar", "r": r, - "m80": int(np.searchsorted(cumulative_energy(sig_t), 0.80)+1) if len(sig_t) > 0 else 0, - "N": sig_t.size, - "sigma_top3": [float(sig_t[i]) for i in range(min(3,len(sig_t)))], - "special_mechanism": is_special, - } - # Overlap: dq_ctl vs dq_tar - ck = f"{st}_dqctl_force_fy_r{r}" - if ck in W_dict: - Wc = W_dict[ck] - n = min(Wc.shape[1], Wt.shape[1], 5) - for k in range(n): - ov = float(abs( - Wc[:, k] / (np.linalg.norm(Wc[:, k])+1e-12) @ - Wt[:, k] / (np.linalg.norm(Wt[:, k])+1e-12) - )) - all_results[f"{st}_O_dqctl_vs_dqtar_r{r}_mode{k+1}"] = { - "overlap": ov, "mode": k+1, "r": r - } - if k == 0: - print(f" O(dqctl, dqtar) mode1={ov:.4f}", flush=True) - - # Overlap dqctl_target vs dqctl_illusion at r=6 - print(f" Modal overlaps r=6:", flush=True) - ovs = compute_modal_overlap(W_dict, st, 6, "force_fy") - for ov in ovs: - print(f" O({ov['case_a']}, {ov['case_b']}) mode{ov['mode']} = {ov['O']:.4f}", flush=True) - - # Save - ccd_path = os.path.join(out_dir, "correction_ccd_results.json") - with open(ccd_path, "w") as f: - json.dump(all_results, f, indent=2) - print(f"\nSaved {len(all_results)} entries to {ccd_path}", flush=True) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/correction_analysis/diagnose_corrections.py b/src/CCD_analysis/correction_analysis/diagnose_corrections.py deleted file mode 100644 index b388871..0000000 --- a/src/CCD_analysis/correction_analysis/diagnose_corrections.py +++ /dev/null @@ -1,377 +0,0 @@ -"""Phase 2: Baseline diagnostics — mean/RMS/vorticity + zone metrics for correction fields. - -For each available scene type: -1. Mean/RMS/vorticity of dq_blk, dq_ctl, dq_tar -2. Three-zone spatial metrics -3. Figures saved to data/figures/ - -Usage: - conda run -n pycuda_3_10 python correction_analysis/diagnose_corrections.py -""" -from __future__ import annotations - -import json -import os -import sys -from typing import Optional - -import numpy as np - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR, NX, NY, L0, CENTER_Y -from CCD_analysis.utils.resampling import load_aligned_fields -from CCD_analysis.correction_analysis.compute_correction_fields import ( - compute_correction, dict_to_field_matrix, -) - -FIG_DIR = os.path.join(DATA_DIR, "figures") -os.makedirs(FIG_DIR, exist_ok=True) - -# Scene types to process -SCENE_TYPES = [ - "illusion_0.75L", - "illusion_1.0L", - "illusion_1.5L", - "steady_cloak", - "karman_re100", - "vortex_lamb", - "vortex_taylor", -] - - -# --------------------------------------------------------------------------- -# Three-zone masks (unified geometry: pinball center at 613 px, sensors at 800 px) -# --------------------------------------------------------------------------- - -def define_zones() -> dict: - """Define three-zone masks for all scenes (unified geometry, 2026-06-28). - - All scenes now use the same pinball/sensor positions after unified collection. - Zone ranges: - near_body: 580-720 px (around pinball at x≈613) - body_wake: 720-850 px (near wake downstream) - sensor_zone: 780-850 px (around sensors at x=800) - """ - zones = {} - mask = np.zeros((NY, NX), dtype=bool) - mask[:, 580:720] = True - zones["near_body"] = mask - - mask = np.zeros((NY, NX), dtype=bool) - mask[:, 720:850] = True - zones["body_wake"] = mask - - mask = np.zeros((NY, NX), dtype=bool) - mask[:, 780:850] = True - zones["sensor_zone"] = mask - - return zones - - -# --------------------------------------------------------------------------- -# Field computation helpers -# --------------------------------------------------------------------------- - -def mean_field(ux: np.ndarray, uy: np.ndarray) -> tuple: - """Compute mean velocity field from snapshots.""" - return np.mean(ux, axis=0), np.mean(uy, axis=0) - - -def rms_field(ux: np.ndarray, uy: np.ndarray) -> np.ndarray: - """Compute RMS magnitude field.""" - ux_rms = np.std(ux, axis=0) - uy_rms = np.std(uy, axis=0) - return np.sqrt(ux_rms**2 + uy_rms**2) - - -def vorticity_field(ux: np.ndarray, uy: np.ndarray) -> np.ndarray: - """Compute mean z-vorticity from mean velocity field.""" - ux_m = np.mean(ux, axis=0) - uy_m = np.mean(uy, axis=0) - return np.gradient(uy_m, axis=1) - np.gradient(ux_m, axis=0) - - -# --------------------------------------------------------------------------- -# Zone metrics -# --------------------------------------------------------------------------- - -def zone_metrics(dq: dict, zones: dict, label: str) -> dict: - """Compute per-zone metrics for a correction field dict. - - Parameters - ---------- - dq : dict with 'ux' (N, NY, NX), 'uy' (N, NY, NX) - zones : dict of (NY, NX) boolean masks - label : str for printing - - Returns - ------- - metrics : dict with per-zone stats - """ - if dq is None: - print(f" {label}: None, skipping zone metrics") - return {} - - ux = dq["ux"] - uy = dq["uy"] - N = ux.shape[0] - - # Mean kinetic energy field (per snapshot, averaged) - ke_field = 0.5 * np.mean(ux**2 + uy**2, axis=0) # (NY, NX) - - # Vorticity field (from mean velocity) - ux_m, uy_m = mean_field(ux, uy) - vor = np.gradient(uy_m, axis=1) - np.gradient(ux_m, axis=0) - enstrophy_field = vor**2 - - metrics = {} - total_ke = ke_field.sum() - - for zname, zmask in zones.items(): - n_pts = zmask.sum() - if n_pts == 0: - continue - - zone_ke = ke_field[zmask].mean() - zone_enstrophy = enstrophy_field[zmask].mean() - zone_ke_frac = ke_field[zmask].sum() / total_ke if total_ke > 0 else 0.0 - - # Centreline asymmetry: ux mean above vs below centreline - cy = int(CENTER_Y) - y_indices = np.where(zmask.any(axis=1))[0] - if len(y_indices) > 0: - y_min, y_max = y_indices.min(), y_indices.max() - above = zmask[y_min:cy, :].sum() - below = zmask[cy:y_max, :].sum() - else: - above = below = 1 - - mask_correction = f"_{label.replace(' ', '_')}" - - metrics[zname] = { - "n_points": int(n_pts), - "mean_KE": float(zone_ke), - "mean_enstrophy": float(zone_enstrophy), - "KE_fraction": float(zone_ke_frac), - } - - print(f" {zname:15s}: KE={zone_ke:.6e}, " - f"ens={zone_enstrophy:.6e}, " - f"KE_frac={zone_ke_frac:.4f}") - - return metrics - - -# --------------------------------------------------------------------------- -# Plotting helpers -# --------------------------------------------------------------------------- - -def plot_mean_rms(dq: dict, label: str, prefix: str, zones: Optional[dict] = None): - """Plot mean ux, mean uy, RMS magnitude for a correction field.""" - if dq is None: - return - - ux = dq["ux"] - uy = dq["uy"] - ux_m, uy_m = mean_field(ux, uy) - rms = rms_field(ux, uy) - - fig, axes = plt.subplots(1, 3, figsize=(18, 5)) - - extent = (0, NX - 1, 0, NY - 1) - - # Mean ux - vmax = max(abs(ux_m).max(), 1e-12) - axes[0].imshow(ux_m, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", extent=extent) - axes[0].set_title(f"{label}: mean ux") - - # Mean uy - vmax = max(abs(uy_m).max(), 1e-12) - axes[1].imshow(uy_m, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", extent=extent) - axes[1].set_title(f"{label}: mean uy") - - # RMS magnitude - axes[2].imshow(rms, cmap="viridis", origin="lower", - aspect="equal", extent=extent) - axes[2].set_title(f"{label}: RMS magnitude") - - # Overlay zone boundaries if provided - if zones is not None: - # simple boundary: first/last column of each zone mask - for zname, zmask in zones.items(): - for ax in axes: - # Find leftmost and rightmost columns with True - cols = np.where(zmask.any(axis=0))[0] - if len(cols) > 1: - ax.axvline(cols[0], color="white", linewidth=0.5, alpha=0.5) - ax.axvline(cols[-1], color="white", linewidth=0.5, alpha=0.5) - - plt.tight_layout() - path = os.path.join(FIG_DIR, f"{prefix}_{label.replace(' ', '_')}.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - -def plot_vorticity(dq: dict, label: str, prefix: str): - """Plot mean vorticity field.""" - if dq is None: - return - - ux = dq["ux"] - uy = dq["uy"] - vor = vorticity_field(ux, uy) - - fig, ax = plt.subplots(figsize=(10, 4)) - vmax = max(np.percentile(abs(vor), 99), 1e-12) - ax.imshow(vor, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", - extent=(0, NX - 1, 0, NY - 1)) - ax.set_title(f"{label}: mean vorticity") - plt.tight_layout() - path = os.path.join(FIG_DIR, f"{prefix}_vorticity_{label.replace(' ', '_')}.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def run(): - print("=" * 60, flush=True) - print("Phase 2: Baseline Diagnostics (correction fields)", flush=True) - print("=" * 60, flush=True) - - zones = define_zones() - - all_metrics = {} - - for scene_type in SCENE_TYPES: - print(f"\n{'=' * 60}", flush=True) - print(f"Scene: {scene_type}", flush=True) - print(f"{'=' * 60}", flush=True) - - try: - corr = compute_correction(scene_type) - except (FileNotFoundError, KeyError, AssertionError, ValueError) as e: - print(f" SKIP: {e}", flush=True) - continue - - if corr["N"] == 0: - print(f" SKIP: no valid data (N=0)", flush=True) - continue - - # Unified geometry — same zones for all scenes - for dq_key, dq_label in [ - ("dq_blk", "dq_blk (pinball blockage)"), - ("dq_ctl", "dq_ctl (control correction)"), - ]: - dq = corr.get(dq_key) - if dq is None: - continue - - prefix = f"corr_{scene_type}" - plot_mean_rms(dq, dq_label, prefix, zones) - plot_vorticity(dq, dq_label, prefix) - - print(f" Zone metrics for {dq_label}:", flush=True) - metrics = zone_metrics(dq, zones, dq_label) - all_metrics[f"{scene_type}_{dq_key}"] = metrics - - # For scenes with a target, also plot dq_tar if available - if dq_key == "dq_ctl" and corr.get("dq_tar") is not None: - dq_tar = corr.get("dq_tar") - if dq_tar is not None: - plot_mean_rms(dq_tar, "dq_tar (target correction)", prefix, zones) - plot_vorticity(dq_tar, "dq_tar (target correction)", prefix) - - # dq_ctl vs dq_tar side-by-side comparison - fig, axes = plt.subplots(2, 2, figsize=(14, 8)) - extent = (0, NX - 1, 0, NY - 1) - - # Row 0: mean ux for dq_ctl and dq_tar - ux_ctl, _ = mean_field(dq["ux"], dq["uy"]) - ux_tar, _ = mean_field(dq_tar["ux"], dq_tar["uy"]) - vmax = max(abs(ux_ctl).max(), abs(ux_tar).max(), 1e-12) - - axes[0, 0].imshow(ux_ctl, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", extent=extent) - axes[0, 0].set_title("dq_ctl mean ux") - - axes[0, 1].imshow(ux_tar, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", extent=extent) - axes[0, 1].set_title("dq_tar mean ux") - - # Row 1: RMS - rms_ctl = rms_field(dq["ux"], dq["uy"]) - rms_tar = rms_field(dq_tar["ux"], dq_tar["uy"]) - rmax = max(rms_ctl.max(), rms_tar.max(), 1e-12) - - axes[1, 0].imshow(rms_ctl, cmap="viridis", vmin=0, vmax=rmax, - origin="lower", aspect="equal", extent=extent) - axes[1, 0].set_title("dq_ctl RMS") - - axes[1, 1].imshow(rms_tar, cmap="viridis", vmin=0, vmax=rmax, - origin="lower", aspect="equal", extent=extent) - axes[1, 1].set_title("dq_tar RMS") - - plt.suptitle(f"{scene_type}: dq_ctl vs dq_tar comparison") - plt.tight_layout() - path = os.path.join(FIG_DIR, f"corr_{scene_type}_ctl_vs_tar.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - # Steady cloak specific: dq_ctl + dq_blk check - if scene_type == "steady_cloak": - dq_b = corr.get("dq_blk") - dq_c = corr.get("dq_ctl") - if dq_b is not None and dq_c is not None: - ux_b = np.mean(dq_b["ux"], axis=0) - ux_c = np.mean(dq_c["ux"], axis=0) - ux_cancel = ux_c + ux_b - - fig, axes = plt.subplots(1, 3, figsize=(18, 4)) - extent = (0, NX - 1, 0, NY - 1) - vmax = max(abs(ux_b).max(), abs(ux_c).max(), abs(ux_cancel).max(), 1e-12) - - axes[0].imshow(ux_b, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", extent=extent) - axes[0].set_title("dq_blk mean ux (blockage)") - - axes[1].imshow(ux_c, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", extent=extent) - axes[1].set_title("dq_ctl mean ux (correction)") - - axes[2].imshow(ux_cancel, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", extent=extent) - axes[2].set_title("dq_ctl + dq_blk (cancel test)") - - plt.suptitle(f"Steady cloak: cancellation test") - plt.tight_layout() - path = os.path.join(FIG_DIR, "steady_cloak_cancel_test.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - # Save zone metrics - metrics_path = os.path.join(DATA_DIR, "ccd", "zone_metrics.json") - with open(metrics_path, "w") as f: - json.dump(all_metrics, f, indent=2) - print(f"\nZone metrics saved to {metrics_path}", flush=True) - print("\nDone.", flush=True) - - -if __name__ == "__main__": - run() diff --git a/src/CCD_analysis/correction_analysis/process_legacy_steady.py b/src/CCD_analysis/correction_analysis/process_legacy_steady.py deleted file mode 100644 index 00ececc..0000000 --- a/src/CCD_analysis/correction_analysis/process_legacy_steady.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Load legacy fields.npz format for steady scenes (steady_cloak, target_channel). - -Converts to the same convention as load_aligned_fields(): - - Transposes fields from (N, NX, NY) -> (N, NY, NX) - - Loads telemetry from sensors.npz - - Returns dict with identical key structure -""" -from __future__ import annotations - -import json -import os -from typing import Any - -import numpy as np - -from CCD_analysis.configs import DATA_DIR, NX, NY - - -def load_legacy_steady(scene_name: str) -> dict: - """Load steady scene from legacy fields.npz format. - - Parameters - ---------- - scene_name : str — one of 'steady_cloak' or 'target_channel' - - Returns - ------- - dict with same keys as load_aligned_fields(): - ux, uy : (N, NY, NX) ndarray - forces : None (no force telemetry in legacy sensors) - sensors : (N, 6) ndarray or None - actions : None (open-loop) - meta : dict with scene info - step_indices : list of int - """ - scene_dir = os.path.join(DATA_DIR, scene_name, scene_name) - if not os.path.isdir(scene_dir): - raise FileNotFoundError(f"Scene directory not found: {scene_dir}") - - # -- fields.npz (native simulation order: NX first) -- - fields_path = os.path.join(scene_dir, "fields.npz") - if not os.path.isfile(fields_path): - raise FileNotFoundError(f"{fields_path} not found") - - fd = np.load(fields_path) - ux_raw = fd["ux"] # (N, NX, NY) - uy_raw = fd["uy"] - N = ux_raw.shape[0] - fd.close() - - # Transpose (N, NX, NY) -> (N, NY, NX) to match load_aligned_fields convention - ux = np.ascontiguousarray(ux_raw.transpose(0, 2, 1)) - uy = np.ascontiguousarray(uy_raw.transpose(0, 2, 1)) - - # -- sensors.npz (telemetry) -- - sensors_path = os.path.join(scene_dir, "sensors.npz") - sensors = None - if os.path.isfile(sensors_path): - sd = np.load(sensors_path) - if "sensors" in sd: - sensors = sd["sensors"] # (N, 6) - assert sensors.shape[0] == N, ( - f"sensors ({sensors.shape[0]}) != fields ({N})" - ) - sd.close() - - # -- meta.json -- - meta = {"scene": scene_name, "scene_id": scene_name, "source": "legacy_steady"} - meta_path = os.path.join(scene_dir, "meta.json") - if os.path.isfile(meta_path): - with open(meta_path) as f: - meta.update(json.load(f)) - - result: dict[str, Any] = { - "ux": ux, - "uy": uy, - "forces": None, # no forces in legacy steady telemetry - "actions": None, # open-loop - "sensors": sensors, - "meta": meta, - "step_indices": list(range(N)), - } - - return result - - -# --------------------------------------------------------------------------- -# Diagnostic helpers -# --------------------------------------------------------------------------- - -def _print_field_stats(label: str, ux: np.ndarray, uy: np.ndarray) -> None: - """Print mean velocity statistics for a set of fields.""" - ux_mean = ux.mean() - uy_mean = uy.mean() - ux_std = ux.std() - uy_std = uy.std() - print(f" {label}:") - print(f" shape = {ux.shape}") - print(f" ux_mean = {ux_mean:.6f} (expect ~U0={0.01} for channel)") - print(f" uy_mean = {uy_mean:.6f} (expect near 0)") - print(f" ux_rms = {ux_std:.6f}") - print(f" uy_rms = {uy_std:.6f}") - - -# --------------------------------------------------------------------------- -# Main (test) -# --------------------------------------------------------------------------- - -if __name__ == "__main__": - print("=" * 60) - print("Testing load_legacy_steady()") - print("=" * 60) - - for scene in ["steady_cloak", "target_channel"]: - print(f"\n--- {scene} ---") - data = load_legacy_steady(scene) - ux = data["ux"] - uy = data["uy"] - sensors = data["sensors"] - N = ux.shape[0] - - _print_field_stats(scene, ux, uy) - - print(f" N_frames = {N}") - print(f" NY x NX = {ux.shape[1]} x {ux.shape[2]}") - print(f" sensors = {sensors.shape if sensors is not None else None}") - print(f" forces = {data['forces']}") - print(f" actions = {data['actions']}") - print(f" step_range = [{data['step_indices'][0]}, {data['step_indices'][-1]}]") - - # Physical reasonableness checks - ux_max = ux.max() - uy_max = abs(uy).max() - print(f" ux_max = {ux_max:.4f} (expect order 0.01)") - print(f" |uy|_max = {uy_max:.4f} (expect < ux_max)") - print(f" metadata = {list(data['meta'].keys())}") - - # Quick: verify convention matches load_aligned_fields - print("\n--- Convention check: transpose correctness ---") - # Load raw from steady_cloak to verify ravel order - raw = np.load( - os.path.join(DATA_DIR, "steady_cloak", "steady_cloak", "fields.npz") - ) - raw_ux = raw["ux"][0] # (NX, NY) - loaded = load_legacy_steady("steady_cloak") - loaded_ux = loaded["ux"][0] # (NY, NX) - - # raw_ux[NX, NY] should == loaded_ux[NY, NX] after transpose - match = np.allclose(raw_ux.T, loaded_ux) - print(f" Transpose (raw.T == loaded): {match}") - raw.close() - - print("\nDone.") diff --git a/src/CCD_analysis/correction_analysis/run_15L_correction.py b/src/CCD_analysis/correction_analysis/run_15L_correction.py deleted file mode 100644 index 9e44e17..0000000 --- a/src/CCD_analysis/correction_analysis/run_15L_correction.py +++ /dev/null @@ -1,366 +0,0 @@ -"""1.5L correction-field CCD: force-CCD, action-CCD, signature-CCD on dq_ctl. - -Extends the Phase 2 pipeline to the 1.5L "special mechanism" case. -Target-only POD basis, Q_delay=6, r=[6, 8, 10]. - -Usage: - conda run -n pycuda_3_10 python correction_analysis/run_15L_correction.py -""" -from __future__ import annotations - -import json -import os -import sys -import time - -import numpy as np - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR, NX, NY, CENTER_Y -from CCD_analysis.utils.resampling import ( - compute_pod, cumulative_energy, e95_index, - compute_reduced_ccd, make_force_obs, -) -from CCD_analysis.correction_analysis.compute_correction_fields import ( - compute_correction, dict_to_field_matrix, -) - -R_CANDIDATES = [6, 8, 10] -CCD_Q = 6 -SCENE_TYPE = "illusion_1.5L" - -# --------------------------------------------------------------------------- -# Zone masks for illusion layout (sensors at x=30*L0=600) -# --------------------------------------------------------------------------- - -def _define_zones() -> dict: - """Define body_wake and sensor_zone masks for 1.5L illusion.""" - zones = {} - # body_wake: immediate downstream of pinball, x=[500, 700) - mask_bw = np.zeros((NY, NX), dtype=bool) - mask_bw[:, 500:700] = True - zones["body_wake"] = mask_bw - # sensor_zone: around sensors at x=600, x=[580, 650) - mask_sz = np.zeros((NY, NX), dtype=bool) - mask_sz[:, 580:650] = True - zones["sensor_zone"] = mask_sz - return zones - - -def _zone_ke_ratio(dq: dict, zones: dict) -> dict: - """Compute correction energy ratio body_wake / sensor_zone.""" - ux, uy = dq["ux"], dq["uy"] - ke_field = 0.5 * np.mean(ux**2 + uy ** 2, axis=0) # (NY, NX) - body_ke = ke_field[zones["body_wake"]].sum() - sensor_ke = ke_field[zones["sensor_zone"]].sum() - ratio = body_ke / sensor_ke if sensor_ke > 0 else float("inf") - return { - "body_wake_KE": float(body_ke), - "sensor_zone_KE": float(sensor_ke), - "ratio_bw_over_sz": float(ratio), - } - - -# --------------------------------------------------------------------------- -# Signature-CCD helpers -# --------------------------------------------------------------------------- - -def compute_tau_corr(a_ctl: np.ndarray, e_s: np.ndarray, - max_lag: int = 12) -> int: - """Find tau that maximises |cross-correlation| between a1 and sensor error. - - Computes average absolute cross-correlation across sensor channels, - returns the lag (in snapshot steps) with the strongest correlation. - """ - a1 = a_ctl[0, :] # leading POD coefficient - n = len(a1) - # Normalise - a1_z = (a1 - a1.mean()) / (a1.std() + 1e-12) - # Average absolute correlation across sensor channels - corr_avg = np.zeros(2 * max_lag + 1) - for ch in range(e_s.shape[0]): - ech = e_s[ch, :n] - ech_z = (ech - ech.mean()) / (ech.std() + 1e-12) - c = np.correlate(a1_z, ech_z, mode="full") - c_mid = len(c) // 2 - seg = c[c_mid - max_lag:c_mid + max_lag + 1] - corr_avg += np.abs(seg) - corr_avg /= e_s.shape[0] - best_lag = np.argmax(corr_avg) - max_lag - return int(best_lag) - - -def _scene_to_target_name(scene_type: str) -> str | None: - if "illusion" in scene_type: - parts = scene_type.split("_") - if len(parts) >= 2: - return f"target_cylinder_{parts[1]}" - return None - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - print("=" * 60, flush=True) - print("1.5L Correction-field CCD — dq_ctl", flush=True) - print("=" * 60, flush=True) - - out_dir = os.path.join(DATA_DIR, "ccd") - os.makedirs(out_dir, exist_ok=True) - all_results = {} - W_dict = {} - - # ---- 1. Load correction fields for 1.5L ---- - print(f"\n--- Loading correction: {SCENE_TYPE} ---", flush=True) - t0 = time.time() - corr = compute_correction(SCENE_TYPE) - dq_ctl = corr["dq_ctl"] - dq_tar = corr["dq_tar"] - diam = corr.get("diam") - t_elapsed = time.time() - t0 - - if dq_ctl is None: - print(" dq_ctl is None — cannot proceed.", flush=True) - return 1 - - print(f" dq_ctl: {dq_ctl['ux'].shape[0]} frames, " - f"forces={'✓' if dq_ctl['forces'] is not None else '✗'}, " - f"actions={'✓' if dq_ctl['actions'] is not None else '✗'}, " - f"sensors={'✓' if dq_ctl['sensors'] is not None else '✗'}, " - f"{t_elapsed:.1f}s", flush=True) - - # ---- 2. Phase drift: zone energy ratio ---- - print(f"\n--- Phase drift: zone energy ratio (body_wake / sensor_zone) ---", - flush=True) - zones = _define_zones() - ze = _zone_ke_ratio(dq_ctl, zones) - print(f" body_wake KE = {ze['body_wake_KE']:.4e}", flush=True) - print(f" sensor_zone KE = {ze['sensor_zone_KE']:.4e}", flush=True) - print(f" ratio (bw/sz) = {ze['ratio_bw_over_sz']:.4f}", flush=True) - all_results["zone_energy_ratio"] = ze - - # ---- 3. POD: target-only basis ---- - print(f"\n--- POD: target-only basis ---", flush=True) - Q_ctl = dict_to_field_matrix(dq_ctl) - N = Q_ctl.shape[1] - print(f" dq_ctl: shape={Q_ctl.shape}", flush=True) - - if dq_tar is not None: - Q_tar = dict_to_field_matrix(dq_tar) - print(f" dq_tar: shape={Q_tar.shape}", flush=True) - mf_tar, modes_tar, sv_tar, coeffs_tar = compute_pod(Q_tar) - # Project dq_ctl into target basis - dc = dq_ctl - q_proj = np.column_stack([ - np.concatenate([dc["ux"][s].ravel(), dc["uy"][s].ravel()]) - for s in range(N) - ]) - a_ctl = modes_tar.T @ (q_proj - mf_tar[:, None]).astype(np.float64) - a_tar = coeffs_tar - print(f" POD: target-only basis " - f"(E95={e95_index(cumulative_energy(sv_tar))})", flush=True) - else: - print(" dq_tar is None — cannot proceed.", flush=True) - return 1 - - # ---- 4. Sensor error for signature line ---- - sensors_ctl = dq_ctl.get("sensors") # (N, 6) — illusion sensors - sensors_tar = dq_tar.get("sensors") # (N, 6) — target sensors - if sensors_ctl is not None and sensors_tar is not None: - # Both have 6 sensor channels: use all 6 dimensions - n_min = min(sensors_ctl.shape[0], sensors_tar.shape[0], N) - e_s_full = (sensors_ctl[:n_min] - sensors_tar[:n_min]).T # (6, N) - print(f" Sensor error e_s: shape={e_s_full.shape}", flush=True) - else: - print(" Sensor data incomplete — signature-CCD skipped.", flush=True) - e_s_full = None - - # ---- 5. Force-CCD, action-CCD, target-CCD ---- - for r in R_CANDIDATES: - a_r = a_ctl[:r, :] - Nv = a_r.shape[1] - print(f"\n r={r}: N={Nv}", flush=True) - - # Force-CCD - frc = dq_ctl.get("forces") - if frc is not None: - for fmode, flabel in [("fy", "force_fy"), - ("fx", "force_fx"), - ("joint", "force_joint")]: - y = make_force_obs(frc, SCENE_TYPE, mode=fmode)[:, :Nv] - W, sig, _, _, _, _ = compute_reduced_ccd( - a_r, y, Q_delay=CCD_Q) - en = cumulative_energy(sig) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 0 - key = f"illusion_1.5L_dqctl_{flabel}_r{r}" - W_dict[key] = W - all_results[key] = { - "scene": SCENE_TYPE, "diam": diam, "obs": flabel, "r": r, - "m80": m80, "N": sig.size, - "sigma_top3": [ - float(sig[i]) for i in range(min(3, len(sig))) - ], - } - if fmode == "fy": - print(f" {key}: m80={m80} " - f"s1={float(sig[0]):.4f}", flush=True) - - # Action-CCD - act = dq_ctl.get("actions") - if act is not None: - y_a = act.T[:, :Nv] - W, sig, _, _, _, _ = compute_reduced_ccd( - a_r, y_a, Q_delay=CCD_Q) - en = cumulative_energy(sig) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 0 - key = f"illusion_1.5L_dqctl_action_r{r}" - W_dict[key] = W - all_results[key] = { - "scene": SCENE_TYPE, "diam": diam, "obs": "action", "r": r, - "m80": m80, "N": sig.size, - "sigma_top3": [ - float(sig[i]) for i in range(min(3, len(sig))) - ], - } - print(f" {key}: m80={m80} s1={float(sig[0]):.4f}", flush=True) - - # Target force-CCD reference - if dq_tar is not None: - a_tr = a_tar[:r, :Nv] - frc_t = dq_tar.get("forces") - if frc_t is not None: - tname = _scene_to_target_name(SCENE_TYPE) or f"{SCENE_TYPE}_tar" - y_t = make_force_obs(frc_t[:Nv], tname, mode="fy") - Wt, sig_t, _, _, _, _ = compute_reduced_ccd( - a_tr, y_t, Q_delay=CCD_Q) - kt = f"illusion_1.5L_dqtar_force_fy_r{r}" - W_dict[kt] = Wt - all_results[kt] = { - "scene": SCENE_TYPE, "diam": diam, - "obs": "force_fy_tar", "r": r, - "m80": int(np.searchsorted( - cumulative_energy(sig_t), 0.80) + 1 - ) if len(sig_t) > 0 else 0, - "N": sig_t.size, - "sigma_top3": [ - float(sig_t[i]) for i in range(min(3, len(sig_t))) - ], - } - # Overlap: dq_ctl vs dq_tar - ck = f"illusion_1.5L_dqctl_force_fy_r{r}" - if ck in W_dict: - Wc = W_dict[ck] - n = min(Wc.shape[1], Wt.shape[1], 5) - for k in range(n): - ov = float(abs( - Wc[:, k] / (np.linalg.norm(Wc[:, k]) + 1e-12) @ - Wt[:, k] / (np.linalg.norm(Wt[:, k]) + 1e-12) - )) - all_results[ - f"illusion_1.5L_O_dqctl_vs_dqtar_r{r}_mode{k+1}" - ] = {"overlap": ov, "mode": k + 1, "r": r} - if k == 0: - print( - f" O(dqctl, dqtar) mode1={ov:.4f}", - flush=True - ) - - # ---- 6. Overlap at r=6 (comparison anchor) ---- - # Print explicit comparison with 0.75L (0.564) and 1.0L (0.913) - key_r6 = "illusion_1.5L_O_dqctl_vs_dqtar_r6_mode1" - ov_r6 = all_results.get(key_r6, {}).get("overlap") - if ov_r6 is not None: - verdict = "lower=special" if ov_r6 < 0.7 else "higher=normal" - print( - f"\n 1.5L O(dqctl, dqtar) = {ov_r6:.4f} " - f"(0.75L: 0.564, 1.0L: 0.913 → {verdict})", - flush=True, - ) - - # ---- 7. Signature-CCD ---- - print(f"\n--- Signature-CCD (future sensor error e_s(t+tau)) ---", - flush=True) - if e_s_full is not None: - # tau candidates - tau_geom = 3 # geometric advection delay (snapshot steps) - tau_corr = compute_tau_corr(a_ctl, e_s_full, max_lag=12) - tau_candidates = [("tau_0", 0), ("tau_geom", tau_geom), - ("tau_corr", tau_corr)] - print(f" tau_geom={tau_geom}, tau_corr={tau_corr}", flush=True) - - for tau_label, tau in tau_candidates: - print(f"\n --- tau={tau} ({tau_label}) ---", flush=True) - for r in R_CANDIDATES: - a_r = a_ctl[:r, :] - Nv = a_r.shape[1] - # Shift observable forward by tau - if tau >= 0: - y_sig = e_s_full[:, tau: tau + Nv] - # Also shift POD coefficients to align: use a_r[:, :-tau] - a_r_aligned = a_r[:, :Nv - tau] if tau > 0 else a_r - y_sig_aligned = y_sig[:, :a_r_aligned.shape[1]] - else: - # Negative tau: shift backward - y_sig = e_s_full[:, :Nv + tau] - a_r_aligned = a_r[:, -tau:] - y_sig_aligned = y_sig[:, :a_r_aligned.shape[1]] - - if y_sig_aligned.shape[1] < CCD_Q: - print(f" r={r}: too few samples ({y_sig_aligned.shape[1]}), skipping", - flush=True) - continue - - W, sig, _, _, _, _ = compute_reduced_ccd( - a_r_aligned, y_sig_aligned, Q_delay=CCD_Q) - en = cumulative_energy(sig) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 0 - key = f"illusion_1.5L_dqctl_signature_{tau_label}_r{r}" - W_dict[key] = W - all_results[key] = { - "scene": SCENE_TYPE, "diam": diam, - "obs": f"signature_{tau_label}", "r": r, - "tau": tau, "m80": m80, "N": sig.size, - "sigma_top3": [ - float(sig[i]) for i in range(min(3, len(sig))) - ], - } - print(f" {key}: m80={m80} " - f"s1={float(sig[0]):.4f}", flush=True) - else: - print(" Skipping: sensor error not available.", flush=True) - - # ---- 8. Save ---- - ccd_path = os.path.join(out_dir, "15L_correction_results.json") - with open(ccd_path, "w") as f: - json.dump(all_results, f, indent=2) - print(f"\nSaved {len(all_results)} entries to {ccd_path}", flush=True) - - # ---- 9. Summary ---- - print("\n" + "=" * 60, flush=True) - print("1.5L Correction-field CCD — Summary", flush=True) - print("=" * 60, flush=True) - print(f" Zone KE ratio (body_wake/sensor_zone): {ze['ratio_bw_over_sz']:.4f}", - flush=True) - if ov_r6 is not None: - print(f" O(dqctl, dqtar) r=6 mode1: {ov_r6:.4f}", flush=True) - - for r in R_CANDIDATES: - print(f"\n r={r}:", flush=True) - for obs in ["force_fy", "force_fx", "force_joint", "action"]: - k = f"illusion_1.5L_dqctl_{obs}_r{r}" - if k in all_results: - d = all_results[k] - print(f" {obs:12s}: m80={d['m80']}, " - f"s1={d['sigma_top3'][0]:.4f}", flush=True) - - print(f"\nDone. Results saved.", flush=True) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/correction_analysis/run_signature_line.py b/src/CCD_analysis/correction_analysis/run_signature_line.py deleted file mode 100644 index 4a31a55..0000000 --- a/src/CCD_analysis/correction_analysis/run_signature_line.py +++ /dev/null @@ -1,563 +0,0 @@ -"""Signature-line CCD on dq_ctl: which correction structures determine future sensor mismatch. - -Force/action line CCD on dq_ctl is complete (Phase 1-2). Now we need the -SIGNATURE LINE — answering which correction structures most determine future -sensor error (rather than instantaneous force). - -Key idea: the observable for signature-CCD is the FUTURE sensor error -e(t+tau) = sensors_ctl(t+tau) - sensors_tar(t+tau). - -Usage: - conda run -n pycuda_3_10 python correction_analysis/run_signature_line.py -""" -from __future__ import annotations - -import json -import os -import sys -import time - -import numpy as np - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR, SCENES -from CCD_analysis.utils.resampling import ( - compute_pod, cumulative_energy, e95_index, - compute_reduced_ccd, make_force_obs, -) -from CCD_analysis.correction_analysis.compute_correction_fields import ( - compute_correction, dict_to_field_matrix, -) - -R_LIST = [6, 8, 10] -CCD_Q = 6 -SCENE_TYPES = ["illusion_0.75L", "illusion_1.0L"] -TAU_GEOM = 3 - - -# --------------------------------------------------------------------------- -# Signature observable -# --------------------------------------------------------------------------- - -def make_signature_obs(sensors_ctl: np.ndarray, sensors_tar: np.ndarray, - step_indices: list, tau: int = 0) -> np.ndarray: - """Construct signature observable: future sensor error e(t+tau). - - Parameters - ---------- - sensors_ctl : (N_total_raw, 6) — full raw sensor telemetry from controlled data. - sensors_tar : (N_total_raw, 6) — full raw sensor telemetry from target data. - step_indices : list of int — absolute frame indices (from dq_ctl step_indices). - tau : int — future shift in simulation steps. - - Returns - ------- - e : (6, N_valid) — sensor error at shifted indices (6 channels). - """ - si = np.asarray(step_indices, dtype=int) - max_idx = min(len(sensors_ctl), len(sensors_tar)) - 1 - shifted = np.clip(si + tau, 0, max_idx) - e = sensors_ctl[shifted] - sensors_tar[shifted] - return e.T # (6, N_valid) - - -# --------------------------------------------------------------------------- -# Tau computation -# --------------------------------------------------------------------------- - -def compute_tau_corr(sensors_ctl: np.ndarray, sensors_tar: np.ndarray, - step_indices: list, max_lag: int = 50) -> int: - """Compute optimal tau via cross-correlation of target/illusion sensor[:,3]. - - Cross-correlates the target cylinder sensor[:,3] with the illusion - sensor[:,3] at the snapshot-aligned times. Returns the absolute lag - (in steps) that maximises cross-correlation. - - Parameters - ---------- - sensors_ctl : (N_total_raw, 6) — full raw sensors from controlled data. - sensors_tar : (N_total_raw, 6) — full raw sensors from target data. - step_indices : list of int — absolute frame indices. - max_lag : int — maximum lag to consider (in steps). - - Returns - ------- - tau : int — optimal lag in steps (non-negative). - """ - si = np.asarray(step_indices, dtype=int) - s_ctl = sensors_ctl[si, 3] - s_tar = sensors_tar[si, 3] - - n = len(s_ctl) - ctl = s_ctl - np.mean(s_ctl) - tar = s_tar - np.mean(s_tar) - - xcorr = np.correlate(tar, ctl, mode='same') - mid = n // 2 - lags = np.arange(-mid, mid + 1) - if n % 2 == 0: - lags = lags[:-1] - - valid = np.abs(lags) <= max_lag - if not np.any(valid): - return 0 - - best_idx = np.argmax(xcorr[valid]) - tau = lags[valid][best_idx] - return int(abs(tau)) - - -# --------------------------------------------------------------------------- -# Raw sensor loader (full telemetry, before step-index subsampling) -# --------------------------------------------------------------------------- - -def _load_raw_sensors(scene_name: str) -> np.ndarray: - """Load full raw sensor telemetry (before step-index subsampling). - - Returns - ------- - sensors : (N_total_raw, 6) ndarray — the full sensor time series. - """ - cfg = SCENES.get(scene_name) - if cfg is None: - raise KeyError(f"Unknown scene: {scene_name}") - - scene_id = cfg["scene_id"] - sd = os.path.join(DATA_DIR, scene_id, scene_name) - - tele_path = None - for p in [os.path.join(sd, "controlled.npz"), os.path.join(sd, "sensors.npz")]: - if os.path.isfile(p): - tele_path = p - break - if tele_path is None: - raise FileNotFoundError(f"No telemetry (*.npz) found in {sd}") - - td = np.load(tele_path) - sensors = td["sensors"] - td.close() - return sensors - - -# --------------------------------------------------------------------------- -# Modal overlap -# --------------------------------------------------------------------------- - -def modal_overlap(W_a: np.ndarray, W_b: np.ndarray, n_modes: int = 5) -> list: - """Pairwise modal overlap between two CCD direction matrices. - - Returns list of {mode, O} dicts. - """ - n = min(W_a.shape[1], W_b.shape[1], n_modes) - results = [] - for k in range(n): - u_a = W_a[:, k] / (np.linalg.norm(W_a[:, k]) + 1e-12) - u_b = W_b[:, k] / (np.linalg.norm(W_b[:, k]) + 1e-12) - ov = float(abs(u_a @ u_b)) - results.append({"mode": k + 1, "O": ov}) - return results - - -# --------------------------------------------------------------------------- -# LOCO validation helpers -# --------------------------------------------------------------------------- - -def r2_score(y_true: np.ndarray, y_pred: np.ndarray) -> float: - """Coefficient of determination.""" - ss_r = np.sum((y_true - y_pred) ** 2) - ss_t = np.sum((y_true - np.mean(y_true)) ** 2) - return float(1.0 - ss_r / (ss_t + 1e-12)) - - -def reconstruct_from_ccd(W, sigma, R, a_test, y_train, CCD_Q, m_obs): - """Reconstruct observable from CCD modes. Returns dict with 'mode1' and 'm80'.""" - am = np.mean(a_test, axis=1, keepdims=True) - as_ = np.std(a_test, axis=1, keepdims=True) + 1e-12 - a_test_z = (a_test - am) / as_ - z_test = W.T @ a_test_z - - ym = np.mean(y_train, axis=1, keepdims=True) - ys = np.std(y_train, axis=1, keepdims=True) + 1e-12 - half = CCD_Q // 2 - - en = cumulative_energy(sigma) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 1 - - results = {} - - # Mode-1 - if R.shape[1] >= 1: - pz_1 = R[:, :1] * sigma[:1] @ z_test[:1, :] - yp_1 = pz_1[half * m_obs:(half + 1) * m_obs, :] * ys + ym - results["mode1"] = yp_1 - else: - results["mode1"] = np.zeros_like(y_train[:, :a_test.shape[1]]) - - # M80 - n_rm = min(m80, R.shape[1]) - if n_rm >= 1: - pz_m = R[:, :n_rm] * sigma[:n_rm] @ z_test[:n_rm, :] - yp_m = pz_m[half * m_obs:(half + 1) * m_obs, :] * ys + ym - results["m80"] = yp_m - else: - results["m80"] = np.zeros_like(y_train[:, :a_test.shape[1]]) - - return results - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - print("=" * 60, flush=True) - print("Signature-line CCD on dq_ctl", flush=True) - print("=" * 60, flush=True) - - out_dir = os.path.join(DATA_DIR, "ccd") - os.makedirs(out_dir, exist_ok=True) - all_results = {} - W_dict = {} # CCD direction matrices keyed by label - pod_basis_cache = {} # (scene_type, r) -> (mean_field, modes_r) - raw_sensors_cache = {} # scene_name -> raw sensors - - # ---- 1. Load correction fields ---- - print("\n--- Step 1: Loading correction fields ---", flush=True) - cache = {} - for st in SCENE_TYPES: - t0 = time.time() - try: - corr = compute_correction(st) - cache[st] = corr - dq = corr["dq_ctl"] - if dq is not None: - print(f" {st}: dq_ctl {dq['ux'].shape[0]} frames, " - f"sensors={'✓' if corr['q_ctl'] is not None else '✗'}, " - f"forces={'✓' if dq['forces'] is not None else '✗'}, " - f"{time.time()-t0:.1f}s", flush=True) - except Exception as e: - print(f" {st}: FAILED — {e}", flush=True) - - # ---- 2. Load raw sensor data for all needed scenes ---- - print("\n--- Step 2: Loading raw sensor telemetry ---", flush=True) - for st in SCENE_TYPES: - corr = cache.get(st) - if corr is None: - continue - diam = corr.get("diam") - tar_name = f"target_cylinder_{diam}L" - for name in [st, tar_name]: - if name not in raw_sensors_cache: - try: - raw_sensors_cache[name] = _load_raw_sensors(name) - print(f" {name}: raw sensors {raw_sensors_cache[name].shape}", flush=True) - except Exception as e: - print(f" {name}: FAILED — {e}", flush=True) - - # ---- 3. For each scene, pre-compute POD basis ---- - print("\n--- Step 3: Building target-only POD basis ---", flush=True) - for st in SCENE_TYPES: - corr = cache.get(st) - if corr is None: - continue - dq_tar = corr["dq_tar"] - if dq_tar is None: - print(f" {st}: dq_tar is None, skipping", flush=True) - continue - - Q_tar = dict_to_field_matrix(dq_tar) - mf_tar, modes_tar, sv_tar, _ = compute_pod(Q_tar) - e95 = e95_index(cumulative_energy(sv_tar)) - print(f" {st}: target-only POD E95={e95}", flush=True) - - for r in R_LIST: - pod_basis_cache[(st, r)] = (mf_tar, modes_tar[:, :r]) - - # ---- 4. Compute tau_corr for each scene ---- - print("\n--- Step 4: Computing tau values ---", flush=True) - tau_config = {} # scene_type -> list of tau - for st in SCENE_TYPES: - corr = cache.get(st) - if corr is None: - continue - raw_ctl = raw_sensors_cache.get(st) - diam = corr.get("diam") - tar_name = f"target_cylinder_{diam}L" - raw_tar = raw_sensors_cache.get(tar_name) - step_idx = corr["q_ctl"].get("step_indices", []) - if step_idx is None or len(step_idx) == 0: - step_idx = list(range(corr["q_ctl"]["ux"].shape[0])) - - if raw_ctl is not None and raw_tar is not None: - tau_corr = compute_tau_corr(raw_ctl, raw_tar, step_idx) - else: - tau_corr = TAU_GEOM - - taus = sorted(set([0, TAU_GEOM, tau_corr])) - tau_config[st] = taus - print(f" {st}: tau_corr={tau_corr}, taus={taus}", flush=True) - - # ---- 5. Signature-CCD and Force-CCD for each (scene, r, tau) ---- - print("\n--- Step 5: Running CCD ---", flush=True) - for st in SCENE_TYPES: - corr = cache.get(st) - if corr is None: - continue - dq_ctl = corr["dq_ctl"] - dq_tar = corr["dq_tar"] - if dq_ctl is None or dq_tar is None: - continue - - diam = corr.get("diam") - taus = tau_config.get(st, [0, TAU_GEOM]) - step_idx = corr["q_ctl"].get("step_indices", []) - if step_idx is None or len(step_idx) == 0: - step_idx = list(range(dq_ctl["ux"].shape[0])) - - raw_ctl = raw_sensors_cache.get(st) - diam = corr.get("diam") - tar_name = f"target_cylinder_{diam}L" - raw_tar = raw_sensors_cache.get(tar_name) - - Q_ctl = dict_to_field_matrix(dq_ctl) - N = Q_ctl.shape[1] - print(f"\n --- {st} (diam={diam}) ---", flush=True) - - for r in R_LIST: - mf_r, modes_r = pod_basis_cache[(st, r)] - a_r = modes_r.T @ (Q_ctl - mf_r[:, None]).astype(np.float64) - Nv = a_r.shape[1] - print(f" r={r}: N={Nv}", flush=True) - - # -- Signature-CCD -- - if raw_ctl is not None and raw_tar is not None and step_idx is not None: - for tau in taus: - e_sig = make_signature_obs(raw_ctl, raw_tar, step_idx, tau=tau) - # Trim to match a_r length - Ne = e_sig.shape[1] - a_r_use = a_r[:, :Ne] if Ne < Nv else a_r - e_use = e_sig[:, :Nv] if Nv < Ne else e_sig - N_use = min(Nv, Ne) - - W, sig, Rmat, z, N_orig, N_valid = compute_reduced_ccd( - a_r_use[:, :N_use], e_use[:, :N_use], Q_delay=CCD_Q - ) - en = cumulative_energy(sig) - m80 = int(np.searchsorted(en, 0.80) + 1) if len(en) > 0 else 0 - - key = f"{st}_sig_tau{tau}_r{r}" - W_dict[key] = W - all_results[key] = { - "scene": st, "diam": diam, "obs": f"sig_tau{tau}", "r": r, - "tau": tau, "m80": m80, "N": sig.size, "N_valid": N_valid, - "N_original": N_orig, - "sigma_top3": [float(sig[i]) for i in range(min(3, len(sig)))], - } - print(f" {key}: m80={m80} " - f"s1={float(sig[0]):.4f} N_valid={N_valid}", flush=True) - - # -- Force-CCD reference (SigmaFy, tau=0) -- - frc = dq_ctl.get("forces") - if frc is not None: - y_f = make_force_obs(frc, st, mode="fy")[:, :Nv] - W_f, sig_f, _, _, N_orig_f, N_valid_f = compute_reduced_ccd( - a_r, y_f, Q_delay=CCD_Q - ) - en_f = cumulative_energy(sig_f) - m80_f = int(np.searchsorted(en_f, 0.80) + 1) if len(en_f) > 0 else 0 - key_f = f"{st}_force_fy_r{r}" - W_dict[key_f] = W_f - all_results[key_f] = { - "scene": st, "diam": diam, "obs": "force_fy", "r": r, - "tau": 0, "m80": m80_f, "N": sig_f.size, "N_valid": N_valid_f, - "N_original": N_orig_f, - "sigma_top3": [float(sig_f[i]) for i in range(min(3, len(sig_f)))], - } - print(f" {key_f}: m80={m80_f} " - f"s1={float(sig_f[0]):.4f} N_valid={N_valid_f}", flush=True) - - # ---- 6. Force vs Signature modal overlap comparison (r=6) ---- - print("\n\n--- Step 6: Force vs Signature modal overlap (r=6) ---", flush=True) - for st in SCENE_TYPES: - corr = cache.get(st) - if corr is None: - continue - diam = corr.get("diam") - taus = tau_config.get(st, [0, TAU_GEOM]) - - force_key = f"{st}_force_fy_r{6}" - if force_key not in W_dict: - print(f" {st}: no force key, skipping overlap", flush=True) - continue - - W_force = W_dict[force_key] - print(f"\n {st} (diam={diam}):", flush=True) - for tau in taus: - sig_key = f"{st}_sig_tau{tau}_r{6}" - if sig_key not in W_dict: - continue - W_sig = W_dict[sig_key] - ovs = modal_overlap(W_force, W_sig, n_modes=5) - for ov in ovs: - key = f"{st}_O_force_vs_sig_tau{tau}_r6_mode{ov['mode']}" - all_results[key] = { - "scene": st, "diam": diam, "r": 6, - "tau_sig": tau, "mode": ov["mode"], - "overlap": ov["O"], - } - ov_str = ", ".join([f"mode{ov['mode']}={ov['O']:.4f}" for ov in ovs]) - print(f" O(force, sig_tau{tau}) r=6: {ov_str}", flush=True) - - # ---- 7. LOCO validation (signature observable) ---- - N_PTS = 24 - N_CYCLES = 4 - - print("\n\n--- Step 7: LOCO validation (signature observable, r=6) ---", flush=True) - loco_results = {} - for st in SCENE_TYPES: - corr = cache.get(st) - if corr is None: - continue - dq_ctl = corr["dq_ctl"] - dq_tar = corr["dq_tar"] - if dq_ctl is None or dq_tar is None: - continue - - diam = corr.get("diam") - taus = tau_config.get(st, [0, TAU_GEOM]) - step_idx = corr["q_ctl"].get("step_indices", []) - if step_idx is None or len(step_idx) == 0: - step_idx = list(range(dq_ctl["ux"].shape[0])) - - tar_name = f"target_cylinder_{diam}L" - raw_ctl = raw_sensors_cache.get(st) - raw_tar = raw_sensors_cache.get(tar_name) - if raw_ctl is None or raw_tar is None: - print(f" {st}: raw sensors missing, skipping LOCO", flush=True) - continue - - Q_ctl = dict_to_field_matrix(dq_ctl) - N_total = Q_ctl.shape[1] - - for tau in taus: - # Build the full signature observable - e_full = make_signature_obs(raw_ctl, raw_tar, step_idx, tau=tau) - - r = 6 - fold_r2_m1, fold_r2_m80 = [], [] - - for fold in range(N_CYCLES): - test_cyc = fold - train_cyc = [c for c in range(N_CYCLES) if c != test_cyc] - train_idx = sorted([c * N_PTS + p for c in train_cyc for p in range(N_PTS)]) - test_idx = sorted([c * N_PTS + p for c in [test_cyc] for p in range(N_PTS)]) - - # Trim to valid range - train_idx = [i for i in train_idx if i < N_total] - test_idx = [i for i in test_idx if i < N_total] - if len(train_idx) < N_PTS or len(test_idx) < N_PTS // 2: - continue - - # Build LOCO POD basis from target-only data - Q_tar = dict_to_field_matrix(dq_tar) - Q_ref = Q_tar[:, train_idx] - mf = np.mean(Q_ref, axis=1) - U, _, _ = np.linalg.svd(Q_ref - mf[:, None], full_matrices=False) - modes_r = U[:, :r] - - a_train = modes_r.T @ (Q_ctl[:, train_idx] - mf[:, None]) - a_test = modes_r.T @ (Q_ctl[:, test_idx] - mf[:, None]) - - y_train = e_full[:, train_idx] - y_test = e_full[:, test_idx] - - # Handle length mismatches - na = a_train.shape[1] - ny = y_train.shape[1] - n_min = min(na, ny) - a_train = a_train[:, :n_min] - y_train = y_train[:, :n_min] - - try: - W, sigma, Rmat, _, _, _ = compute_reduced_ccd(a_train, y_train, Q_delay=CCD_Q) - except Exception as exc: - print(f" LOCO fold {fold}: CCD failed — {exc}", flush=True) - continue - - recon = reconstruct_from_ccd(W, sigma, Rmat, a_test, y_train, CCD_Q, m_obs=6) - - na_test = a_test.shape[1] - ny_test = y_test.shape[1] - n_test = min(na_test, ny_test) - ch_m1 = [r2_score(y_test[c, :n_test], recon["mode1"][c, :n_test]) - for c in range(min(y_test.shape[0], recon["mode1"].shape[0]))] - ch_m80 = [r2_score(y_test[c, :n_test], recon["m80"][c, :n_test]) - for c in range(min(y_test.shape[0], recon["m80"].shape[0]))] - fold_r2_m1.append(float(np.mean(ch_m1))) - fold_r2_m80.append(float(np.mean(ch_m80))) - - if fold_r2_m1: - key = f"{st}_LOCO_sig_tau{tau}_r{r}" - loco_results[key] = { - "scene": st, "diam": diam, "tau": tau, "r": r, - "mode1": { - "mean": float(np.mean(fold_r2_m1)), - "std": float(np.std(fold_r2_m1)), - }, - "m80": { - "mean": float(np.mean(fold_r2_m80)), - "std": float(np.std(fold_r2_m80)), - }, - } - print(f" {key}: R2_m1={loco_results[key]['mode1']['mean']:.4f}+-" - f"{loco_results[key]['mode1']['std']:.4f} " - f"R2_m80={loco_results[key]['m80']['mean']:.4f}+-" - f"{loco_results[key]['m80']['std']:.4f}", flush=True) - else: - print(f" {st} tau={tau}: LOCO skipped (no valid folds)", flush=True) - - all_results["_loco"] = loco_results - - # ---- 8. Save ---- - out_path = os.path.join(out_dir, "signature_ccd_results.json") - with open(out_path, "w") as f: - json.dump(all_results, f, indent=2) - print(f"\nSaved {len(all_results)} entries to {out_path}", flush=True) - - # ---- 9. Summary ---- - print("\n" + "=" * 60, flush=True) - print("SUMMARY", flush=True) - print("=" * 60, flush=True) - - for st in SCENE_TYPES: - corr = cache.get(st) - if corr is None: - continue - diam = corr.get("diam") - taus = tau_config.get(st, [0, TAU_GEOM]) - print(f"\n {st} (diam={diam}):", flush=True) - - # Signature R2_m80 from LOCO (r=6) - for tau in taus: - lk = f"{st}_LOCO_sig_tau{tau}_r{6}" - if lk in loco_results: - r2_m80 = loco_results[lk]["m80"]["mean"] - flag = "✓" if r2_m80 >= 0.4 else "✗" - print(f" LOCO sig_tau{tau} R2_m80={r2_m80:.4f} {flag}", flush=True) - - # Overlaps - for tau in taus: - ok = f"{st}_O_force_vs_sig_tau{tau}_r6_mode1" - if ok in all_results: - ov = all_results[ok]["overlap"] - cat = "shared" if ov > 0.8 else ("partial" if ov > 0.5 else "separated") - print(f" O(force, sig_tau{tau}) mode1={ov:.4f} ({cat})", flush=True) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/correction_analysis/run_steady_metrics.py b/src/CCD_analysis/correction_analysis/run_steady_metrics.py deleted file mode 100644 index 88e0ff9..0000000 --- a/src/CCD_analysis/correction_analysis/run_steady_metrics.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 -"""Compute quantitative metrics for steady cloak. - -Measures: - - Mean wake restoration (downstream ux profile) - - Fluctuation (RMS) suppression ratio - - Recirculation zone length (centreline ux < 0) - - dq_ctl + dq_blk cancellation quality - - Force / power bookkeeping (if forces available) - -Usage: - conda run -n pycuda_3_10 python correction_analysis/run_steady_metrics.py -""" -from __future__ import annotations - -import json -import os -import sys -from typing import Any - -import numpy as np - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR, NX, NY, L0, CENTER_Y, U0, SCENES -from CCD_analysis.correction_analysis.compute_correction_fields import ( - compute_correction, -) -from CCD_analysis.correction_analysis.process_legacy_steady import ( - load_legacy_steady, -) - -# --------------------------------------------------------------------------- -# Sensor / geometry constants (steady_cloak layout) -# --------------------------------------------------------------------------- -# pinball_front_x = 30.0 * L0 = 600 -# pinball_rear_x = 31.3 * L0 = 626 -# sensor_x = 40.0 * L0 = 800 -SENSOR_X_PX = int(SCENES["steady_cloak"]["sensor_x"] * L0) # ~800 -FRONT_X_PX = int(SCENES["steady_cloak"]["pinball_front_x"] * L0) # ~600 -REAR_X_PX = int(SCENES["steady_cloak"]["pinball_rear_x"] * L0) # ~626 -CY = int(round(CENTER_Y)) # centreline row index - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _crop_sensor_zone(ux: np.ndarray) -> np.ndarray: - """Crop ux to sensor-zone column range around SENSOR_X_PX. - - Handles both 2D (NY, NX) and 3D (N, NY, NX) arrays by always - cropping the last (x) axis. - """ - half = int(NX * 0.1) # ~10 % of total width on each side - x0 = max(0, SENSOR_X_PX - half) - x1 = min(NX, SENSOR_X_PX + half) - # Ellipsis crops the last axis regardless of dimensionality - return ux[..., x0:x1] - - -def _crop_streamwise_centreline(ux_mean: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """Extract centreline ux profile from mean field, trimmed to downstream region. - - Returns - ------- - x_vals : (NX_trim,) pixel indices - ux_cl : (NX_trim,) centreline ux values - """ - # Downstream region: from body trailing edge to domain end - x0 = REAR_X_PX - 20 # start a bit before body - x_end = min(NX, int(NX * 0.95)) - ux_cl = ux_mean[CY, x0:x_end] - x_vals = np.arange(x0, x_end) - return x_vals, ux_cl - - -# --------------------------------------------------------------------------- -# Metrics -# --------------------------------------------------------------------------- - -def compute_mean_wake_restoration( - dq_blk: dict, dq_ctl: dict, q_in: dict, q_ctl_raw: dict -) -> dict[str, Any]: - """Compare downstream ux profiles for blocked vs controlled flow. - - A perfect steady cloak restores the wake to the uniform-channel profile. - """ - # Mean fields from correction differences - ux_blk_mean = np.mean(dq_blk["ux"], axis=0) # (NY, NX) — blockage perturbation - ux_ctl_mean = np.mean(dq_ctl["ux"], axis=0) # (NY, NX) — control perturbation - - # Actual mean fields - # q_in (target_channel) is the ideal undisturbed profile - ux_in_mean = np.mean(q_in["ux"], axis=0) # (NY, NX) - - # q_ctl raw = steady_cloak mean - ux_sc_mean = np.mean(q_ctl_raw["ux"], axis=0) # (NY, NX) - - # Centreline profiles - x_vals, ux_cl_in = _crop_streamwise_centreline(ux_in_mean) - _, ux_cl_sc = _crop_streamwise_centreline(ux_sc_mean) - _, ux_cl_blk = _crop_streamwise_centreline(ux_blk_mean) - _, ux_cl_ctl = _crop_streamwise_centreline(ux_ctl_mean) - - # Sensor-zone averaged ux (over the full field, not just centreline) - sz_blk = np.mean(_crop_sensor_zone(dq_blk["ux"])) - sz_ctl = np.mean(_crop_sensor_zone(dq_ctl["ux"])) - - # Wake restoration metric: RMS deviation from target channel in sensor zone - _, ux_sz_in = _crop_sensor_zone(ux_in_mean), None # not used for deviation - ux_sz_sc = _crop_sensor_zone(ux_sc_mean) - ux_sz_tc = _crop_sensor_zone(ux_in_mean) - dev_sc = np.std(ux_sz_sc - ux_sz_tc) - - return { - "sensor_zone_mean_ux_blk": float(np.mean(sz_blk)), - "sensor_zone_mean_ux_ctl": float(np.mean(sz_ctl)), - "centreline_ux_blk_mean": float(np.mean(ux_cl_blk)), - "centreline_ux_ctl_mean": float(np.mean(ux_cl_ctl)), - "sensor_zone_deviation_from_channel": float(dev_sc), - } - - -def compute_rms_suppression(dq_blk: dict, dq_ctl: dict) -> dict[str, Any]: - """Compute RMS fluctuation suppression ratio. - - suppression_ratio = 1 - RMS(rms_ctl) / RMS(rms_blk) - where rms is computed per-pixel over the snapshot dimension. - - A value of 1.0 = perfect suppression, 0.0 = no suppression. - """ - rms_blk = np.std(dq_blk["ux"], axis=0) # (NY, NX) - rms_ctl = np.std(dq_ctl["ux"], axis=0) # (NY, NX) - - global_rms_blk = np.sqrt(np.mean(rms_blk**2)) - global_rms_ctl = np.sqrt(np.mean(rms_ctl**2)) - - suppression_ratio = 1.0 - global_rms_ctl / max(global_rms_blk, 1e-15) - - # Sensor-zone specific - sz_blk = _crop_sensor_zone(rms_blk) - sz_ctl = _crop_sensor_zone(rms_ctl) - sz_suppression = 1.0 - np.mean(sz_ctl) / max(np.mean(sz_blk), 1e-15) - - return { - "global_RMS_blk": float(global_rms_blk), - "global_RMS_ctl": float(global_rms_ctl), - "suppression_ratio": float(suppression_ratio), - "sensor_zone_RMS_blk": float(np.mean(sz_blk)), - "sensor_zone_RMS_ctl": float(np.mean(sz_ctl)), - "sensor_zone_suppression": float(sz_suppression), - } - - -def compute_recirculation_zone(q_ctl_raw: dict) -> dict[str, Any]: - """Find recirculation zone length from mean ux of steady_cloak field. - - Recirculation length: streamwise distance from body trailing edge - to the point where centreline ux recovers to >= 0. - """ - ux_mean = np.mean(q_ctl_raw["ux"], axis=0) # (NY, NX) - x_vals, ux_cl = _crop_streamwise_centreline(ux_mean) - - # Find first point (downstream of body) where ux returns to >= 0 - neg = ux_cl < 0 - if not np.any(neg): - recirc_len = 0.0 - x_recovery = None - else: - # Find the last negative index in this trimmed region - neg_indices = np.where(neg)[0] - last_neg = neg_indices[-1] - x_recovery = int(x_vals[last_neg]) - # Distance from rear cylinder in pixel units, convert to L0 - recirc_len = (x_recovery - REAR_X_PX) / L0 - - # Also report min centreline ux - min_ux = float(np.min(ux_cl)) - - return { - "recirculation_length_L0": float(recirc_len) if recirc_len is not None else 0.0, - "recirculation_x_recovery_px": x_recovery, - "centreline_min_ux": min_ux, - } - - -def compute_cancellation_quality(dq_blk: dict, dq_ctl: dict) -> dict[str, Any]: - """Compute residual cancellation quality. - - For perfect steady cloak: dq_ctl ≈ -dq_blk (control cancels blockage). - measured by: cancellation_ratio = RMS(dq_ctl + dq_blk) / RMS(dq_blk) - (lower is better, 0.0 = perfect cancellation) - """ - residual_ux = dq_ctl["ux"] + dq_blk["ux"] # (N, NY, NX) - rms_residual = np.std(residual_ux) - rms_blk = np.std(dq_blk["ux"]) - - cancel_ratio = rms_residual / max(rms_blk, 1e-15) - - # Sensor-zone specific - sz_res = _crop_sensor_zone(residual_ux) - sz_blk_rms = np.std(_crop_sensor_zone(dq_blk["ux"])) - sz_cancel = np.std(sz_res) / max(sz_blk_rms, 1e-15) - - return { - "cancellation_ratio": float(cancel_ratio), - "sensor_zone_cancellation_ratio": float(sz_cancel), - "residual_RMS": float(rms_residual), - "blockage_RMS": float(rms_blk), - } - - -def compute_force_bookkeeping( - dq_blk: dict, dq_ctl: dict -) -> dict[str, Any]: - """Estimate drag from field data. - - Since steady_cloak sensors.npz does not contain force telemetry, - we estimate drag proxy from the momentum deficit in the wake. - - drag_proxy = integral of (U0 - ux) across a wake profile - (qualitative comparison only, not calibrated to actual drag) - """ - # Use mean ux from blockage (pinball - channel) and control (steady_cloak - pinball) - ux_blk_m = np.mean(dq_blk["ux"], axis=0) # blockage perturbation - ux_ctl_m = np.mean(dq_ctl["ux"], axis=0) # control perturbation - - # Mean flow = blockage + channel for pinball; control restores toward channel - # Take a wake profile at sensor_x location - # Drag proxy: momentum deficit across channel height - # Positive deficit means flow slower than free-stream - deficit_blk = float(np.trapz(-ux_blk_m[:, SENSOR_X_PX])) if SENSOR_X_PX < NX else 0.0 - deficit_ctl = float(np.trapz(-ux_ctl_m[:, SENSOR_X_PX])) if SENSOR_X_PX < NX else 0.0 - deficit_ratio = deficit_ctl / max(abs(deficit_blk), 1e-15) - - return { - "drag_proxy_blockage": deficit_blk, - "drag_proxy_control": deficit_ctl, - "drag_proxy_ratio": deficit_ratio, - "note": "drag proxy from ux deficit at sensor plane; no actual force telemetry available", - } - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def run() -> dict[str, Any]: - print("=" * 60) - print("Steady Cloak Quantitative Metrics") - print("=" * 60) - - # -- 1. Load correction fields -- - print("\n--- Loading correction fields ---") - corr = compute_correction("steady_cloak") - dq_blk = corr["dq_blk"] # q_blk - q_in (pinball blockage) - dq_ctl = corr["dq_ctl"] # q_ctl - q_blk (control correction) - q_in = corr["q_in"] # target_channel (undisturbed) - N = corr["N"] - print(f" Aligned frames: {N}") - - # -- 2. Load steady_cloak raw data (for true mean field) -- - print("\n--- Loading steady_cloak raw fields ---") - sc_raw = load_legacy_steady("steady_cloak") - print(f" ux shape: {sc_raw['ux'].shape}") - - # -- 3. Compute metrics -- - print("\n--- Computing wake restoration ---") - wake = compute_mean_wake_restoration(dq_blk, dq_ctl, q_in, sc_raw) - - print("\n--- Computing RMS suppression ---") - rms = compute_rms_suppression(dq_blk, dq_ctl) - - print("\n--- Computing recirculation zone ---") - recirc = compute_recirculation_zone(sc_raw) - - print("\n--- Computing cancellation quality ---") - cancel = compute_cancellation_quality(dq_blk, dq_ctl) - - print("\n--- Computing drag bookkeeping ---") - drag = compute_force_bookkeeping(dq_blk, dq_ctl) - - # -- 4. Assemble -- - metrics = { - "scene": "steady_cloak", - "N_frames": int(N), - "N_raw": int(sc_raw["ux"].shape[0]), - "recirculation_zone": recirc, - "wake_restoration": wake, - "rms_suppression": rms, - "cancellation_quality": cancel, - "force_bookkeeping": drag, - } - - # -- 5. Save -- - os.makedirs(os.path.join(DATA_DIR, "ccd"), exist_ok=True) - out_path = os.path.join(DATA_DIR, "ccd", "steady_metrics.json") - with open(out_path, "w") as f: - json.dump(metrics, f, indent=2) - print(f"\nMetrics saved to {out_path}") - - # -- 6. Print summary -- - _print_summary(metrics) - - return metrics - - -def _print_summary(m: dict): - r = m["recirculation_zone"] - w = m["wake_restoration"] - rms = m["rms_suppression"] - c = m["cancellation_quality"] - d = m["force_bookkeeping"] - - print("\n" + "=" * 60) - print("=== Steady Cloak Metrics ===") - print("=" * 60) - - # Drag bookkeeping - print(f"Drag proxy (deficit area): blockage={d['drag_proxy_blockage']:.4f}, " - f"control={d['drag_proxy_control']:.4f}") - - # Fluctuation suppression - print(f"Fluctuation suppression (global): {rms['suppression_ratio']*100:.1f}%") - print(f"Fluctuation suppression (sensor zone): {rms['sensor_zone_suppression']*100:.1f}%") - - # Recirculation - print(f"Recirculation length: {r['recirculation_length_L0']:.2f} (L0 units)") - print(f"Centreline min ux: {r['centreline_min_ux']:.6f}") - - # Cancellation - print(f"dq_ctl + dq_blk cancellation ratio: {c['cancellation_ratio']:.4f}") - print(f"Sensor-zone cancellation ratio: {c['sensor_zone_cancellation_ratio']:.4f}") - - # Wake restoration - print(f"Sensor-zone deviation from channel (RMS): {w['sensor_zone_deviation_from_channel']:.6f}") - print(f"Note: {d['note']}") - - -if __name__ == "__main__": - run() diff --git a/src/CCD_analysis/correction_analysis/run_zone_ccd.py b/src/CCD_analysis/correction_analysis/run_zone_ccd.py deleted file mode 100644 index 90a4ad9..0000000 --- a/src/CCD_analysis/correction_analysis/run_zone_ccd.py +++ /dev/null @@ -1,365 +0,0 @@ -"""Zone-wise CCD: force-CCD and signature-CCD per spatial zone. - -Processes each of three spatial zones separately for illusion 0.75L and 1.0L -(unified geometry: pinball center ~613px, sensors at 800px): - - near_body: x[580:720] (envelope around pinball) - - body_wake: x[720:850] (body-connected near wake) - - sensor_zone: x[780:850] (around sensor plane at x=40*L0=800) - -For each zone: mask the snapshot matrix to keep only grid points in the zone, -build a target-only POD basis, project correction fields, and compute -force-CCD (SigmaFy) and signature-CCD (tau=0, tau=tau_corr) at r=6, Q_delay=6. - -Usage: - conda run -n pycuda_3_10 python correction_analysis/run_zone_ccd.py -""" -from __future__ import annotations - -import json -import os -import sys -import time - -import numpy as np - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR, NX, NY -from CCD_analysis.utils.resampling import ( - compute_pod, - cumulative_energy, - compute_reduced_ccd, - make_force_obs, -) -from CCD_analysis.correction_analysis.compute_correction_fields import ( - compute_correction, - dict_to_field_matrix, -) - -CCD_Q = 6 -R = 6 - - -# --------------------------------------------------------------------------- -# Zone masks -# --------------------------------------------------------------------------- - -def _define_zones() -> dict: - """Define three-zone masks for unified geometry (pinball center ~613px, sensors at 800px). - - Updated 2026-06-28 to match unified geometry after all scenes were re-collected - with pinball at front x=30*L0=600, rear x=31.3*L0=626 (center ~613px), sensors at 40*L0=800. - Same zones as diagnose_corrections.py:define_zones(). - """ - zones = {} - - # near_body: envelope around pinball (center ~613px) - mask = np.zeros((NY, NX), dtype=bool) - mask[:, 580:720] = True - zones["near_body"] = mask - - # body_wake: near wake downstream of pinball - mask = np.zeros((NY, NX), dtype=bool) - mask[:, 720:850] = True - zones["body_wake"] = mask - - # sensor_zone: around sensors at x=800 (40*L0) - mask = np.zeros((NY, NX), dtype=bool) - mask[:, 780:850] = True - zones["sensor_zone"] = mask - - return zones - - -# --------------------------------------------------------------------------- -# Masking helper -# --------------------------------------------------------------------------- - -def mask_field_matrix(Q_full: np.ndarray, ny: int, nx: int, - mask: np.ndarray) -> tuple[np.ndarray, np.ndarray]: - """Zero out all grid points outside the mask. - - Parameters - ---------- - Q_full : (2*nx*ny, N) ndarray - Snapshot matrix — first half = ux, second half = uy. - mask : (ny, nx) ndarray - Boolean mask, True = keep. - - Returns - ------- - Q_masked : (2*n_sum, N) ndarray - Masked snapshot matrix. - ux_idx : (n_sum,) ndarray - Indices into the original ux ravel for kept points. - """ - mask_flat = mask.ravel() # (ny*nx,) - ux_idx = np.where(mask_flat)[0] - uy_idx = ux_idx + nx * ny - all_idx = np.concatenate([ux_idx, uy_idx]) - return Q_full[all_idx, :], ux_idx - - -# --------------------------------------------------------------------------- -# tau_corr heuristic (from run_15L_correction.py) -# --------------------------------------------------------------------------- - -def compute_tau_corr(a_ctl: np.ndarray, e_s: np.ndarray, - max_lag: int = 12) -> int: - """Find tau that maximises |cross-correlation| between a1 and sensor error.""" - a1 = a_ctl[0, :] - n = len(a1) - a1_z = (a1 - a1.mean()) / (a1.std() + 1e-12) - corr_avg = np.zeros(2 * max_lag + 1) - for ch in range(e_s.shape[0]): - ech = e_s[ch, :n] - ech_z = (ech - ech.mean()) / (ech.std() + 1e-12) - c = np.correlate(a1_z, ech_z, mode="full") - c_mid = len(c) // 2 - seg = c[c_mid - max_lag:c_mid + max_lag + 1] - corr_avg += np.abs(seg) - corr_avg /= e_s.shape[0] - best_lag = np.argmax(corr_avg) - max_lag - return int(best_lag) - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - print("=" * 60, flush=True) - print("Zone-wise CCD: force + signature per spatial zone", flush=True) - print("=" * 60, flush=True) - - out_dir = os.path.join(DATA_DIR, "ccd") - os.makedirs(out_dir, exist_ok=True) - all_results = {} - - scene_types = ["illusion_0.75L", "illusion_1.0L"] - zones = _define_zones() - - for scene_type in scene_types: - print(f"\n{'=' * 60}", flush=True) - print(f"Processing: {scene_type}", flush=True) - print(f"{'=' * 60}", flush=True) - - # ---- 1. Load correction fields ---- - t0 = time.time() - corr = compute_correction(scene_type) - dq_ctl = corr["dq_ctl"] - dq_tar = corr["dq_tar"] - diam = corr.get("diam") - t_elapsed = time.time() - t0 - - if dq_ctl is None: - print(" dq_ctl is None — cannot proceed.", flush=True) - continue - if dq_tar is None: - print(" dq_tar is None — cannot proceed.", flush=True) - continue - - N = dq_ctl["ux"].shape[0] - print(f" N={N}, diam={diam}, load_time={t_elapsed:.1f}s", flush=True) - - # ---- 2. Build full snapshot matrices ---- - Q_ctl = dict_to_field_matrix(dq_ctl) # (2*NX*NY, N) - Q_tar = dict_to_field_matrix(dq_tar) # (2*NX*NY, N_tar) - print(f" Q_ctl: {Q_ctl.shape}, Q_tar: {Q_tar.shape}", flush=True) - - # ---- 3. Extract global observables (unmasked) ---- - # Force observable (SigmaFy) - frc = dq_ctl.get("forces") - if frc is None: - print(" No force data — skipping.", flush=True) - continue - y_force = make_force_obs(frc, scene_type, mode="fy")[:, :N] # (1, N) - - # Sensor error e_s = sensors_ctl - sensors_tar - sensors_ctl = dq_ctl.get("sensors") - sensors_tar = dq_tar.get("sensors") - if sensors_ctl is None or sensors_tar is None: - print(" Sensor data incomplete — skipping.", flush=True) - continue - n_min = min(sensors_ctl.shape[0], sensors_tar.shape[0], N) - e_s = (sensors_ctl[:n_min] - sensors_tar[:n_min]).T # (6, N) - - print(f" y_force: {y_force.shape}, e_s: {e_s.shape}", flush=True) - - # ---- 4. Process each zone ---- - for zname, zmask in zones.items(): - n_masked = int(zmask.sum()) - print(f"\n --- Zone: {zname} (N_grid={n_masked}) ---", flush=True) - - # Mask snapshot matrices - Q_ctl_m, _ = mask_field_matrix(Q_ctl, NY, NX, zmask) - Q_tar_m, _ = mask_field_matrix(Q_tar, NY, NX, zmask) - n_field = Q_ctl_m.shape[0] - print(f" Masked field dim: {n_field}", flush=True) - - # Build target-only POD basis from masked dq_tar - mf_tar, modes_tar, sv_tar, _ = compute_pod(Q_tar_m) - en_tar = cumulative_energy(sv_tar) - e95 = int(np.searchsorted(en_tar, 0.95) + 1) if len(en_tar) > 0 else 0 - print(f" Target POD: E95={e95}, " - f"N_modes={len(sv_tar)}", flush=True) - - # Project masked dq_ctl into masked target basis - proj_mean = mf_tar[:, None] - a_ctl_all = modes_tar.T @ (Q_ctl_m - proj_mean).astype(np.float64) - a_r = a_ctl_all[:R, :] # (R, N) - - # ---- Force-CCD (SigmaFy) ---- - W_f, sig_f, _, _, _, _ = compute_reduced_ccd( - a_r, y_force, Q_delay=CCD_Q) - en_f = cumulative_energy(sig_f) - m80_f = int(np.searchsorted(en_f, 0.80) + 1) if len(en_f) > 0 else 0 - frc_key = f"{scene_type}_{zname}_force_fy_r{R}" - all_results[frc_key] = { - "scene": scene_type, - "zone": zname, - "r": R, - "N_masked_grid": n_masked, - "m80": m80_f, - "N_modes": int(sig_f.size), - "sigma_top3": [float(sig_f[i]) - for i in range(min(3, len(sig_f)))], - } - s1_f = sig_f[0] - s2_f = sig_f[1] if len(sig_f) > 1 else float('nan') - s3_f = sig_f[2] if len(sig_f) > 2 else float('nan') - print(f" force_fy r={R}: m80={m80_f}, " - f"s1={s1_f:.4f}, s2={s2_f:.4f}, s3={s3_f:.4f}", - flush=True) - - # ---- Compute tau_corr for this zone ---- - tau_corr = compute_tau_corr(a_ctl_all, e_s, max_lag=12) - tau_candidates = [("tau0", 0), ("tau_corr", tau_corr)] - print(f" tau_corr = {tau_corr}", flush=True) - - # ---- Signature-CCD ---- - for tau_label, tau in tau_candidates: - # Shift sensor error forward by tau - if tau >= 0: - y_sig = e_s[:, tau: tau + N] - a_r_aligned = a_r[:, :N - tau] if tau > 0 else a_r - else: - y_sig = e_s[:, :N + tau] - a_r_aligned = a_r[:, -tau:] - - y_sig_aligned = y_sig[:, :a_r_aligned.shape[1]] - - if y_sig_aligned.shape[1] < CCD_Q: - print(f" tau={tau}: too few samples " - f"({y_sig_aligned.shape[1]}), skipping", flush=True) - continue - - W_s, sig_s, _, _, _, _ = compute_reduced_ccd( - a_r_aligned, y_sig_aligned, Q_delay=CCD_Q) - - en_s = cumulative_energy(sig_s) - m80_s = (int(np.searchsorted(en_s, 0.80) + 1) - if len(en_s) > 0 else 0) - sig_key = f"{scene_type}_{zname}_sig_{tau_label}_r{R}" - all_results[sig_key] = { - "scene": scene_type, - "zone": zname, - "r": R, - "tau": tau, - "N_masked_grid": n_masked, - "m80": m80_s, - "N_modes": int(sig_s.size), - "sigma_top3": [float(sig_s[i]) - for i in range(min(3, len(sig_s)))], - } - s1_s = sig_s[0] - s2_s = sig_s[1] if len(sig_s) > 1 else float('nan') - s3_s = sig_s[2] if len(sig_s) > 2 else float('nan') - print(f" sig_{tau_label}: m80={m80_s}, " - f"s1={s1_s:.4f}, s2={s2_s:.4f}, s3={s3_s:.4f}", - flush=True) - - # ---- Overlap O(force, sig) ---- - w_f0 = W_f[:, 0] / (np.linalg.norm(W_f[:, 0]) + 1e-12) - w_s0 = W_s[:, 0] / (np.linalg.norm(W_s[:, 0]) + 1e-12) - overlap = float(abs(w_f0 @ w_s0)) - - ov_key = f"{scene_type}_{zname}_O_force_vs_sig_{tau_label}_r{R}" - all_results[ov_key] = { - "scene": scene_type, - "zone": zname, - "r": R, - "tau": tau, - "overlap": overlap, - } - print(f" O(force, sig)_{tau_label}: {overlap:.4f}", - flush=True) - - # ---- 5. Save results ---- - ccd_path = os.path.join(out_dir, "zone_ccd_results.json") - with open(ccd_path, "w") as f: - json.dump(all_results, f, indent=2) - print(f"\nSaved {len(all_results)} entries to {ccd_path}", flush=True) - - # ---- 6. Print summary table ---- - print("\n" + "=" * 80, flush=True) - print("SUMMARY: Zone CCD Results", flush=True) - print("=" * 80, flush=True) - - for scene_type in scene_types: - print(f"\n{'=' * 70}", flush=True) - print(f" {scene_type}", flush=True) - print(f"{'=' * 70}", flush=True) - header = ( - f" {'Zone':<15s} | {'N_masked':>9s} | " - f"{'force_fy':>20s} | {'sig_tau0':>20s} | {'sig_tau_corr':>22s} | " - f"{'O_0':>6s} | {'O_corr':>6s}" - ) - sep = " " + "-" * (15 + 9 + 20 + 20 + 22 + 6 + 6 + 12) - print(header, flush=True) - print(sep, flush=True) - - for zname in zones: - n_pts = all_results.get( - f"{scene_type}_{zname}_force_fy_r{R}", {} - ).get("N_masked_grid", 0) - - fd = all_results.get(f"{scene_type}_{zname}_force_fy_r{R}", {}) - sd0 = all_results.get(f"{scene_type}_{zname}_sig_tau0_r{R}", {}) - sdc = all_results.get(f"{scene_type}_{zname}_sig_tau_corr_r{R}", {}) - od0 = all_results.get( - f"{scene_type}_{zname}_O_force_vs_sig_tau0_r{R}", {}) - odc = all_results.get( - f"{scene_type}_{zname}_O_force_vs_sig_tau_corr_r{R}", {}) - - def fmt_ccd(d): - m = d.get("m80", "-") - s1 = d.get("sigma_top3", ["-"])[0] - if isinstance(s1, float): - return f"m80={m} s1={s1:.4f}" - return f"m80={m} s1={s1}" - - def fmt_ov(d): - v = d.get("overlap", "-") - if isinstance(v, float): - return f"{v:.4f}" - return f"{v}" - - print( - f" {zname:<15s} | {n_pts:>9d} | " - f"{fmt_ccd(fd):>20s} | {fmt_ccd(sd0):>20s} | " - f"{fmt_ccd(sdc):>22s} | {fmt_ov(od0):>6s} | " - f"{fmt_ov(odc):>6s}", - flush=True, - ) - - print(f"\n{'=' * 80}", flush=True) - print("Done. Zone CCD analysis complete.", flush=True) - print(f"{'=' * 80}", flush=True) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/correction_analysis/visualize_action_ccd.py b/src/CCD_analysis/correction_analysis/visualize_action_ccd.py deleted file mode 100644 index 1cf2e13..0000000 --- a/src/CCD_analysis/correction_analysis/visualize_action_ccd.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Action-CCD mode 1 visualization for cloak scenes. - -Action-CCD finds correction-field structures most correlated with cylinder -rotation speeds. For cloak scenes (steady/karman/vortex), this should reveal -the structures the controller directly modulates — clean velocity deficit -compensation and cylinder dipoles, excluding upstream disturbance structures. - -Usage: - conda run -n pycuda_3_10 python correction_analysis/visualize_action_ccd.py -""" -from __future__ import annotations - -import os -import sys - -import numpy as np -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR, NX, NY, L0 -from CCD_analysis.utils.resampling import ( - compute_pod, cumulative_energy, e95_index, compute_reduced_ccd, - unstack_velocity_modes, -) -from CCD_analysis.correction_analysis.compute_correction_fields import ( - compute_correction, dict_to_field_matrix, -) - -FIG_DIR = os.path.join(DATA_DIR, "figures") -os.makedirs(FIG_DIR, exist_ok=True) - -CLOAK_SCENES = ["steady_cloak", "vortex_lamb", "vortex_taylor"] -# karman_re100 excluded due to 72 vs 96 frame mismatch - -R = 10 -CCD_Q = 6 -CROP_X0, CROP_X1 = 300, 1100 - - -def main(): - print("=" * 60) - print("Action-CCD Mode 1: Cloak scenes") - print("=" * 60) - - for st in CLOAK_SCENES: - print(f"\n--- {st} ---", flush=True) - - # Load correction fields - corr = compute_correction(st) - dq_ctl = corr.get("dq_ctl") - if dq_ctl is None or dq_ctl.get("actions") is None: - print(f" SKIP: no dq_ctl or no actions") - continue - - # Build snapshot matrix and compute POD - Q = dict_to_field_matrix(dq_ctl) - N = Q.shape[1] - mf, modes, sv, coeffs = compute_pod(Q) - e95 = e95_index(cumulative_energy(sv)) - print(f" POD: E95={e95}, N_modes={len(sv)}") - - # Action-CCD: find structures correlated with actions - a_r = coeffs[:R, :] - actions = dq_ctl["actions"][:N].T # (3, N) - W, sigma, _, _, _, _ = compute_reduced_ccd(a_r, actions, Q_delay=CCD_Q) - - print(f" Action-CCD: sigma[0]={sigma[0]:.4f}, sigma_top3={sigma[:3]}") - - # Reconstruct CCD mode 1 in physical space - # z1 = W[:, 0] @ A_z → CCD temporal coefficient - # CCD mode = sum over POD modes of (CCD direction weights * POD mode) - w1 = W[:, 0] / (np.linalg.norm(W[:, 0]) + 1e-12) - ccd_mode1 = modes[:, :R] @ w1 # (2*NX*NY,) - - # Unstack into ux, uy - half = NX * NY - ux_mode = ccd_mode1[:half].reshape(NY, NX) - uy_mode = ccd_mode1[half:].reshape(NY, NX) - - # Plot mode 1: ux + uy + vorticity, cropped - vor = np.gradient(uy_mode, axis=1) - np.gradient(ux_mode, axis=0) - - fig, axes = plt.subplots(1, 3, figsize=(14, 4)) - extent = (CROP_X0, CROP_X1, 0, NY - 1) - - # ux - vmax_ux = max(abs(ux_mode).max(), 1e-12) - axes[0].imshow(ux_mode[:, CROP_X0:CROP_X1], cmap="RdBu_r", - vmin=-vmax_ux, vmax=vmax_ux, - origin="lower", aspect="equal", extent=extent) - axes[0].set_title(f"{st}: Action-CCD mode 1 ux") - - # uy - vmax_uy = max(abs(uy_mode).max(), 1e-12) - axes[1].imshow(uy_mode[:, CROP_X0:CROP_X1], cmap="RdBu_r", - vmin=-vmax_uy, vmax=vmax_uy, - origin="lower", aspect="equal", extent=extent) - axes[1].set_title(f"{st}: Action-CCD mode 1 uy") - - # vorticity - vmax_vor = max(np.percentile(abs(vor), 99), 1e-12) - axes[2].imshow(vor[:, CROP_X0:CROP_X1], cmap="RdBu_r", - vmin=-vmax_vor, vmax=vmax_vor, - origin="lower", aspect="equal", extent=extent) - axes[2].set_title(f"{st}: Action-CCD mode 1 vorticity") - - for ax in axes: - ax.tick_params(left=False, right=False, labelleft=False, - bottom=False, top=False, labelbottom=False) - - plt.tight_layout() - path = os.path.join(FIG_DIR, f"action_ccd_mode1_{st}.png") - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - print(f" Saved: {path}") - - print("\nDone!") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/direct_dq/__init__.py b/src/CCD_analysis/direct_dq/__init__.py new file mode 100644 index 0000000..2c7f21b --- /dev/null +++ b/src/CCD_analysis/direct_dq/__init__.py @@ -0,0 +1,13 @@ +"""CPU-only, strict same-time direct correction-field analysis.""" +from .analysis import ( + CLOSURE_ATOL, MIN_ANALYSIS_FLUID_POINTS, analysis_mask, compute_estimands, + parse_station_token, phase_conditioned_outputs, profile_metrics, segmented_trapezoid, select_relative_interval, station_index, + vorticity, weighted_vector_rms, +) +from .io import load_acquisition_artifact, load_matched_inputs, load_result, load_result_metadata_unverified + +__all__ = [ + "CLOSURE_ATOL", "MIN_ANALYSIS_FLUID_POINTS", "analysis_mask", "compute_estimands", + "parse_station_token", "phase_conditioned_outputs", "profile_metrics", "segmented_trapezoid", "select_relative_interval", "station_index", + "vorticity", "weighted_vector_rms", "load_acquisition_artifact", "load_matched_inputs", "load_result", "load_result_metadata_unverified", +] diff --git a/src/CCD_analysis/direct_dq/__main__.py b/src/CCD_analysis/direct_dq/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/src/CCD_analysis/direct_dq/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/src/CCD_analysis/direct_dq/analysis.py b/src/CCD_analysis/direct_dq/analysis.py new file mode 100644 index 0000000..f781f01 --- /dev/null +++ b/src/CCD_analysis/direct_dq/analysis.py @@ -0,0 +1,217 @@ +"""Pure NumPy direct correction-field arithmetic and physical summaries.""" +from __future__ import annotations +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from typing import Any, Mapping, Sequence +import numpy as np + +CLOSURE_ATOL = 1.0e-6 +MIN_ANALYSIS_FLUID_POINTS = 4 + + +def coordinate_weights(coordinate: np.ndarray) -> np.ndarray: + value = np.asarray(coordinate) + if value.ndim != 1 or value.size < 2 or not np.isfinite(value).all() or np.any(np.diff(value) <= 0): + raise ValueError("coordinate must be finite, increasing, and contain at least two points") + weights = np.empty(value.size, dtype=np.float64) + weights[0] = (value[1] - value[0]) / 2 + weights[-1] = (value[-1] - value[-2]) / 2 + if value.size > 2: + weights[1:-1] = (value[2:] - value[:-2]) / 2 + return weights + + +def analysis_mask(masks: Sequence[np.ndarray]) -> np.ndarray: + values = [np.asarray(mask) for mask in masks] + if len(values) != 3 or any(value.dtype != np.bool_ or value.ndim != 2 for value in values) or len({value.shape for value in values}) != 1: + raise ValueError("exactly three matching solver-derived boolean masks are required") + common = values[0] & values[1] & values[2] + occupied_x = np.flatnonzero(common.any(axis=1)) + occupied_y = np.flatnonzero(common.any(axis=0)) + if int(common.sum()) < MIN_ANALYSIS_FLUID_POINTS or occupied_x.size < 2 or occupied_y.size < 2: + raise ValueError(f"solver-fluid intersection is too small; require >= {MIN_ANALYSIS_FLUID_POINTS} points spanning two x and y coordinates") + return common + + +def compute_estimands(q_target: np.ndarray, q_blk: np.ndarray, q_ctl: np.ndarray, *, closure_atol: float = CLOSURE_ATOL) -> dict[str, np.ndarray]: + values = [np.asarray(item) for item in (q_target, q_blk, q_ctl)] + if any(item.dtype != np.float32 or item.ndim != 4 or item.shape[1] != 2 or not np.isfinite(item).all() for item in values) or len({item.shape for item in values}) != 1: + raise ValueError("q fields must be finite matching float32 (time,vector,x,y)") + target, blocked, controlled = values + e_target = controlled - target + dq_ctl = controlled - blocked + dq_tar = target - blocked + residual = e_target - (dq_ctl - dq_tar) + maximum = float(np.max(np.abs(residual))) + if maximum > closure_atol: + raise ValueError(f"pointwise direct-dq closure failed: {maximum} > {closure_atol}") + return {"e_target": e_target, "dq_ctl": dq_ctl, "dq_tar": dq_tar, "closure_residual": residual} + + +def _valid_runs(mask: np.ndarray) -> list[np.ndarray]: + indices = np.flatnonzero(mask) + if not indices.size: + return [] + return [run for run in np.split(indices, np.flatnonzero(np.diff(indices) > 1) + 1) if run.size] + + +def segmented_trapezoid(values: np.ndarray, coordinate: np.ndarray, mask: np.ndarray) -> float: + value, position, valid = np.asarray(values), np.asarray(coordinate), np.asarray(mask) + if value.shape != position.shape or valid.shape != position.shape or valid.dtype != np.bool_ or value.ndim != 1 or not np.isfinite(value).all(): + raise ValueError("segmented quadrature profile contract is invalid") + coordinate_weights(position) + result = 0.0 + integrator = np.trapezoid + for run in _valid_runs(valid): + if run.size >= 2: + result += float(integrator(value[run], position[run])) + return result + + +@dataclass(frozen=True) +class StationRequest: + token: str + value: np.float32 + + +def parse_station_token(token: str) -> StationRequest: + if not isinstance(token, str) or not token or token != token.strip(): + raise ValueError("station token must be a nonempty canonical decimal string") + try: + decimal = Decimal(token) + except InvalidOperation as exc: + raise ValueError(f"invalid station decimal token: {token!r}") from exc + if not decimal.is_finite(): + raise ValueError("station token must be finite") + value = np.float32(str(decimal)) + if not np.isfinite(value): + raise ValueError("station token is outside finite float32 range") + return StationRequest(token, value) + + +def station_index(x_D: np.ndarray, requested: StationRequest | str) -> int: + coordinate = np.asarray(x_D) + request = parse_station_token(requested) if isinstance(requested, str) else requested + if coordinate.dtype != np.float32 or coordinate.ndim != 1 or not isinstance(request, StationRequest): + raise ValueError("station request and x_D coordinate are invalid") + matches = np.flatnonzero(coordinate == request.value) + if matches.size != 1: + raise ValueError(f"requested x/D station token {request.token!r} is absent or ambiguous after canonical float32 conversion; nearest/tolerance substitution is forbidden") + return int(matches[0]) + + +def weighted_vector_rms(vector: np.ndarray, x_D: np.ndarray, y_D: np.ndarray, mask: np.ndarray) -> float: + field, valid = np.asarray(vector), np.asarray(mask) + if field.shape != (2, valid.shape[0], valid.shape[1]) or not np.isfinite(field).all(): + raise ValueError("weighted vector RMS field contract is invalid") + weights = coordinate_weights(x_D)[:, None] * coordinate_weights(y_D)[None, :] + denominator = float(weights[valid].sum()) + return float(np.sqrt(np.sum(weights[valid] * np.sum(field[:, valid] ** 2, axis=0)) / denominator)) + + +def profile_metrics(target_ux: np.ndarray, controlled_ux: np.ndarray, y_D: np.ndarray, mask: np.ndarray) -> dict[str, float | None]: + target, controlled, y, valid = map(np.asarray, (target_ux, controlled_ux, y_D, mask)) + if target.shape != y.shape or controlled.shape != y.shape or valid.shape != y.shape or valid.dtype != np.bool_ or not np.isfinite(target).all() or not np.isfinite(controlled).all(): + raise ValueError("profile contract is invalid") + signed = target - controlled + positive = np.maximum(signed, 0.0) + signed_integral = segmented_trapezoid(signed, y, valid) + momentum = segmented_trapezoid(target * signed, y, valid) + area = segmented_trapezoid(positive, y, valid) + centroid = None + width = None + if area > 0: + centroid = segmented_trapezoid(positive * y, y, valid) / area + variance = segmented_trapezoid(positive * (y - centroid) ** 2, y, valid) / area + width = float(np.sqrt(max(variance, 0.0))) + return {"signed_target_relative_ux_deficit_integral": signed_integral, "momentum_flux_proxy_incomplete": momentum, "positive_deficit_area": area, "positive_deficit_centroid_y_D": centroid, "positive_deficit_width_D": width} + + +def _masked_axis_derivative(field: np.ndarray, coordinate: np.ndarray, mask: np.ndarray, axis: int) -> tuple[np.ndarray, np.ndarray]: + values = np.asarray(field, dtype=np.float64) + positions = np.asarray(coordinate, dtype=np.float64) + valid = np.asarray(mask) + derivative = np.zeros_like(values, dtype=np.float64) + derivative_valid = np.zeros_like(valid) + outer = values.shape[1 - axis] + for index in range(outer): + line = values[:, index] if axis == 0 else values[index, :] + line_mask = valid[:, index] if axis == 0 else valid[index, :] + for run in _valid_runs(line_mask): + if run.size < 2: + continue + edge_order = 2 if run.size >= 3 else 1 + gradient = np.gradient(line[run], positions[run], edge_order=edge_order) + if axis == 0: + derivative[run, index] = gradient + derivative_valid[run, index] = True + else: + derivative[index, run] = gradient + derivative_valid[index, run] = True + return derivative, derivative_valid + + +def vorticity(ux: np.ndarray, uy: np.ndarray, x_D: np.ndarray, y_D: np.ndarray, mask: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + u, v, valid = np.asarray(ux), np.asarray(uy), np.asarray(mask) + if u.shape != valid.shape or v.shape != valid.shape or valid.dtype != np.bool_ or not np.isfinite(u).all() or not np.isfinite(v).all(): + raise ValueError("vorticity field/mask contract is invalid") + coordinate_weights(x_D) + coordinate_weights(y_D) + dv_dx, valid_x = _masked_axis_derivative(v, x_D, valid, 0) + du_dy, valid_y = _masked_axis_derivative(u, y_D, valid, 1) + omega_valid = valid_x & valid_y + omega = np.zeros_like(u, dtype=np.float32) + omega[omega_valid] = (dv_dx - du_dy)[omega_valid].astype(np.float32) + return omega, omega_valid + + +def phase_conditioned_outputs(*args: Any, **kwargs: Any) -> None: + raise NotImplementedError("phase-conditioned direct-dq is fail-closed until independent cross-role physical-phase equality is proven") + + +def select_relative_interval(timeline: np.ndarray, *, start_after_relative_step: int, end_at_relative_step: int | None = None) -> tuple[np.ndarray, np.ndarray]: + """Select exact samples satisfying start < physical step <= optional end.""" + steps = np.asarray(timeline) + if steps.dtype != np.int64 or steps.ndim != 1 or not steps.size or np.any(np.diff(steps) <= 0): + raise ValueError("selection requires a nonempty strictly increasing int64 relative timeline") + if type(start_after_relative_step) is not int or (end_at_relative_step is not None and type(end_at_relative_step) is not int): + raise ValueError("relative-step selection bounds must be explicit integers") + if end_at_relative_step is not None and end_at_relative_step <= start_after_relative_step: + raise ValueError("end-at-relative-step must be greater than start-after-relative-step") + selected = steps > start_after_relative_step + if end_at_relative_step is not None: + selected &= steps <= end_at_relative_step + indices = np.flatnonzero(selected).astype(np.int64) + if not indices.size: + raise ValueError("relative-step selection is empty") + if end_at_relative_step is not None and int(steps[indices[-1]]) != end_at_relative_step: + raise ValueError("end-at-relative-step must be the terminal selected acquisition step") + return indices, steps[indices].copy() + + +def analyze_loaded(artifacts: Mapping[str, Any], *, station_tokens: Sequence[str], window_sizes: Sequence[int], start_after_relative_step: int, end_at_relative_step: int | None = None) -> tuple[dict[str, np.ndarray], dict[str, Any]]: + fields = {role: artifacts[role].fields for role in ("q_target", "q_blk", "q_ctl")} + original_timeline = fields["q_target"]["acquisition_relative_lattice_steps"] + selected_indices, selected_timeline = select_relative_interval(original_timeline, start_after_relative_step=start_after_relative_step, end_at_relative_step=end_at_relative_step) + q = {role: np.stack((value["ux"][selected_indices], value["uy"][selected_indices]), axis=1) for role, value in fields.items()} + estimands = compute_estimands(q["q_target"], q["q_blk"], q["q_ctl"]) + mask = analysis_mask([fields[role]["fluid_mask"] for role in ("q_target", "q_blk", "q_ctl")]) + means = {**{role: value.mean(axis=0, dtype=np.float64).astype(np.float32) for role, value in q.items()}, **{key: value.mean(axis=0, dtype=np.float64).astype(np.float32) for key, value in estimands.items() if key != "closure_residual"}} + x, y = fields["q_target"]["x_D"], fields["q_target"]["y_D"] + station_requests = [parse_station_token(token) for token in station_tokens] + station_indices = np.asarray([station_index(x, request) for request in station_requests], np.int64) + profile_mask = np.stack([mask[index] for index in station_indices]) if station_indices.size else np.empty((0, y.size), bool) + arrays = {"x_D": x, "y_D": y, "q_target_solver_fluid_mask": fields["q_target"]["fluid_mask"], "q_blk_solver_fluid_mask": fields["q_blk"]["fluid_mask"], "q_ctl_solver_fluid_mask": fields["q_ctl"]["fluid_mask"], "analysis_fluid_mask": mask, "original_acquisition_relative_lattice_steps": original_timeline, "selected_acquisition_relative_lattice_steps": selected_timeline, "selected_timeline_indices": selected_indices, "station_indices": station_indices, "station_x_D": x[station_indices], "profile_analysis_mask": profile_mask, "convergence_window_sizes": np.asarray(window_sizes, np.int64), "q_target_instantaneous": q["q_target"], "q_blk_instantaneous": q["q_blk"], "q_ctl_instantaneous": q["q_ctl"], "e_target_instantaneous": estimands["e_target"], "dq_ctl_instantaneous": estimands["dq_ctl"], "dq_tar_instantaneous": estimands["dq_tar"]} + for name, value in means.items(): + arrays[f"{name}_mean"] = value + omega, omega_valid = vorticity(value[0], value[1], x, y, mask) + arrays[f"{name}_mean_vorticity"] = omega + arrays[f"{name}_mean_vorticity_valid_mask"] = omega_valid + arrays[f"{name}_mean_ux_profiles"] = value[0, station_indices, :] if station_indices.size else np.empty((0, y.size), np.float32) + profile_summaries = [] + for request, index in zip(station_requests, station_indices): + profile_summaries.append({"requested_x_D_token": request.token, "canonical_float32_x_D": float(request.value), "exact_x_D": float(x[index]), **profile_metrics(means["q_target"][0, index], means["q_ctl"][0, index], y, mask[index])}) + from .convergence import convergence_report + convergence = convergence_report(q, estimands, x_D=x, y_D=y, mask=mask, window_sizes=window_sizes) + summary = {"schema_id": "ccd-direct-dq-summary/v2", "sample_count": int(q["q_target"].shape[0]), "selection": {"start_after_relative_step": start_after_relative_step, "end_at_relative_step": end_at_relative_step, "selected_count": int(selected_indices.size), "first_selected_relative_step": int(selected_timeline[0]), "last_selected_relative_step": int(selected_timeline[-1])}, "analysis_fluid_mask_definition": "exact intersection of q_target, q_blk, and q_ctl solver-derived fluid masks; no coordinate-generated mask, crop, or translation", "analysis_fluid_point_count": int(mask.sum()), "time_aggregation": "time mean only; acquisition-relative same-time matching is exact; physical-phase equality is not claimed", "closure": {"identity": "e_target = dq_ctl - dq_tar", "absolute_tolerance": CLOSURE_ATOL, "maximum_absolute_residual": float(np.max(np.abs(estimands["closure_residual"])))}, "shared_baseline_caveat": "dq_ctl and dq_tar share the same -q_blk term; agreement is not mechanism or causation evidence", "weighted_vector_rms_target_error": weighted_vector_rms(means["e_target"], x, y, mask), "profiles": profile_summaries, "momentum_flux_proxy_warning": "incomplete proxy integral u_target*(u_target-u_ctl) dy; excludes pressure, viscous, transverse-flux, and control-surface terms", "convergence": convergence, "uncertainty_claim": "none; prefix/suffix windows from one record are convergence diagnostics, not independent realizations", "phase_conditioned_outputs": "unsupported until independent cross-role physical-phase equality is proven"} + return arrays, summary diff --git a/src/CCD_analysis/direct_dq/cli.py b/src/CCD_analysis/direct_dq/cli.py new file mode 100644 index 0000000..ff44422 --- /dev/null +++ b/src/CCD_analysis/direct_dq/cli.py @@ -0,0 +1,35 @@ +"""Command line entry point for strict CPU-only direct-dq analysis.""" +from __future__ import annotations +import argparse +import json +from pathlib import Path +from typing import Sequence +from .analysis import analyze_loaded +from .io import ResultTransaction, load_matched_inputs + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Strict same-time direct-dq analysis (CPU only)") + parser.add_argument("--case", required=True, choices=("karman_re100", "illusion_1.0L")) + parser.add_argument("--q-target", required=True, type=Path) + parser.add_argument("--q-blk", required=True, type=Path) + parser.add_argument("--q-ctl", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--station-x-D", action="append", required=True, type=str, dest="station_tokens") + parser.add_argument("--window-size", action="append", required=True, type=int, dest="windows") + parser.add_argument("--start-after-relative-step", required=True, type=int) + parser.add_argument("--end-at-relative-step", type=int) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + artifacts = load_matched_inputs(q_target=args.q_target, q_blk=args.q_blk, q_ctl=args.q_ctl, case_id=args.case) + arrays, summary = analyze_loaded(artifacts, station_tokens=args.station_tokens, window_sizes=args.windows, start_after_relative_step=args.start_after_relative_step, end_at_relative_step=args.end_at_relative_step) + config = {"schema_id": "ccd-direct-dq-config/v2", "case_id": args.case, "inputs": {role: str(artifacts[role].path.resolve()) for role in artifacts}, "station_x_D_tokens": list(args.station_tokens), "window_sizes": list(args.windows), "selection": {"start_after_relative_step": args.start_after_relative_step, "end_at_relative_step": args.end_at_relative_step}, "alignment": "exact full acquisition-relative lattice timeline matching followed by exact physical-step inequality selection and exact float32 coordinates; no index trim, nearest-time, phase guess, crop, or translation"} + input_hashes = {role: artifacts[role].input_identity for role in artifacts} + with ResultTransaction(args.output) as transaction: + transaction.write(arrays=arrays, summary=summary, config=config, input_hashes=input_hashes) + result = transaction.publish() + print(json.dumps({"result": str(result), "summary": summary}, sort_keys=True)) + return 0 diff --git a/src/CCD_analysis/direct_dq/convergence.py b/src/CCD_analysis/direct_dq/convergence.py new file mode 100644 index 0000000..a016679 --- /dev/null +++ b/src/CCD_analysis/direct_dq/convergence.py @@ -0,0 +1,35 @@ +"""Declared nested-window convergence diagnostics for one acquisition record.""" +from __future__ import annotations +from typing import Mapping, Sequence +import numpy as np +from .analysis import weighted_vector_rms + +WINDOW_SEMANTICS = "declared nested prefix and suffix sample-count windows from one time record" +INDEPENDENT_REALIZATION_UNCERTAINTY = False + + +def validate_window_sizes(window_sizes: Sequence[int], sample_count: int) -> tuple[int, ...]: + values = tuple(window_sizes) + if not values or any(type(value) is not int or value < 1 or value > sample_count for value in values): + raise ValueError("window sizes must be explicit positive integers no larger than sample count") + if tuple(sorted(set(values))) != values or values[-1] != sample_count: + raise ValueError("window sizes must be unique increasing nested counts ending at the full record") + return values + + +def convergence_report(q: Mapping[str, np.ndarray], estimands: Mapping[str, np.ndarray], *, x_D: np.ndarray, y_D: np.ndarray, mask: np.ndarray, window_sizes: Sequence[int]) -> dict: + sample_count = int(q["q_target"].shape[0]) + windows = validate_window_sizes(window_sizes, sample_count) + series = {key: value for key, value in estimands.items() if key in ("e_target", "dq_ctl", "dq_tar")} + full_means = {key: value.mean(axis=0, dtype=np.float64) for key, value in series.items()} + records = [] + for size in windows: + entry = {"sample_count": size, "prefix": {}, "suffix": {}} + for label, selection in (("prefix", slice(0, size)), ("suffix", slice(sample_count - size, sample_count))): + for key, value in series.items(): + mean = value[selection].mean(axis=0, dtype=np.float64) + entry[label][f"{key}_mean_deviation_weighted_vector_rms"] = weighted_vector_rms((mean - full_means[key]).astype(np.float32), x_D, y_D, mask) + mean_error = series["e_target"][selection].mean(axis=0, dtype=np.float64).astype(np.float32) + entry[label]["mean_target_error_weighted_vector_rms"] = weighted_vector_rms(mean_error, x_D, y_D, mask) + records.append(entry) + return {"window_semantics": WINDOW_SEMANTICS, "window_sizes": list(windows), "records": records, "independent_realization_uncertainty": INDEPENDENT_REALIZATION_UNCERTAINTY} diff --git a/src/CCD_analysis/direct_dq/io.py b/src/CCD_analysis/direct_dq/io.py new file mode 100644 index 0000000..8e61d19 --- /dev/null +++ b/src/CCD_analysis/direct_dq/io.py @@ -0,0 +1,164 @@ +"""Strict readers and immutable result transactions for direct-dq.""" +from __future__ import annotations +from dataclasses import dataclass +from hashlib import sha256 +import json +import os +from pathlib import Path +import shutil +import uuid +from typing import Any, Mapping +import numpy as np +from CCD_analysis.acquisition.artifacts import file_sha256, rename_noreplace +from CCD_analysis.acquisition.contracts import ARTIFACT_SCHEMA_ID, ROLES, canonical_json +from CCD_analysis.acquisition.validation import MANIFEST_KEYS, validate_acquisition_semantics +from .schema import RESULT_SCHEMA_ID, canonical_array_sha256, validate_result_science + +RESULT_FILES = {"arrays.npz", "summary.json", "config.json", "input_hashes.json"} + + +def _fsync_file(path: Path) -> None: + with path.open("rb") as stream: os.fsync(stream.fileno()) + + +def _read_canonical_json(path: Path) -> dict[str, Any]: + try: + raw = path.read_bytes(); value = json.loads(raw) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid JSON: {path}") from exc + if not isinstance(value, dict) or raw != canonical_json(value): + raise ValueError(f"canonical JSON object required: {path}") + return value + + + +@dataclass(frozen=True) +class AcquisitionArtifact: + path: Path + role: str + case_id: str + config: dict[str, Any] + manifest: dict[str, Any] + fields: Mapping[str, np.ndarray] + input_identity: dict[str, Any] + + +def load_acquisition_artifact(path: str | Path, *, expected_case: str, expected_role: str) -> AcquisitionArtifact: + root = Path(path) + if expected_role not in ROLES or not root.is_dir(): raise ValueError("invalid expected role or acquisition artifact directory") + expected_files = {"manifest.json", "config.json", "fields.npz", "controller_state.npz"} + if {item.name for item in root.iterdir() if item.is_file()} != expected_files: raise ValueError("artifact directory file inventory is not exact") + manifest = _read_canonical_json(root / "manifest.json") + if set(manifest) != MANIFEST_KEYS or manifest.get("schema_id") != ARTIFACT_SCHEMA_ID or manifest.get("complete") is not True or set(manifest.get("files", {})) != {"config.json", "fields.npz", "controller_state.npz"}: raise ValueError("artifact manifest schema/inventory is not exact") + for name, digest in manifest["files"].items(): + if file_sha256(root / name) != digest: raise ValueError(f"artifact file hash mismatch: {name}") + config = _read_canonical_json(root / "config.json") + if sha256((root / "config.json").read_bytes()).hexdigest() != manifest["config_sha256"]: raise ValueError("artifact config hash mismatch") + with np.load(root / "fields.npz", allow_pickle=False) as archive: + fields = {key: archive[key].copy() for key in archive.files} + with np.load(root / "controller_state.npz", allow_pickle=False) as archive: + state = {key: archive[key].copy() for key in archive.files} + fields, _ = validate_acquisition_semantics(arrays=fields, config=config, state=state, manifest=manifest, expected_case=expected_case, expected_role=expected_role) + role_instantaneous = np.stack((fields["ux"], fields["uy"]), axis=1) + identity = {"path": str(root.resolve()), "manifest_sha256": file_sha256(root / "manifest.json"), "config_sha256": manifest["config_sha256"], "files": dict(sorted(manifest["files"].items())), "role_instantaneous_sha256": canonical_array_sha256(role_instantaneous)} + return AcquisitionArtifact(root, expected_role, expected_case, config, manifest, fields, identity) + + +def load_matched_inputs(*, q_target: str | Path, q_blk: str | Path, q_ctl: str | Path, case_id: str) -> dict[str, AcquisitionArtifact]: + artifacts = {role: load_acquisition_artifact(path, expected_case=case_id, expected_role=role) for role, path in (("q_target", q_target), ("q_blk", q_blk), ("q_ctl", q_ctl))} + reference = artifacts["q_target"].fields + for role in ("q_blk", "q_ctl"): + fields = artifacts[role].fields + if not np.array_equal(fields["x_D"], reference["x_D"]) or not np.array_equal(fields["y_D"], reference["y_D"]): raise ValueError(f"common grid exact-equality failure for {role}") + if fields["ux"].shape != reference["ux"].shape: raise ValueError(f"field shape mismatch for {role}; silent trimming is forbidden") + if not np.array_equal(fields["acquisition_relative_lattice_steps"], reference["acquisition_relative_lattice_steps"]): raise ValueError(f"acquisition-relative timeline exact-equality failure for {role}") + return artifacts + + +class ResultTransaction: + def __init__(self, destination: str | Path): + self.destination = Path(destination) + self.partial = self.destination.with_name(f".{self.destination.name}.partial.{os.getpid()}.{uuid.uuid4().hex}") + self.active = False + + def __enter__(self) -> "ResultTransaction": + if self.destination.exists(): raise FileExistsError(self.destination) + self.destination.parent.mkdir(parents=True, exist_ok=True) + self.partial.mkdir(); self.active = True + return self + + def write(self, *, arrays: Mapping[str, Any], summary: dict[str, Any], config: dict[str, Any], input_hashes: dict[str, Any]) -> None: + if not self.active: raise RuntimeError("transaction inactive") + data = validate_result_science(arrays=arrays, summary=summary, config=config, input_hashes=input_hashes) + np.savez_compressed(self.partial / "arrays.npz", **data) + for name, value in (("summary.json", summary), ("config.json", config), ("input_hashes.json", input_hashes)): + (self.partial / name).write_bytes(canonical_json(value)) + for path in self.partial.iterdir(): _fsync_file(path) + files = {path.name: file_sha256(path) for path in sorted(self.partial.iterdir())} + (self.partial / "manifest.json").write_bytes(canonical_json({"schema_id": RESULT_SCHEMA_ID, "complete": True, "files": files})) + _fsync_file(self.partial / "manifest.json") + + def publish(self) -> Path: + load_result(self.partial) + rename_noreplace(self.partial, self.destination) + directory_fd = os.open(self.destination.parent, os.O_RDONLY) + try: os.fsync(directory_fd) + finally: os.close(directory_fd) + self.active = False + load_result(self.destination) + return self.destination + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None: + if self.active: shutil.rmtree(self.partial, ignore_errors=True); self.active = False + + +def load_result_metadata_unverified(path: str | Path) -> dict[str, Any]: + """Load and internally validate a result without provenance validation. + + This explicitly unverified API does not reread acquisition inputs and must not + be used for publication, CLI success, or scientific provenance claims. + """ + root = Path(path) + manifest = _read_canonical_json(root / "manifest.json") + if set(manifest) != {"schema_id", "complete", "files"} or manifest.get("schema_id") != RESULT_SCHEMA_ID or manifest.get("complete") is not True or set(manifest.get("files", {})) != RESULT_FILES: raise ValueError("result manifest schema/inventory is not exact") + if {item.name for item in root.iterdir() if item.is_file()} != RESULT_FILES | {"manifest.json"}: raise ValueError("result directory file inventory is not exact") + for name, digest in manifest["files"].items(): + if file_sha256(root / name) != digest: raise ValueError(f"result file hash mismatch: {name}") + with np.load(root / "arrays.npz", allow_pickle=False) as archive: arrays = {key: archive[key].copy() for key in archive.files} + summary, config, input_hashes = (_read_canonical_json(root / name) for name in ("summary.json", "config.json", "input_hashes.json")) + arrays = validate_result_science(arrays=arrays, summary=summary, config=config, input_hashes=input_hashes) + return {"arrays": arrays, "summary": summary, "config": config, "input_hashes": input_hashes, "manifest": manifest, "provenance_validation": "UNVERIFIED: acquisition inputs were not reread"} + + +def _validate_live_result_inputs(result: Mapping[str, Any]) -> None: + arrays, config, recorded = result["arrays"], result["config"], result["input_hashes"] + case_id = config["case_id"] + live: dict[str, AcquisitionArtifact] = {} + for role in ROLES: + path = recorded[role]["path"] + if config["inputs"][role] != path: + raise ValueError(f"{role} configured path contradicts recorded input identity") + try: + artifact = load_acquisition_artifact(path, expected_case=case_id, expected_role=role) + except (OSError, ValueError) as exc: + raise ValueError(f"live acquisition input validation failed for {role}: {path}") from exc + if artifact.input_identity != recorded[role]: + raise ValueError(f"live acquisition manifest/config/file identities changed for {role}") + role_field = np.stack((artifact.fields["ux"], artifact.fields["uy"]), axis=1) + selected_indices = arrays["selected_timeline_indices"] + if not np.array_equal(role_field[selected_indices], arrays[f"{role}_instantaneous"]): + raise ValueError(f"persisted {role} instantaneous field differs from live validated acquisition") + if not np.array_equal(artifact.fields["fluid_mask"], arrays[f"{role}_solver_fluid_mask"]): + raise ValueError(f"persisted {role} mask differs from live validated acquisition") + live[role] = artifact + target = live["q_target"].fields + if not np.array_equal(target["x_D"], arrays["x_D"]) or not np.array_equal(target["y_D"], arrays["y_D"]) or not np.array_equal(target["acquisition_relative_lattice_steps"], arrays["original_acquisition_relative_lattice_steps"]): + raise ValueError("persisted result grid/timeline differs from live validated acquisitions") + + +def load_result(path: str | Path) -> dict[str, Any]: + """Load a result with mandatory live acquisition provenance validation.""" + result = load_result_metadata_unverified(path) + _validate_live_result_inputs(result) + result["provenance_validation"] = "VERIFIED: all live acquisition artifacts reread and matched" + return result diff --git a/src/CCD_analysis/direct_dq/schema.py b/src/CCD_analysis/direct_dq/schema.py new file mode 100644 index 0000000..18c7410 --- /dev/null +++ b/src/CCD_analysis/direct_dq/schema.py @@ -0,0 +1,136 @@ +"""Exact cross-file scientific schema for direct-dq results.""" +from __future__ import annotations +from hashlib import sha256 +from typing import Any, Mapping +import numpy as np +from CCD_analysis.acquisition.contracts import CASES, ROLES, canonical_json +from CCD_analysis.acquisition.validation import require_sha256 +from .analysis import CLOSURE_ATOL, parse_station_token, profile_metrics, station_index, vorticity, weighted_vector_rms +from .convergence import convergence_report, validate_window_sizes + +RESULT_SCHEMA_ID = "ccd-direct-dq-result/v2" +SUMMARY_SCHEMA_ID = "ccd-direct-dq-summary/v2" +CONFIG_SCHEMA_ID = "ccd-direct-dq-config/v2" +BASE_ARRAY_KEYS = {"x_D", "y_D", "q_target_solver_fluid_mask", "q_blk_solver_fluid_mask", "q_ctl_solver_fluid_mask", "analysis_fluid_mask", "original_acquisition_relative_lattice_steps", "selected_acquisition_relative_lattice_steps", "selected_timeline_indices", "station_indices", "station_x_D", "profile_analysis_mask", "convergence_window_sizes", "q_target_instantaneous", "q_blk_instantaneous", "q_ctl_instantaneous", "e_target_instantaneous", "dq_ctl_instantaneous", "dq_tar_instantaneous"} +FIELD_NAMES = ("q_target", "q_blk", "q_ctl", "e_target", "dq_ctl", "dq_tar") +DERIVED_ARRAY_KEYS = {f"{name}_{suffix}" for name in FIELD_NAMES for suffix in ("mean", "mean_vorticity", "mean_vorticity_valid_mask", "mean_ux_profiles")} +ARRAY_KEYS = BASE_ARRAY_KEYS | DERIVED_ARRAY_KEYS +SUMMARY_KEYS = {"schema_id", "sample_count", "selection", "analysis_fluid_mask_definition", "analysis_fluid_point_count", "time_aggregation", "closure", "shared_baseline_caveat", "weighted_vector_rms_target_error", "profiles", "momentum_flux_proxy_warning", "convergence", "uncertainty_claim", "phase_conditioned_outputs"} +CONFIG_KEYS = {"schema_id", "case_id", "inputs", "station_x_D_tokens", "window_sizes", "selection", "alignment"} +IDENTITY_KEYS = {"path", "manifest_sha256", "config_sha256", "files", "role_instantaneous_sha256"} +INPUT_FILES = {"config.json", "fields.npz", "controller_state.npz"} + + +def _finite_number(value: Any, label: str) -> float: + if type(value) not in (int, float) or not np.isfinite(value): raise ValueError(f"{label} must be finite number") + return float(value) + + +def canonical_array_sha256(value: np.ndarray) -> str: + """Hash exact dtype, shape, and contiguous bytes for scientific identity.""" + array = np.ascontiguousarray(value) + header = canonical_json({"dtype": array.dtype.str, "shape": list(array.shape)}) + digest = sha256(); digest.update(header); digest.update(array.tobytes()) + return digest.hexdigest() + + +def validate_result_science(*, arrays: Mapping[str, Any], summary: Mapping[str, Any], config: Mapping[str, Any], input_hashes: Mapping[str, Any]) -> dict[str, np.ndarray]: + if not isinstance(summary, Mapping) or set(summary) != SUMMARY_KEYS or summary["schema_id"] != SUMMARY_SCHEMA_ID: + raise ValueError("direct-dq summary schema is not exact") + if not isinstance(config, Mapping) or set(config) != CONFIG_KEYS or config["schema_id"] != CONFIG_SCHEMA_ID or config["case_id"] not in CASES: + raise ValueError("direct-dq config schema is not exact") + if not isinstance(config["inputs"], Mapping) or set(config["inputs"]) != set(ROLES) or any(not isinstance(value, str) or not value for value in config["inputs"].values()): raise ValueError("direct-dq input path schema is not exact") + if not isinstance(config["alignment"], str) or not isinstance(config["station_x_D_tokens"], list) or not all(isinstance(token, str) for token in config["station_x_D_tokens"]): raise ValueError("direct-dq station/alignment config is invalid") + if not isinstance(config["window_sizes"], list): raise ValueError("window sizes must be a list") + selection_config = config["selection"] + if not isinstance(selection_config, Mapping) or set(selection_config) != {"start_after_relative_step", "end_at_relative_step"}: + raise ValueError("selection config schema is not exact") + start = selection_config["start_after_relative_step"]; end = selection_config["end_at_relative_step"] + if type(start) is not int or (end is not None and type(end) is not int) or (end is not None and end <= start): + raise ValueError("selection bounds are invalid") + if not isinstance(input_hashes, Mapping) or set(input_hashes) != set(ROLES): raise ValueError("input hash roles are not exact") + for role, identity in input_hashes.items(): + if not isinstance(identity, Mapping) or set(identity) != IDENTITY_KEYS or identity["path"] != config["inputs"][role] or set(identity["files"]) != INPUT_FILES: raise ValueError(f"input identity structure is invalid for {role}") + require_sha256(identity["manifest_sha256"], f"{role}.manifest_sha256"); require_sha256(identity["config_sha256"], f"{role}.config_sha256"); require_sha256(identity["role_instantaneous_sha256"], f"{role}.role_instantaneous_sha256") + for name, digest in identity["files"].items(): require_sha256(digest, f"{role}.files[{name}]") + + if set(arrays) != ARRAY_KEYS: raise ValueError("direct-dq arrays schema is not exact") + data = {key: np.asarray(value) for key, value in arrays.items()} + x, y = data["x_D"], data["y_D"] + original_timeline = data["original_acquisition_relative_lattice_steps"] + timeline = data["selected_acquisition_relative_lattice_steps"] + selected_indices = data["selected_timeline_indices"] + if x.dtype != np.float32 or y.dtype != np.float32 or x.ndim != 1 or y.ndim != 1 or x.size < 2 or y.size < 2 or not np.isfinite(x).all() or not np.isfinite(y).all() or np.any(np.diff(x) <= 0) or np.any(np.diff(y) <= 0): raise ValueError("result coordinates are invalid") + if original_timeline.dtype != np.int64 or original_timeline.ndim != 1 or original_timeline.size < 1 or np.any(np.diff(original_timeline) <= 0): raise ValueError("original result timeline is invalid") + if timeline.dtype != np.int64 or timeline.ndim != 1 or timeline.size < 1 or np.any(np.diff(timeline) <= 0): raise ValueError("selected result timeline is invalid") + if selected_indices.dtype != np.int64 or selected_indices.ndim != 1 or selected_indices.size != timeline.size or np.any(np.diff(selected_indices) <= 0) or np.any(selected_indices < 0) or np.any(selected_indices >= original_timeline.size): raise ValueError("selected timeline indices are invalid") + expected_indices = np.flatnonzero((original_timeline > start) & ((original_timeline <= end) if end is not None else True)).astype(np.int64) + if not np.array_equal(selected_indices, expected_indices) or not np.array_equal(timeline, original_timeline[selected_indices]): raise ValueError("persisted selection contradicts exact inequality selection") + if end is not None and int(timeline[-1]) != end: raise ValueError("selection terminal step contradicts requested end") + nt, nx, ny, ns = timeline.size, x.size, y.size, len(config["station_x_D_tokens"]) + persisted_windows = data["convergence_window_sizes"] + if persisted_windows.dtype != np.int64 or persisted_windows.ndim != 1 or persisted_windows.tolist() != config["window_sizes"]: raise ValueError("convergence window sizes contradict persisted arrays") + if summary["sample_count"] != nt: raise ValueError("summary sample count contradicts arrays") + expected_selection_summary = {"start_after_relative_step": start, "end_at_relative_step": end, "selected_count": nt, "first_selected_relative_step": int(timeline[0]), "last_selected_relative_step": int(timeline[-1])} + if summary["selection"] != expected_selection_summary: raise ValueError("summary selection contradicts arrays/config") + masks = [] + for role in ROLES: + mask = data[f"{role}_solver_fluid_mask"] + if mask.dtype != np.bool_ or mask.shape != (nx, ny) or not mask.any(): raise ValueError(f"{role} solver mask invalid") + masks.append(mask) + analysis = data["analysis_fluid_mask"] + if analysis.dtype != np.bool_ or analysis.shape != (nx, ny) or not np.array_equal(analysis, masks[0] & masks[1] & masks[2]) or summary["analysis_fluid_point_count"] != int(analysis.sum()): raise ValueError("analysis mask/summary is inconsistent") + indices, station_values, profile_mask = data["station_indices"], data["station_x_D"], data["profile_analysis_mask"] + if indices.dtype != np.int64 or indices.shape != (ns,) or station_values.dtype != np.float32 or station_values.shape != (ns,) or profile_mask.dtype != np.bool_ or profile_mask.shape != (ns, ny): raise ValueError("station arrays are invalid") + expected_station_indices = np.asarray([station_index(x, parse_station_token(token)) for token in config["station_x_D_tokens"]], np.int64) + if not np.array_equal(indices, expected_station_indices) or not np.array_equal(station_values, x[indices]) or not np.array_equal(profile_mask, analysis[indices]): raise ValueError("station arrays contradict exact token/grid mapping") + if not isinstance(summary["profiles"], list) or len(summary["profiles"]) != ns: raise ValueError("summary profiles count invalid") + + instantaneous = {} + for name in FIELD_NAMES: + value = data[f"{name}_instantaneous"] + if value.dtype != np.float32 or value.shape != (nt, 2, nx, ny) or not np.isfinite(value).all(): raise ValueError(f"{name} instantaneous array invalid") + instantaneous[name] = value + expected_estimands = { + "e_target": instantaneous["q_ctl"] - instantaneous["q_target"], + "dq_ctl": instantaneous["q_ctl"] - instantaneous["q_blk"], + "dq_tar": instantaneous["q_target"] - instantaneous["q_blk"], + } + for name, expected in expected_estimands.items(): + if not np.array_equal(instantaneous[name], expected): raise ValueError(f"persisted {name} instantaneous differs from absolute-role recomputation") + residual = expected_estimands["e_target"] - (expected_estimands["dq_ctl"] - expected_estimands["dq_tar"]) + maximum = float(np.max(np.abs(residual))) + closure = summary["closure"] + if not isinstance(closure, Mapping) or set(closure) != {"identity", "absolute_tolerance", "maximum_absolute_residual"} or closure["identity"] != "e_target = dq_ctl - dq_tar" or closure["absolute_tolerance"] != CLOSURE_ATOL or maximum > CLOSURE_ATOL or closure["maximum_absolute_residual"] != maximum: raise ValueError("reloaded pointwise closure validation failed") + for name in FIELD_NAMES: + mean, omega, omega_mask, profiles = (data[f"{name}_{suffix}"] for suffix in ("mean", "mean_vorticity", "mean_vorticity_valid_mask", "mean_ux_profiles")) + if mean.dtype != np.float32 or mean.shape != (2, nx, ny) or not np.isfinite(mean).all() or omega.dtype != np.float32 or omega.shape != (nx, ny) or not np.isfinite(omega).all() or omega_mask.dtype != np.bool_ or omega_mask.shape != (nx, ny) or profiles.dtype != np.float32 or profiles.shape != (ns, ny) or not np.isfinite(profiles).all() or not np.array_equal(profiles, mean[0, indices, :]): raise ValueError(f"{name} mean/vorticity/profile schema invalid") + expected_omega, expected_omega_mask = vorticity(mean[0], mean[1], x, y, analysis) + if not np.array_equal(omega, expected_omega) or not np.array_equal(omega_mask, expected_omega_mask): raise ValueError(f"{name} vorticity contradicts mean field/mask") + for name in FIELD_NAMES: + expected = instantaneous[name].mean(axis=0, dtype=np.float64).astype(np.float32) + if not np.array_equal(data[f"{name}_mean"], expected): raise ValueError(f"{name} mean contradicts persisted instantaneous fields") + expected_rms = weighted_vector_rms(data["e_target_mean"], x, y, analysis) + if summary["weighted_vector_rms_target_error"] != expected_rms: raise ValueError("summary weighted target error contradicts arrays") + for item, token, index in zip(summary["profiles"], config["station_x_D_tokens"], indices): + expected_keys = {"requested_x_D_token", "canonical_float32_x_D", "exact_x_D", "signed_target_relative_ux_deficit_integral", "momentum_flux_proxy_incomplete", "positive_deficit_area", "positive_deficit_centroid_y_D", "positive_deficit_width_D"} + if not isinstance(item, Mapping) or set(item) != expected_keys or item["requested_x_D_token"] != token or item["canonical_float32_x_D"] != float(parse_station_token(token).value) or item["exact_x_D"] != float(x[index]): raise ValueError("summary station token mapping is inconsistent") + metrics = profile_metrics(data["q_target_mean"][0, index], data["q_ctl_mean"][0, index], y, analysis[index]) + if any(item[key] != value for key, value in metrics.items()): raise ValueError("summary profile metrics contradict arrays") + convergence = summary["convergence"] + recomputed_convergence = convergence_report({"q_target": instantaneous["q_target"]}, expected_estimands, x_D=x, y_D=y, mask=analysis, window_sizes=config["window_sizes"]) + if convergence != recomputed_convergence: + raise ValueError("convergence report differs from deterministic recomputation") + fixed_summary = { + "analysis_fluid_mask_definition": "exact intersection of q_target, q_blk, and q_ctl solver-derived fluid masks; no coordinate-generated mask, crop, or translation", + "time_aggregation": "time mean only; acquisition-relative same-time matching is exact; physical-phase equality is not claimed", + "shared_baseline_caveat": "dq_ctl and dq_tar share the same -q_blk term; agreement is not mechanism or causation evidence", + "momentum_flux_proxy_warning": "incomplete proxy integral u_target*(u_target-u_ctl) dy; excludes pressure, viscous, transverse-flux, and control-surface terms", + "uncertainty_claim": "none; prefix/suffix windows from one record are convergence diagnostics, not independent realizations", + "phase_conditioned_outputs": "unsupported until independent cross-role physical-phase equality is proven", + } + if any(summary[key] != value for key, value in fixed_summary.items()): raise ValueError("fixed direct-dq semantic strings differ from authority") + expected_alignment = "exact full acquisition-relative lattice timeline matching followed by exact physical-step inequality selection and exact float32 coordinates; no index trim, nearest-time, phase guess, crop, or translation" + if config["alignment"] != expected_alignment: raise ValueError("fixed alignment semantics differ from authority") + canonical_json(summary); canonical_json(config); canonical_json(input_hashes) + return data diff --git a/src/CCD_analysis/evidence/acquisition_contract_reviews.json b/src/CCD_analysis/evidence/acquisition_contract_reviews.json new file mode 100644 index 0000000..a431f5e --- /dev/null +++ b/src/CCD_analysis/evidence/acquisition_contract_reviews.json @@ -0,0 +1,291 @@ +{ + "schema_id": "ccd-acquisition-independent-gate-remediation/v1", + "reviewed_at_utc": "2026-08-03T18:14:30Z", + "prior_gate": { + "status": "FAILED", + "high_findings": [ + "role CLI built runtime and exited without acquisition", + "Legacy end_control_interval did not persist final EMA into self.action" + ], + "medium_findings": "digest, object identity, controller/telemetry/artifact validation and end-to-end CPU coverage incomplete" + }, + "remediation": [ + { + "name": "solver_lifecycle", + "status": "fixed_cpu_regression", + "evidence": "final EMA/raw step persisted at boundary; next interval starts from prior final EMA" + }, + { + "name": "complete_runner", + "status": "fixed_cpu_end_to_end", + "evidence": "runner initializes, collects exact dual clocks, updates FIFO/clocks, captures mask/grid/state and atomically publishes; CLI invokes runner" + }, + { + "name": "identity_binding", + "status": "fixed_static", + "evidence": "expected SHA256 bound for two models/two configs/action formula; normalization/harmonics content persisted and hashed; policy archive spaces checked" + }, + { + "name": "object_identity", + "status": "fixed_cpu", + "evidence": "exact IDs/order/types/centers/radii/action width verified and persisted" + }, + { + "name": "validation", + "status": "fixed_cpu", + "evidence": "strict telemetry, controller, harmonic, normalization, action, timeline, DDF, clocks, hash and fsync/revalidation contracts" + }, + { + "name": "verification", + "status": "passed_cpu_only", + "evidence": "19 full CCD acquisition tests passed; strengthened preflight PASS; no CFD run" + } + ], + "re_review": { + "status": "ready_focused_static_review", + "runtime_success_claimed": false + }, + "second_re_review": { + "status": "FAILED", + "findings": [ + "normalization trajectory was not isolated by exact post-stabilization restore", + "Illusion PPO incorrectly consumed newly generated +11D target harmonics/normalization", + "case-specific first policy observations were not exact", + "field schedule was not prevalidated" + ] + }, + "second_remediation": [ + { + "name": "checkpoint_sequence", + "status": "fixed_cpu", + "evidence": "stabilize -> full solver checkpoint -> zero norm trajectory -> exact restore -> FIFO warmup; restored EMA/action exactly zero" + }, + { + "name": "illusion_training_reference", + "status": "fixed_exact_hash", + "evidence": "q_ctl binds SR_analysis/data/illusion/illusion_1L norm 9ec5... and two target-force harmonics f135...; +11D target eight-channel harmonics are measured phase evidence only" + }, + { + "name": "policy_initial_state", + "status": "fixed_cpu", + "evidence": "Karman first 12D input bitwise zero; Illusion first input is warmup boundary normalization plus frozen training harmonic phase zero; next-input update tested" + }, + { + "name": "schedule", + "status": "fixed_prebuild", + "evidence": "nonempty terminal-inclusive schedule and horizon divisibility rejected before runtime construction" + }, + { + "name": "verification", + "status": "passed_cpu_only", + "evidence": "24 pinball_math tests and strengthened preflight PASS; no CFD" + } + ], + "third_review": { + "status": "FAILED", + "finding": "solver absolute, acquisition-relative, and policy harmonic phase clocks were conflated" + }, + "third_remediation": [ + { + "name": "public_solver_clocks", + "status": "fixed_cpu", + "evidence": "solver_clock_state exposes absolute lattice/control counters; full checkpoint/restore preserves both" + }, + { + "name": "domain_separation", + "status": "fixed_cpu", + "evidence": "runtime/state/config distinguish solver_absolute_lattice_clock, solver_absolute_control_clock, acquisition_relative_lattice/control, policy_harmonic_phase_index" + }, + { + "name": "artifact_lineage", + "status": "fixed_cpu", + "evidence": "lattice_steps/sample_ids are solver absolute; acquisition_relative_lattice_steps added; control_indices rollout-relative; solver_absolute_control_indices added" + }, + { + "name": "verification", + "status": "passed_cpu_only", + "evidence": "25 tests including nonzero solver origin and per-interval assertions; no CFD" + } + ], + "final_static_review": { + "status": "FAILED", + "finding": "DualClockCollector called boundary-only solver_clock_state during active split snapshots" + }, + "final_static_remediation": [ + { + "name": "active_accessor", + "status": "fixed", + "evidence": "Legacy active_step_clock_state valid only with active split and >=1 completed step; reports absolute lattice and last fully completed control count/active interval index" + }, + { + "name": "collector_lifecycle", + "status": "fixed_cpu", + "evidence": "snapshots call active_step_clock_state; initialization and post-end checks call boundary-only solver_clock_state" + }, + { + "name": "verification", + "status": "passed_cpu_only", + "evidence": "strict fake flow rejects wrong accessor lifecycle; 26 tests and preflight PASS; no CFD" + } + ], + "final_independent_pre_cfd_static_gate": { + "status": "PASS", + "decision": "PASS after four fail-back reviews", + "fail_back_reviews": 4, + "verification": { + "environment": "pinball_math", + "cpu_tests": 26, + "result": "passed" + }, + "unresolved_static_findings": { + "high": 0, + "medium": 0 + }, + "runtime_authorization": { + "authorized": true, + "scope": "minimal sequential fresh-process pycuda smoke only", + "production_authorized": false + }, + "remaining_requirement": "Strict +11D Illusion replay/history compatibility remains required before production.", + "runtime_success_claimed": false + }, + "runtime_smoke_attempt_2026_08_03": { + "status": "FAILED_NO_ARTIFACT", + "scope": "authorized minimal sequential fresh-process pycuda smoke; karman_re100 q_target", + "finding": "GPU execution reached the scheduled field snapshot, then velocity extraction rejected zero density at solver-flagged solid cells because the decoder incorrectly required valid density over the full lattice.", + "artifact_status": "none published; transactional output remained absent", + "scientific_diagnosis": "Legacy FLUID is bit 0 (0b00000001) in driver.py and kernels/macros.h; collision executes on FLUID cells while solid-cell populations are not a valid macroscopic-field domain.", + "remediation": [ + "public Legacy current_step_velocity_field now copies completed solver flags and passes them to a pure mask-aware D2Q9 decoder", + "decoder accepts only an exact boolean (x,y) mask or exact uint8 Legacy flags, validates finite nonzero density only on FLUID cells, and emits exact float32 zero on all nonfluid cells", + "unused solid populations, including zero and nonfinite garbage, are ignored for field extraction; invalid fluid populations/density fail closed", + "artifact validation requires finite full velocity fields, a saved nonempty solver fluid mask, and exact-zero solid velocities" + ], + "verification": { + "environment": "pinball_math", + "focused_acquisition_tests": 30, + "full_active_tests": 109, + "result": "passed", + "lints": "none" + }, + "runtime_success_claimed": false, + "readiness": "READY_TO_RETRY_SAME_MINIMAL_KARMAN_Q_TARGET_SMOKE", + "production_authorized": false + }, + "runtime_smoke_attempt_2026_08_03_coordinate_contract": { + "status": "FAILED_NO_ARTIFACT", + "scope": "authorized minimal sequential fresh-process pycuda smoke; karman_re100 q_target", + "finding": "Second smoke reached artifact semantic validation; np.diff-based uniform float32 spacing rejected canonical x_D=np.arange(1280,dtype=float32)/20 because representable differences vary by ULP.", + "artifact_status": "none published; transactional output remained absent", + "remediation": [ + "runtime config persists exact coordinate schema, axis order, float32 dtype, per-axis count/origin/spacing, and lattice reference length", + "one shared helper analytically generates float32 coordinates and validation requires strict finite float32 1D bitwise array equality; no tolerance, nearest matching, or np.diff uniformity inference", + "regressions accept nx=1280,D=20 despite ULP-varying differences and reject shifted, alternate-cast/noncanonical, skipped, nonmonotonic, wrong-dtype, non-1D, and nonfinite arrays", + "direct-dq retains exact cross-role x_D/y_D array equality" + ], + "verification": { + "environment": "pinball_math", + "focused_acquisition_and_direct_dq_tests": 69, + "full_active_tests": 111, + "result": "passed", + "lints": "none" + }, + "runtime_success_claimed": false, + "readiness": "READY_TO_RETRY_SAME_MINIMAL_KARMAN_Q_TARGET_SMOKE", + "production_authorized": false + }, + "runtime_smoke_attempt_2026_08_04_q_ctl_context": { + "status": "REMEDIATED_READY_TO_RETRY", + "scope": "authorized minimal sequential fresh-process pycuda smoke; karman_re100 q_target, q_blk, q_ctl", + "passed_roles": { + "q_target": "artifact published and manifest complete", + "q_blk": "artifact published and manifest complete" + }, + "failed_role": { + "role": "q_ctl", + "finding": "Stable Baselines PPO loaded on cuda:0; the subsequent Legacy PyCUDA kernel failed with invalid resource handle.", + "artifact_status": "none published; transactional q_ctl output remained absent" + }, + "remediation": [ + "_load_policy accepts only policy_device='cpu' and PPO.load receives device='cpu'", + "build/runtime provenance persists policy_device=cpu separately from cfd_device logical 0", + "policy verification rejects a non-CPU loaded policy device", + "preflight continues to inspect the bound policy archive without importing PPO, Torch, PyCUDA, or initializing CUDA", + "authoritative role, object/action, model, formula, and source identities remain unchanged" + ], + "numerical_note": "Deterministic policy inference may have tiny backend-dependent floating-point differences; the retry smoke validates actual chosen production CPU inference rather than asserting CUDA/CPU bit identity.", + "verification": { + "environment": "pinball_math", + "focused_acquisition_tests": 35, + "full_active_tests": 114, + "result": "passed", + "lints": "none" + }, + "runtime_success_claimed_for_q_ctl": false, + "readiness": "READY_TO_RETRY_MINIMAL_KARMAN_Q_CTL_SMOKE", + "production_authorized": false + }, + "decoder_incident_2026_08_04": { + "severity": "HIGH", + "status": "REMEDIATED_CPU_VERIFIED_PRODUCTION_BLOCKED", + "finding": "Active decoder divided D2Q9 momentum by density, while archived verified Legacy cfd_interface divides both momentum components by u0.", + "impact": "All six smoke-20260804 artifacts are withdrawn and invalid under current schema; apparent 1e3-1e5 speeds were decoder artifacts, not current scientific evidence.", + "withdrawn_artifacts": [ + "src/CCD_analysis/evidence/smoke-20260804/karman_re100/q_target", + "src/CCD_analysis/evidence/smoke-20260804/karman_re100/q_blk", + "src/CCD_analysis/evidence/smoke-20260804/karman_re100/q_ctl", + "src/CCD_analysis/evidence/smoke-20260804/illusion_1.0L/q_target", + "src/CCD_analysis/evidence/smoke-20260804/illusion_1.0L/q_blk", + "src/CCD_analysis/evidence/smoke-20260804/illusion_1.0L/q_ctl" + ], + "immutability": "do not overwrite; retain only as invalid-decoder negative evidence", + "remediation": [ + "exact archived q/U0 formula on solver FLUID cells", + "positive finite u0 and finite fluid-population checks", + "nonfluid exact zero while ignoring nonfluid garbage", + "field and point probe share the same ux/uy q/U0 decoder", + "contract/artifact schema v2 plus decoder schema/formula SHA validation rejects old artifacts", + "no arbitrary scientific magnitude bound in artifact validator" + ], + "production_authorized": false, + "next_path": "src/CCD_analysis/evidence/smoke-20260804-q-over-u0-v1", + "verification": { + "environment": "pinball_math", + "full_active_tests": 114, + "result": "passed", + "withdrawn_artifact_loader_check": "all six rejected", + "lints": "none" + } + }, + "control_boundary_lineage_remediation_2026_08_04": { + "status": "CPU_VERIFIED_READY_FOR_FRESH_PILOT_RETRY", + "finding": "artifact v2 retained only terminal FIFO and field-time telemetry, so sustained q_ctl policy/action lineage was not independently reconstructable when field cadence differed from control cadence", + "schema": "ccd-acquisition-artifact/v3", + "persisted": [ + "initial_fifo_history exact pre-first-action (150,12)", + "boundary_observation_history interval averages (control_count,12)", + "policy_source_observation_history and per-row SHA256", + "policy_input_observation_history (control_count,s_dim)", + "policy_harmonic_phase_indices zero-origin", + "complete requested normalized/physical control histories" + ], + "validator": [ + "terminal FIFO exact rolling append(initial,boundaries)", + "policy sources exact initial-last then preceding boundaries for q_ctl; explicit zero/not-applicable for other roles", + "policy inputs exact frozen normalization/harmonic reconstruction including Karman initial zero", + "field-time requested telemetry indexes complete control histories", + "solver/acquisition/policy clocks exact" + ], + "verification": { + "environment": "pinball_math", + "focused_acquisition_tests": 36, + "full_active_tests": 116, + "result": "passed", + "cfd_executed": false + }, + "immutability": "all schema-v2 pilot artifacts remain immutable and are not current production evidence", + "production_authorized": false, + "readiness": "READY_FOR_FRESH_NO_CLOBBER_PILOT_RETRY", + "next_path": "src/CCD_analysis/evidence/smoke-20260804-q-over-u0-lineage-v3" + } +} diff --git a/src/CCD_analysis/evidence/archive_boundary_check.json b/src/CCD_analysis/evidence/archive_boundary_check.json new file mode 100644 index 0000000..0b66b3d --- /dev/null +++ b/src/CCD_analysis/evidence/archive_boundary_check.json @@ -0,0 +1,45 @@ +{ + "active_archive_import_scan_matches": 0, + "active_imports": { + "src.CCD_analysis": "src/CCD_analysis/__init__.py", + "src.CCD_analysis.acquisition": "src/CCD_analysis/acquisition/__init__.py", + "src.CCD_analysis.direct_dq": "src/CCD_analysis/direct_dq/__init__.py", + "src.CCD_analysis.evidence": "src/CCD_analysis/evidence/__init__.py", + "src.CCD_analysis.original_ccd": "src/CCD_analysis/original_ccd/__init__.py", + "src.CCD_analysis.tests": "src/CCD_analysis/tests/__init__.py" + }, + "active_legacy_surface_scan_matches": 0, + "archive_import_blocked": true, + "archive_import_error": "src.CCD_analysis.archive is historical and non-importable", + "environment": "pinball_math", + "errors": [], + "manifest_sha256": "8e36f79112d8dfe017259c28ec69add89412880d8622b254392c87c632a19bb3", + "payload_counts": { + "directories": 718, + "file_bytes": 92230101305, + "files": 1331, + "symlinks": 16 + }, + "schema_version": 1, + "status": "passed", + "independent_post_archive_gate": { + "reviewer": "independent", + "status": "passed", + "verified": { + "all_files_hash_verified": true, + "directories": 718, + "files": 1331, + "symlinks": 16, + "omissions": 0, + "archive_nesting_leaks": 0, + "active_import_leaks": 0 + }, + "findings": { + "high": [], + "medium": [], + "low_operational_concerns": [ + "The 92.23 GB archived payload remains inside the working tree; never force-add the ignored payload." + ] + } + } +} diff --git a/src/CCD_analysis/evidence/direct_dq_core_reviews.json b/src/CCD_analysis/evidence/direct_dq_core_reviews.json new file mode 100644 index 0000000..a4e0d1c --- /dev/null +++ b/src/CCD_analysis/evidence/direct_dq_core_reviews.json @@ -0,0 +1 @@ +{"blockers":["Production CFD acquisition remains unauthorized.","No real direct-dq result can be produced until three completed role artifacts pass strict validation."],"downstream_authorization":"authorized only for unique next entry: derive-original-ccd pre-todo","final_provenance_remediation":{"evidence":"Result now persists q_target/q_blk/q_ctl instantaneous vector fields. Loader hashes validated acquisition ux/uy-derived role arrays independently into input_hashes, verifies persisted role content against those identities, recomputes all instantaneous estimands, all six means, all six vorticities/masks/profiles, physical summary metrics, and convergence. Fully rehashed nonconstant common role shift with updated role means/vorticity/profiles and result manifest is rejected by acquisition-bound content identity.","status":"FIXED"},"independent_review_history":[{"finding_count":4,"findings":["acquisition reader trusted hash-bound state without shared full semantic validation","result publication used overwrite-capable ordinary rename under race","result files lacked one exact shared cross-file scientific validator","station CLI float64 parsing rejected ordinary decimal tokens such as 31.3"],"severity":"HIGH_MEDIUM","status":"FAIL"},{"remediated_at_utc":"2026-08-03T19:05:00Z","status":"REMEDIATED_PENDING_INDEPENDENT_RE_REVIEW","verification":"40 active pinball_math tests passed; lints clean; no CFD"},{"finding_count":2,"findings":["runtime role/action/source semantics were still self-authenticating rather than compared to dependency-safe frozen authority","result convergence structure was validated but every metric was not deterministically recomputed"],"severity":"HIGH_MEDIUM","status":"SECOND_REVIEW_FAIL"},{"remediated_at_utc":"2026-08-03T19:50:00Z","status":"SECOND_REVIEW_REMEDIATED_PENDING_RE_REVIEW","verification":"58 active pinball_math tests passed; lints clean; no CFD"},{"finding_count":1,"findings":["absolute role instantaneous fields were not persisted/bound, permitting common-role shifts invisible to estimands"],"status":"FINAL_REVIEW_MEDIUM_FAIL"},{"remediated_at_utc":"2026-08-03T20:10:00Z","status":"FINAL_MEDIUM_REMEDIATED_PENDING_FOCUSED_RE_REVIEW","verification":"59 active pinball_math tests passed; lints clean; no CFD"},{"finding_count":1,"findings":["result role hashes remained packaged metadata rather than a live independent acquisition anchor"],"status":"FINAL_PROVENANCE_OPTION2_REQUIRED"},{"remediated_at_utc":"2026-08-03T20:30:00Z","status":"OPTION2_REMEDIATED_PENDING_FINAL_RE_REVIEW","verification":"63 active pinball_math tests passed; lints clean; no CFD"},{"finding_count":0,"reviewed_at_utc":"2026-08-03T20:35:00Z","status":"FINAL_INDEPENDENT_PASS","verification":"63 active pinball_math tests passed; lints clean; no CFD; no real direct-dq result claimed; live-input provenance is mandatory and limits portability"}],"live_input_provenance_remediation":{"evidence":"load_result now mandates rereading all recorded acquisition paths, full artifact hash/semantic validation, exact recorded manifest/config/file/content identities, and exact role fields/masks/grid/timeline equality. ResultTransaction publish invokes this before rename and after publication. Explicit load_result_metadata_unverified is labelled UNVERIFIED and is not used for publication/CLI success. Regressions cover fully rehashed common shift plus updated input_hashes/result manifest, missing input, moved input, changed input, and disappearance before publish.","status":"FIXED"},"portability_limitation":"Provenance-validated results require the immutable acquisition directories at their recorded absolute paths; standalone/moved result loading is unverified only.","real_dataset_result":null,"real_dataset_status":"not produced: no real completed three-role acquisition artifacts are available/authorized; synthetic active-writer artifacts only","remediations":[{"evidence":"acquisition.validation.validate_acquisition_semantics is authoritative and called by ArtifactTransaction.write, ArtifactTransaction.publish, and direct_dq loader; exact config/runtime/clocks/fields/state/action semantics and identities are checked.","finding":1,"status":"FIXED"},{"evidence":"Result publication calls Linux renameat2(RENAME_NOREPLACE) through ctypes and fails closed if unavailable; multiprocess test proves exactly one winner and preserves existing sentinels.","finding":2,"status":"FIXED"},{"evidence":"direct_dq.schema.validate_result_science defines exact arrays/summary/config/input schema and validates before write and on load, including canonical JSON, mask/station/profile/mean/vorticity/convergence consistency and reloaded pointwise closure.","finding":3,"status":"FIXED"},{"evidence":"station decimal tokens are preserved, parsed with Decimal, canonically converted to float32, and compared by exact float32 equality; 31.3 present/absent tests pass without nearest/tolerance logic.","finding":4,"status":"FIXED"}],"reviewed_at_utc":"2026-08-04T01:15:00Z","reviews":[{"evidence":"Active direct_dq is correction-field arithmetic only; forbidden legacy concepts/imports absent from active Python; no original-CCD implementation or comparison claim.","id":1,"name":"scope_review","status":"PASS"},{"evidence":"Outputs distinguish exact acquisition-relative same-time matching from unproven physical phase; phase API fails closed; shared -q_blk and incomplete momentum proxy caveats are explicit; no causation claim.","id":2,"name":"scientific_review","status":"PASS"},{"evidence":"No archive imports, silent trim, nearest-time/station substitution, phase guessing, coordinate-generated mask, crop, or geometry translation. Grid/timeline/station matches are exact or rejected.","id":3,"name":"code_review","status":"PASS"},{"evidence":"pinball_math full active suite: 63 passed; IDE lints clean; CFD not run; no real direct-dq result produced or claimed.","id":4,"name":"verification","status":"PASS"},{"evidence":"Each acquisition manifest/file/state-array/config hash is revalidated. Result publication is immutable, no-clobber, partial-cleaning, atomic, hash-bound, and reload-validated with arrays/summary/config/input identities.","id":5,"name":"artifact_review","status":"PASS"},{"evidence":"Durable decisions are limited to strict acquisition-relative matching, solver-mask intersection, time-mean-only phase fail-closed behavior, and single-record convergence semantics.","id":6,"name":"memory_checkpoint","status":"PASS"}],"schema_id":"ccd-direct-dq-post-todo-reviews/v1","second_review_remediations":[{"evidence":"contracts.py is now CPU-safe authority for RoleSpec/object/observation/controller/harmonic/model path, ACTION_FORMULA, config/model paths+SHA, and Illusion training reference paths+SHA. runtime and validator import the same authority. Validator requires exact authoritative values rather than rehashed self-inventories. Forged-rehashed writer, publish, and load regressions cover formula, observation slices, harmonic counts, model path/hash, config path/hash, and all four training-reference path/hash fields.","finding":1,"status":"FIXED"},{"evidence":"result schema calls shared convergence_report from persisted instantaneous estimands, grid weights, mask, and persisted/config windows, then compares the complete report exactly. Fixed semantics/flags are authoritative. Hash-consistent tamper regressions cover prefix metric, suffix metric, window semantics, window sizes, independent-realization flag, and per-record sample count.","finding":2,"status":"FIXED"}],"status":"FINAL_INDEPENDENT_PASS","strict_relative_step_selection":{"contract":"full timelines match exactly before exclusive-start/inclusive-end physical-step inequality selection; optional end is terminal; interval nonempty; no index trim or nearest boundary","production_preflight":{"canonical_float32_station_multiplicity":{"35":1,"40":1,"45":1},"illusion_1.0L":{"burn_after":90000,"selected_count":144,"stations":[35,40,45],"windows":[36,72,108,144]},"karman_re100":{"burn_after":120000,"selected_count":120,"stations":[35,40,45],"windows":[30,60,90,120]}},"provenance":"original live artifacts remain fully validated; selected persisted role arrays are compared exactly to selected live columns on verified reload","schema":"ccd-direct-dq result/config/summary v2","status":"IMPLEMENTED_FULL_SUITE_PASS","verification":"41 focused direct-dq tests and 125 full active tests passed; lints clean; no CFD/result execution"},"todo":"direct-dq-core","todo_status":"completed","unique_next_entry":"derive-original-ccd pre-todo"} diff --git a/src/CCD_analysis/evidence/final_results_review.json b/src/CCD_analysis/evidence/final_results_review.json new file mode 100644 index 0000000..e6f0fca --- /dev/null +++ b/src/CCD_analysis/evidence/final_results_review.json @@ -0,0 +1 @@ +{"authoritative_evidence":{"illusion_acquisition":"src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-illusion-authorized-450-fi1250","illusion_compatibility_certificate":"src/CCD_analysis/evidence/illusion-compatibility-certificate-v1.json","illusion_result":"src/CCD_analysis/evidence/direct-dq-illusion-authorized-burn90000","karman_acquisition":"src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-karman-450-fi2000","karman_result":"src/CCD_analysis/evidence/direct-dq-karman-burn120000"},"claim_boundary":["same-time direct differences only","no physical-phase equality","no independent-realization uncertainty","no causal or mechanism claim","no CCD-versus-POD superiority claim","no real-case CCD analysis"],"high_findings":[],"low_findings":["working tree contains unrelated changes; authoritative artifacts remain hash-bound"],"medium_findings":[],"schema_id":"ccd-final-results-review/v1","status":"PASS","verification":{"diff_check":"passed","illusion_live_provenance_reload":"passed","karman_live_provenance_reload":"passed","lints":"no diagnostics","pinball_math_tests":"128 passed"}} diff --git a/src/CCD_analysis/evidence/illusion-compatibility-certificate-v1.json b/src/CCD_analysis/evidence/illusion-compatibility-certificate-v1.json new file mode 100644 index 0000000..309a0c1 --- /dev/null +++ b/src/CCD_analysis/evidence/illusion-compatibility-certificate-v1.json @@ -0,0 +1 @@ +{"checks":{"boundary_history_shape_exact":true,"control_count_exact":true,"field_bound":true,"field_interval_exact":true,"initial_fifo_shape_exact":true,"normalized_action_bound":true,"physical_action_bound":true,"policy_input_bound":true,"policy_input_shape_exact":true,"policy_source_hash_count_exact":true,"raw_observation_bound":true},"claim_boundary":"runtime compatibility only; not accuracy, stability, physical-phase, or mechanism validation","contract":{"acceptance":"all active acquisition semantics pass; exact 150-control FIFO/policy lineage; all listed finite declared bounds pass","case_id":"illusion_1.0L","field_q_over_u0_absolute_limit":5.0,"normalized_action_absolute_limit":1.0,"physical_action_absolute_limit":0.1,"policy_input_absolute_limit":1.0,"raw_observation_absolute_limit":5.0,"required_control_count":150,"required_field_interval":600,"required_policy_input_width":14,"role":"q_ctl","schema_id":"ccd-illusion-policy-compatibility/v1"},"contract_sha256":"e3ee5ec16a4da986c4da56a9cb769547be316ad49fdba752be995cd0d495d633","metrics":{"field_q_over_u0_max_abs":2.4387996196746826,"normalized_action_max_abs":0.2324981540441513,"physical_action_max_abs":0.013582179322838783,"policy_input_max_abs":0.4606127142906189,"raw_observation_max_abs":1.4082750082015991},"pilot_manifest_sha256":"fdd3e9826dea167f78727bd00ed2da968cf407c2f6302c33969fc2bf815457a2","pilot_path":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/smoke-20260804-q-over-u0-lineage-v3/illusion_1.0L/q_ctl-pilot150","production_authorized":true,"schema_id":"ccd-illusion-policy-compatibility/v1","status":"PASS"} diff --git a/src/CCD_analysis/evidence/karman_dynamic_final_review.json b/src/CCD_analysis/evidence/karman_dynamic_final_review.json new file mode 100644 index 0000000..e3a95f0 --- /dev/null +++ b/src/CCD_analysis/evidence/karman_dynamic_final_review.json @@ -0,0 +1 @@ +{"authoritative_publication":"src/CCD_analysis/data/karman-dynamic/karman-dynamic-v1-production/publication-v2","claim_review":"PASS","code_review":"PASS","dense_deletion":"RETAIN_ALL: compact products do not suffice for every source-level downstream recomputation; zero has no compact fields","initial_finding":{"finding":"publication-v1 labeled the deliverable four-role but omitted the target reference bar","resolution":"published immutable publication-v2 with target, zero, constant_mean, and DRL bars","severity":"medium"},"remediation_cycles":1,"rounds":2,"schema_id":"ccd-karman-dynamic-final-review/v1","science_review":"PASS","scope":"all karman_dynamic code, tests, immutable result artifacts, final publication, claims, provenance, deletion decision","unresolved_high":[],"unresolved_medium":[],"verified":["fresh live parent reloads","manifest and canonical array hashes","four-role mean recomputation and decomposition closure","temporal and phase-domain essential recomputation","old real_ccd/direct_dq loader isolation","zero phase output prohibition","focused and full tests","changed-file lints","git diff whitespace"]} diff --git a/src/CCD_analysis/evidence/original_ccd_derivation_reviews.json b/src/CCD_analysis/evidence/original_ccd_derivation_reviews.json new file mode 100644 index 0000000..8fec7db --- /dev/null +++ b/src/CCD_analysis/evidence/original_ccd_derivation_reviews.json @@ -0,0 +1,98 @@ +{ + "schema_id": "ccd-original-ccd-derivation-post-todo-reviews/v1", + "todo": "derive-original-ccd", + "todo_status": "completed", + "reviews": [ + { + "id": 1, + "name": "scope_review", + "status": "PASS", + "evidence": "Only active contract docs, private _reference.py, tests, evidence, README, and checkpoint changed; archive remained read-only; no CFD, production API, CLI, or artifact implementation." + }, + { + "id": 2, + "name": "equation_review", + "status": "PASS", + "evidence": "Contract states U in C^(M x N), P in C^(LQ x N), A=P U^dagger/(N sqrt(LQ)), A=R Sigma V^dagger, and reproduces Lyu equations (3.1)-(3.2), Q=128, Delta tau=2pi/128. A literal materialized P U^dagger test at noninteger-cycle length equals the factorized accumulator across five chunk sizes to tight tolerance." + }, + { + "id": 3, + "name": "weighted_variational_review", + "status": "PASS", + "evidence": "Computation dtype includes U, P, and W before safe casts; exact W shape, finite, Hermitian, and PD checks precede square roots. Real U/P with genuinely complex non-diagonal HPD W is checked against direct A and Phi^H W Phi." + }, + { + "id": 4, + "name": "timing_preprocessing_review", + "status": "PASS", + "evidence": "Pair uniqueness is by (block,timestamp); repeated local clocks and interleaved blocks are valid; per-block local clocks must be strictly ordered; integer offsets use independent per-block sequences; duplicate delays are allowed in declared order; no crossing/wrap/nearest." + }, + { + "id": 5, + "name": "pod_equivalence_review", + "status": "PASS", + "evidence": "Complex full-rank weighted equivalence verifies singular values, phase-aligned simple modes, degenerate projectors, coefficients, and reconstruction. Truncation tests cover simple containment, partial degenerate intersection with equal value but incomplete projector, and strict loss without intersection. Standardization correction states unequal-scale non-equivalence and exact-degenerate isotropic invariance." + }, + { + "id": 6, + "name": "verification_and_gate_review", + "status": "PASS", + "evidence": "Final independent math review passed after remediation. The focused suite has 20 passing tests and the full active suite has 83 passing tests; the published noisy case checks the traveling-pair projector and sign/phase-invariant individual mode ordering, and literal materialized P U^dagger equals the factorized accumulator across five chunk sizes." + } + ], + "production_api_available": false, + "cfd_run": false, + "independent_math_gate": { + "required": true, + "status": "FINAL_INDEPENDENT_PASS", + "downstream_blocked": false, + "next_todo_after_pass": "implement-original-ccd" + }, + "verification": { + "environment": "pinball_math", + "focused": "20 passed", + "full_active_suite": "83 passed", + "lints": "clean", + "cfd": "not run", + "commit": "not created", + "scoped_diff_check": "clean" + }, + "independent_math_gate_history": [ + { + "status": "FAIL", + "severity": "HIGH_MEDIUM_WITH_LISTED_LOW_GAPS", + "finding_count": 7, + "findings": [ + "W dtype excluded before casting and complex W could be lost", + "lag uniqueness/order semantics incorrect for repeated local clocks and interleaving", + "false exact-degenerate standardization rotation-sensitivity claim", + "truncated equality conditions did not distinguish value/vector/projector", + "full-rank complex weighted equivalence coverage incomplete", + "published N=10000 noise-100 example not executed", + "Q1/odd/duplicate delays, complex multiobservable, left relation, centering and scalar gaps" + ] + }, + { + "status": "REMEDIATED_PENDING_INDEPENDENT_RE_REVIEW", + "evidence": "Reference, contract, and 18 focused tests address every listed high/medium/low finding; production remains blocked." + }, + { + "status": "FOLLOWUP_MEDIUM_LOW_REMEDIATED_PENDING_RE_REVIEW", + "findings": [ + "published noisy test lacked individual sign/phase-invariant mode-order checks", + "factorized/chunked accumulator lacked literal materialized P U^dagger equivalence on noninteger cycles", + "optional low fail-closed and chunk-invariance gaps" + ], + "evidence": "Published frozen-seed overlaps are now predeclared with observed values and bounded tolerances; literal P/U equality holds across chunks 1,17,128,257,4096 at noninteger-cycle n=421 with identical RNG; duplicate field pairs, invalid delay kind/noninteger index, nonfinite U/P/W, and exact-only interpolation are tested." + }, + { + "status": "FINAL_INDEPENDENT_PASS", + "evidence": "Independent math re-review passed after all recorded FAIL and remediation rounds; 20 focused tests and 83 full active tests pass. Production implementation is authorized, while CFD execution and empirical/CFD claims remain unauthorized." + } + ], + "production_implementation_authorized": true, + "cfd_claims_authorized": false, + "empirical_claims_authorized": false, + "unique_next_entry": "implement-original-ccd pre-todo", + "production_cfd_authorized": false +} diff --git a/src/CCD_analysis/evidence/original_ccd_implementation_reviews.json b/src/CCD_analysis/evidence/original_ccd_implementation_reviews.json new file mode 100644 index 0000000..006f6c6 --- /dev/null +++ b/src/CCD_analysis/evidence/original_ccd_implementation_reviews.json @@ -0,0 +1,92 @@ +{ + "schema_id": "ccd-original-ccd-implementation-post-todo-reviews/v1", + "todo": "implement-original-ccd", + "todo_status": "COMPLETED_FINAL_INDEPENDENT_PASS_AFTER_REMEDIATION", + "reviews": [ + { + "id": 1, + "name": "scope_review", + "status": "PASS", + "evidence": "Only active production CCD code/tests/docs/evidence/checkpoint changed; no plan/archive edit, CFD, real-case CCD, artifact writer, POD comparison, empirical claim, or commit." + }, + { + "id": 2, + "name": "scientific_review", + "status": "PASS_AFTER_REMEDIATION", + "evidence": "Initial independent review FAIL findings were remediated; independent re-review passed with no remaining severity-rated findings or new contradictions." + }, + { + "id": 3, + "name": "code_review", + "status": "PASS_AFTER_REMEDIATION", + "evidence": "Initial independent review FAIL findings were remediated; independent re-review passed with no new high- or medium-severity regression." + }, + { + "id": 4, + "name": "verification", + "status": "PASS", + "evidence": "Final pinball_math suite passed 105 active tests, including 42 focused production+derivation tests; edited-file IDE lints and scoped diff checks are clean; CFD and real-case CCD were not run." + }, + { + "id": 5, + "name": "artifact_review", + "status": "NOT_APPLICABLE_PASS", + "evidence": "This todo implements an in-memory algorithm/reconstruction API only; the plan does not request an immutable result writer or persisted result schema." + }, + { + "id": 6, + "name": "memory_checkpoint", + "status": "PASS", + "evidence": "One durable original-CCD implementation completion memory was saved after both independent reviews reached final PASS." + } + ], + "production_api_available": true, + "independent_code_review": { + "status": "FINAL_PASS", + "prior_findings": [ + "Public LaggedObservables metadata was not cross-validated against its matrix shape and delays.", + "Diagonal and dense encodings of the same HPD metric used inconsistent numerical conditioning thresholds." + ], + "remediation": "fit validates finite nonempty LQxN matrix, positive integer L/Q, exact row/delay counts and unique valid field mapping before reshaping to validated (L,Q,N); diagonal and dense W share one relative conditioning threshold.", + "re_review": "PASS; no new high- or medium-severity regression." + }, + "independent_science_review": { + "status": "FINAL_PASS", + "prior_findings": [ + "Authoritative math contract still declared the production API unavailable after implementation.", + "Authoritative centering contract described one joint switch while production explicitly supports independent U and P centering." + ], + "remediation": "Updated ORIGINAL_CCD_MATH.md to record the passed implementation gate and available public API, and to define all four independent snapshot/observable centering regimes.", + "re_review": "PASS; no remaining severity-rated findings or new contradictions." + }, + "verification": { + "environment": "pinball_math", + "focused": "42 production+derivation tests passed", + "full_active_suite": "105 passed", + "lints": "clean", + "cfd": "not run", + "real_case_ccd": "not run", + "commit": "not created" + }, + "latest_production_reviews": { + "initial_status": "FAIL", + "finding_groups": { + "A": "HPD conditioning was not scale invariant or dtype-epsilon documented.", + "B": "LaggedObservables invariant was deferred to fit and row ordering was not explicit.", + "C": "Left-function empirical identity, conjugation convention, direct-sum normalization and LQ view were incomplete.", + "D": "No analytic N_valid endpoint-drop denominator test.", + "E": "No selected-only centering regression with dropped outlier.", + "F": "No fail-closed nonzero degenerate-block/null-vector reconstruction policy." + }, + "remediation_status": "FINAL_INDEPENDENT_CODE_AND_SCIENCE_PASS", + "remediation_evidence": [ + "One min/max HPD helper uses eps(real dtype)*M for diagonal and dense weights; 1e-20/1e20 scale tests pass.", + "Frozen LaggedObservables validates finite LQxN, L/Q, delay count, explicit channel-major/delay-minor metadata and unique indices in __post_init__.", + "left_functions_lq and direct complex P@a.conj()/N_valid identity are documented and tested.", + "Analytic endpoint drop distinguishes N_valid=3 from wrong original N=4 and reports selected indices/count.", + "Centering after selection excludes a dropped 1e12 field outlier and validates both selected means.", + "Declared tolerances classify nonzero blocks/null modes; default reconstruction excludes nulls and refuses split blocks; config rank cannot clip a block; physical weighted projector invariance is tested." + ], + "verification": "105 full active tests and 42 focused production+derivation tests passed; edited-file lints clean; scoped diff check clean" + } +} diff --git a/src/CCD_analysis/evidence/real_ccd_contract_review.json b/src/CCD_analysis/evidence/real_ccd_contract_review.json new file mode 100644 index 0000000..119e4ee --- /dev/null +++ b/src/CCD_analysis/evidence/real_ccd_contract_review.json @@ -0,0 +1,72 @@ +{ + "claim_boundary": { + "forbidden_claims": [ + "CCD>POD", + "causal", + "mechanism", + "response-time", + "same-phase", + "independent-realization", + "observable prediction" + ], + "mean_context_outside_ccd": [ + "mean effective actions", + "authoritative mean dq_ctl" + ], + "sigma_label": "cross-correlation strength" + }, + "contract": { + "centering": { + "action_channels": true, + "field_rows": true, + "implicit": false + }, + "field_estimand": "centered full-resolution dq_ctl=q_ctl-q_blk on persisted analysis_fluid_mask", + "flatten_order": "component-major ux then uy; each component uses C-order (x-major,y-minor) analysis-mask order", + "observable_source": "q_ctl effective_applied_action[-3:] at each exact admitted field time, native physical units", + "observables": [ + "front_ccw_positive", + "upper_ccw_positive", + "lower_ccw_positive" + ], + "path": "src/CCD_analysis/original_ccd/REAL_CASE_CCD_CONTRACT.md", + "prohibited": [ + "POD pre-reduction", + "whitening", + "standardization", + "nearest-time matching", + "interpolation", + "phase guessing", + "silent trimming" + ], + "publication": "future per-case fsync-backed atomic no-replace immutable schema with mandatory live-provenance reload", + "streaming": "future provenance-validated mask-compressed two/three-pass implementation with explicit RAM/scratch budgets", + "weighting": "coordinate trapezoid cell-area weights repeated by component; W^(1/2) coordinates; no area normalization" + }, + "review": { + "evidence": "Contract cross-checked against active original_ccd math/API, acquisition v3 effective-action semantics and lineage, and direct-dq v2 selection/mask/provenance contracts.", + "status": "PASS", + "verification": [ + "canonical JSON parse and checkpoint invariant assertions passed", + "documentation whitespace checks passed", + "git diff --check passed", + "no code tests run because this todo changes contract/checkpoint documentation only" + ] + }, + "schema_id": "ccd-real-ccd-contract-review/v1", + "scope": { + "cases": [ + "karman_re100", + "illusion_1.0L" + ], + "cfd_run": false, + "data_or_artifacts_modified": false, + "q": 1, + "real_case_ccd_run": false, + "streaming_implemented": false, + "tau": 0 + }, + "todo": "real-ccd-contract", + "todo_status": "COMPLETED_CONTRACT_ONLY", + "unique_next_entry": "real-ccd-streaming" +} diff --git a/src/CCD_analysis/evidence/real_ccd_karman_figures_review.json b/src/CCD_analysis/evidence/real_ccd_karman_figures_review.json new file mode 100644 index 0000000..6dcb557 --- /dev/null +++ b/src/CCD_analysis/evidence/real_ccd_karman_figures_review.json @@ -0,0 +1 @@ +{"checks":["public load_result is sole data entry","verified transient centered snapshots only","no full MxN persistence","full 1280x512 grid","correct sigma/sigma^2 labels","Q=1 bars not lag curves","exact acquisition-relative steps","fixed-parity mismatch diagnostics are descriptive only","no causal/mechanism/response-time/CCD>POD claims"],"code_review":"PASS","initial_finding":{"finding":"single oversized reconstruction plate was impractical to render and did not initially include uy rank views","resolution":"split by selected timestamp and included ux/uy truth, reconstruction, and residual for ranks 1/2/3","severity":"medium"},"remaining_scope":"Illusion awaits user authorization; governing todo remains in_progress","remediation_cycles":1,"rounds":1,"schema_id":"ccd-real-ccd-figures-review/v1","science_review":"PASS","scope":"Karman-only subset of real-ccd-figures-review"} diff --git a/src/CCD_analysis/evidence/real_ccd_karman_result_review.json b/src/CCD_analysis/evidence/real_ccd_karman_result_review.json new file mode 100644 index 0000000..e05fe18 --- /dev/null +++ b/src/CCD_analysis/evidence/real_ccd_karman_result_review.json @@ -0,0 +1 @@ +{"Q":1,"action_means":{"front":-0.004750394590640402,"lower":0.03969304291531443,"upper":-0.0417400509895136},"artifact_manifest_sha256":"97d5cb300d642295bd3dffedd85d940fde06c8cc1c236fade390db78c3e3810e","artifact_path":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/real-ccd-karman-q1-tau0-burn120000-v1","canonical_array_hashes":{"action_mean":"67b3145d45ae4d618e4e4c39137d9fe0257ccd43557f3ce30d63bdf424cb985a","analysis_fluid_mask":"d8f661bea050bd3b60146b2051c8ec12a0491760599f885964306dd3089a842f","authoritative_mean_dq_ctl":"a45034707ee2db79a1c3af0c128b0bb8440d971e70567900b71ec79dd494778c","coefficients":"80e9f3b8b8475fc3ebbc952ab5a33f764f1af27eb0678a951b38a1087f6824a3","coordinate_weights":"56519c10c3b8a8d95b45244f6ca2cf48ca2e50bd6e8c594c7ad6db56d02ff2bc","cross_correlation":"7b8914e18344c044f4a5f4a58445eea502a38f4bc35a1fb327c35e9540b3a6cb","effective_actions":"64767cef6691c3cfedd06e400922e03b5403077bad30275b883ec801b64752a5","field_mean":"df4c0c131f787c3783ee1e1c6a900e5da08eb41fc984d66d181c68738a7963ba","identifiable_mode_mask":"75dd88e3c492f98b627e0d14e5d13716012b068ad46a1f0f7e26c4c24e8c6c0f","left_functions":"b896e02f82679cd1accca07debeea6b4c619f0d52a32fbc07ef0098f50067d97","physical_modes":"d2b06939cbf232fe08e571b2e03d3f12ac89cfdd588a73b84a8578ef98505f9f","q_blk_solver_fluid_mask":"d8f661bea050bd3b60146b2051c8ec12a0491760599f885964306dd3089a842f","q_ctl_solver_fluid_mask":"d8f661bea050bd3b60146b2051c8ec12a0491760599f885964306dd3089a842f","q_target_solver_fluid_mask":"0af572839be6a3ea6cc7c8d88eb2799295c2dc69aa4a4d12899ff7fd76252915","residual_block_boundaries":"b20bfed6cc98fd7eda53c0783d8c9ce4c66153b87c29a88d4fcdeab0155613b7","selected_acquisition_relative_lattice_steps":"644479c9c190d9513bd165e2e312fcccda45b2be211b4e24b8b16bf5a3d82597","selected_q_ctl_absolute_lattice_steps":"271e000641fa32c9eeda1f2518c2dfbd156548da542eaccbcb101badd6f1cdde","selected_timeline_indices":"696ea7f9b8802dae14563504cd0a6156c62ba77b68554b032c07ee0ea0e56d56","singular_values":"04c2776f76cd78ce086e799d3a9e92013cbaa590655728d5c3cd95c5000ffe8a","weighted_relative_residuals":"23ea5f9d0392775f9a69094c77e2f29c23dc44ee2fa43aa7670867352450545d","x_D":"96baa892f5c98be62214f1b500ba9a8b1dbc7c29d395544563420060bc29ec72","y_D":"1437d8d261b01d8da05550aad70a7de0ab40aed2d6ea9ed4f37eb82c070609f2"},"case_id":"karman_re100","cfd_run":false,"chunk_size":8,"decision":"PASS","degenerate_singular_blocks":[],"dimensions":{"coefficients":[3,120],"cross_correlation":[3,1299184],"modes":[1299184,3],"samples":120,"spatial_dof":1299184},"direct_dq_manifest_sha256":"f4c835d3da947e3a675815ec9884fcaeb737552ce19406d9b9e08c36eeb77814","direct_dq_path":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/direct-dq-karman-burn120000","essential_identity_recompute":"PASS: means, weights, cross-correlation, weighted orthonormality, SVD factorization, and coefficients recomputed by fresh loader","illusion_run":false,"immutable_artifact_modified":false,"interpretation":"Singular values are cross-correlation strengths, not field energy, explained variance, or canonical coefficients. Means are outside centered CCD.","memory":{"admission_basis":"all-fluid M estimate before authoritative loader; tight mask estimate also passed","conservative_all_fluid_estimated_peak_ram_bytes":16986938700,"decision":"PASS","estimated_peak_ram_bytes":16982785740,"estimated_scratch_bytes":0,"formula":"ceil(safety_margin * sum(terms)); loader residency explicitly included; no MxM or full float64 MxN term","ram_budget_bytes":47033803776,"raw_peak_ram_bytes":13586228592,"raw_scratch_bytes":0,"safety_margin":1.25,"scratch_budget_bytes":0,"terms_bytes":{"actions_coefficients_and_small_svd":6000,"chunk_float64_working_set":249443328,"cross_and_modes_float64":93541248,"field_mean_float64":10393472,"loader_decompression_and_copy_allowance":6606028800,"validated_direct_result_float32_fields":3774873600,"validated_live_acquisition_float32_fields":2831155200,"weights_and_roots_float64":20786944}},"null_tolerance":1.4676751183519247e-11,"numerical_rank":3,"provenance_reload":"VERIFIED: direct-dq and live acquisition inputs reread and essential identities recomputed","residual_block_boundaries":[1,2,3],"schema_id":"ccd-real-ccd-karman-review/v1","scientific_boundaries":"no CCD>POD, causal, mechanism, response-time, same-phase, independent-realization, uncertainty, or observable-prediction claim","singular_values":[0.14676751183519246,0.05920814023903697,0.0063884442355860915],"squared_cross_correlation_strengths":[0.02154070253029336,0.003505603870565469,4.081221975119316e-05],"tau":0,"unresolved_high_findings":[],"weighted_relative_residuals":[0.7823805118116768,0.4698397615223296,0.45542496362972573]} diff --git a/src/CCD_analysis/evidence/real_ccd_preflight_tests_review.json b/src/CCD_analysis/evidence/real_ccd_preflight_tests_review.json new file mode 100644 index 0000000..282f9fe --- /dev/null +++ b/src/CCD_analysis/evidence/real_ccd_preflight_tests_review.json @@ -0,0 +1 @@ +{"preflight_reports":{"illusion_1.0L":{"actions":{"channel_order":["front","upper","lower"],"identities":["front_ccw_positive","upper_ccw_positive","lower_ccw_positive"],"ranges":{"front":[-0.0033282283693552017,0.009038125164806843],"lower":[0.0037760159466415644,0.011678713373839855],"upper":[-0.01109163649380207,-0.005904059391468763]},"units":"native solver angular-velocity command units"},"case_id":"illusion_1.0L","decision":"PASS","dimensions":{"L":3,"M":1301196,"N":144,"Nx":1280,"Ny":512,"Q":1,"analysis_fluid_points":650598},"direct_dq_root":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/direct-dq-illusion-authorized-burn90000","direct_manifest_sha256":"b0370194bcbbc52225c77742a1fb0dc765dd3a15061e2b5c5fb2074f87ec28d5","host_memory":{"available_host_ram_bytes":62506463232,"ram_budget_bytes":46879847424,"safety_reserve_bytes":15626615808},"live_acquisition_roots":{"q_blk":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-illusion-authorized-450-fi1250/q_blk","q_ctl":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-illusion-authorized-450-fi1250/q_ctl","q_target":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-illusion-authorized-450-fi1250/q_target"},"memory":{"admission_basis":"all-fluid M estimate before authoritative loader; tight mask estimate also passed","conservative_all_fluid_estimated_peak_ram_bytes":20289954540,"decision":"PASS","estimated_peak_ram_bytes":20286525900,"estimated_scratch_bytes":0,"formula":"ceil(safety_margin * sum(terms)); loader residency explicitly included; no MxM or full float64 MxN term","ram_budget_bytes":46879847424,"raw_peak_ram_bytes":16229220720,"raw_scratch_bytes":0,"safety_margin":1.25,"scratch_budget_bytes":0,"terms_bytes":{"actions_coefficients_and_small_svd":7152,"chunk_float64_working_set":249829632,"cross_and_modes_float64":93686112,"field_mean_float64":10409568,"loader_decompression_and_copy_allowance":7927234560,"validated_direct_result_float32_fields":4529848320,"validated_live_acquisition_float32_fields":3397386240,"weights_and_roots_float64":20819136}},"read_only":true,"schema_id":"ccd-real-ccd-preflight/v1","scratch_note":"No disk scratch or result directory is used by preflight.","selection":{"first_index":72,"first_q_ctl_absolute_step":693250,"first_relative_step":91250,"last_index":215,"last_q_ctl_absolute_step":872000,"last_relative_step":270000}},"karman_re100":{"actions":{"channel_order":["front","upper","lower"],"identities":["front_ccw_positive","upper_ccw_positive","lower_ccw_positive"],"ranges":{"front":[-0.027850069105625153,0.014855165034532547],"lower":[0.01888139545917511,0.07301757484674454],"upper":[-0.07204609364271164,-0.008392715826630592]},"units":"native solver angular-velocity command units"},"case_id":"karman_re100","decision":"PASS","dimensions":{"L":3,"M":1299184,"N":120,"Nx":1280,"Ny":512,"Q":1,"analysis_fluid_points":649592},"direct_dq_root":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/direct-dq-karman-burn120000","direct_manifest_sha256":"f4c835d3da947e3a675815ec9884fcaeb737552ce19406d9b9e08c36eeb77814","host_memory":{"available_host_ram_bytes":62899654656,"ram_budget_bytes":47174740992,"safety_reserve_bytes":15724913664},"live_acquisition_roots":{"q_blk":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-karman-450-fi2000/q_blk","q_ctl":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-karman-450-fi2000/q_ctl","q_target":"/home/frank14f/DynamisLab/src/CCD_analysis/evidence/production-20260804-q-over-u0-v3-karman-450-fi2000/q_target"},"memory":{"admission_basis":"all-fluid M estimate before authoritative loader; tight mask estimate also passed","conservative_all_fluid_estimated_peak_ram_bytes":16986938700,"decision":"PASS","estimated_peak_ram_bytes":16982785740,"estimated_scratch_bytes":0,"formula":"ceil(safety_margin * sum(terms)); loader residency explicitly included; no MxM or full float64 MxN term","ram_budget_bytes":47174740992,"raw_peak_ram_bytes":13586228592,"raw_scratch_bytes":0,"safety_margin":1.25,"scratch_budget_bytes":0,"terms_bytes":{"actions_coefficients_and_small_svd":6000,"chunk_float64_working_set":249443328,"cross_and_modes_float64":93541248,"field_mean_float64":10393472,"loader_decompression_and_copy_allowance":6606028800,"validated_direct_result_float32_fields":3774873600,"validated_live_acquisition_float32_fields":2831155200,"weights_and_roots_float64":20786944}},"read_only":true,"schema_id":"ccd-real-ccd-preflight/v1","scratch_note":"No disk scratch or result directory is used by preflight.","selection":{"first_index":60,"first_q_ctl_absolute_step":754000,"first_relative_step":122000,"last_index":179,"last_q_ctl_absolute_step":992000,"last_relative_step":360000}}},"review":{"cycle_count":1,"final_status":"PASS","initial_findings":[{"finding":"live q_ctl absolute timestamps were not rebound on reload","remediation":"exact live q_ctl lattice-step equality plus hash-consistent tamper regression","severity":"high","status":"CLOSED"},{"finding":"scientific summary labels and claim boundary could be hash-consistently relabeled","remediation":"freeze summary strings in immutable schema plus relabel regressions","severity":"high","status":"CLOSED"},{"finding":"preflight corruption was not tested at its public entry point","remediation":"corrupted-live-root preflight no-output regression","severity":"medium","status":"CLOSED"}],"unresolved_high_findings":[]},"schema_id":"ccd-real-ccd-preflight-tests-review/v1","scope":{"archive_or_existing_artifact_edited":false,"cfd_run":false,"plan_edited":false,"preflight_result_directories_created":false,"real_case_ccd_run":false,"real_results_published":false},"status":"PASS_AFTER_ONE_REMEDIATION_REVIEW","synthetic_coverage":["streaming equals original_ccd dense literal API/reference across chunk sizes 1,2,3,4,7","cross-correlation, singular values, left/right subspaces, coefficients, weighted residuals","solver mask intersection, component-major flattening, quadrature weights","action order, identities, native units, exact timestamps/indices","memory budget fail-closed, schema/hash tamper, no-clobber, reload, missing/corrupted provenance/live roots"],"todo":"real-ccd-preflight-tests","verification":{"focused_real_ccd_tests":"22 passed","full_active_pinball_math_tests":"150 passed","git_diff_check":"scoped todo diff check passed; full workspace diff check reports unrelated pre-existing src/steady_pinball_theory/tests/test_core.py:582 blank line at EOF","lints":"no errors in real_ccd package and focused tests"}} diff --git a/src/CCD_analysis/evidence/real_ccd_streaming_review.json b/src/CCD_analysis/evidence/real_ccd_streaming_review.json new file mode 100644 index 0000000..74af0c5 --- /dev/null +++ b/src/CCD_analysis/evidence/real_ccd_streaming_review.json @@ -0,0 +1 @@ +{"blockers":["Real-data preflight/equivalence gate intentionally deferred to real-ccd-preflight-tests; no real artifact was opened or published."],"review":{"cycle_count":1,"evidence":["three-pass mask-compressed Q=1/tau=0 implementation","mandatory direct-dq and live q_ctl provenance on load","atomic fsync no-replace transaction","canonical schema/hash/inventory validation","focused synthetic tests"],"findings":[],"remediation":"Initial focused-test defect corrected before review: RealCCDInput action/mean positional construction now preserves declared field ordering. No additional review/remediation cycle used.","status":"PASS"},"schema_id":"ccd-real-ccd-streaming-review/v1","scope":{"archive_or_data_modified":false,"cfd_run":false,"package":"src/CCD_analysis/real_ccd","real_artifact_preflight":false,"real_case_ccd_run":false,"real_results_published":false},"todo":"real-ccd-streaming","todo_status":"IMPLEMENTED_SYNTHETIC_TESTED_NO_REAL_PREFLIGHT_OR_RUN","unique_next_entry":"real-ccd-preflight-tests","verification":{"focused_real_ccd_tests":"5 passed","full_active_pinball_math_tests":"133 passed","git_diff_check":"passed","lints":"no errors"}} diff --git a/src/CCD_analysis/karman_dynamic/__init__.py b/src/CCD_analysis/karman_dynamic/__init__.py new file mode 100644 index 0000000..9c387f2 --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/__init__.py @@ -0,0 +1,10 @@ +"""Karman DRL-vs-constant-mean campaign; CPU-safe imports.""" +from .artifacts import load_role_artifact +from .contracts import CONTRACT,ROLES,constant_mean_provenance,contract_snapshot,effective_action_mean,verify_decomposition +from .phase import evaluate_gate,load_phase_compact,publish_phase_compact,recover_phase +from .dynamic_increment import load_dynamic_increment,publish_dynamic_increment +from .temporal_ccd import TemporalConfig,TemporalTransaction,decompose_temporal,load_temporal_input,load_temporal_result +from .phase_domain_ccd import PhaseDomainTransaction,decompose_phase_domain,load_phase_domain_result +from .publication import load_dynamic_publication,publish_dynamic_figures +from .orchestration import CampaignSchedule,orchestrate,role_command +__all__=["CONTRACT","ROLES","CampaignSchedule","constant_mean_provenance","contract_snapshot","effective_action_mean","load_role_artifact","recover_phase","evaluate_gate","publish_phase_compact","load_phase_compact","publish_dynamic_increment","load_dynamic_increment","TemporalConfig","TemporalTransaction","load_temporal_input","decompose_temporal","load_temporal_result","PhaseDomainTransaction","decompose_phase_domain","load_phase_domain_result","publish_dynamic_figures","load_dynamic_publication","orchestrate","role_command","verify_decomposition"] diff --git a/src/CCD_analysis/karman_dynamic/__main__.py b/src/CCD_analysis/karman_dynamic/__main__.py new file mode 100644 index 0000000..13be635 --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/__main__.py @@ -0,0 +1,2 @@ +from .cli import main +raise SystemExit(main()) diff --git a/src/CCD_analysis/karman_dynamic/artifacts.py b/src/CCD_analysis/karman_dynamic/artifacts.py new file mode 100644 index 0000000..db2cd5a --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/artifacts.py @@ -0,0 +1,33 @@ +"""Immutable campaign wrappers around unchanged schema-v3 payloads.""" +import json +from pathlib import Path +import numpy as np +from CCD_analysis.acquisition.artifacts import file_sha256,rename_noreplace +from CCD_analysis.acquisition.validation import validate_acquisition_semantics +from .contracts import * +def _legacy(path,role): + manifest=json.loads((path/"manifest.json").read_text()); config=json.loads((path/"config.json").read_text()) + for n,d in manifest["files"].items(): + if file_sha256(path/n)!=d: raise ValueError("legacy payload hash mismatch") + with np.load(path/"fields.npz",allow_pickle=False) as f,np.load(path/"controller_state.npz",allow_pickle=False) as s: arrays={k:f[k] for k in f.files}; state={k:s[k] for k in s.files} + validate_acquisition_semantics(arrays=arrays,config=config,state=state,manifest=manifest,expected_case=CASE_ID,expected_role=LEGACY_ROLE[role]); return arrays +def publish_wrapper(staging,destination,*,role,campaign_id,warmup_intervals,collect_boundaries,constant_mean_provenance=None,smoke=False): + staging,destination=Path(staging),Path(destination); arrays=_legacy(staging/"payload",role); count=len(arrays["sensors"]); stop=warmup_intervals+collect_boundaries + if countlen(telemetry["center_sensor_uy"]): raise ValueError("invalid retained slice") + if meta["role"]=="constant_mean" and not isinstance(meta["constant_mean_provenance"],dict): raise ValueError("constant_mean provenance missing") + return {"path":path,"metadata":meta,"telemetry":telemetry,"legacy_arrays":arrays} diff --git a/src/CCD_analysis/karman_dynamic/cli.py b/src/CCD_analysis/karman_dynamic/cli.py new file mode 100644 index 0000000..1c3881f --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/cli.py @@ -0,0 +1,38 @@ +import argparse +import json +from pathlib import Path +from .contracts import ROLES +from .orchestration import CampaignSchedule,orchestrate +from .runtime import execute_role +from .phase import publish_phase_compact +from .temporal_ccd import TemporalConfig, TemporalTransaction, decompose_temporal, load_temporal_input +from .phase_domain_ccd import PhaseDomainTransaction,decompose_phase_domain +from .publication import publish_dynamic_figures +def main(argv=None): + p=argparse.ArgumentParser(); s=p.add_subparsers(dest="command",required=True); common=argparse.ArgumentParser(add_help=False); common.add_argument("--campaign-id",required=True); common.add_argument("--warmup-intervals",type=int,default=480); common.add_argument("--collect-boundaries",type=int,default=360); common.add_argument("--launch-delay-seconds",type=float,default=120) + role=s.add_parser("role",parents=[common]); role.add_argument("--role",choices=ROLES,required=True); role.add_argument("--output",type=Path,required=True); role.add_argument("--drl-artifact",type=Path); role.add_argument("--phase-artifact",type=Path); role.add_argument("--smoke",action="store_true") + orch=s.add_parser("orchestrate",parents=[common]); orch.add_argument("--root",type=Path,required=True); orch.add_argument("--execute",action="store_true"); orch.add_argument("--smoke",action="store_true") + phase=s.add_parser("phase-gate"); phase.add_argument("--role",choices=ROLES,default="drl"); phase.add_argument("--role-artifact",type=Path); phase.add_argument("--drl-artifact",type=Path); phase.add_argument("--output",type=Path,required=True) + temporal=s.add_parser("temporal-ccd"); temporal.add_argument("--drl-artifact",type=Path,required=True); temporal.add_argument("--phase-artifact",type=Path,required=True); temporal.add_argument("--output",type=Path,required=True); temporal.add_argument("--chunk-size",type=int,default=8); temporal.add_argument("--ram-budget-bytes",type=int,required=True) + phase_ccd=s.add_parser("phase-domain-ccd"); phase_ccd.add_argument("--drl-phase",type=Path,required=True); phase_ccd.add_argument("--constant-mean-phase",type=Path,required=True); phase_ccd.add_argument("--dynamic-increment",type=Path,required=True); phase_ccd.add_argument("--output",type=Path,required=True) + publication=s.add_parser("publication"); publication.add_argument("--dynamic-increment",type=Path,required=True); publication.add_argument("--temporal-ccd",type=Path,required=True); publication.add_argument("--phase-domain-ccd",type=Path,required=True); publication.add_argument("--output",type=Path,required=True) + a=p.parse_args(argv) + if a.command=="publication": + published=publish_dynamic_figures(a.dynamic_increment,a.temporal_ccd,a.phase_domain_ccd,a.output); print(json.dumps({"result":str(published.resolve())},sort_keys=True)); return 0 + if a.command=="phase-domain-ccd": + result=decompose_phase_domain(a.drl_phase,a.constant_mean_phase,a.dynamic_increment) + with PhaseDomainTransaction(a.output) as tx: tx.write(result); published=tx.publish() + print(json.dumps({"result":str(published.resolve()),"summary":result.summary},sort_keys=True)); return 0 + if a.command=="temporal-ccd": + inp=load_temporal_input(a.drl_artifact,a.phase_artifact); result=decompose_temporal(inp,streaming_config=TemporalConfig(a.chunk_size,a.ram_budget_bytes)); + with TemporalTransaction(a.output) as tx: tx.write(result); published=tx.publish() + print(json.dumps({"result":str(published.resolve()),"summary":result.summary},sort_keys=True)); return 0 + if a.command=="phase-gate": + source=a.role_artifact or a.drl_artifact + if source is None: p.error("phase-gate requires --role-artifact (or legacy --drl-artifact)") + result=publish_phase_compact(source,a.output,role=a.role); print(result["summary"]); return 0 + schedule=CampaignSchedule(a.campaign_id,a.warmup_intervals,a.collect_boundaries,a.launch_delay_seconds) + if a.command=="role": + execute_role(role=a.role,output=a.output,campaign_id=a.campaign_id,warmup_intervals=a.warmup_intervals,collect_boundaries=a.collect_boundaries,drl_artifact=a.drl_artifact,phase_artifact=a.phase_artifact,launch_delay_seconds=a.launch_delay_seconds,smoke=a.smoke); return 0 + for c in orchestrate(root=a.root,schedule=schedule,execute=a.execute,smoke=a.smoke): print(" ".join(c)) + return 0 diff --git a/src/CCD_analysis/karman_dynamic/contracts.py b/src/CCD_analysis/karman_dynamic/contracts.py new file mode 100644 index 0000000..d02faa9 --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/contracts.py @@ -0,0 +1,31 @@ +"""CPU-only Karman dynamic-increment campaign contract.""" +from dataclasses import asdict,dataclass +from hashlib import sha256 +import json +from pathlib import Path +import numpy as np +SCHEMA_ID="ccd-karman-dynamic-campaign/v1"; ARTIFACT_SCHEMA_ID="ccd-karman-dynamic-role/v1"; CASE_ID="karman_re100" +ROLES=("target","zero","drl","constant_mean"); EXECUTION_ORDER=("drl","constant_mean","target","zero"); LEGACY_ROLE={"target":"q_target","zero":"q_blk","drl":"q_ctl","constant_mean":"q_ctl"} +CONTROL_INTERVAL=800; DEFAULT_WARMUP_INTERVALS=480; DEFAULT_COLLECT_BOUNDARIES=360; MIN_LAUNCH_COOLDOWN_SECONDS=30.; DEFAULT_LAUNCH_DELAY_SECONDS=120. +CENTER_SENSOR_UY_COLUMN=3; EXPECTED_OPTANE_MOUNT=Path("/home/frank14f/optane"); DEFAULT_OPTANE_ROOT=EXPECTED_OPTANE_MOUNT/"DynamisLab/ccd/karman-dynamic"; DEFAULT_REPO_MAPPING=Path(__file__).resolve().parents[1]/"data/karman-dynamic" +LEASE_PATH=Path(__file__).resolve().parents[3]/".runtime/ccd-karman-dynamic.lock"; COOLDOWN_PATH=LEASE_PATH.with_name("ccd-karman-dynamic-cooldown.json") +@dataclass(frozen=True) +class CampaignContract: + case_id:str=CASE_ID; code_reynolds:int=100; physical_re_D:float=50.; roles:tuple=ROLES; control_interval:int=CONTROL_INTERVAL; phase_signal:str="center sensor uy=sensors[:,3]"; comparison:str="drl minus constant_mean" +CONTRACT=CampaignContract() +def canonical_json(v): return (json.dumps(v,sort_keys=True,separators=(",",":"),allow_nan=False)+"\n").encode() +def contract_snapshot(): + v={"schema_id":SCHEMA_ID,"contract":asdict(CONTRACT),"legacy_role_mapping":LEGACY_ROLE,"claims":{"zero":"passive baseline only","target":"cloaking-error reference only","dynamic_comparison":"DRL minus fresh effective-action-mean constant control","phase_difference":"independent phase-conditioned means; not pointwise counterfactual or causal"}}; v["contract_sha256"]=sha256(canonical_json(v)).hexdigest(); return v +def effective_action_mean(effective,retained_start): + v=np.asarray(effective) + if v.dtype!=np.float32 or v.ndim!=2 or v.shape[1]<3 or not np.isfinite(v).all(): raise ValueError("effective actions must be finite float32 (boundary, >=3)") + if type(retained_start)is not int or retained_start<0 or retained_start>=len(v): raise ValueError("retained_start must select a nonempty fresh DRL interval") + return np.mean(v[retained_start:,-3:],axis=0,dtype=np.float64).astype(np.float32) +def constant_mean_provenance(*,drl_manifest_sha256,effective,retained_start): + if not isinstance(drl_manifest_sha256,str) or len(drl_manifest_sha256)!=64: raise ValueError("DRL manifest SHA256 required") + int(drl_manifest_sha256,16); mean=effective_action_mean(effective,retained_start); source=np.ascontiguousarray(np.asarray(effective)[retained_start:,-3:]) + return {"schema_id":"ccd-karman-constant-mean-provenance/v1","drl_manifest_sha256":drl_manifest_sha256,"retained_start":retained_start,"source_effective_action_sha256":sha256(source.tobytes()).hexdigest(),"constant_mean_physical_action":mean.tolist(),"symmetrized":False,"source":"fresh DRL retained effective_applied_action mean"} +def verify_decomposition(q_d,q_c): + d,c=np.asarray(q_d,dtype=np.float64),np.asarray(q_c,dtype=np.float64) + if d.shape!=c.shape or d.ndim<1 or not np.isfinite(d).all() or not np.isfinite(c).all(): raise ValueError("matching finite arrays required") + return bool(np.allclose(d-c,(d.mean(0)-c.mean(0))+((d-d.mean(0))-(c-c.mean(0))),rtol=1e-12,atol=1e-12)) diff --git a/src/CCD_analysis/karman_dynamic/dynamic_increment.py b/src/CCD_analysis/karman_dynamic/dynamic_increment.py new file mode 100644 index 0000000..f354f2e --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/dynamic_increment.py @@ -0,0 +1,89 @@ +"""Immutable four-role Karman dynamic-increment statistics and phase differences.""" +from __future__ import annotations +import json, shutil, tempfile +from pathlib import Path +import numpy as np +from CCD_analysis.acquisition.artifacts import file_sha256, rename_noreplace +from CCD_analysis.direct_dq.analysis import coordinate_weights +from .artifacts import load_role_artifact +from .contracts import canonical_json, verify_decomposition +from .phase import load_phase_compact + +SCHEMA_ID="ccd-karman-dynamic-increment/v1" +ROLES=("drl","constant_mean","target","zero") + +def _wrms(field, weights): + return float(np.sqrt(np.sum(weights*np.sum(np.asarray(field,dtype=np.float64)**2,axis=0))/np.sum(weights))) + +def _dense_statistics(role_path, common, weights): + d=load_role_artifact(role_path); start,stop=d["metadata"]["retained_slice"]; a=d["legacy_arrays"] + q=np.stack((a["ux"][start:stop][:,common],a["uy"][start:stop][:,common]),axis=1) + mean=np.mean(q,axis=0,dtype=np.float64).astype(np.float32) + fluct=_wrms(np.sqrt(np.mean((q-mean[None])**2,axis=0,dtype=np.float64)),weights) + action=d["telemetry"]["effective_applied_action"][start:stop,-3:] + stats={"retained_boundary_count":int(stop-start),"fluctuation_weighted_vector_rms":fluct,"effective_action_mean":np.mean(action,axis=0,dtype=np.float64).tolist(),"effective_action_rms":np.sqrt(np.mean(action.astype(np.float64)**2,axis=0)).tolist(),"effective_action_total_rms":float(np.sqrt(np.mean(action.astype(np.float64)**2)))} + return mean,stats + +def _phase_mean(path, common): + info=load_phase_compact(path); s=info["summary"] + if not s["gate_passed"]: return None + with np.load(Path(path)/"compact.npz",allow_pickle=False) as z: + role_mask=z["fluid_mask"]; fields=np.mean(z["cycle_bin_fields"],axis=0,dtype=np.float64).astype(np.float32) + full=np.zeros((10,2,*role_mask.shape),np.float32); full[:,:,role_mask]=fields + return full[:,:,common] + +def publish_dynamic_increment(role_paths,phase_paths,output): + output=Path(output) + if output.exists(): raise FileExistsError(output) + if set(role_paths)!=set(ROLES) or set(phase_paths)!=set(ROLES): raise ValueError("exactly four role and phase paths required") + loaded={r:load_role_artifact(role_paths[r],expected_role=r) for r in ROLES} + x=loaded["drl"]["legacy_arrays"]["x_D"]; y=loaded["drl"]["legacy_arrays"]["y_D"] + for r,d in loaded.items(): + a=d["legacy_arrays"] + if not np.array_equal(a["x_D"],x) or not np.array_equal(a["y_D"],y): raise ValueError(f"{r} grid mismatch") + common=np.logical_and.reduce([loaded[r]["legacy_arrays"]["fluid_mask"] for r in ROLES]) + if common.sum()<4: raise ValueError("four-mask intersection too small") + weights=(coordinate_weights(x)[:,None]*coordinate_weights(y)[None,:])[common] + means={}; stats={} + for r in ROLES: means[r],stats[r]=_dense_statistics(Path(role_paths[r]),common,weights) + for r in ("zero","constant_mean","drl"): + stats[r]["mean_target_error_weighted_vector_rms"]=_wrms(means[r]-means["target"],weights) + stats["target"]["mean_target_error_weighted_vector_rms"]=0.0 + zero_constant=means["constant_mean"]-means["zero"]; constant_drl=means["drl"]-means["constant_mean"] + phase={r:_phase_mean(phase_paths[r],common) for r in ROLES}; phase_available=phase["drl"] is not None and phase["constant_mean"] is not None + arrays={"x_D":x,"y_D":y,"four_role_fluid_mask":common,"quadrature_weights":weights,"mean_drl":means["drl"],"mean_constant_mean":means["constant_mean"],"mean_target":means["target"],"mean_zero":means["zero"],"mean_increment_zero_to_constant":zero_constant,"mean_increment_constant_to_drl":constant_drl} + phase_metrics={} + if phase_available: + total=phase["drl"]-phase["constant_mean"]; centered=(phase["drl"]-means["drl"])-(phase["constant_mean"]-means["constant_mean"]) + arrays.update({"phase_mean_drl":phase["drl"],"phase_mean_constant_mean":phase["constant_mean"],"phase_difference_total_drl_minus_constant":total,"phase_difference_centered_drl_minus_constant":centered}) + for r in ("target","zero"): + if phase[r] is not None: arrays[f"phase_mean_{r}"]=phase[r] + if phase["target"] is not None: + for r in ("zero","constant_mean","drl"): + if phase[r] is not None: + vals=[_wrms(phase[r][b]-phase["target"][b],weights) for b in range(10)]; phase_metrics[r]={"target_error_by_bin_weighted_vector_rms":vals,"target_error_cycle_mean_weighted_vector_rms":float(np.mean(vals))} + closure=total-((means["drl"]-means["constant_mean"])[None]+centered); closure_max=float(np.max(np.abs(closure))) + if closure_max>2e-6 or not verify_decomposition(phase["drl"],phase["constant_mean"]): raise ValueError("phase decomposition closure failed") + else: closure_max=None + benefits={"zero_to_constant_overall_mean_control_benefit_target_error_reduction":stats["zero"]["mean_target_error_weighted_vector_rms"]-stats["constant_mean"]["mean_target_error_weighted_vector_rms"],"constant_to_drl_dynamic_increment_target_error_reduction":stats["constant_mean"]["mean_target_error_weighted_vector_rms"]-stats["drl"]["mean_target_error_weighted_vector_rms"],"zero_to_drl_total_target_error_reduction":stats["zero"]["mean_target_error_weighted_vector_rms"]-stats["drl"]["mean_target_error_weighted_vector_rms"]} + benefit_closure=benefits["zero_to_constant_overall_mean_control_benefit_target_error_reduction"]+benefits["constant_to_drl_dynamic_increment_target_error_reduction"]-benefits["zero_to_drl_total_target_error_reduction"] + parents={r:{"role_path":str(Path(role_paths[r]).resolve()),"role_campaign_manifest_sha256":file_sha256(Path(role_paths[r])/"campaign_manifest.json"),"phase_path":str(Path(phase_paths[r]).resolve()),"phase_manifest_sha256":file_sha256(Path(phase_paths[r])/"manifest.json"),"phase_gate_passed":phase[r] is not None} for r in ROLES} + summary={"schema_id":SCHEMA_ID,"complete":True,"drl_constant_phase_differences_available":phase_available,"phase_gate_passed_by_role":{r:phase[r] is not None for r in ROLES},"phase_blockers":[r for r in ROLES if phase[r] is None],"mask_definition":"exact intersection of all four solver-derived fluid masks","analysis_fluid_point_count":int(common.sum()),"quadrature_rule":"coordinate_weights(x_D)*coordinate_weights(y_D) on common mask; not area-normalized","statistics":stats,"phase_target_error_metrics":phase_metrics,"benefits":benefits,"closure":{"benefit_additivity_absolute_residual":float(abs(benefit_closure)),"mean_increment_max_absolute_residual":float(np.max(np.abs((means['drl']-means['zero'])-(zero_constant+constant_drl)))),"phase_decomposition_max_absolute_residual":closure_max},"claims":"independent phase-conditioned trajectory means; not pointwise counterfactual, response, or causal effect","dense_fields_deleted":False,"parents":parents} + output.parent.mkdir(parents=True,exist_ok=True); stage=Path(tempfile.mkdtemp(prefix=f".{output.name}.partial-",dir=output.parent)) + try: + np.savez_compressed(stage/"arrays.npz",**arrays); (stage/"summary.json").write_bytes(canonical_json(summary)); files={p.name:file_sha256(p) for p in stage.iterdir() if p.is_file()}; (stage/"manifest.json").write_bytes(canonical_json({"schema_id":SCHEMA_ID,"complete":True,"files":files})); rename_noreplace(stage,output); return load_dynamic_increment(output) + except Exception: shutil.rmtree(stage,ignore_errors=True); raise + +def load_dynamic_increment(path): + path=Path(path); manifest=json.loads((path/"manifest.json").read_text()); summary=json.loads((path/"summary.json").read_text()) + if manifest.get("schema_id")!=SCHEMA_ID or not manifest.get("complete") or summary.get("schema_id")!=SCHEMA_ID: raise ValueError("dynamic increment schema/incomplete") + for n,h in manifest["files"].items(): + if file_sha256(path/n)!=h: raise ValueError("dynamic increment file hash mismatch") + for r,p in summary["parents"].items(): + load_role_artifact(p["role_path"],expected_role=r); load_phase_compact(p["phase_path"]) + if file_sha256(Path(p["role_path"])/"campaign_manifest.json")!=p["role_campaign_manifest_sha256"] or file_sha256(Path(p["phase_path"])/"manifest.json")!=p["phase_manifest_sha256"]: raise ValueError("dynamic increment live provenance mismatch") + with np.load(path/"arrays.npz",allow_pickle=False) as z: + required={"x_D","y_D","four_role_fluid_mask","quadrature_weights","mean_drl","mean_constant_mean","mean_target","mean_zero","mean_increment_zero_to_constant","mean_increment_constant_to_drl"} + if not required.issubset(z.files): raise ValueError("dynamic increment arrays incomplete") + if np.max(np.abs((z["mean_drl"]-z["mean_zero"])-(z["mean_increment_zero_to_constant"]+z["mean_increment_constant_to_drl"])))>2e-6: raise ValueError("mean increment closure mismatch") + return {"path":path,"summary":summary} diff --git a/src/CCD_analysis/karman_dynamic/orchestration.py b/src/CCD_analysis/karman_dynamic/orchestration.py new file mode 100644 index 0000000..5a5bbea --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/orchestration.py @@ -0,0 +1,31 @@ +"""Fresh-child campaign planning and fail-closed state.""" +from dataclasses import dataclass +import json,os,subprocess,sys +from pathlib import Path +from .artifacts import load_role_artifact +from .contracts import * +from .safety import enforce_cooldown,exclusive_lease,require_execution_environment,validate_optane_storage +@dataclass(frozen=True) +class CampaignSchedule: + campaign_id:str; warmup_intervals:int=DEFAULT_WARMUP_INTERVALS; collect_boundaries:int=DEFAULT_COLLECT_BOUNDARIES; launch_delay_seconds:float=DEFAULT_LAUNCH_DELAY_SECONDS + def __post_init__(self): + if not self.campaign_id or self.warmup_intervals<1 or self.collect_boundaries<1 or self.launch_delay_seconds<30: raise ValueError("campaign id, positive counts, and delay >=30 required") +def role_command(*,role,output,schedule,drl_artifact=None,smoke=False): + if role not in ROLES: raise ValueError("invalid role") + c=[sys.executable,"-m","CCD_analysis.karman_dynamic","role","--campaign-id",schedule.campaign_id,"--role",role,"--output",str(output),"--warmup-intervals",str(schedule.warmup_intervals),"--collect-boundaries",str(schedule.collect_boundaries)] + if drl_artifact is not None:c += ["--drl-artifact",str(drl_artifact)] + if smoke:c.append("--smoke") + return c +def mark_campaign(root,*,campaign_id,status,failed_role=None,error=None): + if status not in {"READY","RUNNING","QUARANTINED","FAILED","COMPLETE"}: raise ValueError("invalid status") + root=Path(root); root.mkdir(parents=True,exist_ok=True); state={"schema_id":"ccd-karman-dynamic-state/v1","campaign_id":campaign_id,"status":status,"failed_role":failed_role,"error":error}; tmp=root/f".campaign_state.{os.getpid()}.tmp"; tmp.write_text(json.dumps(state,sort_keys=True)+"\n"); os.replace(tmp,root/"campaign_state.json"); return state +def orchestrate(*,root,schedule,execute=False,smoke=False): + root=Path(root); commands=[role_command(role=r,output=root/r,schedule=schedule,drl_artifact=(root/"drl" if r=="constant_mean" else None),smoke=smoke) for r in EXECUTION_ORDER] + if not execute:return commands + require_execution_environment(); backing=validate_optane_storage(repo_mapping=DEFAULT_REPO_MAPPING,optane_root=DEFAULT_OPTANE_ROOT,mount=EXPECTED_OPTANE_MOUNT) + if backing!=root and backing not in root.resolve(strict=False).parents: raise ValueError("campaign root must be below validated Optane mapping") + mark_campaign(root,campaign_id=schedule.campaign_id,status="RUNNING") + for role,command in zip(EXECUTION_ORDER,commands): + try: subprocess.run(command,check=True,env=dict(os.environ)); load_role_artifact(root/role,expected_role=role) + except Exception as exc: mark_campaign(root,campaign_id=schedule.campaign_id,status="QUARANTINED",failed_role=role,error=repr(exc)); raise + mark_campaign(root,campaign_id=schedule.campaign_id,status="COMPLETE"); return commands diff --git a/src/CCD_analysis/karman_dynamic/phase.py b/src/CCD_analysis/karman_dynamic/phase.py new file mode 100644 index 0000000..25849fb --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/phase.py @@ -0,0 +1,72 @@ +"""Fail-closed phase/stationarity gate and compact cycle-balanced fields.""" +from __future__ import annotations +import json, shutil, tempfile +from pathlib import Path +import numpy as np +from CCD_analysis.acquisition.artifacts import file_sha256, rename_noreplace +from .artifacts import load_role_artifact +from .contracts import canonical_json, constant_mean_provenance +PHASE_SCHEMA_ID="ccd-karman-dynamic-phase-compact/v1" +LIMITS={"minimum_complete_cycles":10,"maximum_period_cv":0.05,"maximum_amplitude_cv":0.10,"minimum_cycle_amplitude_fraction":0.25,"maximum_prefix_suffix_period_shift":0.05,"maximum_prefix_suffix_amplitude_shift":0.10,"maximum_double_crossing_fraction":0.0,"maximum_field_sensitivity_relative_rms":0.15} +def _crossings(s,rising): + s=np.asarray(s,np.float64); q=(s[:-1]<0)&(s[1:]>=0) if rising else (s[:-1]>0)&(s[1:]<=0); i=np.flatnonzero(q); return i-s[i]/(s[i+1]-s[i]) +def recover_phase(signal): + y=np.asarray(signal) + if y.dtype!=np.float32 or y.ndim!=1 or len(y)<3 or not np.isfinite(y).all(): raise ValueError("phase signal must be finite float32 vector") + rising,falling=_crossings(y,True),_crossings(y,False); ids=np.full(len(y),-1,np.int64); phase=np.full(len(y),np.nan); amps=[]; double=[] + for c,(left,right) in enumerate(zip(rising[:-1],rising[1:])): + idx=np.flatnonzero((np.arange(len(y))>=left)&(np.arange(len(y))left)&(falling1 else float("inf"); ac=float(np.std(a,ddof=1)/np.mean(a)) if n>1 else float("inf"); low=a1 else None,"period_cv":pc,"period_min_boundaries":float(np.min(p)) if n else None,"period_max_boundaries":float(np.max(p)) if n else None,"amplitude_mean":float(np.mean(a)) if n else None,"amplitude_cv":ac,"minimum_cycle_amplitude":float(np.min(a)) if n else None,"low_amplitude_cycle_count":int(np.sum(low)),"double_crossing_cycle_count":int(np.sum(r["double_crossing"])),"double_crossing_fraction":df,"prefix_suffix_period_shift":ps,"prefix_suffix_amplitude_shift":ass,"signal_mean":float(np.mean(signal)),"signal_std":float(np.std(signal)),"signal_peak_to_peak":float(np.ptp(signal))} + tests=((n>=10,f"complete cycle count {n} < 10"),(pc<=.05,f"period CV {pc:.6g} > 0.05"),(ac<=.10,f"amplitude CV {ac:.6g} > 0.10"),(not np.any(low),f"low-amplitude cycles {int(np.sum(low))} > 0"),(df<=0,f"double-crossing fraction {df:.6g} > 0"),(ps<=.05,f"prefix/suffix period shift {ps:.6g} > 0.05"),(ass<=.10,f"prefix/suffix amplitude shift {ass:.6g} > 0.10")); return r,m,[msg for ok,msg in tests if not ok] +def _bins(ux,uy,actions,mask,phase,ids,bins,origin): + cycles=np.unique(ids[ids>=0]); fields=np.empty((len(cycles),bins,2,int(mask.sum())),np.float32); acts=np.empty((len(cycles),bins,3),np.float32); counts=np.zeros((len(cycles),bins),np.int64); bid=np.full(len(phase),-1,np.int64); valid=np.isfinite(phase); bid[valid]=np.floor(np.mod(phase[valid]-origin,2*np.pi)*bins/(2*np.pi)).astype(np.int64) + for ci,c in enumerate(cycles): + for b in range(bins): + s=(ids==c)&(bid==b); counts[ci,b]=s.sum() + if not s.any(): raise ValueError(f"empty cycle/bin {c}/{b} for {bins} bins") + fields[ci,b,0]=np.mean(ux[s][:,mask],axis=0,dtype=np.float64); fields[ci,b,1]=np.mean(uy[s][:,mask],axis=0,dtype=np.float64); acts[ci,b]=np.mean(actions[s,-3:],axis=0,dtype=np.float64) + return fields,acts,counts +def _curve_distance(a,b,b_origin_fraction=0.): + def weights(n,origin): + u=((np.arange(120)+.5)/120-origin)*n-.5; lo=np.floor(u).astype(int); return lo%n,(lo+1)%n,(u-lo)[:,None] + ai0,ai1,aw=weights(len(a),0.); bi0,bi1,bw=weights(len(b),b_origin_fraction); af=a.reshape(len(a),-1); bf=b.reshape(len(b),-1); err=energy=0.; count=0 + for left in range(0,af.shape[1],4096): + aa=(1-aw)*af[ai0,left:left+4096]+aw*af[ai1,left:left+4096]; bb=(1-bw)*bf[bi0,left:left+4096]+bw*bf[bi1,left:left+4096]; err+=float(np.sum((aa-bb)**2)); energy+=float(np.sum((aa-aa.mean(0))**2)); count+=aa.size + return float(np.sqrt(err/max(count,1))/max(np.sqrt(energy/max(count,1)),np.finfo(float).eps)) +def publish_phase_compact(role_path,output,*,role="drl"): + if role not in {"drl","constant_mean","target","zero"}: raise ValueError("invalid phase role") + role_path,output=Path(role_path),Path(output); d=load_role_artifact(role_path,expected_role=role); start,stop=d["metadata"]["retained_slice"]; a=d["legacy_arrays"]; signal=d["telemetry"]["center_sensor_uy"][start:stop]; r,m,blockers=evaluate_gate(signal); parent=file_sha256(role_path/"campaign_manifest.json") + if output.exists(): raise FileExistsError(output) + output.parent.mkdir(parents=True,exist_ok=True); stage=Path(tempfile.mkdtemp(prefix=f".{output.name}.partial-",dir=output.parent)); provenance=None + try: + if not blockers: + ux,uy=a["ux"][start:stop],a["uy"][start:stop]; actions=d["telemetry"]["effective_applied_action"][start:stop]; mask=a["fluid_mask"]; primary,pacts,counts=_bins(ux,uy,actions,mask,r["phase"],r["cycle_id"],10,0.); ensembles={}; all_counts={} + for bins,origin,name in ((8,0.,"bins8"),(12,0.,"bins12"),(10,np.pi/10,"bins10_half_shift")): + q,_,c=_bins(ux,uy,actions,mask,r["phase"],r["cycle_id"],bins,origin); ensembles[name]=np.mean(q,axis=0,dtype=np.float64).astype(np.float32); all_counts[name]=c + n=len(primary); h=n//2; base=np.mean(primary,axis=0,dtype=np.float64); splits={"odd_cycles":np.mean(primary[::2],axis=0,dtype=np.float64),"even_cycles":np.mean(primary[1::2],axis=0,dtype=np.float64),"prefix_cycles":np.mean(primary[:h],axis=0,dtype=np.float64),"suffix_cycles":np.mean(primary[-h:],axis=0,dtype=np.float64)}; sensitivity={k:_curve_distance(base,v,.05 if k=="bins10_half_shift" else 0.) for k,v in {**ensembles,**splits}.items()}; m["field_sensitivity_relative_rms"]=sensitivity + unstable={k:v for k,v in sensitivity.items() if v>LIMITS["maximum_field_sensitivity_relative_rms"]} + if unstable: blockers.append(f"field sensitivity relative RMS exceeds 0.15: {unstable}") + if not blockers: + np.savez_compressed(stage/"compact.npz",fluid_mask=mask,x_D=a["x_D"],y_D=a["y_D"],cycle_bin_fields=primary,cycle_bin_effective_actions=pacts,cycle_bin_counts=counts,cycle_ids=np.unique(r["cycle_id"][r["cycle_id"]>=0]),rising_crossings=r["rising_crossings"],periods=r["periods"],cycle_amplitudes=r["cycle_amplitudes"],**{f"ensemble_{k}":v for k,v in ensembles.items()},**{f"counts_{k}":v for k,v in all_counts.items()}) + if role=="drl": + provenance=constant_mean_provenance(drl_manifest_sha256=parent,effective=d["telemetry"]["effective_applied_action"],retained_start=start); (stage/"constant_mean_provenance.json").write_bytes(canonical_json(provenance)) + summary={"schema_id":PHASE_SCHEMA_ID,"complete":True,"gate_passed":not blockers,"blockers":blockers,"quality_limits":LIMITS,"metrics":m,"phase_definition":"linear phase between independent rising zero crossings of retained center sensor uy","primary_bins":10,"sensitivity":{"bin_counts":[8,10,12],"origin":"zero and half-bin for 10","cycle_splits":["odd/even","prefix/suffix"]},"role":role,"source":{"absolute_path":str(role_path.resolve()),"campaign_manifest_sha256":parent,"payload_manifest_sha256":file_sha256(role_path/"payload/manifest.json"),"retained_slice":[start,stop]},"dense_fields_deleted":False,"constant_mean_provenance":provenance}; (stage/"summary.json").write_bytes(canonical_json(summary)); files={p.name:file_sha256(p) for p in stage.iterdir() if p.is_file()}; (stage/"manifest.json").write_bytes(canonical_json({"schema_id":PHASE_SCHEMA_ID,"complete":True,"files":files})); rename_noreplace(stage,output); return load_phase_compact(output) + except Exception: shutil.rmtree(stage,ignore_errors=True); raise +def load_phase_compact(path): + path=Path(path); manifest=json.loads((path/"manifest.json").read_text()); summary=json.loads((path/"summary.json").read_text()) + if manifest.get("schema_id")!=PHASE_SCHEMA_ID or not manifest.get("complete") or summary.get("schema_id")!=PHASE_SCHEMA_ID: raise ValueError("phase compact schema/incomplete") + for n,h in manifest["files"].items(): + if file_sha256(path/n)!=h: raise ValueError("phase compact file hash mismatch") + role=summary.get("role","drl"); source=Path(summary["source"]["absolute_path"]); load_role_artifact(source,expected_role=role) + if file_sha256(source/"campaign_manifest.json")!=summary["source"]["campaign_manifest_sha256"] or file_sha256(source/"payload/manifest.json")!=summary["source"]["payload_manifest_sha256"]: raise ValueError("phase compact live source provenance mismatch") + if summary["gate_passed"]: + with np.load(path/"compact.npz",allow_pickle=False) as z: + required={"fluid_mask","x_D","y_D","cycle_bin_fields","cycle_bin_effective_actions","cycle_bin_counts","cycle_ids","rising_crossings","periods","cycle_amplitudes","ensemble_bins8","ensemble_bins12","ensemble_bins10_half_shift","counts_bins8","counts_bins12","counts_bins10_half_shift"} + if set(z.files)!=required or z["cycle_bin_fields"].dtype!=np.float32 or z["cycle_bin_fields"].shape[1:3]!=(10,2) or np.any(z["cycle_bin_counts"]<=0): raise ValueError("phase compact arrays invalid") + elif "constant_mean_provenance.json" in manifest["files"] or (path/"compact.npz").exists(): raise ValueError("failed gate published forbidden products") + if role!="drl" and (summary.get("constant_mean_provenance") is not None or "constant_mean_provenance.json" in manifest["files"]): raise ValueError("non-DRL phase artifact contains DRL mean provenance") + return {"path":path,"summary":summary} diff --git a/src/CCD_analysis/karman_dynamic/phase_domain_ccd.py b/src/CCD_analysis/karman_dynamic/phase_domain_ccd.py new file mode 100644 index 0000000..f3323a6 --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/phase_domain_ccd.py @@ -0,0 +1,119 @@ +"""Exploratory circular phase-domain CCD for DRL minus constant-mean dynamics.""" +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path +from typing import Any +import json, shutil, tempfile +import numpy as np +from CCD_analysis.acquisition.artifacts import file_sha256, rename_noreplace +from CCD_analysis.direct_dq.schema import canonical_array_sha256 +from .contracts import canonical_json +from .dynamic_increment import load_dynamic_increment +from .phase import load_phase_compact + +SCHEMA_ID="ccd-karman-phase-domain-exploratory/v1" +CHANNELS=("front","upper","lower") +VARIANTS=(("bins8",8,0.0),("primary",10,0.0),("bins12",12,0.0),("bins10_half_shift",10,0.5)) +HARMONIC_ORDERS=(1,2,3) +CLAIM_BOUNDARY="exploratory circular phase co-variation only; phase offsets are not time-response lags, causality, mechanism, uncertainty, or observable prediction" + +@dataclass(frozen=True) +class PhaseDomainResult: + arrays:dict[str,np.ndarray]; config:dict[str,Any]; summary:dict[str,Any]; input_hashes:dict[str,Any] + +def _periodic_resample(curve,n,origin_fraction=0.0,order=None): + x=np.asarray(curve,np.float64); old_n=len(x); old=(np.arange(old_n)+.5)/old_n; new=(np.arange(n)+.5+origin_fraction)/n + if order is None: order=min((old_n-1)//2,4) + out=np.broadcast_to(x.mean(0),(n,)+x.shape[1:]).copy(); flat=x.reshape(old_n,-1); outf=out.reshape(n,-1) + for k in range(1,order+1): + c=(2/old_n)*np.sum(flat*np.cos(2*np.pi*k*old)[:,None],axis=0); s=(2/old_n)*np.sum(flat*np.sin(2*np.pi*k*old)[:,None],axis=0) + outf+=np.cos(2*np.pi*k*new)[:,None]*c+np.sin(2*np.pi*k*new)[:,None]*s + return out + +def _decompose(u,p,w): + u=np.asarray(u,np.float64); p=np.asarray(p,np.float64); w=np.asarray(w,np.float64); n=u.shape[1] + uc=u-u.mean(1,keepdims=True); pc=p-p.mean(1,keepdims=True); roots=np.sqrt(w) + a=pc@(uc*roots[:,None]).T/(n*np.sqrt(3.0)); left,s,vh=np.linalg.svd(a,full_matrices=False); weighted=vh.T + for k in range(weighted.shape[1]): + pivot=int(np.argmax(np.abs(weighted[:,k]))) + if weighted[pivot,k]<0: weighted[:,k]*=-1; left[:,k]*=-1 + rank=int(np.sum(s>1e-10*s[0])) if len(s) and s[0]>0 else 0 + return {"cross":a,"left":left,"singular":s,"modes":weighted/roots[:,None],"rank":rank} + +def _comparison(primary,other,w): + r=min(primary["rank"],other["rank"],3); out={"rank":other["rank"],"leading_spectrum":other["singular"][:3].tolist()} + if r: + root=np.sqrt(w)[:,None]; vp=primary["modes"][:,:r]*root; vo=other["modes"][:,:r]*root + out["projector_principal_cosines"]=np.linalg.svd(vp.T@vo,compute_uv=False).tolist(); out["minimum_projector_cosine"]=float(min(out["projector_principal_cosines"])); out["left_function_absolute_cosines"]=[float(abs(primary["left"][:,k]@other["left"][:,k])) for k in range(r)] + else: out.update(projector_principal_cosines=[],minimum_projector_cosine=None,left_function_absolute_cosines=[]) + return out + +def _load_inputs(drl_phase,constant_phase,dynamic): + dp,cp=load_phase_compact(drl_phase),load_phase_compact(constant_phase); di=load_dynamic_increment(dynamic) + if not dp["summary"]["gate_passed"] or dp["summary"].get("role","drl")!="drl" or not cp["summary"]["gate_passed"] or cp["summary"].get("role","drl")!="constant_mean": raise ValueError("passing DRL and constant_mean phase artifacts required") + with np.load(Path(drl_phase)/"compact.npz",allow_pickle=False) as dz, np.load(Path(constant_phase)/"compact.npz",allow_pickle=False) as cz, np.load(Path(dynamic)/"arrays.npz",allow_pickle=False) as iz: + if not np.array_equal(dz["fluid_mask"],cz["fluid_mask"]): raise ValueError("phase masks differ") + common=iz["four_role_fluid_mask"].copy(); selector=common[dz["fluid_mask"]] + if int(selector.sum())!=len(iz["quadrature_weights"]): raise ValueError("dynamic weights/common mask mismatch") + means={r:iz[f"mean_{r}"].copy() for r in ("drl","constant_mean")}; weights=np.concatenate((iz["quadrature_weights"],iz["quadrature_weights"])).astype(np.float64); fields={} + for name,_,_ in VARIANTS: + kd="cycle_bin_fields" if name=="primary" else f"ensemble_{name}"; dd=np.mean(dz[kd],axis=0,dtype=np.float64) if name=="primary" else dz[kd].astype(np.float64); cc=np.mean(cz[kd],axis=0,dtype=np.float64) if name=="primary" else cz[kd].astype(np.float64) + fields[name]=(dd[:,:,selector]-means["drl"][None])-(cc[:,:,selector]-means["constant_mean"][None]) + actions=np.mean(dz["cycle_bin_effective_actions"],axis=0,dtype=np.float64) + return dp,cp,di,fields,actions,weights + +def decompose_phase_domain(drl_phase,constant_phase,dynamic_increment)->PhaseDomainResult: + dp,cp,di,fields,actions,w=_load_inputs(drl_phase,constant_phase,dynamic_increment); results={}; arrays={"coordinate_weights":w,"primary_phase_action_curve":actions} + for name,n,origin in VARIANTS: + u=fields[name].transpose(1,2,0).reshape(len(w),n); p=_periodic_resample(actions,n,origin_fraction=origin).T; r=_decompose(u,p,w); results[name]=r + for key in ("cross","left","singular","modes"): arrays[f"{name}_{key}"]=r[key] + primary=results["primary"]; comparisons={name:_comparison(primary,r,w) for name,r in results.items() if name!="primary"}; harmonic={} + for order in HARMONIC_ORDERS: + u0=fields["primary"].transpose(1,2,0).reshape(len(w),10).T; r=_decompose(_periodic_resample(u0,10,order=order).T,_periodic_resample(actions,10,order=order).T,w); harmonic[str(order)]=_comparison(primary,r,w) + for key in ("cross","left","singular","modes"): arrays[f"harmonic_{order}_{key}"]=r[key] + offsets=[] + for off in range(10): + r=_decompose(fields["primary"].transpose(1,2,0).reshape(len(w),10),np.roll((actions-actions.mean(0)).T,off,axis=1),w); signed=off if off<=5 else off-10 + offsets.append({"phase_offset_bins":signed,"phase_offset_radians":float(2*np.pi*signed/10),"leading_spectrum":r["singular"][:3].tolist(),"rank":r["rank"]}) + checks=list(comparisons.values())+list(harmonic.values()); stable=[x["minimum_projector_cosine"] for x in checks if x.get("minimum_projector_cosine") is not None]; left=[min(x["left_function_absolute_cosines"]) for x in checks if x["left_function_absolute_cosines"]]; rank_stable=all(x["rank"]==primary["rank"] for x in checks) + decision="PASS_EXPLORATORY" if primary["rank"]>0 and rank_stable and min(stable,default=0)>=0.9 and min(left,default=0)>=0.8 else "DOWNGRADE" + config={"schema_id":SCHEMA_ID,"field_estimand":"separately_centered_phase_coherent_difference_delta_q_prime_phase(phi)=(q_DRL(phi)-mean_q_DRL)-(q_constant_mean(phi)-mean_q_constant_mean)","observable_estimand":"DRL phase-conditioned three-channel effective_applied_action fluctuation","operator":"A=P(W^(1/2)U)^T/(N*sqrt(3Q)); Q=1","primary_bins":10,"sensitivity_bins":[8,12],"half_bin_origin":True,"harmonic_orders":list(HARMONIC_ORDERS),"action_grid_sensitivity":"periodic Fourier interpolation of immutable primary 10-bin DRL action curve, maximum order 4; harmonic tests truncate both U and P","phase_offset_semantics":"circular phase offsets only; not time-response lags","center_snapshots":True,"center_observables":True,"standardization":False,"whitening":False,"pod":False,"claim_boundary":CLAIM_BOUNDARY} + summary={"schema_id":SCHEMA_ID,"decision":decision,"N":10,"Q":1,"M":len(w),"numerical_rank":primary["rank"],"primary_singular_values":primary["singular"].tolist(),"primary_squared_singular_values":(primary["singular"]**2).tolist(),"primary_left_functions":primary["left"].tolist(),"sensitivity":{"bin_and_origin":comparisons,"harmonic_order":harmonic},"circular_phase_offsets":offsets,"stability_gate":{"rank_stable":rank_stable,"minimum_projector_principal_cosine":min(stable,default=None),"minimum_matched_left_function_absolute_cosine":min(left,default=None),"thresholds":{"projector_cosine":0.9,"left_function_absolute_cosine":0.8}},"spectrum_label":"cross-correlation strength; not field energy or explained variance","claim_boundary":CLAIM_BOUNDARY,"scientific_contract_explicit":True} + parents={"drl_phase":{"path":str(Path(drl_phase).resolve()),"manifest_sha256":file_sha256(Path(drl_phase)/"manifest.json")},"constant_mean_phase":{"path":str(Path(constant_phase).resolve()),"manifest_sha256":file_sha256(Path(constant_phase)/"manifest.json")},"dynamic_increment":{"path":str(Path(dynamic_increment).resolve()),"manifest_sha256":file_sha256(Path(dynamic_increment)/"manifest.json")}}; hashes={"parents":parents,"canonical_arrays":{k:canonical_array_sha256(v) for k,v in arrays.items()}} + validate_phase_domain(arrays,config,summary,hashes); return PhaseDomainResult(arrays,config,summary,hashes) + +def validate_phase_domain(arrays,config,summary,hashes): + if config.get("schema_id")!=SCHEMA_ID or summary.get("schema_id")!=SCHEMA_ID or config.get("field_estimand")!="separately_centered_phase_coherent_difference_delta_q_prime_phase(phi)=(q_DRL(phi)-mean_q_DRL)-(q_constant_mean(phi)-mean_q_constant_mean)" or config.get("operator")!="A=P(W^(1/2)U)^T/(N*sqrt(3Q)); Q=1": raise ValueError("phase-domain scientific contract contradicted") + if config.get("phase_offset_semantics")!="circular phase offsets only; not time-response lags" or any(config.get(k) is not False for k in ("standardization","whitening","pod")): raise ValueError("phase-domain preprocessing/offset contract contradicted") + d={k:np.asarray(v) for k,v in arrays.items()}; m=len(d["coordinate_weights"]) + if summary.get("N")!=10 or summary.get("Q")!=1 or summary.get("M")!=m or d["primary_cross"].shape!=(3,m) or d["primary_left"].shape!=(3,3): raise ValueError("phase-domain dimensions invalid") + if set(hashes)!={"parents","canonical_arrays"} or set(hashes["canonical_arrays"])!=set(d) or any(hashes["canonical_arrays"][k]!=canonical_array_sha256(v) for k,v in d.items()): raise ValueError("phase-domain hashes invalid") + canonical_json(config); canonical_json(summary); canonical_json(hashes) + +class PhaseDomainTransaction: + def __init__(self,destination): self.destination=Path(destination); self.stage=None + def __enter__(self): + if self.destination.exists(): raise FileExistsError(self.destination) + self.destination.parent.mkdir(parents=True,exist_ok=True); self.stage=Path(tempfile.mkdtemp(prefix=f".{self.destination.name}.partial-",dir=self.destination.parent)); return self + def write(self,result): + validate_phase_domain(result.arrays,result.config,result.summary,result.input_hashes); np.savez_compressed(self.stage/"arrays.npz",**result.arrays) + for n,v in (("config.json",result.config),("summary.json",result.summary),("input_hashes.json",result.input_hashes)): (self.stage/n).write_bytes(canonical_json(v)) + files={p.name:file_sha256(p) for p in self.stage.iterdir()}; (self.stage/"manifest.json").write_bytes(canonical_json({"schema_id":SCHEMA_ID,"complete":True,"files":files})) + def publish(self): + load_phase_domain_result(self.stage,recompute=False); rename_noreplace(self.stage,self.destination); self.stage=None; load_phase_domain_result(self.destination,recompute=True); return self.destination + def __exit__(self,*args): + if self.stage is not None: shutil.rmtree(self.stage,ignore_errors=True) + +def load_phase_domain_result(path,recompute=True): + root=Path(path); manifest=json.loads((root/"manifest.json").read_text()) + if manifest.get("schema_id")!=SCHEMA_ID or not manifest.get("complete") or set(manifest.get("files",{}))!={"arrays.npz","config.json","summary.json","input_hashes.json"}: raise ValueError("phase-domain manifest invalid") + for n,h in manifest["files"].items(): + if file_sha256(root/n)!=h: raise ValueError("phase-domain file hash mismatch") + with np.load(root/"arrays.npz",allow_pickle=False) as z: arrays={k:z[k].copy() for k in z.files} + config=json.loads((root/"config.json").read_text()); summary=json.loads((root/"summary.json").read_text()); hashes=json.loads((root/"input_hashes.json").read_text()); validate_phase_domain(arrays,config,summary,hashes) + for parent in hashes["parents"].values(): + if file_sha256(Path(parent["path"])/"manifest.json")!=parent["manifest_sha256"]: raise ValueError("phase-domain live parent identity changed") + if recompute: + p=hashes["parents"]; fresh=decompose_phase_domain(p["drl_phase"]["path"],p["constant_mean_phase"]["path"],p["dynamic_increment"]["path"]) + for k in arrays: np.testing.assert_allclose(arrays[k],fresh.arrays[k],rtol=2e-11,atol=2e-12) + return {"arrays":arrays,"config":config,"summary":summary,"input_hashes":hashes,"manifest":manifest,"provenance_validation":"VERIFIED: live compact and dynamic-increment parents reread; essential decomposition recomputed" if recompute else "VERIFIED_HASHES_AND_LIVE_PARENTS"} diff --git a/src/CCD_analysis/karman_dynamic/publication.py b/src/CCD_analysis/karman_dynamic/publication.py new file mode 100644 index 0000000..8cc4043 --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/publication.py @@ -0,0 +1,72 @@ +"""Deterministic artifact-only publication for the Karman dynamic campaign.""" +from __future__ import annotations +import json, os, shutil, uuid +from pathlib import Path +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from CCD_analysis.acquisition.artifacts import file_sha256, rename_noreplace +from .contracts import canonical_json +from .dynamic_increment import load_dynamic_increment +from .temporal_ccd import load_temporal_result, CHANNELS +from .phase_domain_ccd import load_phase_domain_result +SCHEMA_ID="ccd-karman-dynamic-publication/v1" +STEMS=("01_four_role_mean_performance","02_drl_constant_phase_difference","03_temporal_ccd_spectrum_sensitivity","04_temporal_ccd_leading_modes","05_temporal_ccd_left_lag_functions","06_phase_domain_ccd_downgrade") +def _save(fig,root,stem): + names=[] + for ext in ("png","pdf"): + q=root/f"{stem}.{ext}"; fig.savefig(q,dpi=300 if ext=="png" else None,bbox_inches="tight",metadata={"Creator":"CCD_analysis.karman_dynamic.publication"}); names.append(q.name) + plt.close(fig); return names +def _wrms(field,w): + a=np.asarray(field,float); return float(np.sqrt(np.sum(w*np.sum(a*a,axis=0))/np.sum(w))) +def _components(v,mask): + n=int(mask.sum()); out=[] + for a in (v[:n],v[n:]): + f=np.full(mask.shape,np.nan); f[mask]=a; out.append(f) + return out +def _panel(ax,f,x,y,mask,lim,title): + ax.pcolormesh(x,y,f.T,shading="nearest",cmap="RdBu_r",vmin=-lim,vmax=lim,rasterized=True); solid=np.ma.masked_where(mask,np.ones(mask.shape)); ax.pcolormesh(x,y,solid.T,shading="nearest",cmap="Greys",vmin=0,vmax=1); ax.set(title=title,xlabel="x/D",ylabel="y/D"); ax.set_aspect("equal") +def _source(root): return {"path":str(root.resolve()),"manifest_sha256":file_sha256(root/"manifest.json")} +def publish_dynamic_figures(dynamic_root,temporal_root,phase_domain_root,output): + dynamic_root,temporal_root,phase_domain_root=map(Path,(dynamic_root,temporal_root,phase_domain_root)); destination=Path(output) + if destination.exists(): raise FileExistsError(destination) + dynamic=load_dynamic_increment(dynamic_root); temporal=load_temporal_result(temporal_root,recompute=True); phase=load_phase_domain_result(phase_domain_root,recompute=True) + if phase["summary"]["decision"]!="DOWNGRADE": raise ValueError("frozen phase-domain decision must be DOWNGRADE") + partial=destination.with_name(f".{destination.name}.partial.{os.getpid()}.{uuid.uuid4().hex}"); partial.mkdir(parents=True) + try: + with np.load(dynamic_root/"arrays.npz",allow_pickle=False) as z: inc={k:z[k].copy() for k in z.files} + ds=dynamic["summary"]; ta,ts=temporal["arrays"],temporal["summary"]; ps=phase["summary"]; files=[] + roles=("target","zero","constant_mean","drl"); errors=[ds["statistics"][r]["mean_target_error_weighted_vector_rms"] for r in roles]; b=ds["benefits"] + fig,ax=plt.subplots(figsize=(6.4,3.8),layout="constrained"); bars=ax.bar(("Target","Zero","Constant mean","DRL"),errors,color=("#59A14F","#777777","#4C78A8","#E45756")); ax.bar_label(bars,fmt="%.4f",padding=3); ax.annotate(f"-{b['zero_to_constant_overall_mean_control_benefit_target_error_reduction']:.4f}",xy=(2,errors[2]),xytext=(1,errors[1]+.012),arrowprops={"arrowstyle":"->"},ha="center"); ax.annotate(f"-{b['constant_to_drl_dynamic_increment_target_error_reduction']:.4f}",xy=(3,errors[3]),xytext=(2,errors[2]+.012),arrowprops={"arrowstyle":"->"},ha="center"); ax.set(ylabel="Mean-field target error (weighted vector RMS)",ylim=(0,max(errors)*1.25),title="Mean performance decomposition"); ax.grid(axis="y",alpha=.25); files+=_save(fig,partial,STEMS[0]) + bins=np.arange(1,11); cp=ds["phase_target_error_metrics"]["constant_mean"]["target_error_by_bin_weighted_vector_rms"]; dp=ds["phase_target_error_metrics"]["drl"]["target_error_by_bin_weighted_vector_rms"]; centered=inc["phase_difference_centered_drl_minus_constant"]; w=inc["quadrature_weights"]; cr=[_wrms(centered[i],w) for i in range(10)] + fig,axs=plt.subplots(1,2,figsize=(10,3.7),layout="constrained"); axs[0].plot(bins,cp,"o-",label="Constant mean"); axs[0].plot(bins,dp,"o-",label="DRL"); axs[0].set(ylabel="Phase target error (weighted vector RMS)",title="Independent 10-bin phase means"); axs[0].legend(); axs[1].plot(bins,cr,"o-",color="#7A5195"); axs[1].set(ylabel="Centered DRL-constant difference (weighted vector RMS)",title="Phase-coherent increment"); [a.set(xlabel="Phase bin",xticks=bins) for a in axs]; [a.grid(alpha=.25) for a in axs]; fig.suptitle("DRL versus constant mean; zero omitted because its phase gate failed"); files+=_save(fig,partial,STEMS[1]) + sigma=ta["primary_singular_values"]; sens=ts["sensitivity"]; fig,axs=plt.subplots(1,2,figsize=(10,3.7),layout="constrained"); axs[0].semilogy(np.arange(1,len(sigma)+1),sigma,"o-",ms=3); axs[0].set(xlabel="Mode",ylabel="Cross-correlation strength",title="Negative-lag CCD spectrum"); spectra=[sigma[:3]]+[np.asarray(v["common_support_vs_primary"]["leading_spectrum"]) for v in sens]; labels=["-17...0"]+[f"{v['lags'][0]}...0" for v in sens] + for vals,label in zip(spectra,labels): axs[1].plot((1,2,3),vals/sigma[:3],"o-",label=label) + axs[1].axhline(1,color="black",lw=.7); axs[1].set(xlabel="Leading mode",ylabel="Strength / primary strength",xticks=(1,2,3),title="Common-support sensitivity (N=21)"); axs[1].legend(title="Lag window"); [a.grid(alpha=.25) for a in axs]; fig.suptitle("Closed-loop temporal co-variation; not causality or response time"); files+=_save(fig,partial,STEMS[2]) + mask,x,y=ta["fluid_mask"],ta["x_D"],ta["y_D"]; comps=[_components(ta["primary_physical_modes"][:,j],mask) for j in range(3)]; fig,axs=plt.subplots(3,2,figsize=(12,7),sharex=True,sharey=True,layout="constrained") + for j,c in enumerate(comps): + lim=float(np.percentile(np.abs(np.concatenate([q[np.isfinite(q)] for q in c])),99)) or 1 + for ax,q,label in zip(axs[j],c,("ux","uy")): _panel(ax,q,x,y,mask,lim,f"Mode {j+1} {label}") + fig.suptitle("Leading temporal CCD physical modes (full grid; per-mode symmetric scale)"); files+=_save(fig,partial,STEMS[3]) + left=ta["primary_left_functions"].reshape(3,len(ta["primary_lags"]),-1); fig,axs=plt.subplots(1,3,figsize=(11,3.4),sharey=True,layout="constrained") + for j,ax in enumerate(axs): + for ch,name in enumerate(CHANNELS): ax.plot(ta["primary_lags"],left[ch,:,j],"o-",ms=3,label=name) + peak=ts["primary_left_lag_metrics"]["modes"][j]["peak_lag_boundaries"]; ax.axvline(peak,color="black",ls="--",lw=.8); ax.set(title=f"Mode {j+1}; energy peak {peak}",xlabel="tau / 800 lattice steps"); ax.grid(alpha=.25) + axs[0].set_ylabel("Left lag-function component"); axs[-1].legend(); fig.suptitle("tau < 0 means action precedes field; lag structure is descriptive"); files+=_save(fig,partial,STEMS[4]) + primary=np.asarray(ps["primary_singular_values"][:3]); variants={"10 bins":primary,"8 bins":ps["sensitivity"]["bin_and_origin"]["bins8"]["leading_spectrum"],"12 bins":ps["sensitivity"]["bin_and_origin"]["bins12"]["leading_spectrum"],"half-bin":ps["sensitivity"]["bin_and_origin"]["bins10_half_shift"]["leading_spectrum"],"harmonic 1":ps["sensitivity"]["harmonic_order"]["1"]["leading_spectrum"]}; fig,axs=plt.subplots(1,2,figsize=(10,3.7),layout="constrained"); axs[0].bar((1,2,3),primary,color="#F58518"); axs[0].set(xlabel="Mode",ylabel="Cross-correlation strength",title="Primary 10-bin spectrum") + for name,vals in variants.items(): axs[1].plot((1,2,3),np.asarray(vals)/primary,"o-",label=name) + axs[1].set(xlabel="Mode",ylabel="Strength / primary strength",xticks=(1,2,3),title="Resolution and harmonic sensitivity"); axs[1].legend(fontsize=8); [a.grid(alpha=.25) for a in axs]; fig.suptitle("PHASE-DOMAIN CCD - DOWNGRADE: first-harmonic rank is 2, not 3"); files+=_save(fig,partial,STEMS[5]) + report={"schema_id":SCHEMA_ID,"artifact_only":True,"sources":{"dynamic_increment":_source(dynamic_root),"temporal_ccd":_source(temporal_root),"phase_domain_ccd":_source(phase_domain_root)},"source_reload":{"dynamic_increment":"VERIFIED live four-role and phase parents","temporal_ccd":temporal["provenance_validation"],"phase_domain_ccd":phase["provenance_validation"]},"figure_files":files,"mean_target_errors":dict(zip(roles,errors)),"mean_target_error_reductions":b,"phase_target_error_cycle_means":{r:ds["phase_target_error_metrics"][r]["target_error_cycle_mean_weighted_vector_rms"] for r in ("constant_mean","drl")},"centered_phase_difference_weighted_vector_rms_by_bin":cr,"zero_phase_output":"PROHIBITED: zero failed the phase gate; no zero phase result is plotted or claimed","temporal_leading_singular_values":sigma[:3].tolist(),"temporal_common_support_sensitivity":sens,"phase_domain_decision":ps["decision"],"phase_domain_leading_singular_values":primary.tolist(),"phase_domain_stability_gate":ps["stability_gate"],"claim_boundary":"Independent trajectory statistics and closed-loop co-variation only; no pointwise counterfactual, causal, mechanism, response-time, uncertainty, CCD>POD, or explained-variance claim."}; (partial/"RESULTS.json").write_bytes(canonical_json(report)); fraction=100*b["constant_to_drl_dynamic_increment_target_error_reduction"]/b["zero_to_drl_total_target_error_reduction"] + lines=["# Karman dynamic-increment publication results","","All outputs were generated from fresh verified reloads of immutable artifacts; no CFD or CUDA was used.","","## Mean performance","",f"Mean target error decreases from zero `{errors[1]:.7f}` to constant mean `{errors[2]:.7f}` (reduction `{b['zero_to_constant_overall_mean_control_benefit_target_error_reduction']:.7f}`), then to DRL `{errors[3]:.7f}` (additional reduction `{b['constant_to_drl_dynamic_increment_target_error_reduction']:.7f}`). The latter is about {fraction:.1f}% of the total zero-to-DRL reduction.","","## Phase and CCD boundary","",f"The cycle-mean 10-bin target error is `{report['phase_target_error_cycle_means']['constant_mean']:.7f}` for constant mean and `{report['phase_target_error_cycle_means']['drl']:.7f}` for DRL. Zero is absent because it failed the phase gate.","",f"Temporal leading strengths are `{report['temporal_leading_singular_values']}`. Common-support comparisons preserve the leading-three subspace; native-support changes are not timing evidence.","",f"Phase-domain CCD is **{ps['decision']}**. Primary rank is 3, but first-harmonic rank is `{ps['sensitivity']['harmonic_order']['1']['rank']}`; only exploratory circular co-variation is supported.","",f"Claim boundary: {report['claim_boundary']}",""]; (partial/"RESULTS.md").write_text("\n".join(lines)); hashes={q.name:file_sha256(q) for q in partial.iterdir() if q.is_file()}; (partial/"manifest.json").write_bytes(canonical_json({"schema_id":SCHEMA_ID,"complete":True,"files":hashes})); rename_noreplace(partial,destination); return destination + except Exception: shutil.rmtree(partial,ignore_errors=True); raise +def load_dynamic_publication(path): + root=Path(path); manifest=json.loads((root/"manifest.json").read_text()) + if manifest.get("schema_id")!=SCHEMA_ID or not manifest.get("complete"): raise ValueError("publication manifest invalid") + for name,h in manifest.get("files",{}).items(): + if file_sha256(root/name)!=h: raise ValueError("publication file hash mismatch") + report=json.loads((root/"RESULTS.json").read_text()) + if report.get("schema_id")!=SCHEMA_ID or report.get("phase_domain_decision")!="DOWNGRADE" or not report.get("zero_phase_output","").startswith("PROHIBITED"): raise ValueError("publication claim contract invalid") + for source in report["sources"].values(): + if file_sha256(Path(source["path"])/"manifest.json")!=source["manifest_sha256"]: raise ValueError("publication source identity changed") + return {"manifest":manifest,"report":report,"provenance_validation":"VERIFIED publication hashes and immutable source manifest identities"} diff --git a/src/CCD_analysis/karman_dynamic/runtime.py b/src/CCD_analysis/karman_dynamic/runtime.py new file mode 100644 index 0000000..9faa5b7 --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/runtime.py @@ -0,0 +1,46 @@ +"""Campaign role adapter; solver/PPO imports remain lazy until guarded execution.""" +from pathlib import Path +import tempfile,shutil +import numpy as np +from CCD_analysis.acquisition.artifacts import file_sha256 +from CCD_analysis.acquisition.runtime import build_role_runtime,initialize_role,run_role_acquisition +from .artifacts import load_role_artifact,publish_wrapper +from .phase import load_phase_compact +from .contracts import CASE_ID,LEGACY_ROLE,constant_mean_provenance,DEFAULT_REPO_MAPPING,DEFAULT_OPTANE_ROOT,EXPECTED_OPTANE_MOUNT,LEASE_PATH,COOLDOWN_PATH +from .safety import require_execution_environment,validate_optane_storage,exclusive_lease,enforce_cooldown +class FixedPolicy: + def __init__(self,normalized): self.normalized=np.asarray(normalized,np.float32); self.device="cpu" + def predict(self,obs,deterministic=True): return self.normalized.copy(),None +def execute_role(*,role,output,campaign_id,warmup_intervals,collect_boundaries,drl_artifact=None,phase_artifact=None,launch_delay_seconds=120,smoke=False): + require_execution_environment(); output=Path(output) + backing=validate_optane_storage(repo_mapping=DEFAULT_REPO_MAPPING,optane_root=DEFAULT_OPTANE_ROOT,mount=EXPECTED_OPTANE_MOUNT) + if backing!=output.parent.resolve(strict=False) and backing not in output.parent.resolve(strict=False).parents: raise ValueError("role output must be below validated Optane campaign root") + if output.exists(): raise FileExistsError(output) + if smoke and (warmup_intervals,collect_boundaries)!=(1,2): raise ValueError("smoke requires exactly --warmup-intervals 1 --collect-boundaries 2") + relative_parent=output.parent.resolve(strict=False).relative_to(backing) + current=backing + for component in relative_parent.parts: + current=current/component + if current.is_symlink(): raise ValueError("campaign output parents must not be symlinks") + current.mkdir(exist_ok=True) + if output.parent.resolve(strict=True).relative_to(backing)!=relative_parent: raise ValueError("campaign output parent escaped validated Optane root") + if output.exists(): raise FileExistsError(output) + total=warmup_intervals+collect_boundaries; staging=Path(tempfile.mkdtemp(prefix=f".{role}.campaign-",dir=output.parent)); payload=staging/"payload"; provenance=None + try: + with exclusive_lease(LEASE_PATH,campaign_id=campaign_id,role=role): + enforce_cooldown(COOLDOWN_PATH,delay_seconds=launch_delay_seconds) + legacy=LEGACY_ROLE[role]; runtime=None; initializer=initialize_role + if role=="constant_mean": + if drl_artifact is None or phase_artifact is None: raise ValueError("constant_mean requires --drl-artifact and --phase-artifact") + gate=load_phase_compact(phase_artifact) + if not gate["summary"]["gate_passed"]: raise ValueError("constant_mean blocked by failed phase/stationarity gate") + drl=load_role_artifact(drl_artifact,expected_role="drl"); start,_=drl["metadata"]["retained_slice"]; provenance=constant_mean_provenance(drl_manifest_sha256=file_sha256(Path(drl_artifact)/"campaign_manifest.json"),effective=drl["telemetry"]["effective_applied_action"],retained_start=start) + if provenance != gate["summary"]["constant_mean_provenance"]: raise ValueError("phase gate constant_mean provenance mismatch") + mean=np.asarray(provenance["constant_mean_physical_action"],np.float32); normalized=((mean/.01)-np.asarray([0.,-4.,4.],np.float32))/8 + if np.any(normalized < -1) or np.any(normalized > 1): raise ValueError("fresh DRL mean cannot be represented by frozen policy action bounds") + runtime=build_role_runtime(case=CASE_ID,role="q_ctl") + def initializer(current,**kwargs): initialize_role(current,**kwargs); current.policy=FixedPolicy(normalized) + run_role_acquisition(case=CASE_ID,role=legacy,output=payload,control_count=total,field_interval=800,runtime=runtime,initializer=initializer) + return publish_wrapper(staging,output,role=role,campaign_id=campaign_id,warmup_intervals=warmup_intervals,collect_boundaries=collect_boundaries,constant_mean_provenance=provenance,smoke=smoke) + except Exception: + shutil.rmtree(staging,ignore_errors=True); raise diff --git a/src/CCD_analysis/karman_dynamic/safety.py b/src/CCD_analysis/karman_dynamic/safety.py new file mode 100644 index 0000000..4ec7436 --- /dev/null +++ b/src/CCD_analysis/karman_dynamic/safety.py @@ -0,0 +1,34 @@ +"""GPU, Optane, lease, and cooldown guards.""" +from contextlib import contextmanager +import json,os,socket,time +from pathlib import Path +from .contracts import MIN_LAUNCH_COOLDOWN_SECONDS +def require_execution_environment(env=None): + env=os.environ if env is None else env + if env.get("CONDA_DEFAULT_ENV")!="pycuda_3_10": raise RuntimeError("requires CONDA_DEFAULT_ENV=pycuda_3_10") + visible=env.get("CUDA_VISIBLE_DEVICES") + if visible is None or not visible.strip() or "," in visible: raise RuntimeError("CUDA_VISIBLE_DEVICES must expose exactly one GPU") + return visible.strip() +def validate_optane_storage(*,repo_mapping,optane_root,mount,is_mount=os.path.ismount,stat=os.stat): + mapping,root,mount=Path(repo_mapping),Path(optane_root).resolve(),Path(mount).resolve() + if not is_mount(mount): raise ValueError("Optane backing path is not a mount") + if not mapping.is_symlink() or mapping.resolve(strict=True)!=root: raise ValueError("repository mapping must be a stable symlink to exact Optane root") + if mount!=root and mount not in root.parents: raise ValueError("campaign root must be under Optane mount") + if stat(root).st_dev!=stat(mount).st_dev: raise ValueError("campaign root is not Optane-backed") + return root +@contextmanager +def exclusive_lease(path,*,campaign_id,role): + path=Path(path); path.parent.mkdir(parents=True,exist_ok=True); payload={"schema_id":"ccd-karman-dynamic-lease/v1","pid":os.getpid(),"host":socket.gethostname(),"campaign_id":campaign_id,"role":role} + try: fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) + except FileExistsError: raise RuntimeError(f"campaign CFD lease already exists; never steal it: {path}") from None + with os.fdopen(fd,"w") as s: json.dump(payload,s,sort_keys=True); s.write("\n"); s.flush(); os.fsync(s.fileno()) + try: yield payload + finally: + if not path.exists() or json.loads(path.read_text())!=payload: raise RuntimeError("campaign lease ownership changed or disappeared") + path.unlink() +def enforce_cooldown(path,*,delay_seconds,monotonic=time.monotonic,sleep=time.sleep): + if delay_seconds=1 required") + +@dataclass(frozen=True) +class TemporalResult: + arrays:dict[str,np.ndarray]; config:dict[str,Any]; summary:dict[str,Any]; input_hashes:dict[str,Any] + +def _memory(inp:TemporalInput,cfg:TemporalConfig)->dict[str,Any]: + n=len(inp.actions); m=2*int(inp.mask.sum()); q=max(len(PRIMARY_LAGS),*(len(x) for x in NEIGHBOR_LAG_WINDOWS)); c=min(n,cfg.chunk_size) + terms={"loaded_role_fields_float32":int(inp.fields.nbytes),"loaded_actions_clocks_masks":int(inp.actions.nbytes+inp.relative_steps.nbytes+inp.absolute_steps.nbytes+inp.cycle_ids.nbytes+inp.mask.nbytes),"field_mean_weights_float64":3*m*8,"largest_cross_modes_float64":3*(3*q)*m*8,"chunk_float64":m*c*8,"small_factors":(3*q*n+3*q*3*q)*8} + raw=sum(terms.values()); peak=int(np.ceil(raw*cfg.safety_margin)); out={"terms_bytes":terms,"raw_peak_ram_bytes":raw,"estimated_peak_ram_bytes":peak,"ram_budget_bytes":cfg.ram_budget_bytes,"safety_margin":cfg.safety_margin,"scratch_bytes":0,"decision":"PASS" if peak<=cfg.ram_budget_bytes else "FAIL","formula":"ceil(safety_margin*declared terms); no MxM or full float64 MxN"} + if peak>cfg.ram_budget_bytes: raise MemoryError(f"temporal CCD memory admission failed: {out}") + return out + +def load_temporal_input(role_root:str|Path,phase_root:str|Path)->TemporalInput: + role_root,phase_root=Path(role_root).resolve(),Path(phase_root).resolve(); d=load_role_artifact(role_root,expected_role="drl"); phase=load_phase_compact(phase_root) + if not phase["summary"]["gate_passed"]: raise ValueError("passing DRL phase artifact required") + if phase["summary"]["source"]["campaign_manifest_sha256"]!=file_sha256(role_root/"campaign_manifest.json"): raise ValueError("phase/role parent mismatch") + start,stop=d["metadata"]["retained_slice"]; a=d["legacy_arrays"]; t=d["telemetry"] + if d["metadata"]["contract"]["contract"]["control_interval"]!=800: raise ValueError("exact 800-lattice-step cadence required") + rel=t["acquisition_relative_lattice_steps"][start:stop].copy(); absolute=a["lattice_steps"][start:stop].copy() + if rel.dtype!=np.int64 or absolute.dtype!=np.int64 or not np.all(np.diff(rel)==800) or not np.all(np.diff(absolute)==800): raise ValueError("exact 800-step clocks required") + r=recover_phase(t["center_sensor_uy"][start:stop]); ids=r["cycle_id"].copy() + with np.load(phase_root/"compact.npz",allow_pickle=False) as z: + if not np.array_equal(z["cycle_ids"],np.unique(ids[ids>=0])) or not np.array_equal(z["fluid_mask"],a["fluid_mask"]): raise ValueError("phase blocks/mask differ from live role") + fields=np.stack((a["ux"][start:stop],a["uy"][start:stop]),axis=1) + actions=t["effective_applied_action"][start:stop,-3:].copy() + if fields.dtype!=np.float32 or actions.dtype!=np.float32 or fields.shape[0]!=len(ids) or actions.shape!=(len(ids),3): raise ValueError("full-resolution DRL fields/actions invalid") + return TemporalInput(role_root,phase_root,file_sha256(role_root/"campaign_manifest.json"),file_sha256(phase_root/"manifest.json"),a["x_D"].copy(),a["y_D"].copy(),a["fluid_mask"].copy(),fields,actions,rel,absolute,ids) + +def _admit(ids:np.ndarray,lags:tuple[int,...],support:np.ndarray|None=None)->tuple[np.ndarray,np.ndarray]: + if not lags or any(type(x) is not int or x>0 for x in lags) or tuple(sorted(set(lags)))!=lags: raise ValueError("lags must be unique increasing nonpositive integers") + local={}; positions={} + for i,b in enumerate(ids): + if b>=0: positions[i]=len(local.setdefault(int(b),[])); local[int(b)].append(i) + field=[]; obs=[] + for i in range(len(ids)): + if i not in positions: continue + seq=local[int(ids[i])]; pos=positions[i]; targets=[pos+lag for lag in lags] + if min(targets)<0: continue + if support is not None and i not in support: continue + field.append(i); obs.append([seq[j] for j in targets]) + if not field: raise ValueError("no complete block-local lag columns") + return np.asarray(field,np.int64),np.asarray(obs,np.int64) + +def _chunks(inp:TemporalInput,indices:np.ndarray,chunk:int)->Iterator[tuple[slice,np.ndarray]]: + mask=inp.mask + for s in range(0,len(indices),chunk): + e=min(s+chunk,len(indices)); raw=inp.fields[indices[s:e]]; u=np.concatenate((raw[:,0][:,mask],raw[:,1][:,mask]),axis=1).T.astype(np.float64); yield slice(s,e),u + +def _decompose(inp:TemporalInput,lags:tuple[int,...],cfg:TemporalConfig,field_mean:np.ndarray,weights:np.ndarray,support:np.ndarray|None=None)->dict[str,np.ndarray]: + fi,oi=_admit(inp.cycle_ids,lags,support); p=inp.actions[oi].transpose(2,1,0).reshape(3*len(lags),len(fi)).astype(np.float64); pm=p.mean(1); roots=np.sqrt(weights); cross=np.zeros((len(pm),len(weights))) + for sl,u in _chunks(inp,fi,cfg.chunk_size): cross+=(p[:,sl]-pm[:,None])@((u-field_mean[:,None])*roots[:,None]).T + cross/=len(fi)*np.sqrt(3*len(lags)); left,s,vh=np.linalg.svd(cross,full_matrices=False); weighted=vh.T + for k in range(weighted.shape[1]): + pivot=int(np.argmax(np.abs(weighted[:,k]))) + if weighted[pivot,k]<0: weighted[:,k]*=-1; left[:,k]*=-1 + modes=weighted/roots[:,None]; coeff=np.empty((len(s),len(fi))); total=0. + for sl,u in _chunks(inp,fi,cfg.chunk_size): x=(u-field_mean[:,None])*roots[:,None]; coeff[:,sl]=weighted.T@x; total+=float(np.sum(x*x)) + residual=np.sqrt(np.maximum(total-np.cumsum(np.sum(coeff*coeff,axis=1)),0)/max(total,np.finfo(float).tiny)) + return {"lags":np.asarray(lags,np.int64),"field_indices":fi,"observable_indices":oi,"observable_mean":pm,"cross_correlation":cross,"left_functions":left,"singular_values":s,"physical_modes":modes,"coefficients":coeff,"weighted_relative_residuals":residual} + +def _left_metrics(result:dict[str,np.ndarray])->dict[str,Any]: + lags=result["lags"]; left=result["left_functions"].reshape(3,len(lags),-1); out=[] + for k in range(left.shape[2]): + lag_energy=np.sum(left[:,:,k]**2,axis=0); lag_energy/=lag_energy.sum(); channel=np.sum(left[:,:,k]**2,axis=1) + out.append({"mode":k+1,"peak_lag_boundaries":int(lags[int(np.argmax(lag_energy))]),"peak_lag_lattice_steps":800*int(lags[int(np.argmax(lag_energy))]),"lag_energy_centroid_boundaries":float(np.sum(lags*lag_energy)),"channel_squared_norms":{CHANNELS[j]:float(channel[j]) for j in range(3)},"coefficient_rms":float(np.sqrt(np.mean(result["coefficients"][k]**2)))}) + return {"modes":out,"lag_energy_by_mode":np.sum(left*left,axis=0).T.tolist()} + +def _compare(a:dict[str,np.ndarray],b:dict[str,np.ndarray],weights:np.ndarray)->dict[str,Any]: + r=min(3,len(a["singular_values"]),len(b["singular_values"])); roots=np.sqrt(weights)[:,None]; va=a["physical_modes"][:,:r]*roots; vb=b["physical_modes"][:,:r]*roots + overlap=np.linalg.svd(va.T@vb,compute_uv=False) + return {"leading_spectrum":b["singular_values"][:r].tolist(),"leading_spectrum_relative_change":((b["singular_values"][:r]-a["singular_values"][:r])/np.maximum(a["singular_values"][:r],np.finfo(float).tiny)).tolist(),"leading_weighted_subspace_principal_cosines":overlap.tolist()} + +def decompose_temporal(inp:TemporalInput,*,streaming_config:TemporalConfig)->TemporalResult: + mem=_memory(inp,streaming_config); mask=inp.mask; point=(coordinate_weights(inp.x_D)[:,None]*coordinate_weights(inp.y_D)[None,:])[mask]; weights=np.concatenate((point,point)); all_idx=np.arange(len(inp.actions),dtype=np.int64) + fsum=np.zeros(len(weights)); count=0 + for _,u in _chunks(inp,all_idx,streaming_config.chunk_size): fsum+=u.sum(1); count+=u.shape[1] + field_mean=fsum/count + primary=_decompose(inp,PRIMARY_LAGS,streaming_config,field_mean,weights); common=primary["field_indices"] + neighbors=[]; common_neighbors=[] + for lags in NEIGHBOR_LAG_WINDOWS: + neighbors.append(_decompose(inp,lags,streaming_config,field_mean,weights)); common_neighbors.append(_decompose(inp,lags,streaming_config,field_mean,weights,support=common)) + arrays={"x_D":inp.x_D,"y_D":inp.y_D,"fluid_mask":mask,"coordinate_weights":weights,"full_run_field_mean":field_mean,"full_run_action_mean":inp.actions.astype(np.float64).mean(0),"effective_actions":inp.actions,"relative_steps":inp.relative_steps,"absolute_steps":inp.absolute_steps,"cycle_ids":inp.cycle_ids} + for prefix,r in [("primary",primary)]+[(f"neighbor_{i}",v) for i,v in enumerate(neighbors)]+[(f"common_neighbor_{i}",v) for i,v in enumerate(common_neighbors)]: + for k,v in r.items(): arrays[f"{prefix}_{k}"]=v + sensitivity=[] + for i,(n,c) in enumerate(zip(neighbors,common_neighbors)): + sensitivity.append({"lags":n["lags"].tolist(),"native_N":int(len(n["field_indices"])),"common_support_N":int(len(c["field_indices"])),"native_vs_primary":_compare(primary,n,weights),"common_support_vs_primary":_compare(primary,c,weights)}) + config={"schema_id":"ccd-karman-temporal-lagged-config/v1","case_id":"karman_re100","role":"drl","primary_lags_boundaries":list(PRIMARY_LAGS),"neighbor_lag_windows_boundaries":[list(x) for x in NEIGHBOR_LAG_WINDOWS],"lag_sign":"tau<0 means action precedes field","cadence_lattice_steps":800,"row_order":ROW_ORDER,"observable_channels":list(CHANNELS),"action_identities":list(ACTION_IDENTITIES),"field_estimand":"full-resolution mask-compressed q_DRL(t)-mean_over_all_360_retained_q_DRL","observable_estimand":"exact same-boundary three-channel effective_applied_action fluctuation; per-lag-row admitted-support mean","operator":"A=P(W^(1/2)U)^T/(N*sqrt(3Q))","center_snapshots":True,"center_observables":True,"standardization":False,"whitening":False,"interpolation":False,"nearest":False,"wrap":False,"block_definition":"independent complete rising-zero-crossing phase cycles","chunk_size":streaming_config.chunk_size,"memory":mem,"claim_boundary":CLAIM_BOUNDARY} + summary={"schema_id":"ccd-karman-temporal-lagged-summary/v1","N":int(len(primary["field_indices"])),"Q":len(PRIMARY_LAGS),"M":len(weights),"cycle_count":int(len(np.unique(inp.cycle_ids[inp.cycle_ids>=0]))),"numerical_rank":int(np.sum(primary["singular_values"]>1e-10*primary["singular_values"][0])),"spectrum_label":"cross-correlation strength; not field energy or explained variance","primary_singular_values":primary["singular_values"].tolist(),"primary_squared_singular_values":(primary["singular_values"]**2).tolist(),"primary_left_lag_metrics":_left_metrics(primary),"sensitivity":sensitivity,"claim_boundary":CLAIM_BOUNDARY,"provenance_status":"VERIFIED_LIVE_ROLE_AND_PHASE_REQUIRED_ON_LOAD"} + hashes={"parents":{"role":{"path":str(inp.role_root),"campaign_manifest_sha256":inp.role_manifest_sha256},"phase":{"path":str(inp.phase_root),"manifest_sha256":inp.phase_manifest_sha256}},"canonical_arrays":{k:canonical_array_sha256(v) for k,v in arrays.items()}} + validate_temporal_result(arrays,config,summary,hashes); return TemporalResult(arrays,config,summary,hashes) + +def validate_temporal_result(arrays,config,summary,hashes): + if config.get("schema_id")!="ccd-karman-temporal-lagged-config/v1" or summary.get("schema_id")!="ccd-karman-temporal-lagged-summary/v1" or config.get("primary_lags_boundaries")!=list(PRIMARY_LAGS) or config.get("row_order")!=ROW_ORDER or config.get("claim_boundary")!=CLAIM_BOUNDARY or summary.get("claim_boundary")!=CLAIM_BOUNDARY: raise ValueError("temporal CCD frozen schema contradicted") + if config.get("field_estimand")!="full-resolution mask-compressed q_DRL(t)-mean_over_all_360_retained_q_DRL" or config.get("observable_channels")!=list(CHANNELS) or config.get("cadence_lattice_steps")!=800 or any(config.get(k) is not False for k in ("standardization","whitening","interpolation","nearest","wrap")): raise ValueError("temporal CCD estimand contradicted") + d={k:np.asarray(v) for k,v in arrays.items()}; n=len(d["effective_actions"]); m=2*int(d["fluid_mask"].sum()) + if summary.get("N")!=len(d["primary_field_indices"]) or summary.get("Q")!=18 or summary.get("M")!=m or d["primary_cross_correlation"].shape!=(54,m) or d["primary_left_functions"].shape[0]!=54 or d["full_run_field_mean"].shape!=(m,): raise ValueError("temporal CCD dimensions invalid") + if d["effective_actions"].shape!=(n,3) or d["cycle_ids"].shape!=(n,) or not np.all(np.diff(d["relative_steps"])==800): raise ValueError("temporal CCD clocks/actions invalid") + if set(hashes)!={"parents","canonical_arrays"} or set(hashes["canonical_arrays"])!=set(d) or any(hashes["canonical_arrays"][k]!=canonical_array_sha256(v) for k,v in d.items()): raise ValueError("temporal CCD hashes invalid") + canonical_json(config); canonical_json(summary); canonical_json(hashes); return d + +class TemporalTransaction: + def __init__(self,destination): self.destination=Path(destination); self.stage=None + def __enter__(self): + if self.destination.exists(): raise FileExistsError(self.destination) + self.destination.parent.mkdir(parents=True,exist_ok=True); self.stage=Path(tempfile.mkdtemp(prefix=f".{self.destination.name}.partial-",dir=self.destination.parent)); return self + def write(self,result): + validate_temporal_result(result.arrays,result.config,result.summary,result.input_hashes); np.savez_compressed(self.stage/"arrays.npz",**result.arrays) + for n,v in (("config.json",result.config),("summary.json",result.summary),("input_hashes.json",result.input_hashes)): (self.stage/n).write_bytes(canonical_json(v)) + files={p.name:file_sha256(p) for p in self.stage.iterdir()}; (self.stage/"manifest.json").write_bytes(canonical_json({"schema_id":SCHEMA_ID,"complete":True,"files":files})) + def publish(self): + load_temporal_result(self.stage,recompute=False); rename_noreplace(self.stage,self.destination); self.stage=None; load_temporal_result(self.destination,recompute=True); return self.destination + def __exit__(self,*args): + if self.stage is not None: shutil.rmtree(self.stage,ignore_errors=True) + +def load_temporal_result(path,recompute=True): + root=Path(path); manifest=json.loads((root/"manifest.json").read_text()) + if manifest.get("schema_id")!=SCHEMA_ID or not manifest.get("complete") or set(manifest.get("files",{}))!={"arrays.npz","config.json","summary.json","input_hashes.json"}: raise ValueError("temporal CCD manifest invalid") + for n,h in manifest["files"].items(): + if file_sha256(root/n)!=h: raise ValueError("temporal CCD file hash mismatch") + with np.load(root/"arrays.npz",allow_pickle=False) as z: arrays={k:z[k].copy() for k in z.files} + config=json.loads((root/"config.json").read_text()); summary=json.loads((root/"summary.json").read_text()); hashes=json.loads((root/"input_hashes.json").read_text()); validate_temporal_result(arrays,config,summary,hashes) + inp=load_temporal_input(hashes["parents"]["role"]["path"],hashes["parents"]["phase"]["path"]) + if inp.role_manifest_sha256!=hashes["parents"]["role"]["campaign_manifest_sha256"] or inp.phase_manifest_sha256!=hashes["parents"]["phase"]["manifest_sha256"]: raise ValueError("temporal CCD live parent identity changed") + if not np.array_equal(inp.actions,arrays["effective_actions"]) or not np.array_equal(inp.relative_steps,arrays["relative_steps"]) or not np.array_equal(inp.cycle_ids,arrays["cycle_ids"]): raise ValueError("temporal CCD live telemetry/blocks changed") + if recompute: + cfg=TemporalConfig(config["chunk_size"],config["memory"]["ram_budget_bytes"],config["memory"]["safety_margin"]); fresh=decompose_temporal(inp,streaming_config=cfg) + for k in arrays: np.testing.assert_allclose(arrays[k],fresh.arrays[k],rtol=2e-11,atol=2e-12) + return {"arrays":arrays,"config":config,"summary":summary,"input_hashes":hashes,"manifest":manifest,"provenance_validation":"VERIFIED: live DRL role and passing phase parents reread; essential decomposition recomputed" if recompute else "VERIFIED_HASHES_AND_LIVE_PARENTS"} diff --git a/src/CCD_analysis/original_ccd/ORIGINAL_CCD_MATH.md b/src/CCD_analysis/original_ccd/ORIGINAL_CCD_MATH.md new file mode 100644 index 0000000..625fdd4 --- /dev/null +++ b/src/CCD_analysis/original_ccd/ORIGINAL_CCD_MATH.md @@ -0,0 +1,154 @@ +# Original full-field CCD mathematical contract + +This active document defines the original canonical correlation decomposition (CCD) from Lyu's formulation. The archived `Lyu23.md` is a read-only historical source, not active code or authority. The private CPU reference freezes the literal equations for tests, and the separate public production API implements this contract after the independent mathematical gate passed. + +## Literal construction + +Let the field snapshots be columns + +\[ +U=[u_1,\ldots,u_N]\in\mathbb C^{M\times N}. +\] + +For one observable sampled at the declared delays \(\tau_1,\ldots,\tau_Q\), let \(p_i=[p(t_i+\tau_1),\ldots,p(t_i+\tau_Q)]^T\) and + +\[ +P=[p_1,\ldots,p_N]\in\mathbb C^{Q\times N}. +\] + +For \(L\) observables, stack their \(Q\)-row blocks in declared order, so \(P\in\mathbb C^{LQ\times N}\). The literal multiobservable normalization used here is + +\[ +A={P U^\dagger\over N\sqrt{LQ}},\qquad +A=R\Sigma V^\dagger. +\] + +For \(L=1\), this is exactly Lyu (2.4), \(A=PU^\dagger/(N\sqrt Q)\). Rows of multiobservable \(P\) are channel-major/delay-minor: row \(lQ+q\) is channel \(l\), declared delay \(q\). The columns of \(V\) are field modes and columns of \(R\) are unit-normalized lag/observable correlation functions in the direct-sum \(\mathbb C^{LQ}\) metric. With coefficients \(a_k=v_k^\dagger X\), the exact empirical identity is + +\[ +{P a_k^\dagger\over N}=\sqrt{LQ}\,\sigma_k r_k. +\] + +Here \(P a_k^\dagger\) uses the complex conjugate of the coefficient row. This is consistent with \(A=PX^\dagger/(N\sqrt{LQ})\); statements of Lyu's correlation as \(R=\langle p^*u +angle\) use the conjugated-observable convention and correspond by complex conjugation, while agreeing for real data. `left_functions_lq()` exposes the same rows as `(L,Q,rank)` without reordering. The values \(\sigma_k^2\) are the discrete variational eigenvalues because + +\[ +A^\dagger A v_k=\sigma_k^2v_k, +\quad +\max_{\|v\|_2=1}\|Av\|_2^2=\sigma_1^2. +\] + +Each observable channel remains in its physical units. Multiplication of all rows by one common complex scalar scales every singular value by its modulus and changes no right-mode projector. Relative channel scaling changes \(P^\dagger P\), hence changes the observable metric and can change modes and ranking; it is never treated as innocuous normalization. + +## Physical weighted inner product + +Let the declared physical inner product be \(\langle f,g\rangle_W=f^\dagger Wg\), where \(W=W^\dagger\succ0\) contains quadrature/component weights. Work in Euclidean coordinates + +\[ +X=W^{1/2}U, +\qquad +A_W={P X^\dagger\over N\sqrt{LQ}} + ={P U^\dagger W^{1/2}\over N\sqrt{LQ}}, +\qquad A_W=R\Sigma V^\dagger. +\] + +Back-transform \(\Phi=W^{-1/2}V\). Then \(\Phi^\dagger W\Phi=I\), and the physical-amplitude coefficients are + +\[ +a=\Phi^\dagger WU=V^\dagger W^{1/2}U=V^\dagger X. +\] + +Thus \(U=\Phi a\) when the complete basis is retained. This square-root coordinate convention is essential: inserting \(W\) directly into the Euclidean SVD would solve a different problem. Diagonal and dense encodings use one scale-invariant HPD conditioning rule: all eigenvalues must be positive and \(\lambda_{\min}/\lambda_{\max}>\epsilon_{ +m real}M\), where \(\epsilon_{ +m real}\) is machine epsilon of the real computation dtype. Consequently a uniform positive rescaling of \(W\) does not change acceptance; singular values scale by the square root of that factor, physical modes by its inverse square root, and the field projector is unchanged. + +## Preprocessing contract + +The original paper's equations apply directly to full field snapshots: there is **no preliminary POD, no row standardization, and no whitening**. Centering is optional declared preprocessing. Production declares snapshot and observable centering independently: `center_snapshots=True` subtracts each row mean from \(U\), and `center_observables=True` subtracts each row mean from \(P\), before constructing \(A\). Either, both, or neither may be selected, and each selection defines the corresponding explicit correlation objective. If a switch is disabled, that input's offsets remain part of the correlation. Centering must never be inferred or silently applied. For exact-lag input, incomplete endpoint columns are dropped first and both declared means are computed only over the resulting \(N_{valid}\) aligned columns. The operator denominator is likewise \(N_{valid}\), and the retained original field indices are returned. + +## Exact clocks, delays, and blocks + +Every CCD column is a physical pairing, not an array roll. The inputs declare either: + +1. explicit physical timestamps and physical delays, requiring exact equality \(t_i+\tau_j=t_k^p\); or +2. an observable index clock and declared integer offsets from an exactly timestamp-matched base sample. + +There is no circular wrap, nearest-time substitution, or crossing of realization/block boundaries. Pair identity is `(block, timestamp)`, so repeated local timestamps in different blocks are valid but duplicate pairs are rejected. Storage may interleave blocks. Integer offsets operate on each block's independent storage subsequence, whose local timestamps must be strictly increasing; unsorted local clocks are rejected rather than silently reordered. A field column is admitted only when all requested lag samples exist in the same block. \(Q\) is any positive integer, including one: odd/even and symmetric/asymmetric delay sets are equally valid. Delay order is preserved. Duplicate delays are allowed deliberately and produce duplicate rows in that declared order. + +Field and observable cadences may differ only under an explicit timestamp pairing/interpolation contract. The derivation reference and its tests authorize exact samples only; nearest matching is forbidden and interpolation remains a future production design decision. + +## Full-rank weighted POD-coordinate equivalence + +Apply the identical declared preprocessing and form \(X=W^{1/2}U\). Let a full-rank POD factorization of the snapshot range be + +\[ +X=\Psi C, +\qquad \Psi^\dagger\Psi=I, +\qquad C=\Psi^\dagger X, +\] + +where `full-rank` means \(\Psi\) spans `range(X)` and \(C\) contains physical-amplitude coefficients—no per-row standardization. Define + +\[ +A_X={PX^\dagger\over N\sqrt{LQ}}, +\qquad +A_C={PC^\dagger\over N\sqrt{LQ}}. +\] + +Since \(X^\dagger=C^\dagger\Psi^\dagger\), \(A_X=A_C\Psi^\dagger\), and therefore + +\[ +A_X^\dagger A_X=\Psi(A_C^\dagger A_C)\Psi^\dagger. +\] + +The nonzero singular values agree and coefficient-space right modes \(z_k\) lift to weighted field modes \(v_k=\Psi z_k\), then physical modes \(\phi_k=W^{-1/2}v_k\). A simple mode agrees up to one complex unit-modulus phase. For an exactly degenerate singular value, individual vectors are not identifiable; equality means equality of the complete degenerate spectral projector. + +For truncated \(\Psi_r\), coordinate CCD solves only + +\[ +\max_{v\in\operatorname{range}(\Psi_r),\ \|v\|_2=1}\|A_Xv\|_2^2, +\] + +namely the constrained optimum in the retained POD subspace. Let \(E_1\) be the top eigenspace of \(A_X^\dagger A_X\) and \(S=\operatorname{range}(\Psi_r)\). The constrained leading value equals the full leading value if and only if \(E_1\cap S\ne\{0\}\). If the top eigenvalue is simple, this says its unique vector (up to complex phase) lies in \(S\). For a degenerate top eigenvalue, a partial intersection preserves the leading value and a shared maximizing vector but not the full top projector. Recovering the complete top projector requires \(E_1\subseteq S\). With no intersection the constrained value is strictly smaller. The same containment statement applies mode cluster by mode cluster. A low-energy but observable-correlated structure can therefore be removed before CCD. This is a subspace statement, not a POD-superiority or CCD-superiority claim. + +If coefficient rows have unequal/non-isotropic scales and are standardized, the transformed matrix is \(D^{-1}C\), not physical-amplitude \(C\), and generally changes the metric and CCD objective. In an exactly degenerate POD subspace the covariance is isotropic, so all admissible unitary rotations have equal row variance and standardization is rotation invariant there; the prior contrary statement was false. For a merely near-degenerate cluster, unequal variances make the operation basis dependent, although the admissibility and numerical meaning of arbitrary rotations then depend on a separately declared tolerance rather than exact spectral degeneracy. + + +## Singular blocks and identifiable reconstruction + +The numerical SVD vectors are retained for transparent algebra, but individual vectors are not identifiable inside a nonzero degenerate singular block, and null/near-zero vectors are arbitrary. `singular_block_rtol` and `singular_block_atol` declare both the nonzero threshold and block clustering tolerance. The result exposes `singular_blocks`, `identifiable_mode_mask`, and `identifiable_rank`; default reconstruction uses only identifiable non-null modes. + +A configured rank or default reconstruction selection may not split a declared nonzero degenerate block. Such a request fails closed. A caller may explicitly set `allow_basis_dependent=True` on reconstruction to select arbitrary returned SVD representatives, but that output is marked basis-dependent and carries no individual-mode identification claim. Rank clipping in `CCDConfig` has no override and is rejected if it cuts a block. In physical coordinates the invariant object for a complete block \(B\) is its weighted projector + +\[ +\Pi_B=\Phi_B\Phi_B^\dagger W, +\] + +not any individual column. Phase convention fixes only a representative phase and does not change this identifiability policy. + +## Lyu published synthetic example (3.1–3.2) + +The archived paper defines + +\[ +\begin{aligned} +u(x,t)={}&2\cos(t-x)+1.5\cos(2t)\cos(2x)+\cos(3t)\cos(3x)\\ +&+0.5\cos(4t)\cos(4x)+\cos(6t)\cos(6x)e^{-0.1(x-\pi)^2}+100r(t,x), +\end{aligned}\tag{3.1} +\] + +where \(r\) is uniform on \([-0.5,0.5]\), and + +\[ +p(t)=\cos(t-\pi/4)+\sin(2t-\pi/3)+\cos(4t)+\cos(6t-\pi/12).\tag{3.2} +\] + +The correlated field structures are the traveling-wave pair at frequency 1 and the frequency 2, 4, and 6 structures. The energetic \(\cos(3t)\cos(3x)\) term and random noise are uncorrelated. The published discretization uses 128 spatial points, \(Q=128\), \(\Delta\tau=2\pi/128\), \(N=10^4\) cycles (1,280,000 field snapshots), and noise amplitude 100, giving an observable Nyquist angular harmonic 64 and harmonic resolution 1 over the \(2\pi\) lag window. The reference test executes those exact published parameters with chunked direct accumulation of \(P U^\dagger\), algebraically identical to materializing the full matrices while avoiding their multi-gigabyte storage. It predeclares and checks the traveling-wave pair followed by frequency-2, localized frequency-6, and frequency-4 structures; the leading \(\sigma^2\) ratio \(4:4:2.25\); frequency-3 suppression; and separation from the noise floor. This is an exact-parameter stochastic reproduction of the equations, not a pixel reproduction of the published figure. A separate noiseless test isolates algebra, and multiple smaller-N seeds test convergence statistically. + +## Frozen real-case specialization + +The first real-data use is now frozen by [`REAL_CASE_CCD_CONTRACT.md`](REAL_CASE_CCD_CONTRACT.md). For `karman_re100` and `illusion_1.0L` separately, it mandates centered full-resolution mask-compressed `dq_ctl=q_ctl-q_blk`, the three centered native-unit exact-field-time q_ctl effective action channels, `Q=1`, `tau=0`, the literal weighted operator above, and no POD/whitening/standardization. It also freezes flattening, quadrature, provenance, OOM-safe streaming, immutable schema, and claim limits. This is a documentation contract only: the streaming adapter and real-case CCD results do not yet exist. + +## Gate and scope + +`_reference.py` remains private, derivation-only, CPU test support and is not imported by `original_ccd/__init__.py`. The independent mathematical gate passed before implementation. The public CPU API now provides the production full-field decomposition, exact lag construction, coefficients, and weighted field-projection reconstruction defined here. It introduces no immutable artifact format, CLI, CFD execution, real-case result, observable prediction, empirical mechanism claim, or CCD-versus-POD superiority claim. diff --git a/src/CCD_analysis/original_ccd/README.md b/src/CCD_analysis/original_ccd/README.md new file mode 100644 index 0000000..b85d861 --- /dev/null +++ b/src/CCD_analysis/original_ccd/README.md @@ -0,0 +1,61 @@ +# Original full-field CCD production API + +This package implements Lyu's original full-field operator +`A = P (W^1/2 U)† / (N sqrt(LQ))` and its direct rectangular SVD. There is no +pre-reduction, whitening, standardization, or inferred centering. + +`U` is `(M,N)`. Supplied `P` may be `(LQ,N)` (declared as `L=1`) or explicit +`(L,Q,N)`. `LaggedObservables` is a validated immutable public value: its finite +2D matrix, positive `L/Q`, `LQ` row count, delay count, channel-major/delay-minor +row order, and unique nonnegative field indices must agree at construction. +`fit()` additionally checks indices against `U`, selects admitted columns first, +and then computes declared `U` and `P` means and the denominator using +`N_valid`. The result reports `selected_field_indices` and `valid_sample_count`. + +```python +from CCD_analysis.original_ccd import CCDConfig, build_lagged_observables, fit + +lagged = build_lagged_observables( + field_times, observable_times, observable_values, delays, + field_blocks=field_blocks, observable_blocks=observable_blocks, + delay_kind="time", +) +result = fit( + U, lagged, weight=quadrature_weights, + config=CCDConfig(center_snapshots=True, center_observables=True, + chunk_size=4096, singular_block_rtol=1e-10), +) +projection = result.reconstruct() # all identifiable non-null blocks +residual = result.residual() +R_lq = result.left_functions_lq() # (L,Q,rank), no row reordering +``` + +For coefficient row `a_k`, the complex empirical identity is +`P @ a_k.conj() / N_valid = sqrt(LQ) * sigma_k * r_k`. Thus left functions are +unit vectors in the channel/direct-delay sum. This `P` convention is conjugate +to writing Lyu's correlation as `` and agrees directly for real data. + +One-dimensional weights are efficient positive diagonal `W`; two-dimensional +weights are dense complex Hermitian HPD. Both use the same scale-invariant +condition `lambda_min/lambda_max > eps(real computation dtype) * M`. Uniformly +rescaling `W` is therefore accepted identically: singular values scale with its +square root while the weighted field projector remains unchanged. + +Returned SVD vectors include transparent null representatives, but +`identifiable_mode_mask` excludes near-zero vectors and `singular_blocks` +records nonzero degenerate clusters under declared tolerances. Default +reconstruction uses `identifiable_rank`. Rank or index selections that split a +nonzero block, or select null vectors, fail closed. Only an explicit +`allow_basis_dependent=True` reconstruction can request arbitrary SVD vectors; +it does not make them identifiable. `CCDConfig.rank` cannot clip through a +block. The invariant physical block object is `Phi_B Phi_B† W`; phase fixing +only chooses representatives. + +Reconstruction is a weighted projection of field snapshots, not observable +prediction. Exact lags never wrap, sort, interpolate, cross blocks, or use a +nearest timestamp. No CFD, real-case result, causal interpretation, or +method-superiority claim is included. + +## Frozen real-case adapter boundary + +`REAL_CASE_CCD_CONTRACT.md` specializes this API for the two authoritative real cases: centered full-resolution `dq_ctl`, exact-time centered q_ctl effective actions in front/upper/lower native units, `Q=1`, `tau=0`, physical coordinate quadrature, provenance-validated mask compression, bounded streaming, and immutable no-clobber results. It is a contract only. The present in-memory API is not itself the real-artifact streaming adapter, and no real-case CCD result has been run. diff --git a/src/CCD_analysis/original_ccd/REAL_CASE_CCD_CONTRACT.md b/src/CCD_analysis/original_ccd/REAL_CASE_CCD_CONTRACT.md new file mode 100644 index 0000000..4ad4b1e --- /dev/null +++ b/src/CCD_analysis/original_ccd/REAL_CASE_CCD_CONTRACT.md @@ -0,0 +1,108 @@ +# Real-case Q=1 original CCD contract + +This document is the active mathematical and adapter contract for the first real-case CCD analyses of `karman_re100` and `illusion_1.0L`. It specializes the literal weighted Lyu operator in `ORIGINAL_CCD_MATH.md`; it does not change that operator. The estimand remains frozen here. The active `real_ccd/` package now implements the streaming adapter and immutable schema; no real-artifact preflight, real-case decomposition run, published result artifact, or empirical result has been performed. + +## Frozen inputs and admitted columns + +Each case is processed independently. The only admissible top-level input is its authoritative direct-dq result together with all three immutable live acquisition roots recorded by that result: + +- `evidence/direct-dq-karman-burn120000/` and its recorded schema-v3 Kármán acquisition roots; +- `evidence/direct-dq-illusion-authorized-burn90000/` and its recorded certificate-authorized schema-v3 Illusion acquisition roots. + +Every preflight, run, and reload must reread both levels. It must validate exact inventories and hashes; case/role/schema and velocity-decoder identity; grid and coordinates; all three solver-derived masks and their persisted intersection; original and selected integer indices/timestamps; selected role fields; and q_ctl action identity, order, units, field-time values, and source/control history lineage. Missing live roots, duplicate or non-increasing selected times, non-finite data, any mismatch, or an incomplete exact join fails closed. A result directory alone is not portable evidence. There is no silent trim, nearest-time match, interpolation, timestamp tolerance, phase guess, crop, translated/rebuilt mask, or block wrap. + +The admitted columns are exactly the direct-dq selected columns, in persisted order: 120 Kármán columns selected by relative lattice step `>120000`, and 144 Illusion columns selected by relative lattice step `>90000`. These counts are input-contract facts, not CCD results. + +## Frozen field estimand, mask, flattening, and weights + +At every admitted exact field time, + +\[ +dq_{ctl}(t_n)=q_{ctl}(t_n)-q_{blk}(t_n). +\] + +Only full-resolution `u_x,u_y` degrees of freedom at `analysis_fluid_mask` points are admitted. That authoritative mask is the exact persisted intersection of the three solver-derived fluid masks. The adapter must neither crop the 1280-by-512 grid nor infer a geometry mask. + +The frozen flattening is component-major, then NumPy C order on `(x,y)`: first `dq_ctl[:,0,:,:][:,analysis_fluid_mask]` in the mask order produced by C-order flattening (`x` major, `y` minor), then the identically ordered `u_y` values. Equivalently, each snapshot column has `M=2*count_nonzero(analysis_fluid_mask)` rows, with all `u_x` rows before all `u_y` rows. The result must persist this declaration, dimensions, coordinates, and a hash of the exact boolean mask. + +Let `w_x=coordinate_weights(x_D)` and `w_y=coordinate_weights(y_D)` use the active endpoint-half/interior-centered coordinate quadrature. For every admitted fluid point `(i,j)`, the diagonal physical weight is `w_x[i]*w_y[j]`, repeated once for `u_x` and once for `u_y` in the frozen flatten order. Weights are positive native `D^2` area weights and are not normalized by mask area. The result must persist the coordinate/weight rule, exact weight vector (or losslessly reproducible coordinates plus its canonical hash), dtype, and hash. Computation uses the literal square-root coordinates `X=W^(1/2)U`; inserting `W` directly into the Euclidean SVD is forbidden. + +For the admitted columns, compute each spatial row mean in stable floating-point accumulation and explicitly form the conceptual centered matrix + +\[ +U_c(:,n)=dq_{ctl}(:,n)-\overline{dq}_{ctl}. +\] + +`center_snapshots=true` is mandatory. The field mean, accumulation dtype/method, count, and centering flag must be persisted. The authoritative direct-dq mean `dq_ctl` remains a separate physical context result; it is reported beside mean actions and must not be called a CCD mode. + +## Frozen observable and mean-action algebra + +The observable has exactly `L=3` channels and `Q=1`, in this immutable order and identity: + +1. `front` / `front_ccw_positive`; +2. `upper` / `upper_ccw_positive`; +3. `lower` / `lower_ccw_positive`. + +For every admitted field column, take the last three entries of q_ctl `effective_applied_action` at that exact field timestamp. These are the solver EMA commands at the completed lattice step, in the artifact's native physical action units. Requested normalized/physical actions, another role's actions, a control-boundary neighbor, interpolated values, reordering, sign changes, or unit conversion are forbidden. The adapter must revalidate the q_ctl field-time-to-control-history lineage before admitting a column. + +Let `P` be the resulting real `3 x N` matrix. Compute and persist each channel mean and explicitly center each row, `P_c=P-mu 1_N^T`; `center_observables=true` is mandatory. There is no channel standardization, whitening, RMS scaling, nondimensionalization, or metric change. Native-unit relative channel scaling is part of the estimand. + +Centering does not discard a constant action contribution to the centered operator. Since `U_c 1_N=0`, + +\[ +P U_c^\dagger=(P_c+\mu 1_N^T)U_c^\dagger +=P_cU_c^\dagger+\mu(U_c1_N)^\dagger=P_cU_c^\dagger. +\] + +Mean effective actions and the authoritative mean `dq_ctl` are therefore reported outside CCD as physical context, never as modes. + +## Frozen Q=1, tau=0 operator + +The first analyses are separate case-wise runs with exactly `Q=1` and `tau=0`. CCD does not require `Q>1`. The only join is the exact common field/action sample at the same admitted timestamp. No response lag is guessed, and no `Q=2`, `Q=12`, period/phase window, or lagged extension is implicitly authorized. + +With `LQ=3`, the literal weighted operator is + +\[ +A_W={P_c(W^{1/2}U_c)^\dagger\over N\sqrt{3}}\in\mathbb R^{3\times M}, +\qquad A_W=R\Sigma V^\dagger, +\qquad \Phi=W^{-1/2}V. +\] + +The implementation must accumulate the equivalent `3 x M` cross-correlation and perform a direct rectangular/thin SVD. It must not perform POD pre-reduction, whitening, action/row standardization, covariance normalization, implicit centering, or materialize an `M x M` operator. `sigma` and `sigma^2` are labeled only as cross-correlation strength in this declared native-action/weighted-field metric; they are not field energy, explained variance, or canonical correlation coefficients. + +Coefficients are `a=V^dagger W^(1/2)U_c`. Full-field modes are persisted in the frozen mask-compressed order with an exact unflattening declaration; optional full-grid `u_x/u_y` views place values only on the authoritative mask. Optional vorticity is derived from modes and labeled as derived, not independently decomposed. + +## Implemented streaming and memory contract + +The active adapter is mask-compressed and OOM-safe: + +1. Pass 1 rereads and validates provenance and exact columns, then computes stable field and three-channel action means, counts, and finite-value checks. +2. Pass 2 rereads chunks, explicitly centers and applies `sqrt(W)`, and accumulates only the `3 x M` cross-correlation before thin SVD. +3. Pass 3 rereads exact columns to stream coefficients, weighted residual scalars at complete singular-block boundaries, and declared spot checks. + +Passes may be safely fused only when numerical equivalence and the same memory bound are demonstrated. The adapter must never hold a full float64 `M x N` copy, multiple full-field duplicates, an `M x M` operator, or default full `M x N` reconstructions. Selected snapshot reconstructions are generated from modes and coefficients only on explicit request and are saved no-clobber. + +Before work begins, preflight must compute conservative peak RAM and scratch estimates from actual `M,N,L,Q`, input and accumulation dtypes, chunk size, resident mode/cross-correlation/coefficient arrays, decompression or loader buffers, temporary arrays, and an explicit safety margin. The formulas, terms, available/allowed budgets, safety margin, and decision are persisted. Missing budgets or estimates exceeding either budget fail closed. This contract does not choose a machine-specific budget; it requires the future run configuration to declare one. + +## Immutable result schema contract + +Each case publishes separately through fsync-backed atomic no-replace semantics. A partial or failed directory must never satisfy the loader. Schema v1 is implemented as canonical `arrays.npz`, `config.json`, `summary.json`, and `input_hashes.json`, covered by a complete hash manifest and atomic no-replace transaction. Schema evolution may not omit these required identities and products: + +- schema ID/version, completeness marker, exact file inventory and SHA256 hashes; +- case, `Q=1`, `tau=0`, `L=3`, row/channel order, all config and numerical tolerances; +- direct-dq result identity, all three absolute live acquisition roots and their manifest/config/file identities, and compatibility-certificate identity where applicable; +- exact original/selected indices and timestamps, selected count, grid/coordinates, three solver masks, analysis mask, flatten order, component order, weights, and hashes; +- exact action identities/order/native units/semantics and q_ctl action/control-history lineage identities; +- field/action means, centering flags, accumulation methods/dtypes, and finite/count checks; +- conservative RAM/scratch estimate, formulas, budget, margin, and pass/chunk strategy; +- weighted full-field `u_x/u_y` modes, singular values, three zero-lag left-function channels, coefficients at exact timestamps, numerical rank, null threshold, and degenerate singular blocks; +- weighted reconstruction residual metrics only at complete nondegenerate/block boundaries, plus any explicitly requested selected reconstructions and their validation; +- claim-boundary strings and loader provenance status. + +Every preflight/run/reload revalidates live provenance. Fresh-process reload must recompute enough identities, dimensions, orthogonality/singular relations, and persisted scalar checks to fail closed on corruption or contradiction. Full reconstructed `M x N` fields are not a default schema member. + +## Deliverables and claim boundary + +Per case, eventual artifact-derived reporting includes mean effective actions and authoritative mean `dq_ctl` outside CCD; `sigma` and `sigma^2` cross-correlation-strength spectra; leading full-field velocity modes; three zero-lag left vectors; exact-time coefficients; and complete-block weighted residual curves. Degenerate blocks are interpreted as subspaces/projectors, not unique individual modes. + +The analyses do not support a CCD-over-POD claim, causality, mechanism, response time, same phase, independent realizations, uncertainty from one record, or an observable-prediction claim. Ordinary exact-sample lag correlations may be computed only after both Q=1 results validate, are descriptive diagnostics rather than CCD or causal evidence, and require stopping for user approval before any lagged-CCD contract or run. diff --git a/src/CCD_analysis/original_ccd/__init__.py b/src/CCD_analysis/original_ccd/__init__.py new file mode 100644 index 0000000..bcda4af --- /dev/null +++ b/src/CCD_analysis/original_ccd/__init__.py @@ -0,0 +1,7 @@ +"""Production original full-field canonical correlation decomposition.""" +from .api import CCDConfig, CCDResult, LaggedObservables, decompose, fit, flatten_fields, unflatten_fields +from .delays import build_lagged_observables + +IMPLEMENTATION_AVAILABLE = True + +__all__ = ["IMPLEMENTATION_AVAILABLE", "CCDConfig", "CCDResult", "LaggedObservables", "build_lagged_observables", "decompose", "fit", "flatten_fields", "unflatten_fields"] diff --git a/src/CCD_analysis/original_ccd/_reference.py b/src/CCD_analysis/original_ccd/_reference.py new file mode 100644 index 0000000..042d934 --- /dev/null +++ b/src/CCD_analysis/original_ccd/_reference.py @@ -0,0 +1,169 @@ +"""Private, derivation-only CPU reference for the original Lyu CCD contract. + +This module is intentionally not imported by :mod:`CCD_analysis.original_ccd`. +It favors literal mathematics and fail-closed timing checks over performance. +""" +from dataclasses import dataclass +from typing import Optional, Sequence + +import numpy as np + + +@dataclass(frozen=True) +class _ReferenceResult: + cross_correlation: np.ndarray + left_functions: np.ndarray + singular_values: np.ndarray + weighted_modes: np.ndarray + physical_modes: np.ndarray + physical_amplitude_coefficients: np.ndarray + preprocessed_snapshots: np.ndarray + + +def _hermitian_square_roots(weight: np.ndarray): + weight = np.asarray(weight) + if weight.ndim != 2 or weight.shape[0] != weight.shape[1]: + raise ValueError("W must be square") + if not np.allclose(weight, weight.conj().T, rtol=1e-12, atol=1e-12): + raise ValueError("W must be Hermitian") + values, vectors = np.linalg.eigh(weight) + scale = max(1.0, float(np.max(np.abs(values)))) + if np.min(values) <= 1e-12 * scale: + raise ValueError("W must be positive definite") + root = (vectors * np.sqrt(values)) @ vectors.conj().T + inverse = (vectors * (1.0 / np.sqrt(values))) @ vectors.conj().T + return root, inverse + + +def _as_observable_rows(observables: np.ndarray, sample_count: int): + values = np.asarray(observables) + if values.ndim == 2: + if values.shape[1] != sample_count: + raise ValueError("P must have N columns") + return values, 1, values.shape[0] + if values.ndim == 3: + if values.shape[2] != sample_count: + raise ValueError("P must have shape (L,Q,N)") + return values.reshape(values.shape[0] * values.shape[1], sample_count), values.shape[0], values.shape[1] + raise ValueError("P must have shape (Q,N) or (L,Q,N)") + + +def _reference_ccd(snapshots: np.ndarray, observables: np.ndarray, *, weight: Optional[np.ndarray] = None, center: bool = False) -> _ReferenceResult: + """Evaluate the literal weighted-coordinate CCD equations without whitening.""" + u = np.asarray(snapshots) + if u.ndim != 2 or u.shape[1] == 0: + raise ValueError("U must have nonempty shape (M,N)") + p, observable_count, delay_count = _as_observable_rows(observables, u.shape[1]) + if delay_count <= 0 or observable_count <= 0: + raise ValueError("L and Q must be positive") + raw_weight = np.eye(u.shape[0]) if weight is None else np.asarray(weight) + dtype = np.result_type(u.dtype, p.dtype, raw_weight.dtype, np.float64) + if not (np.can_cast(u.dtype, dtype, casting="safe") and np.can_cast(p.dtype, dtype, casting="safe") and np.can_cast(raw_weight.dtype, dtype, casting="safe")): + raise TypeError("U, P, and W must cast losslessly to the computation dtype") + u = u.astype(dtype, copy=True) + p = p.astype(dtype, copy=True) + if not np.all(np.isfinite(u)) or not np.all(np.isfinite(p)): + raise ValueError("U and P must be finite") + if center: + u -= u.mean(axis=1, keepdims=True) + p -= p.mean(axis=1, keepdims=True) + m, n = u.shape + if raw_weight.shape != (m, m): + raise ValueError(f"W must have exact shape ({m}, {m})") + w = raw_weight.astype(dtype, copy=True) + if not np.all(np.isfinite(w)): + raise ValueError("W must be finite") + root, inverse = _hermitian_square_roots(w) + x = root @ u + a = p @ x.conj().T / (n * np.sqrt(observable_count * delay_count)) + r, sigma, vh = np.linalg.svd(a, full_matrices=True) + v = vh.conj().T + physical_modes = inverse @ v + coefficients = physical_modes.conj().T @ w @ u + return _ReferenceResult(a, r, sigma, v, physical_modes, coefficients, u) + + +def _literal_cross_correlation(snapshots: np.ndarray, observables: np.ndarray): + u = np.asarray(snapshots) + p, l, q = _as_observable_rows(observables, u.shape[1]) + return p @ u.conj().T / (u.shape[1] * np.sqrt(l * q)) + + +def _build_lag_matrix(field_times: Sequence, observable_times: Sequence, observables: np.ndarray, delays: Sequence, *, field_blocks: Optional[Sequence] = None, observable_blocks: Optional[Sequence] = None, delay_kind: str = "time", interpolation: str = "exact"): + """Build exact lag columns using canonical per-block timestamp order. + + Storage may interleave blocks. Within each block timestamps must be strictly + increasing after selecting their storage subsequence; unsorted local clocks + are rejected rather than silently canonicalized. ``(block, timestamp)`` + pairs must be unique. Duplicate delays are allowed and preserve declaration + order, because they deliberately duplicate rows of P. + """ + ft = np.asarray(field_times) + ot = np.asarray(observable_times) + y = np.asarray(observables) + if y.ndim == 1: + y = y[None, :] + if ft.ndim != 1 or ot.ndim != 1 or y.ndim != 2 or y.shape[1] != ot.size: + raise ValueError("invalid clock or observable shape") + if interpolation != "exact": + raise NotImplementedError("only exact timestamp pairing is authorized") + fb = np.zeros(ft.size, dtype=np.int64) if field_blocks is None else np.asarray(field_blocks) + ob = np.zeros(ot.size, dtype=np.int64) if observable_blocks is None else np.asarray(observable_blocks) + if fb.shape != ft.shape or ob.shape != ot.shape: + raise ValueError("block labels must match clock shape") + delays = list(delays) + if not delays: + raise ValueError("Q must be positive") + + def pair(value): + return value.item() if hasattr(value, "item") else value + + field_pairs = [(pair(fb[i]), pair(ft[i])) for i in range(ft.size)] + observable_pairs = [(pair(ob[i]), pair(ot[i])) for i in range(ot.size)] + if len(set(field_pairs)) != len(field_pairs): + raise ValueError("field (block, timestamp) pairs must be unique") + if len(set(observable_pairs)) != len(observable_pairs): + raise ValueError("observable (block, timestamp) pairs must be unique") + + block_sequences = {} + for storage_index, (block, time) in enumerate(observable_pairs): + block_sequences.setdefault(block, []).append((time, storage_index)) + for block, sequence in block_sequences.items(): + local_times = [item[0] for item in sequence] + if any(not local_times[i] < local_times[i + 1] for i in range(len(local_times) - 1)): + raise ValueError(f"observable timestamps must be strictly increasing within block {block!r}") + + lookup = {key: index for index, key in enumerate(observable_pairs)} + local_positions = { + (block, time): (position, sequence) + for block, sequence in block_sequences.items() + for position, (time, _) in enumerate(sequence) + } + columns, field_indices = [], [] + for i, (block, time) in enumerate(field_pairs): + indices = [] + for delay in delays: + if delay_kind == "time": + index = lookup.get((block, pair(time + delay))) + elif delay_kind == "index": + if not isinstance(delay, (int, np.integer)): + raise ValueError("index delays must be integers") + local = local_positions.get((block, time)) + if local is None: + index = None + else: + position, sequence = local + target = position + int(delay) + index = sequence[target][1] if 0 <= target < len(sequence) else None + else: + raise ValueError("delay_kind must be 'time' or 'index'") + if index is None: + indices = [] + break + indices.append(index) + if indices: + columns.append(y[:, indices].reshape(-1)) + field_indices.append(i) + if not columns: + raise ValueError("no complete exact lag columns") + return np.stack(columns, axis=1), np.asarray(field_indices, dtype=np.int64) diff --git a/src/CCD_analysis/original_ccd/api.py b/src/CCD_analysis/original_ccd/api.py new file mode 100644 index 0000000..5d68f54 --- /dev/null +++ b/src/CCD_analysis/original_ccd/api.py @@ -0,0 +1,362 @@ +"""Production CPU implementation of original full-field Lyu CCD. + +The implementation SVDs the rectangular ``(L*Q) x M`` cross-correlation and +never forms an ``M x M`` operator. Rows are always channel-major/delay-minor. +""" +from dataclasses import dataclass +from typing import Optional, Sequence, Tuple, Union + +import numpy as np + +Array = np.ndarray +ROW_ORDER = "channel-major_delay-minor" + + +def _positive_int(name: str, value: object) -> int: + if not isinstance(value, (int, np.integer)) or isinstance(value, (bool, np.bool_)) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return int(value) + + +def _numeric_finite_array(name: str, value: object) -> Array: + array = np.asarray(value) + if array.dtype.kind not in "iufc" or array.dtype.kind == "b": + raise TypeError(f"{name} must have a real or complex numeric dtype") + if not np.all(np.isfinite(array)): + raise ValueError(f"{name} must be finite") + return array + + +@dataclass(frozen=True) +class CCDConfig: + """Declared preprocessing, rank-identification, and numerical configuration.""" + + center_snapshots: bool = False + center_observables: bool = False + rank: Optional[int] = None + chunk_size: Optional[int] = None + phase_convention: bool = True + singular_block_rtol: float = 1e-10 + singular_block_atol: float = 0.0 + + +@dataclass(frozen=True) +class LaggedObservables: + """Validated exact lag matrix and field-column mapping. + + ``matrix`` rows are channel-major/delay-minor: row ``l*Q + q`` stores + channel ``l`` at declared delay ``q``. + """ + + matrix: Array + field_indices: Array + delays: Tuple[object, ...] + observable_count: int + delay_count: int + row_order: str = ROW_ORDER + + def __post_init__(self) -> None: + matrix = _numeric_finite_array("lagged matrix", self.matrix) + if matrix.ndim != 2 or 0 in matrix.shape: + raise ValueError("lagged matrix must have nonempty shape (LQ,N)") + observable_count = _positive_int("observable_count", self.observable_count) + delay_count = _positive_int("delay_count", self.delay_count) + delays = tuple(self.delays) + if len(delays) != delay_count: + raise ValueError("lagged delays length must equal delay_count") + if matrix.shape[0] != observable_count * delay_count: + raise ValueError("lagged metadata L*Q must equal the matrix row count") + if self.row_order != ROW_ORDER: + raise ValueError(f"row_order must be {ROW_ORDER!r}") + indices = np.asarray(self.field_indices) + if indices.ndim != 1 or not np.issubdtype(indices.dtype, np.integer): + raise ValueError("lagged field_indices must be one-dimensional integers") + indices = indices.astype(np.int64, copy=True) + if indices.size != matrix.shape[1] or np.any(indices < 0): + raise ValueError("lagged field_indices must be nonnegative and match matrix columns") + if np.unique(indices).size != indices.size: + raise ValueError("lagged field_indices must not contain duplicates") + matrix = matrix.copy() + matrix.setflags(write=False) + indices.setflags(write=False) + object.__setattr__(self, "matrix", matrix) + object.__setattr__(self, "field_indices", indices) + object.__setattr__(self, "delays", delays) + object.__setattr__(self, "observable_count", observable_count) + object.__setattr__(self, "delay_count", delay_count) + + def as_lqn(self) -> Array: + """Return the explicit channel-major ``(L,Q,N)`` view.""" + return self.matrix.reshape(self.observable_count, self.delay_count, self.matrix.shape[1]) + + +@dataclass(frozen=True) +class CCDResult: + """Original-CCD factors and weighted field-projection data.""" + + config: CCDConfig + cross_correlation: Array + left_functions: Array + singular_values: Array + weighted_modes: Array + physical_modes: Array + coefficients: Array + snapshot_mean: Array + observable_mean: Array + preprocessed_snapshots: Array + selected_field_indices: Array + observable_count: int + delay_count: int + singular_blocks: Tuple[Tuple[int, int], ...] + identifiable_mode_mask: Array + + @property + def rank(self) -> int: + """Number of retained numerical SVD vectors, including any null vectors.""" + return int(self.singular_values.size) + + @property + def identifiable_rank(self) -> int: + """Number of retained non-null vectors; complete degenerate blocks count fully.""" + return int(np.count_nonzero(self.identifiable_mode_mask)) + + @property + def valid_sample_count(self) -> int: + """Number of admitted aligned columns used in the empirical average.""" + return int(self.preprocessed_snapshots.shape[1]) + + def left_functions_lq(self) -> Array: + """Return unit-normalized left functions as ``(L,Q,rank)``.""" + return self.left_functions.reshape(self.observable_count, self.delay_count, self.rank) + + def _mode_indices(self, rank: Optional[int], mode_indices: Optional[Sequence[int]], allow_basis_dependent: bool) -> Array: + if rank is not None and mode_indices is not None: + raise ValueError("specify rank or mode_indices, not both") + if mode_indices is None: + use_rank = self.identifiable_rank if rank is None else _validate_rank(rank, self.rank) + indices = np.arange(use_rank, dtype=np.int64) + else: + raw = np.asarray(mode_indices) + if raw.ndim != 1 or raw.size == 0 or not np.issubdtype(raw.dtype, np.integer): + raise ValueError("mode_indices must be a nonempty one-dimensional integer sequence") + indices = raw.astype(np.int64, copy=False) + if np.any(indices < 0) or np.any(indices >= self.rank): + raise ValueError("mode index is out of range") + if np.unique(indices).size != indices.size: + raise ValueError("mode_indices must not contain duplicates") + if allow_basis_dependent: + return indices + chosen = set(indices.tolist()) + if any(not self.identifiable_mode_mask[index] for index in indices): + raise ValueError("null/near-zero modes are basis-dependent; pass allow_basis_dependent=True explicitly") + for start, stop in self.singular_blocks: + overlap = chosen.intersection(range(start, stop)) + if overlap and len(overlap) != stop - start: + raise ValueError("selection splits a nonzero degenerate singular block; pass allow_basis_dependent=True explicitly") + return indices + + def reconstruct(self, *, rank: Optional[int] = None, mode_indices: Optional[Sequence[int]] = None, restore_snapshot_mean: bool = False, allow_basis_dependent: bool = False) -> Array: + """Reconstruct a weighted field projection. + + By default this uses all identifiable non-null modes and refuses a rank + or index selection that splits a nonzero degenerate block. Setting + ``allow_basis_dependent=True`` explicitly requests arbitrary SVD vectors. + """ + indices = self._mode_indices(rank, mode_indices, allow_basis_dependent) + reconstructed = self.physical_modes[:, indices] @ self.coefficients[indices] + if restore_snapshot_mean: + reconstructed = reconstructed + self.snapshot_mean + return reconstructed + + def residual(self, *, rank: Optional[int] = None, mode_indices: Optional[Sequence[int]] = None, allow_basis_dependent: bool = False) -> Array: + """Return preprocessed snapshots minus the selected weighted projection.""" + return self.preprocessed_snapshots - self.reconstruct(rank=rank, mode_indices=mode_indices, allow_basis_dependent=allow_basis_dependent) + + +def flatten_fields(fields: Array) -> Tuple[Array, Tuple[int, ...]]: + """Flatten ``(..., N)`` field samples to ``(M, N)`` and return field shape.""" + values = _numeric_finite_array("fields", fields) + if values.ndim < 2 or values.shape[-1] == 0: + raise ValueError("fields must have nonempty shape (..., N)") + field_shape = tuple(values.shape[:-1]) + return values.reshape(int(np.prod(field_shape)), values.shape[-1]), field_shape + + +def unflatten_fields(snapshots: Array, field_shape: Sequence[int]) -> Array: + """Restore a flattened ``(M,N)`` matrix to ``(...,N)``.""" + values = _numeric_finite_array("snapshots", snapshots) + shape = tuple(int(value) for value in field_shape) + if values.ndim != 2 or not shape or any(value <= 0 for value in shape): + raise ValueError("snapshots and field_shape are invalid") + if int(np.prod(shape)) != values.shape[0]: + raise ValueError("field_shape does not match the flattened field dimension") + return values.reshape(shape + (values.shape[1],)) + + +def _observable_rows(observables: Array, sample_count: int) -> Tuple[Array, int, int]: + values = _numeric_finite_array("P", observables) + if values.ndim == 2: + if values.shape[0] == 0 or values.shape[1] != sample_count: + raise ValueError("P must have nonempty shape (LQ,N)") + return values, 1, values.shape[0] + if values.ndim == 3: + if 0 in values.shape or values.shape[2] != sample_count: + raise ValueError("P must have nonempty shape (L,Q,N)") + return values.reshape(values.shape[0] * values.shape[1], sample_count), values.shape[0], values.shape[1] + raise ValueError("P must have shape (LQ,N) or (L,Q,N)") + + +def _validate_config(config: CCDConfig) -> None: + if not isinstance(config, CCDConfig): + raise TypeError("config must be CCDConfig") + if type(config.center_snapshots) is not bool or type(config.center_observables) is not bool or type(config.phase_convention) is not bool: + raise TypeError("centering and phase declarations must be bool") + if config.chunk_size is not None: + _positive_int("chunk_size", config.chunk_size) + for name, value in (("singular_block_rtol", config.singular_block_rtol), ("singular_block_atol", config.singular_block_atol)): + if not isinstance(value, (int, float, np.integer, np.floating)) or isinstance(value, (bool, np.bool_)) or not np.isfinite(value) or value < 0: + raise ValueError(f"{name} must be finite and nonnegative") + + +def _validate_rank(rank: object, maximum: int) -> int: + value = _positive_int("rank", rank) + if value > maximum: + raise ValueError(f"rank must be between 1 and {maximum}") + return value + + +def _conditioning_threshold(dtype: np.dtype, size: int) -> float: + """Relative HPD eigenvalue floor: ``eps(real computation dtype) * size``.""" + real_dtype = np.empty((), dtype=dtype).real.dtype + return float(np.finfo(real_dtype).eps * max(1, size)) + + +def _validate_positive_spectrum(values: Array, dtype: np.dtype, size: int) -> None: + maximum = float(np.max(values)) + minimum = float(np.min(values)) + if maximum <= 0 or minimum <= 0 or minimum / maximum <= _conditioning_threshold(dtype, size): + raise ValueError("W must be positive definite and numerically well-conditioned by min/max ratio") + + +def _weight_transforms(weight: Optional[Array], size: int, dtype: np.dtype): + if weight is None: + diagonal = np.ones(size, dtype=np.float64) + return "diagonal", diagonal, diagonal + raw = _numeric_finite_array("W", weight) + if raw.ndim == 1: + if raw.shape != (size,): + raise ValueError(f"diagonal W must have exact shape ({size},)") + diagonal = raw.astype(dtype, copy=True) + if np.iscomplexobj(diagonal) and not np.all(diagonal.imag == 0): + raise ValueError("diagonal W must be real positive") + diagonal = diagonal.real + _validate_positive_spectrum(diagonal, dtype, size) + return "diagonal", np.sqrt(diagonal), 1.0 / np.sqrt(diagonal) + if raw.ndim != 2 or raw.shape != (size, size): + raise ValueError(f"dense W must have exact shape ({size}, {size})") + dense = raw.astype(dtype, copy=True) + hermitian_scale = float(np.max(np.abs(dense))) + tolerance = _conditioning_threshold(dtype, size) * hermitian_scale + if hermitian_scale == 0 or float(np.max(np.abs(dense - dense.conj().T))) > tolerance: + raise ValueError("W must be Hermitian") + values, vectors = np.linalg.eigh(dense) + _validate_positive_spectrum(values, dtype, size) + root = (vectors * np.sqrt(values)) @ vectors.conj().T + inverse = (vectors * (1.0 / np.sqrt(values))) @ vectors.conj().T + return "dense", root, inverse + + +def _apply_transform(kind: str, transform: Array, values: Array) -> Array: + return transform[:, None] * values if kind == "diagonal" else transform @ values + + +def _phase_fix(left: Array, right: Array) -> Tuple[Array, Array]: + left = left.copy() + right = right.copy() + for column in range(right.shape[1]): + pivot = int(np.argmax(np.abs(right[:, column]))) + value = right[pivot, column] + if value != 0: + phase = np.conj(value) / abs(value) + right[:, column] *= phase + left[:, column] *= phase + return left, right + + +def _classify_singular_values(values: Array, config: CCDConfig) -> Tuple[Array, Tuple[Tuple[int, int], ...]]: + if values.size == 0: + return np.zeros(0, dtype=bool), () + scale = float(values[0]) + null_tolerance = config.singular_block_atol + config.singular_block_rtol * scale + identifiable = values > null_tolerance + blocks = [] + stop_nonzero = int(np.count_nonzero(identifiable)) + start = 0 + while start < stop_nonzero: + stop = start + 1 + while stop < stop_nonzero and abs(float(values[stop] - values[start])) <= config.singular_block_atol + config.singular_block_rtol * max(float(values[start]), float(values[stop])): + stop += 1 + if stop - start > 1: + blocks.append((start, stop)) + start = stop + return identifiable, tuple(blocks) + + +def decompose(snapshots: Array, observables: Array, *, weight: Optional[Array] = None, config: Optional[CCDConfig] = None) -> CCDResult: + """Decompose aligned full-field snapshots and supplied lag observables.""" + cfg = CCDConfig() if config is None else config + _validate_config(cfg) + u_raw = _numeric_finite_array("U", snapshots) + if u_raw.ndim != 2 or 0 in u_raw.shape: + raise ValueError("U must have nonempty shape (M,N)") + p_raw, observable_count, delay_count = _observable_rows(observables, u_raw.shape[1]) + weight_raw = np.ones(u_raw.shape[0]) if weight is None else _numeric_finite_array("W", weight) + dtype = np.result_type(u_raw.dtype, p_raw.dtype, weight_raw.dtype, np.float64) + u = u_raw.astype(dtype, copy=True) + p = p_raw.astype(dtype, copy=True) + snapshot_mean = u.mean(axis=1, keepdims=True) if cfg.center_snapshots else np.zeros((u.shape[0], 1), dtype=dtype) + observable_mean = p.mean(axis=1, keepdims=True) if cfg.center_observables else np.zeros((p.shape[0], 1), dtype=dtype) + u -= snapshot_mean + p -= observable_mean + kind, root, inverse = _weight_transforms(weight, u.shape[0], dtype) + chunk = u.shape[1] if cfg.chunk_size is None else min(int(cfg.chunk_size), u.shape[1]) + cross = np.zeros((p.shape[0], u.shape[0]), dtype=dtype) + for start in range(0, u.shape[1], chunk): + stop = min(start + chunk, u.shape[1]) + cross += p[:, start:stop] @ _apply_transform(kind, root, u[:, start:stop]).conj().T + cross /= u.shape[1] * np.sqrt(observable_count * delay_count) + left, singular_values, vh = np.linalg.svd(cross, full_matrices=False) + weighted_modes = vh.conj().T + if cfg.phase_convention: + left, weighted_modes = _phase_fix(left, weighted_modes) + identifiable, blocks = _classify_singular_values(singular_values, cfg) + maximum_rank = singular_values.size + retained_rank = maximum_rank if cfg.rank is None else _validate_rank(cfg.rank, maximum_rank) + if cfg.rank is not None: + if any(start < retained_rank < stop for start, stop in blocks): + raise ValueError("configured rank splits a nonzero degenerate singular block") + left = left[:, :retained_rank] + singular_values = singular_values[:retained_rank] + weighted_modes = weighted_modes[:, :retained_rank] + identifiable = identifiable[:retained_rank] + blocks = tuple((start, stop) for start, stop in blocks if stop <= retained_rank) + physical_modes = _apply_transform(kind, inverse, weighted_modes) + coefficients = np.empty((retained_rank, u.shape[1]), dtype=dtype) + for start in range(0, u.shape[1], chunk): + stop = min(start + chunk, u.shape[1]) + coefficients[:, start:stop] = weighted_modes.conj().T @ _apply_transform(kind, root, u[:, start:stop]) + return CCDResult(cfg, cross, left, singular_values, weighted_modes, physical_modes, coefficients, snapshot_mean, observable_mean, u, np.arange(u.shape[1], dtype=np.int64), observable_count, delay_count, blocks, identifiable) + + +def fit(snapshots: Array, observables: Union[Array, LaggedObservables], *, weight: Optional[Array] = None, config: Optional[CCDConfig] = None) -> CCDResult: + """Fit CCD, applying a validated exact-lag field-column mapping first.""" + if not isinstance(observables, LaggedObservables): + return decompose(snapshots, observables, weight=weight, config=config) + u = _numeric_finite_array("U", snapshots) + if u.ndim != 2 or 0 in u.shape: + raise ValueError("U must have nonempty shape (M,N)") + indices = observables.field_indices + if np.any(indices >= u.shape[1]): + raise ValueError("lagged field_indices do not map valid U columns") + result = decompose(u[:, indices], observables.as_lqn(), weight=weight, config=config) + return CCDResult(result.config, result.cross_correlation, result.left_functions, result.singular_values, result.weighted_modes, result.physical_modes, result.coefficients, result.snapshot_mean, result.observable_mean, result.preprocessed_snapshots, indices.copy(), observables.observable_count, observables.delay_count, result.singular_blocks, result.identifiable_mode_mask) diff --git a/src/CCD_analysis/original_ccd/delays.py b/src/CCD_analysis/original_ccd/delays.py new file mode 100644 index 0000000..d44b973 --- /dev/null +++ b/src/CCD_analysis/original_ccd/delays.py @@ -0,0 +1,94 @@ +"""Exact, block-local lag construction for original CCD.""" +from typing import Optional, Sequence + +import numpy as np + +from .api import LaggedObservables, _numeric_finite_array + + +def _scalar(value): + return value.item() if hasattr(value, "item") else value + + +def _pairs(times, blocks, label): + pairs = [] + for block, time in zip(blocks, times): + pair = (_scalar(block), _scalar(time)) + try: + hash(pair) + except TypeError as exc: + raise TypeError(f"{label} block/timestamp values must be hashable") from exc + pairs.append(pair) + if len(set(pairs)) != len(pairs): + raise ValueError(f"{label} (block, timestamp) pairs must be unique") + return pairs + + +def build_lagged_observables(field_times: Sequence, observable_times: Sequence, observables: np.ndarray, delays: Sequence, *, field_blocks: Optional[Sequence] = None, observable_blocks: Optional[Sequence] = None, delay_kind: str = "time", interpolation: str = "exact") -> LaggedObservables: + """Construct exact lag columns, preserving delay and field-column order.""" + field_clock = np.asarray(field_times) + observable_clock = np.asarray(observable_times) + values = _numeric_finite_array("observables", observables) + if field_clock.ndim != 1 or observable_clock.ndim != 1: + raise ValueError("field_times and observable_times must be one-dimensional") + if field_clock.dtype.kind not in "iuf" or observable_clock.dtype.kind not in "iuf": + raise TypeError("timestamps must have a real numeric dtype") + if not np.all(np.isfinite(field_clock)) or not np.all(np.isfinite(observable_clock)): + raise ValueError("timestamps must be finite") + if values.ndim == 1: + values = values[None, :] + if values.ndim != 2 or values.shape[0] == 0 or values.shape[1] != observable_clock.size: + raise ValueError("observables must have shape (L, observable_sample_count)") + if interpolation != "exact": + raise NotImplementedError("only exact timestamp pairing is supported") + delay_values = tuple(delays) + if not delay_values: + raise ValueError("Q must be positive") + if delay_kind not in ("time", "index"): + raise ValueError("delay_kind must be 'time' or 'index'") + if delay_kind == "index" and any(not isinstance(value, (int, np.integer)) or isinstance(value, (bool, np.bool_)) for value in delay_values): + raise ValueError("index delays must be integers") + field_block_values = np.zeros(field_clock.size, dtype=np.int64) if field_blocks is None else np.asarray(field_blocks) + observable_block_values = np.zeros(observable_clock.size, dtype=np.int64) if observable_blocks is None else np.asarray(observable_blocks) + if field_block_values.shape != field_clock.shape or observable_block_values.shape != observable_clock.shape: + raise ValueError("block labels must exactly match their clock shape") + field_pairs = _pairs(field_clock, field_block_values, "field") + observable_pairs = _pairs(observable_clock, observable_block_values, "observable") + sequences = {} + for storage_index, (block, time) in enumerate(observable_pairs): + sequences.setdefault(block, []).append((time, storage_index)) + for block, sequence in sequences.items(): + local_times = [entry[0] for entry in sequence] + if any(not local_times[index] < local_times[index + 1] for index in range(len(local_times) - 1)): + raise ValueError(f"observable timestamps must be strictly increasing within block {block!r}") + lookup = {pair: index for index, pair in enumerate(observable_pairs)} + local_positions = {(block, time): (position, sequence) for block, sequence in sequences.items() for position, (time, _) in enumerate(sequence)} + columns = [] + field_indices = [] + for field_index, (block, time) in enumerate(field_pairs): + sample_indices = [] + for delay in delay_values: + if delay_kind == "time": + try: + target_time = _scalar(time + delay) + except (TypeError, ValueError) as exc: + raise TypeError("time delays must be compatible with timestamp dtype") from exc + observable_index = lookup.get((block, target_time)) + else: + local = local_positions.get((block, time)) + if local is None: + observable_index = None + else: + position, sequence = local + target = position + int(delay) + observable_index = sequence[target][1] if 0 <= target < len(sequence) else None + if observable_index is None: + sample_indices = [] + break + sample_indices.append(observable_index) + if sample_indices: + columns.append(values[:, sample_indices].reshape(-1)) + field_indices.append(field_index) + if not columns: + raise ValueError("no complete exact lag columns") + return LaggedObservables(np.stack(columns, axis=1), np.asarray(field_indices, dtype=np.int64), delay_values, values.shape[0], len(delay_values)) diff --git a/src/CCD_analysis/real_ccd/__init__.py b/src/CCD_analysis/real_ccd/__init__.py new file mode 100644 index 0000000..7f9a967 --- /dev/null +++ b/src/CCD_analysis/real_ccd/__init__.py @@ -0,0 +1,5 @@ +"""Active real-case provenance-validated streaming CCD package.""" +from .core import MemoryBudget,RealCCDInput,RealCCDResult,StreamingConfig,decompose_streaming,estimate_memory,inspect_direct_dq_dimensions,load_validated_input +from .io import ResultTransaction,load_result +from .preflight import available_host_memory_bytes,preflight_real_artifact,safe_host_budget +__all__=["MemoryBudget","StreamingConfig","RealCCDInput","RealCCDResult","estimate_memory","inspect_direct_dq_dimensions","load_validated_input","decompose_streaming","ResultTransaction","load_result","available_host_memory_bytes","safe_host_budget","preflight_real_artifact"] diff --git a/src/CCD_analysis/real_ccd/__main__.py b/src/CCD_analysis/real_ccd/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/src/CCD_analysis/real_ccd/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/src/CCD_analysis/real_ccd/cli.py b/src/CCD_analysis/real_ccd/cli.py new file mode 100644 index 0000000..de05bd8 --- /dev/null +++ b/src/CCD_analysis/real_ccd/cli.py @@ -0,0 +1,46 @@ +"""CPU-only CLI for provenance-bound real-case CCD.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Sequence + +from .core import MemoryBudget, StreamingConfig, decompose_streaming, load_validated_input +from .io import ResultTransaction +from .preflight import preflight_real_artifact, safe_host_budget + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Provenance-bound streaming real-case CCD (CPU only)") + subparsers = parser.add_subparsers(dest="command", required=True) + for command in ("preflight", "run"): + subparser = subparsers.add_parser(command) + subparser.add_argument("--direct-dq-root", required=True, type=Path) + subparser.add_argument("--chunk-size", required=True, type=int) + subparser.add_argument("--available-host-bytes", type=int, help="Override detected MemAvailable for deterministic admission tests") + if command == "run": + subparser.add_argument("--output", required=True, type=Path) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + root = args.direct_dq_root.resolve() + if args.command == "preflight": + report = preflight_real_artifact(root, chunk_size=args.chunk_size, available_bytes=args.available_host_bytes) + print(json.dumps(report, sort_keys=True)) + return 0 + + output = args.output.resolve() + if output.exists(): + raise FileExistsError(output) + host = safe_host_budget(available_bytes=args.available_host_bytes) + config = StreamingConfig(args.chunk_size, MemoryBudget(host["ram_budget_bytes"], 0, 1.25)) + inp, memory = load_validated_input(root, streaming_config=config) + result = decompose_streaming(inp, streaming_config=config, memory_estimate=memory) + with ResultTransaction(output) as transaction: + transaction.write(result) + published = transaction.publish() + print(json.dumps({"result": str(published), "case_id": result.config["case_id"], "chunk_size": args.chunk_size, "host_memory": host, "memory": memory, "summary": result.summary}, sort_keys=True)) + return 0 diff --git a/src/CCD_analysis/real_ccd/core.py b/src/CCD_analysis/real_ccd/core.py new file mode 100644 index 0000000..dcdf8a6 --- /dev/null +++ b/src/CCD_analysis/real_ccd/core.py @@ -0,0 +1,147 @@ +"""Provenance-bound, mask-compressed streaming real-case Q=1 CCD.""" +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterator, Mapping +import json +import numpy as np +from CCD_analysis.acquisition.contracts import ACTION_IDENTITIES +from CCD_analysis.direct_dq.analysis import coordinate_weights +from CCD_analysis.direct_dq.io import load_acquisition_artifact, load_result as load_direct_dq_result +from CCD_analysis.acquisition.artifacts import file_sha256 +from CCD_analysis.direct_dq.schema import canonical_array_sha256 + +ACTION_CHANNEL_NAMES=("front","upper","lower") +ACTION_UNITS="native solver angular-velocity command units" +FLATTEN_ORDER="component-major ux then uy; C-order analysis-mask point order" +WEIGHT_RULE="direct_dq.coordinate_weights(x_D)*coordinate_weights(y_D), repeated ux then uy; not area-normalized" + +@dataclass(frozen=True) +class MemoryBudget: + ram_bytes:int + scratch_bytes:int + safety_margin:float=1.25 + def __post_init__(self): + if type(self.ram_bytes) is not int or type(self.scratch_bytes) is not int or self.ram_bytes<=0 or self.scratch_bytes<0 or not np.isfinite(self.safety_margin) or self.safety_margin<1: + raise ValueError("explicit positive RAM, nonnegative scratch, and safety_margin>=1 required") + +@dataclass(frozen=True) +class StreamingConfig: + chunk_size:int + budget:MemoryBudget + singular_block_rtol:float=1e-10 + singular_block_atol:float=0.0 + def __post_init__(self): + if type(self.chunk_size) is not int or self.chunk_size<=0: raise ValueError("chunk_size must be positive") + if min(self.singular_block_rtol,self.singular_block_atol)<0: raise ValueError("singular tolerances must be nonnegative") + +@dataclass(frozen=True) +class RealCCDInput: + case_id:str; direct_dq_root:Path; direct_manifest_sha256:str + acquisition_identities:Mapping[str,Mapping[str,Any]] + x_D:np.ndarray; y_D:np.ndarray; role_masks:Mapping[str,np.ndarray]; analysis_mask:np.ndarray + selected_indices:np.ndarray; selected_relative_steps:np.ndarray + q_ctl_absolute_steps:np.ndarray; dq_ctl:np.ndarray; actions:np.ndarray + authoritative_mean_dq_ctl:np.ndarray + +@dataclass(frozen=True) +class RealCCDResult: + arrays:dict[str,np.ndarray]; config:dict[str,Any]; summary:dict[str,Any]; input_hashes:dict[str,Any] + +def _read_json(path:Path)->dict[str,Any]: + value=json.loads(path.read_text()) + if not isinstance(value,dict): raise ValueError(f"JSON object required: {path}") + return value + +def estimate_memory(*,m:int,n:int,nx:int,ny:int,full_count:int,chunk_size:int,budget:MemoryBudget)->dict[str,Any]: + if min(m,n,nx,ny,full_count,chunk_size)<=0: raise ValueError("memory dimensions must be positive") + c=min(chunk_size,n); grid=nx*ny + terms={ + "validated_direct_result_float32_fields":6*n*2*grid*4, + "validated_live_acquisition_float32_fields":3*full_count*2*grid*4, + "loader_decompression_and_copy_allowance":(6*n+3*full_count)*2*grid*4, + "field_mean_float64":m*8,"weights_and_roots_float64":2*m*8, + "cross_and_modes_float64":(3*m+2*3*m)*8, + "chunk_float64_working_set":3*m*c*8, + "actions_coefficients_and_small_svd":(3*n+3*n+30)*8, + } + raw=sum(terms.values()); peak=int(np.ceil(raw*budget.safety_margin)); scratch_raw=0; scratch=int(np.ceil(scratch_raw*budget.safety_margin)) + decision=peak<=budget.ram_bytes and scratch<=budget.scratch_bytes + out={"formula":"ceil(safety_margin * sum(terms)); loader residency explicitly included; no MxM or full float64 MxN term","terms_bytes":terms,"raw_peak_ram_bytes":raw,"estimated_peak_ram_bytes":peak,"raw_scratch_bytes":scratch_raw,"estimated_scratch_bytes":scratch,"ram_budget_bytes":budget.ram_bytes,"scratch_budget_bytes":budget.scratch_bytes,"safety_margin":budget.safety_margin,"decision":"PASS" if decision else "FAIL"} + if not decision: raise MemoryError(f"real-CCD memory/scratch estimate exceeds explicit budget: {out}") + return out + +def inspect_direct_dq_dimensions(path:str|Path)->dict[str,int]: + root=Path(path); summary=_read_json(root/'summary.json'); config=_read_json(root/'config.json') + if config.get('schema_id')!='ccd-direct-dq-config/v2': raise ValueError('authoritative direct-dq config required') + identities=_read_json(root/'input_hashes.json'); qctl=Path(identities['q_ctl']['path']); acq_manifest=_read_json(qctl/'manifest.json'); acq_config=_read_json(qctl/'config.json') + frame=acq_config['runtime']['coordinate_frame'] + return {"n":int(summary['sample_count']),"nx":int(frame['x']['count']),"ny":int(frame['y']['count']),"full_count":int(acq_manifest['field_count'])} + +def load_validated_input(path:str|Path,*,streaming_config:StreamingConfig)->tuple[RealCCDInput,dict[str,Any]]: + root=Path(path).resolve(); dims=inspect_direct_dq_dimensions(root) + estimate=estimate_memory(m=2*dims['nx']*dims['ny'],chunk_size=streaming_config.chunk_size,budget=streaming_config.budget,**dims) + direct=load_direct_dq_result(root); a=direct['arrays']; cfg=direct['config']; recorded=direct['input_hashes']; case=cfg['case_id'] + mask=a['analysis_fluid_mask']; m=2*int(mask.sum()) + tight=estimate_memory(m=m,chunk_size=streaming_config.chunk_size,budget=streaming_config.budget,**dims) + qctl=load_acquisition_artifact(recorded['q_ctl']['path'],expected_case=case,expected_role='q_ctl') + idx=a['selected_timeline_indices']; rel=a['selected_acquisition_relative_lattice_steps'] + if not np.array_equal(qctl.fields['acquisition_relative_lattice_steps'][idx],rel): raise ValueError('q_ctl exact selected indices/timestamps mismatch') + actions=qctl.fields['effective_applied_action'][idx,-3:].copy() + if actions.dtype!=np.float32 or actions.shape!=(idx.size,3) or not np.isfinite(actions).all(): raise ValueError('q_ctl effective field-time actions invalid') + dq=(a['q_ctl_instantaneous']-a['q_blk_instantaneous']) + if not np.array_equal(dq,a['dq_ctl_instantaneous']): raise ValueError('authoritative direct_dq dq_ctl identity failed') + inp=RealCCDInput(case,root,file_sha256(root/'manifest.json'),recorded,a['x_D'].copy(),a['y_D'].copy(),{r:a[f'{r}_solver_fluid_mask'].copy() for r in ('q_target','q_blk','q_ctl')},mask.copy(),idx.copy(),rel.copy(),qctl.fields['lattice_steps'][idx].copy(),dq,actions,a['dq_ctl_mean'].copy()) + estimate={**tight,"conservative_all_fluid_estimated_peak_ram_bytes":estimate['estimated_peak_ram_bytes'],"admission_basis":"all-fluid M estimate before authoritative loader; tight mask estimate also passed"} + return inp,estimate + +def _chunks(inp:RealCCDInput,chunk:int)->Iterator[tuple[slice,np.ndarray,np.ndarray]]: + mask=inp.analysis_mask + for start in range(0,inp.selected_indices.size,chunk): + stop=min(start+chunk,inp.selected_indices.size); raw=inp.dq_ctl[start:stop] + field=np.concatenate((raw[:,0][:,mask],raw[:,1][:,mask]),axis=1).T.astype(np.float64) + actions=inp.actions[start:stop].T.astype(np.float64) + if not np.isfinite(field).all() or not np.isfinite(actions).all(): raise ValueError('nonfinite streamed column') + yield slice(start,stop),field,actions + +def _classify(s:np.ndarray,rtol:float,atol:float)->tuple[np.ndarray,list[list[int]],list[int],float]: + tol=atol+(rtol*float(s[0]) if s.size else 0.0); identifiable=s>tol; blocks=[]; boundaries=[]; stop=int(identifiable.sum()); start=0 + while start1: blocks.append([start,end]) + boundaries.append(end); start=end + return identifiable,blocks,boundaries,tol + +def decompose_streaming(inp:RealCCDInput,*,streaming_config:StreamingConfig,memory_estimate:dict[str,Any]|None=None)->RealCCDResult: + if inp.actions.shape!=(inp.selected_indices.size,3): raise ValueError('exactly three action channels required') + n=inp.selected_indices.size + for name, values in (("selected indices", inp.selected_indices), ("selected relative timestamps", inp.selected_relative_steps), ("q_ctl absolute timestamps", inp.q_ctl_absolute_steps)): + if values.dtype != np.int64 or values.shape != (n,) or np.any(np.diff(values) <= 0): + raise ValueError(f'{name} must be exact strictly increasing int64 values') + mask=inp.analysis_mask; points=int(mask.sum()); m=2*points + estimate=memory_estimate or estimate_memory(m=m,n=n,nx=inp.x_D.size,ny=inp.y_D.size,full_count=inp.dq_ctl.shape[0],chunk_size=streaming_config.chunk_size,budget=streaming_config.budget) + wx=coordinate_weights(inp.x_D); wy=coordinate_weights(inp.y_D); point_w=(wx[:,None]*wy[None,:])[mask]; weights=np.concatenate((point_w,point_w)); roots=np.sqrt(weights) + fsum=np.zeros(m,np.float64); psum=np.zeros(3,np.float64); count=0 + for _,u,p in _chunks(inp,streaming_config.chunk_size): fsum+=u.sum(axis=1); psum+=p.sum(axis=1); count+=u.shape[1] + if count!=n: raise ValueError('stream pass count mismatch') + fmean=fsum/n; pmean=psum/n; cross=np.zeros((3,m),np.float64) + for _,u,p in _chunks(inp,streaming_config.chunk_size): cross+=(p-pmean[:,None])@((u-fmean[:,None])*roots[:,None]).T + cross/=n*np.sqrt(3.0) + left,s,vh=np.linalg.svd(cross,full_matrices=False); weighted=vh.T + for k in range(weighted.shape[1]): + pivot=int(np.argmax(np.abs(weighted[:,k]))) + if weighted[pivot,k]<0: weighted[:,k]*=-1; left[:,k]*=-1 + modes=weighted/roots[:,None]; identifiable,blocks,boundaries,null_tol=_classify(s,streaming_config.singular_block_rtol,streaming_config.singular_block_atol) + coeff=np.empty((3,n),np.float64); total_sq=0.0 + for sl,u,_ in _chunks(inp,streaming_config.chunk_size): + x=(u-fmean[:,None])*roots[:,None]; coeff[:,sl]=weighted.T@x; total_sq+=float(np.sum(x*x)) + residual=np.asarray([max(total_sq-float(np.sum(coeff[:r]**2)),0.0) for r in boundaries],np.float64) + residual=np.sqrt(residual/max(total_sq,np.finfo(float).tiny)) + arrays={"x_D":inp.x_D,"y_D":inp.y_D,"q_target_solver_fluid_mask":inp.role_masks['q_target'],"q_blk_solver_fluid_mask":inp.role_masks['q_blk'],"q_ctl_solver_fluid_mask":inp.role_masks['q_ctl'],"analysis_fluid_mask":mask,"selected_timeline_indices":inp.selected_indices,"selected_acquisition_relative_lattice_steps":inp.selected_relative_steps,"selected_q_ctl_absolute_lattice_steps":inp.q_ctl_absolute_steps,"coordinate_weights":weights,"field_mean":fmean,"action_mean":pmean,"effective_actions":inp.actions,"cross_correlation":cross,"left_functions":left,"singular_values":s,"physical_modes":modes,"coefficients":coeff,"identifiable_mode_mask":identifiable,"residual_block_boundaries":np.asarray(boundaries,np.int64),"weighted_relative_residuals":residual,"authoritative_mean_dq_ctl":inp.authoritative_mean_dq_ctl} + input_hashes={"direct_dq":{"path":str(inp.direct_dq_root),"manifest_sha256":inp.direct_manifest_sha256},"acquisitions":{k:dict(v) for k,v in inp.acquisition_identities.items()},"canonical_arrays":{k:canonical_array_sha256(v) for k,v in arrays.items()}} + config={"schema_id":"ccd-real-ccd-config/v1","case_id":inp.case_id,"Q":1,"tau":0,"observable_count":3,"channel_names":list(ACTION_CHANNEL_NAMES),"action_identities":list(ACTION_IDENTITIES),"action_units":ACTION_UNITS,"flatten_order":FLATTEN_ORDER,"weight_rule":WEIGHT_RULE,"center_snapshots":True,"center_observables":True,"standardization":False,"whitening":False,"chunk_size":streaming_config.chunk_size,"accumulation_dtype":"float64","input_field_dtype":"float32","singular_block_rtol":streaming_config.singular_block_rtol,"singular_block_atol":streaming_config.singular_block_atol,"memory":estimate,"full_reconstructions_persisted":False} + summary={"schema_id":"ccd-real-ccd-summary/v1","sample_count":n,"spatial_dof_count":m,"numerical_rank":int(identifiable.sum()),"null_tolerance":null_tol,"degenerate_singular_blocks":blocks,"complete_block_boundaries":boundaries,"spectrum_label":"cross-correlation strength; not field energy, explained variance, or canonical coefficient","mean_context":"mean effective actions and authoritative mean dq_ctl are outside CCD","claim_boundary":"no CCD>POD, causal, mechanism, response-time, same-phase, independent-realization, uncertainty, or observable-prediction claim","passes":3,"provenance_status":"VERIFIED_LIVE_INPUTS_REQUIRED_ON_LOAD"} + from .schema import validate_result + arrays=validate_result(arrays=arrays,config=config,summary=summary,input_hashes=input_hashes) + return RealCCDResult(arrays,config,summary,input_hashes) diff --git a/src/CCD_analysis/real_ccd/figures.py b/src/CCD_analysis/real_ccd/figures.py new file mode 100644 index 0000000..7b79dd0 --- /dev/null +++ b/src/CCD_analysis/real_ccd/figures.py @@ -0,0 +1,83 @@ +"""Deterministic artifact-only Karman real-CCD figures.""" +from __future__ import annotations +import argparse,json,os,shutil,uuid +from pathlib import Path +from typing import Sequence +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from CCD_analysis.acquisition.contracts import canonical_json +from .io import load_result +STEMS=("01_cross_correlation_spectrum","02_mean_and_physical_modes","03_action_channel_left_vectors","04_coefficient_time_traces","05_weighted_reconstruction_residual","06_snapshot_reconstructions") +def _components(v,mask): + n=int(mask.sum()); result=[] + for part in (v[:n],v[n:]): + field=np.full(mask.shape,np.nan); field[mask]=part; result.append(field) + return result +def _limit(fields): + v=np.concatenate([np.abs(f[np.isfinite(f)]) for f in fields]); return float(np.percentile(v,99)) or 1. +def _panel(ax,field,x,y,mask,limit,title): + image=ax.pcolormesh(x,y,field.T,shading="nearest",cmap="RdBu_r",vmin=-limit,vmax=limit,rasterized=True) + solid=np.ma.masked_where(mask,np.ones(mask.shape)); ax.pcolormesh(x,y,solid.T,shading="nearest",cmap="Greys",vmin=0,vmax=1) + ax.set(title=title,xlabel="x/D",ylabel="y/D"); ax.set_aspect("equal"); return image +def _save(fig,root,stem): + names=[] + for ext in ("png","pdf"): + p=root/f"{stem}.{ext}"; fig.savefig(p,dpi=300 if ext=="png" else None,bbox_inches="tight",metadata={"Creator":"CCD_analysis.real_ccd.figures"}); names.append(p.name) + plt.close(fig); return names +def _metrics(mode,mask,x,y,w): + n=int(mask.sum()); pw=w[:n]; e=(mode[:n]**2+mode[n:]**2)*pw; xx,yy=np.meshgrid(x,y,indexing="ij"); ux,uy=_components(mode,mask) + def mm(a,sign): + b=sign*np.flip(a,axis=1); valid=np.isfinite(a)&np.isfinite(b); return float(np.linalg.norm((a-b)[valid])/np.linalg.norm(a[valid])) + return {"weighted_ux_norm":float(np.sqrt(np.sum(mode[:n]**2*pw))),"weighted_uy_norm":float(np.sqrt(np.sum(mode[n:]**2*pw))),"energy_centroid_x_D":float(np.sum(xx[mask]*e)/e.sum()),"energy_centroid_y_D":float(np.sum(yy[mask]*e)/e.sum()),"ux_reflection_even_mismatch":mm(ux,1),"uy_reflection_odd_mismatch":mm(uy,-1)} +def publish_karman_figures(result_root:str|Path,output:str|Path,*,snapshot_indices:Sequence[int]|None=None)->Path: + destination=Path(output) + if destination.exists(): raise FileExistsError(destination) + partial=destination.with_name(f".{destination.name}.partial.{os.getpid()}.{uuid.uuid4().hex}"); partial.mkdir(parents=True) + try: + loaded=load_result(result_root,include_centered_snapshots=True); a=loaded["arrays"]; cfg=loaded["config"] + if cfg["case_id"]!="karman_re100": raise ValueError("Karman-only publication requires karman_re100") + x,y,mask=a["x_D"],a["y_D"],a["analysis_fluid_mask"]; modes,coef,steps=a["physical_modes"],a["coefficients"],a["selected_acquisition_relative_lattice_steps"]; snapshots=loaded["centered_snapshots"] + selected=tuple(snapshot_indices or (0,len(steps)//2,len(steps)-1)) + if not selected or len(set(selected))!=len(selected) or min(selected)<0 or max(selected)>=len(steps): raise ValueError("invalid snapshot indices") + files=[]; sigma=a["singular_values"]; ids=np.arange(1,4) + fig,axs=plt.subplots(1,2,figsize=(8,3.2),layout="constrained"); axs[0].bar(ids,sigma); axs[1].bar(ids,sigma**2,color="#D95F02"); axs[0].set_ylabel(r"$\sigma_j$ (cross-correlation strength)"); axs[1].set_ylabel(r"$\sigma_j^2$ (squared cross-correlation strength)") + for ax in axs: ax.set(xlabel="CCD mode j",xticks=ids); ax.grid(axis="y",alpha=.25) + fig.suptitle("Karman Q=1, tau=0 spectrum (not field energy or explained variance)"); files+=_save(fig,partial,STEMS[0]) + mean=[a["authoritative_mean_dq_ctl"][i].astype(float) for i in range(2)]; fields=[_components(modes[:,j],mask) for j in range(3)]; limits=(_limit(mean),_limit([f[0] for f in fields]),_limit([f[1] for f in fields])) + fig,axs=plt.subplots(4,2,figsize=(12,8.5),sharex=True,sharey=True,layout="constrained") + for c in range(2): _panel(axs[0,c],mean[c],x,y,mask,limits[0],f"authoritative mean dq_ctl {'ux' if c==0 else 'uy'}") + for j in range(3): + for c in range(2): _panel(axs[j+1,c],fields[j][c],x,y,mask,limits[c+1],f"mode {j+1} {'ux' if c==0 else 'uy'}") + fig.suptitle("Full-resolution physical fields; black is solver non-fluid mask/geometry"); files+=_save(fig,partial,STEMS[1]) + fig,axs=plt.subplots(1,3,figsize=(9,3.2),sharey=True,layout="constrained") + for j,ax in enumerate(axs): ax.bar(cfg["channel_names"],a["left_functions"][:,j]); ax.axhline(0,color="black",lw=.7); ax.set_title(f"mode {j+1}, zero lag"); ax.tick_params(axis="x",rotation=25) + axs[0].set_ylabel("left-vector component"); fig.suptitle("Q=1 action-channel left vectors (not lag curves)"); files+=_save(fig,partial,STEMS[2]) + fig,ax=plt.subplots(figsize=(8,4),layout="constrained") + for j in range(3): ax.plot(steps,coef[j],label=f"mode {j+1}") + ax.set(xlabel="acquisition-relative lattice step",ylabel="physical-amplitude coefficient"); ax.legend(); ax.grid(alpha=.25); files+=_save(fig,partial,STEMS[3]) + fig,ax=plt.subplots(figsize=(5.5,3.6),layout="constrained"); boundaries=a["residual_block_boundaries"]; residuals=a["weighted_relative_residuals"]; ax.plot(boundaries,residuals,"o-"); ax.set(xlabel="complete retained mode block boundary",ylabel="weighted relative reconstruction residual",xticks=boundaries,ylim=(0,1)); ax.grid(alpha=.25); files+=_save(fig,partial,STEMS[4]) + snapshot_limits=[] + for index in selected: + truth=snapshots[:,index]; vectors=[truth] + for rank in (1,2,3): + recon=modes[:,:rank]@coef[:rank,index]; vectors.extend((recon,truth-recon)) + ff=[_components(v,mask) for v in vectors]; limit=_limit([z for pair in ff for z in pair]); snapshot_limits.append(limit) + fig,axs=plt.subplots(2,7,figsize=(18,5.5),sharex=True,sharey=True,layout="constrained") + labels=["centered snapshot"] + for rank in (1,2,3): labels.extend((f"rank {rank} reconstruction",f"rank {rank} residual")) + for col,(pair,label) in enumerate(zip(ff,labels)): + for component in range(2): _panel(axs[component,col],pair[component],x,y,mask,limit,f"{label} {'ux' if component==0 else 'uy'}") + stem=f"{STEMS[5]}_step{int(steps[index])}"; fig.suptitle(f"On-demand centered dq_ctl reconstruction at acquisition-relative step {int(steps[index])}"); files+=_save(fig,partial,stem) + metrics=[_metrics(modes[:,j],mask,x,y,a["coordinate_weights"]) for j in range(3)]; centered=a["effective_actions"].astype(float)-a["action_mean"]; correlations=[{cfg["channel_names"][k]:float(np.corrcoef(coef[j],centered[:,k])[0,1]) for k in range(3)} for j in range(3)] + report={"schema_id":"ccd-real-ccd-karman-figures/v1","case_id":"karman_re100","source_result":str(Path(result_root).resolve()),"source_provenance":loaded["provenance_validation"],"grid_dimensions":{"nx":int(x.size),"ny":int(y.size),"mask_points":int(mask.sum()),"spatial_dof":int(modes.shape[0]),"downsampling":"none"},"figure_files":files,"snapshot_indices":list(selected),"snapshot_acquisition_relative_lattice_steps":[int(steps[i]) for i in selected],"snapshot_common_robust_limits":snapshot_limits,"robust_scale":"symmetric 99th percentile absolute value over each comparable set","singular_values":sigma.tolist(),"squared_cross_correlation_strengths":(sigma**2).tolist(),"action_means":a["action_mean"].tolist(),"mode_metrics":metrics,"coefficient_action_pearson_correlations":correlations,"residual_block_boundaries":boundaries.tolist(),"weighted_relative_reconstruction_residuals":residuals.tolist(),"claim_boundary":loaded["summary"]["claim_boundary"],"full_reconstructions_persisted":False} + (partial/"KARMAN_INTERPRETATION.json").write_bytes(canonical_json(report)); lines=["# Karman real-CCD figure interpretation","",f"Verified source: `{report['source_result']}`.","",f"Mean actions (front/upper/lower, native units): `{report['action_means']}`. The authoritative mean dq_ctl is context outside centered CCD.",""] + for j,m in enumerate(metrics): lines.append(f"- Mode {j+1}: weighted ux/uy norms {m['weighted_ux_norm']:.6g}/{m['weighted_uy_norm']:.6g}; squared-amplitude centroid (x/D,y/D)=({m['energy_centroid_x_D']:.4f},{m['energy_centroid_y_D']:.4f}); ux-even/uy-odd reflection mismatches {m['ux_reflection_even_mismatch']:.4f}/{m['uy_reflection_odd_mismatch']:.4f}; coefficient/action Pearson correlations `{correlations[j]}`.") + lines += ["",f"Selected acquisition-relative lattice steps: `{report['snapshot_acquisition_relative_lattice_steps']}`. Views are generated on demand; no full MxN reconstruction is persisted.","",f"Complete-block weighted relative residuals: `{report['weighted_relative_reconstruction_residuals']}`. This is the weighted norm of centered field content outside the retained modal projection divided by the weighted centered-field norm; it is not unexplained variance or q_target error.","",f"Claim boundary: {report['claim_boundary']}.",""]; (partial/"KARMAN_INTERPRETATION.md").write_text("\n".join(lines),encoding="ascii") + os.rename(partial,destination); return destination + except Exception: + shutil.rmtree(partial,ignore_errors=True); raise +def main(argv:Sequence[str]|None=None)->int: + parser=argparse.ArgumentParser(); parser.add_argument("--result-root",required=True,type=Path); parser.add_argument("--output",required=True,type=Path); parser.add_argument("--snapshot-index",action="append",type=int); args=parser.parse_args(argv); out=publish_karman_figures(args.result_root,args.output,snapshot_indices=args.snapshot_index); print(json.dumps({"published":str(out.resolve())})); return 0 +if __name__=="__main__": raise SystemExit(main()) diff --git a/src/CCD_analysis/real_ccd/io.py b/src/CCD_analysis/real_ccd/io.py new file mode 100644 index 0000000..7bc398d --- /dev/null +++ b/src/CCD_analysis/real_ccd/io.py @@ -0,0 +1,71 @@ +"""Immutable real-CCD transaction and mandatory verified loader.""" +from __future__ import annotations +import json,os,shutil,uuid +from pathlib import Path +from typing import Any +import numpy as np +from CCD_analysis.acquisition.artifacts import file_sha256,rename_noreplace +from CCD_analysis.acquisition.contracts import canonical_json +from CCD_analysis.direct_dq.io import load_result as load_direct_dq_result +from .core import RealCCDResult +from .schema import RESULT_SCHEMA_ID,validate_result +RESULT_FILES={"arrays.npz","config.json","summary.json","input_hashes.json"} +def _read(path:Path): + raw=path.read_bytes(); value=json.loads(raw) + if not isinstance(value,dict) or raw!=canonical_json(value): raise ValueError(f"canonical JSON object required: {path}") + return value +def _fsync(path:Path): + with path.open('rb') as f: os.fsync(f.fileno()) +class ResultTransaction: + def __init__(self,destination:str|Path): + self.destination=Path(destination); self.partial=self.destination.with_name(f'.{self.destination.name}.partial.{os.getpid()}.{uuid.uuid4().hex}'); self.active=False + def __enter__(self): + if self.destination.exists(): raise FileExistsError(self.destination) + self.destination.parent.mkdir(parents=True,exist_ok=True); self.partial.mkdir(); self.active=True; return self + def write(self,result:RealCCDResult): + if not self.active: raise RuntimeError('transaction inactive') + arrays=validate_result(arrays=result.arrays,config=result.config,summary=result.summary,input_hashes=result.input_hashes) + np.savez_compressed(self.partial/'arrays.npz',**arrays) + for name,value in (("config.json",result.config),("summary.json",result.summary),("input_hashes.json",result.input_hashes)): (self.partial/name).write_bytes(canonical_json(value)) + for p in self.partial.iterdir(): _fsync(p) + files={p.name:file_sha256(p) for p in sorted(self.partial.iterdir())}; (self.partial/'manifest.json').write_bytes(canonical_json({"schema_id":RESULT_SCHEMA_ID,"complete":True,"files":files})); _fsync(self.partial/'manifest.json') + def publish(self): + load_result(self.partial); rename_noreplace(self.partial,self.destination); fd=os.open(self.destination.parent,os.O_RDONLY) + try: os.fsync(fd) + finally: os.close(fd) + self.active=False; load_result(self.destination); return self.destination + def __exit__(self,*args): + if self.active: shutil.rmtree(self.partial,ignore_errors=True); self.active=False +def load_result(path:str|Path, *, include_centered_snapshots:bool=False)->dict[str,Any]: + root=Path(path); manifest=_read(root/'manifest.json') + if set(manifest)!={"schema_id","complete","files"} or manifest.get('schema_id')!=RESULT_SCHEMA_ID or manifest.get('complete') is not True or set(manifest.get('files',{}))!=RESULT_FILES: raise ValueError('real-CCD manifest schema/inventory is not exact') + if {p.name for p in root.iterdir() if p.is_file()}!=RESULT_FILES|{"manifest.json"}: raise ValueError('real-CCD file inventory is not exact') + for name,digest in manifest['files'].items(): + if file_sha256(root/name)!=digest: raise ValueError(f'real-CCD file hash mismatch: {name}') + with np.load(root/'arrays.npz',allow_pickle=False) as z: arrays={k:z[k].copy() for k in z.files} + config,summary,input_hashes=(_read(root/n) for n in ('config.json','summary.json','input_hashes.json')) + arrays=validate_result(arrays=arrays,config=config,summary=summary,input_hashes=input_hashes) + direct=load_direct_dq_result(input_hashes['direct_dq']['path']) + if file_sha256(Path(input_hashes['direct_dq']['path'])/'manifest.json')!=input_hashes['direct_dq']['manifest_sha256']: raise ValueError('live direct-dq identity changed') + da=direct['arrays']; idx=arrays['selected_timeline_indices'] + if not np.array_equal(da['selected_timeline_indices'],idx) or not np.array_equal(da['selected_acquisition_relative_lattice_steps'],arrays['selected_acquisition_relative_lattice_steps']) or not np.array_equal(da['analysis_fluid_mask'],arrays['analysis_fluid_mask']): raise ValueError('live direct-dq selection/mask differs') + qctl_path=input_hashes['acquisitions']['q_ctl']['path']; from CCD_analysis.direct_dq.io import load_acquisition_artifact + qctl=load_acquisition_artifact(qctl_path,expected_case=config['case_id'],expected_role='q_ctl') + if qctl.input_identity!=input_hashes['acquisitions']['q_ctl'] or not np.array_equal(qctl.fields['effective_applied_action'][idx,-3:],arrays['effective_actions']): raise ValueError('live q_ctl effective-action provenance differs') + if not np.array_equal(qctl.fields['lattice_steps'][idx], arrays['selected_q_ctl_absolute_lattice_steps']): raise ValueError('live q_ctl absolute timestamp provenance differs') + from CCD_analysis.direct_dq.analysis import coordinate_weights + mask=arrays['analysis_fluid_mask']; point_w=(coordinate_weights(arrays['x_D'])[:,None]*coordinate_weights(arrays['y_D'])[None,:])[mask]; w=np.concatenate((point_w,point_w)) + np.testing.assert_array_equal(w,arrays['coordinate_weights']) + dq=da['q_ctl_instantaneous']-da['q_blk_instantaneous']; u=np.concatenate((dq[:,0][:,mask],dq[:,1][:,mask]),axis=1).T.astype(np.float64) + p=arrays['effective_actions'].T.astype(np.float64); umean=u.mean(axis=1); pmean=p.mean(axis=1) + np.testing.assert_allclose(arrays['field_mean'],umean,rtol=0,atol=0); np.testing.assert_allclose(arrays['action_mean'],pmean,rtol=0,atol=0) + x=(u-umean[:,None])*np.sqrt(w)[:,None]; cross=(p-pmean[:,None])@x.T/(u.shape[1]*np.sqrt(3.0)) + np.testing.assert_allclose(arrays['cross_correlation'],cross,rtol=1e-12,atol=1e-12) + modes=arrays['physical_modes']; weighted=modes*np.sqrt(w)[:,None] + np.testing.assert_allclose(weighted.T@weighted,np.eye(3),rtol=1e-10,atol=1e-10) + np.testing.assert_allclose(arrays['cross_correlation'],arrays['left_functions']@np.diag(arrays['singular_values'])@weighted.T,rtol=1e-10,atol=1e-10) + np.testing.assert_allclose(arrays['coefficients'],weighted.T@x,rtol=1e-10,atol=1e-10) + loaded={"arrays":arrays,"config":config,"summary":summary,"input_hashes":input_hashes,"manifest":manifest,"provenance_validation":"VERIFIED: direct-dq and live acquisition inputs reread and essential identities recomputed"} + if include_centered_snapshots: + loaded["centered_snapshots"]=(u-umean[:,None]) + return loaded diff --git a/src/CCD_analysis/real_ccd/preflight.py b/src/CCD_analysis/real_ccd/preflight.py new file mode 100644 index 0000000..09777dd --- /dev/null +++ b/src/CCD_analysis/real_ccd/preflight.py @@ -0,0 +1,36 @@ +"""Read-only real-artifact admission report for real-case CCD.""" +from __future__ import annotations +from pathlib import Path +from typing import Any +import numpy as np +from .core import MemoryBudget, StreamingConfig, load_validated_input + +GIB = 1024 ** 3 + +def available_host_memory_bytes() -> int: + values = {} + for line in Path("/proc/meminfo").read_text().splitlines(): + key, value = line.split(":", 1) + values[key] = int(value.strip().split()[0]) * 1024 + available = values.get("MemAvailable", 0) + if available <= 0: + raise RuntimeError("positive /proc/meminfo MemAvailable is required") + return available + +def safe_host_budget(*, available_bytes: int | None = None) -> dict[str, int]: + available = available_host_memory_bytes() if available_bytes is None else available_bytes + if type(available) is not int or available <= 0: + raise ValueError("available_bytes must be a positive integer") + reserve = max(8 * GIB, int(np.ceil(available * 0.25))) + budget = available - reserve + if budget <= 0: + raise MemoryError("available host RAM does not exceed the mandatory safety reserve") + return {"available_host_ram_bytes": available, "safety_reserve_bytes": reserve, "ram_budget_bytes": budget} + +def preflight_real_artifact(path: str | Path, *, chunk_size: int = 8, available_bytes: int | None = None) -> dict[str, Any]: + """Validate all live provenance and return an in-memory report; write nothing.""" + host = safe_host_budget(available_bytes=available_bytes) + cfg = StreamingConfig(chunk_size, MemoryBudget(host["ram_budget_bytes"], 0, 1.25)) + inp, memory = load_validated_input(path, streaming_config=cfg) + action_ranges = {name: [float(inp.actions[:, i].min()), float(inp.actions[:, i].max())] for i, name in enumerate(("front", "upper", "lower"))} + return {"schema_id": "ccd-real-ccd-preflight/v1", "decision": "PASS", "read_only": True, "case_id": inp.case_id, "direct_dq_root": str(inp.direct_dq_root), "direct_manifest_sha256": inp.direct_manifest_sha256, "live_acquisition_roots": {role: identity["path"] for role, identity in inp.acquisition_identities.items()}, "dimensions": {"N": int(inp.selected_indices.size), "Nx": int(inp.x_D.size), "Ny": int(inp.y_D.size), "analysis_fluid_points": int(inp.analysis_mask.sum()), "M": 2 * int(inp.analysis_mask.sum()), "Q": 1, "L": 3}, "selection": {"first_index": int(inp.selected_indices[0]), "last_index": int(inp.selected_indices[-1]), "first_relative_step": int(inp.selected_relative_steps[0]), "last_relative_step": int(inp.selected_relative_steps[-1]), "first_q_ctl_absolute_step": int(inp.q_ctl_absolute_steps[0]), "last_q_ctl_absolute_step": int(inp.q_ctl_absolute_steps[-1])}, "actions": {"identities": ["front_ccw_positive", "upper_ccw_positive", "lower_ccw_positive"], "channel_order": ["front", "upper", "lower"], "units": "native solver angular-velocity command units", "ranges": action_ranges}, "host_memory": host, "memory": memory, "scratch_note": "No disk scratch or result directory is used by preflight."} diff --git a/src/CCD_analysis/real_ccd/schema.py b/src/CCD_analysis/real_ccd/schema.py new file mode 100644 index 0000000..3367e10 --- /dev/null +++ b/src/CCD_analysis/real_ccd/schema.py @@ -0,0 +1,46 @@ +"""Strict immutable schema for real-case streaming CCD.""" +from __future__ import annotations +from typing import Any,Mapping +import numpy as np +from CCD_analysis.acquisition.contracts import ACTION_IDENTITIES,CASES,canonical_json +from CCD_analysis.direct_dq.schema import canonical_array_sha256 +RESULT_SCHEMA_ID='ccd-real-ccd-result/v1' +ARRAY_KEYS={"x_D","y_D","q_target_solver_fluid_mask","q_blk_solver_fluid_mask","q_ctl_solver_fluid_mask","analysis_fluid_mask","selected_timeline_indices","selected_acquisition_relative_lattice_steps","selected_q_ctl_absolute_lattice_steps","coordinate_weights","field_mean","action_mean","effective_actions","cross_correlation","left_functions","singular_values","physical_modes","coefficients","identifiable_mode_mask","residual_block_boundaries","weighted_relative_residuals","authoritative_mean_dq_ctl"} +CONFIG_KEYS={"schema_id","case_id","Q","tau","observable_count","channel_names","action_identities","action_units","flatten_order","weight_rule","center_snapshots","center_observables","standardization","whitening","chunk_size","accumulation_dtype","input_field_dtype","singular_block_rtol","singular_block_atol","memory","full_reconstructions_persisted"} +SUMMARY_KEYS={"schema_id","sample_count","spatial_dof_count","numerical_rank","null_tolerance","degenerate_singular_blocks","complete_block_boundaries","spectrum_label","mean_context","claim_boundary","passes","provenance_status"} +def validate_result(*,arrays:Mapping[str,Any],config:Mapping[str,Any],summary:Mapping[str,Any],input_hashes:Mapping[str,Any])->dict[str,np.ndarray]: + if set(config)!=CONFIG_KEYS or config.get('schema_id')!='ccd-real-ccd-config/v1' or config.get('case_id') not in CASES: raise ValueError('real-CCD config schema is not exact') + fixed=(config['Q']==1 and config['tau']==0 and config['observable_count']==3 and config['channel_names']==['front','upper','lower'] and config['action_identities']==list(ACTION_IDENTITIES) and config['action_units']=='native solver angular-velocity command units' and config['flatten_order']=='component-major ux then uy; C-order analysis-mask point order' and config['weight_rule']=='direct_dq.coordinate_weights(x_D)*coordinate_weights(y_D), repeated ux then uy; not area-normalized' and config['center_snapshots'] is True and config['center_observables'] is True and config['standardization'] is False and config['whitening'] is False and config['full_reconstructions_persisted'] is False and config['accumulation_dtype']=='float64') + if not fixed: raise ValueError('frozen real-CCD estimand/config contradicted') + mem=config['memory'] + if not isinstance(mem,Mapping) or mem.get('decision')!='PASS' or mem.get('estimated_peak_ram_bytes',1)>mem.get('ram_budget_bytes',0) or mem.get('estimated_scratch_bytes',1)>mem.get('scratch_budget_bytes',0): raise ValueError('memory decision must pass explicit budgets') + frozen_summary = { + 'spectrum_label': 'cross-correlation strength; not field energy, explained variance, or canonical coefficient', + 'mean_context': 'mean effective actions and authoritative mean dq_ctl are outside CCD', + 'claim_boundary': 'no CCD>POD, causal, mechanism, response-time, same-phase, independent-realization, uncertainty, or observable-prediction claim', + 'provenance_status': 'VERIFIED_LIVE_INPUTS_REQUIRED_ON_LOAD', + } + if set(summary)!=SUMMARY_KEYS or summary.get('schema_id')!='ccd-real-ccd-summary/v1' or summary.get('passes')!=3 or any(summary.get(k) != v for k, v in frozen_summary.items()): raise ValueError('real-CCD summary schema/claims are not exact') + if set(arrays)!=ARRAY_KEYS: raise ValueError('real-CCD array inventory is not exact') + d={k:np.asarray(v) for k,v in arrays.items()}; n=summary['sample_count']; m=summary['spatial_dof_count'] + if type(n) is not int or type(m) is not int or n<1 or m<1: raise ValueError('invalid result dimensions') + if d['selected_timeline_indices'].dtype!=np.int64 or d['selected_timeline_indices'].shape!=(n,) or np.any(np.diff(d['selected_timeline_indices'])<=0): raise ValueError('selected indices invalid') + for key in ('selected_acquisition_relative_lattice_steps','selected_q_ctl_absolute_lattice_steps'): + if d[key].dtype!=np.int64 or d[key].shape!=(n,) or np.any(np.diff(d[key])<=0): raise ValueError(f'{key} invalid') + mask=d['analysis_fluid_mask']; nx=d['x_D'].size; ny=d['y_D'].size + masks=[] + for role in ('q_target','q_blk','q_ctl'): + v=d[f'{role}_solver_fluid_mask']; masks.append(v) + if v.dtype!=np.bool_ or v.shape!=(nx,ny): raise ValueError('role mask invalid') + if mask.dtype!=np.bool_ or mask.shape!=(nx,ny) or not np.array_equal(mask,masks[0]&masks[1]&masks[2]) or m!=2*int(mask.sum()): raise ValueError('analysis mask/flatten dimension invalid') + shapes={"coordinate_weights":(m,),"field_mean":(m,),"action_mean":(3,),"effective_actions":(n,3),"cross_correlation":(3,m),"left_functions":(3,3),"singular_values":(3,),"physical_modes":(m,3),"coefficients":(3,n),"identifiable_mode_mask":(3,)} + for k,s in shapes.items(): + if d[k].shape!=s or (d[k].dtype.kind in 'fc' and not np.isfinite(d[k]).all()): raise ValueError(f'{k} invalid') + if d['coordinate_weights'].dtype!=np.float64 or np.any(d['coordinate_weights']<=0) or d['field_mean'].dtype!=np.float64 or d['action_mean'].dtype!=np.float64 or d['effective_actions'].dtype!=np.float32: raise ValueError('canonical dtypes invalid') + if d['authoritative_mean_dq_ctl'].dtype!=np.float32 or d['authoritative_mean_dq_ctl'].shape!=(2,nx,ny) or not np.isfinite(d['authoritative_mean_dq_ctl']).all(): raise ValueError('authoritative mean dq_ctl invalid') + if d['residual_block_boundaries'].dtype!=np.int64 or d['weighted_relative_residuals'].dtype!=np.float64 or d['residual_block_boundaries'].shape!=d['weighted_relative_residuals'].shape or np.any(np.diff(d['residual_block_boundaries'])<=0): raise ValueError('block residual schema invalid') + if summary['complete_block_boundaries']!=d['residual_block_boundaries'].tolist() or summary['numerical_rank']!=int(d['identifiable_mode_mask'].sum()): raise ValueError('summary rank/block contradiction') + if set(input_hashes)!={'direct_dq','acquisitions','canonical_arrays'} or set(input_hashes['acquisitions'])!={'q_target','q_blk','q_ctl'} or set(input_hashes['canonical_arrays'])!=ARRAY_KEYS: raise ValueError('input/hash schema invalid') + if any(input_hashes['canonical_arrays'][k]!=canonical_array_sha256(v) for k,v in d.items()): raise ValueError('canonical array hash mismatch') + canonical_json(config); canonical_json(summary); canonical_json(input_hashes) + return d diff --git a/src/CCD_analysis/results/ccd/correction_ccd_results.json b/src/CCD_analysis/results/ccd/correction_ccd_results.json deleted file mode 100644 index f3415da..0000000 --- a/src/CCD_analysis/results/ccd/correction_ccd_results.json +++ /dev/null @@ -1,731 +0,0 @@ -{ - "illusion_0.75L_dqctl_force_fy_r6": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "force_fy", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 0.9363693259668311, - 0.5427124016524685, - 0.02867293614015484 - ], - "special_mechanism": false - }, - "illusion_0.75L_dqctl_force_fx_r6": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "force_fx", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 0.8887476512241682, - 0.7024685939692135, - 0.17373174245109088 - ], - "special_mechanism": false - }, - "illusion_0.75L_dqctl_action_r6": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "action", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 1.4929827205494475, - 1.0564978623336443, - 0.3536240923864859 - ], - "special_mechanism": false - }, - "illusion_0.75L_dqtar_force_fy_r6": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "force_fy_tar", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 0.9557871261173136, - 0.5104093433240032, - 0.00272381506789553 - ], - "special_mechanism": false - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r6_mode1": { - "overlap": 0.38274004938666406, - "mode": 1, - "r": 6 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r6_mode2": { - "overlap": 0.3742735673782987, - "mode": 2, - "r": 6 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r6_mode3": { - "overlap": 0.4160221885366777, - "mode": 3, - "r": 6 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r6_mode4": { - "overlap": 0.7933885479724803, - "mode": 4, - "r": 6 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r6_mode5": { - "overlap": 0.6200548221579631, - "mode": 5, - "r": 6 - }, - "illusion_0.75L_dqctl_force_fy_r8": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "force_fy", - "r": 8, - "m80": 2, - "N": 7, - "sigma_top3": [ - 1.1478269894791648, - 0.652863182963739, - 0.028786077604042823 - ], - "special_mechanism": false - }, - "illusion_0.75L_dqctl_force_fx_r8": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "force_fx", - "r": 8, - "m80": 2, - "N": 7, - "sigma_top3": [ - 0.9786149641068052, - 0.7691113807859654, - 0.188609876636516 - ], - "special_mechanism": false - }, - "illusion_0.75L_dqctl_action_r8": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "action", - "r": 8, - "m80": 2, - "N": 8, - "sigma_top3": [ - 1.8212349775439216, - 1.2876771686498985, - 0.36519455351462626 - ], - "special_mechanism": false - }, - "illusion_0.75L_dqtar_force_fy_r8": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "force_fy_tar", - "r": 8, - "m80": 2, - "N": 7, - "sigma_top3": [ - 0.9559584636516365, - 0.5114433620666231, - 0.020016175925514433 - ], - "special_mechanism": false - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r8_mode1": { - "overlap": 0.32804391166994556, - "mode": 1, - "r": 8 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r8_mode2": { - "overlap": 0.3279264837693497, - "mode": 2, - "r": 8 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r8_mode3": { - "overlap": 0.04479564315361202, - "mode": 3, - "r": 8 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r8_mode4": { - "overlap": 0.345183841487867, - "mode": 4, - "r": 8 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r8_mode5": { - "overlap": 0.4363341302545839, - "mode": 5, - "r": 8 - }, - "illusion_0.75L_dqctl_force_fy_r10": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "force_fy", - "r": 10, - "m80": 2, - "N": 7, - "sigma_top3": [ - 1.2041535893498374, - 0.6687434874320507, - 0.033382686367867884 - ], - "special_mechanism": false - }, - "illusion_0.75L_dqctl_force_fx_r10": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "force_fx", - "r": 10, - "m80": 2, - "N": 7, - "sigma_top3": [ - 1.021082402461413, - 0.7906946910917539, - 0.19569390445830676 - ], - "special_mechanism": false - }, - "illusion_0.75L_dqctl_action_r10": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "action", - "r": 10, - "m80": 2, - "N": 10, - "sigma_top3": [ - 1.873772820479455, - 1.3476153959060446, - 0.3912418253445644 - ], - "special_mechanism": false - }, - "illusion_0.75L_dqtar_force_fy_r10": { - "scene": "illusion_0.75L", - "diam": 0.75, - "obs": "force_fy_tar", - "r": 10, - "m80": 2, - "N": 7, - "sigma_top3": [ - 0.95651717599455, - 0.5116328905682535, - 0.02153808322188994 - ], - "special_mechanism": false - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r10_mode1": { - "overlap": 0.3200853391492714, - "mode": 1, - "r": 10 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r10_mode2": { - "overlap": 0.3169378757210263, - "mode": 2, - "r": 10 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r10_mode3": { - "overlap": 0.07447905724690952, - "mode": 3, - "r": 10 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r10_mode4": { - "overlap": 0.05289143799442159, - "mode": 4, - "r": 10 - }, - "illusion_0.75L_O_dqctl_vs_dqtar_r10_mode5": { - "overlap": 0.3699778630857817, - "mode": 5, - "r": 10 - }, - "illusion_1.0L_dqctl_force_fy_r6": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "force_fy", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 0.9133527075400956, - 0.5492417355984791, - 0.03698984539919565 - ], - "special_mechanism": false - }, - "illusion_1.0L_dqctl_force_fx_r6": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "force_fx", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 0.9627700852255946, - 0.575350111829231, - 0.12210932634955265 - ], - "special_mechanism": false - }, - "illusion_1.0L_dqctl_action_r6": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "action", - "r": 6, - "m80": 3, - "N": 6, - "sigma_top3": [ - 1.1703371479902003, - 0.928425799748123, - 0.6993315776263908 - ], - "special_mechanism": false - }, - "illusion_1.0L_dqtar_force_fy_r6": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "force_fy_tar", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 0.9230800944823152, - 0.5409412328754315, - 0.013881195337352553 - ], - "special_mechanism": false - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r6_mode1": { - "overlap": 0.9255017481198297, - "mode": 1, - "r": 6 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r6_mode2": { - "overlap": 0.9092817930388517, - "mode": 2, - "r": 6 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r6_mode3": { - "overlap": 0.5921722575222331, - "mode": 3, - "r": 6 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r6_mode4": { - "overlap": 0.3695873593868685, - "mode": 4, - "r": 6 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r6_mode5": { - "overlap": 0.49273985058737546, - "mode": 5, - "r": 6 - }, - "illusion_1.0L_dqctl_force_fy_r8": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "force_fy", - "r": 8, - "m80": 2, - "N": 7, - "sigma_top3": [ - 1.0512290431026903, - 0.6038168034405464, - 0.038604687614151825 - ], - "special_mechanism": false - }, - "illusion_1.0L_dqctl_force_fx_r8": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "force_fx", - "r": 8, - "m80": 2, - "N": 7, - "sigma_top3": [ - 1.024212111702677, - 0.6235362523473601, - 0.1398413327047235 - ], - "special_mechanism": false - }, - "illusion_1.0L_dqctl_action_r8": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "action", - "r": 8, - "m80": 3, - "N": 8, - "sigma_top3": [ - 1.289052813250248, - 0.9661024499957929, - 0.7306012213889583 - ], - "special_mechanism": false - }, - "illusion_1.0L_dqtar_force_fy_r8": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "force_fy_tar", - "r": 8, - "m80": 2, - "N": 7, - "sigma_top3": [ - 0.9296477714715806, - 0.5431506822355127, - 0.013939617415588563 - ], - "special_mechanism": false - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r8_mode1": { - "overlap": 0.7335300559150799, - "mode": 1, - "r": 8 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r8_mode2": { - "overlap": 0.7735101257517341, - "mode": 2, - "r": 8 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r8_mode3": { - "overlap": 0.5847457198896681, - "mode": 3, - "r": 8 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r8_mode4": { - "overlap": 0.361282468563919, - "mode": 4, - "r": 8 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r8_mode5": { - "overlap": 0.024744386112383842, - "mode": 5, - "r": 8 - }, - "illusion_1.0L_dqctl_force_fy_r10": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "force_fy", - "r": 10, - "m80": 2, - "N": 7, - "sigma_top3": [ - 1.1068219379848616, - 0.6279999575810425, - 0.03877250537017653 - ], - "special_mechanism": false - }, - "illusion_1.0L_dqctl_force_fx_r10": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "force_fx", - "r": 10, - "m80": 2, - "N": 7, - "sigma_top3": [ - 1.0798028524010137, - 0.6538475361620503, - 0.14841911584738715 - ], - "special_mechanism": false - }, - "illusion_1.0L_dqctl_action_r10": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "action", - "r": 10, - "m80": 3, - "N": 10, - "sigma_top3": [ - 1.3276794140080077, - 1.0190742997211644, - 0.7375547554838907 - ], - "special_mechanism": false - }, - "illusion_1.0L_dqtar_force_fy_r10": { - "scene": "illusion_1.0L", - "diam": 1.0, - "obs": "force_fy_tar", - "r": 10, - "m80": 2, - "N": 7, - "sigma_top3": [ - 0.9303683839417445, - 0.5438911279398368, - 0.015264317923416145 - ], - "special_mechanism": false - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r10_mode1": { - "overlap": 0.6843033711672328, - "mode": 1, - "r": 10 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r10_mode2": { - "overlap": 0.7092374384087569, - "mode": 2, - "r": 10 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r10_mode3": { - "overlap": 0.5408170275775555, - "mode": 3, - "r": 10 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r10_mode4": { - "overlap": 0.2185189595791332, - "mode": 4, - "r": 10 - }, - "illusion_1.0L_O_dqctl_vs_dqtar_r10_mode5": { - "overlap": 0.19476801271238336, - "mode": 5, - "r": 10 - }, - "illusion_1.5L_dqctl_force_fy_r6": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "force_fy", - "r": 6, - "m80": 1, - "N": 6, - "sigma_top3": [ - 0.9147833091150019, - 0.45038325163212484, - 0.03489015873428606 - ], - "special_mechanism": true - }, - "illusion_1.5L_dqctl_force_fx_r6": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "force_fx", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 0.24320738039972734, - 0.21287788693838852, - 0.042222381595628855 - ], - "special_mechanism": true - }, - "illusion_1.5L_dqctl_action_r6": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "action", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 0.20100087148018667, - 0.16474318694556772, - 0.06287962977503929 - ], - "special_mechanism": true - }, - "illusion_1.5L_dqtar_force_fy_r6": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "force_fy_tar", - "r": 6, - "m80": 2, - "N": 6, - "sigma_top3": [ - 0.9652520561509902, - 0.5009040282539201, - 0.03619846551092434 - ], - "special_mechanism": true - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r6_mode1": { - "overlap": 0.9219537578400411, - "mode": 1, - "r": 6 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r6_mode2": { - "overlap": 0.9523902874156159, - "mode": 2, - "r": 6 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r6_mode3": { - "overlap": 0.6142970258676237, - "mode": 3, - "r": 6 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r6_mode4": { - "overlap": 0.6094429047936349, - "mode": 4, - "r": 6 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r6_mode5": { - "overlap": 0.7802610781754543, - "mode": 5, - "r": 6 - }, - "illusion_1.5L_dqctl_force_fy_r8": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "force_fy", - "r": 8, - "m80": 1, - "N": 7, - "sigma_top3": [ - 1.0829392538529714, - 0.5377241276357321, - 0.035957125479337396 - ], - "special_mechanism": true - }, - "illusion_1.5L_dqctl_force_fx_r8": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "force_fx", - "r": 8, - "m80": 2, - "N": 7, - "sigma_top3": [ - 0.24532002823760304, - 0.2176027887532337, - 0.06997228412312347 - ], - "special_mechanism": true - }, - "illusion_1.5L_dqctl_action_r8": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "action", - "r": 8, - "m80": 2, - "N": 8, - "sigma_top3": [ - 0.24569133479372693, - 0.19734109779802844, - 0.07358222383616513 - ], - "special_mechanism": true - }, - "illusion_1.5L_dqtar_force_fy_r8": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "force_fy_tar", - "r": 8, - "m80": 2, - "N": 7, - "sigma_top3": [ - 0.965559410900674, - 0.5011783760115653, - 0.036374759425362196 - ], - "special_mechanism": true - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r8_mode1": { - "overlap": 0.7669082530299124, - "mode": 1, - "r": 8 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r8_mode2": { - "overlap": 0.7846172591640821, - "mode": 2, - "r": 8 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r8_mode3": { - "overlap": 0.5966131594534103, - "mode": 3, - "r": 8 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r8_mode4": { - "overlap": 0.4961899748365384, - "mode": 4, - "r": 8 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r8_mode5": { - "overlap": 0.554083931172919, - "mode": 5, - "r": 8 - }, - "illusion_1.5L_dqctl_force_fy_r10": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "force_fy", - "r": 10, - "m80": 1, - "N": 7, - "sigma_top3": [ - 1.2267278703286486, - 0.583312734476056, - 0.03658838363884156 - ], - "special_mechanism": true - }, - "illusion_1.5L_dqctl_force_fx_r10": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "force_fx", - "r": 10, - "m80": 2, - "N": 7, - "sigma_top3": [ - 0.2567417793308261, - 0.23357924739128508, - 0.10637915056271505 - ], - "special_mechanism": true - }, - "illusion_1.5L_dqctl_action_r10": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "action", - "r": 10, - "m80": 2, - "N": 10, - "sigma_top3": [ - 0.2696699328581466, - 0.23742010933522745, - 0.08104308573268743 - ], - "special_mechanism": true - }, - "illusion_1.5L_dqtar_force_fy_r10": { - "scene": "illusion_1.5L", - "diam": 1.5, - "obs": "force_fy_tar", - "r": 10, - "m80": 2, - "N": 7, - "sigma_top3": [ - 0.9662457501034244, - 0.5015382439664361, - 0.03666891184815278 - ], - "special_mechanism": true - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r10_mode1": { - "overlap": 0.6614436164339873, - "mode": 1, - "r": 10 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r10_mode2": { - "overlap": 0.7185566980291951, - "mode": 2, - "r": 10 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r10_mode3": { - "overlap": 0.5549038455928854, - "mode": 3, - "r": 10 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r10_mode4": { - "overlap": 0.42330974813153344, - "mode": 4, - "r": 10 - }, - "illusion_1.5L_O_dqctl_vs_dqtar_r10_mode5": { - "overlap": 0.4199314352386385, - "mode": 5, - "r": 10 - } -} \ No newline at end of file diff --git a/src/CCD_analysis/results/ccd/zone_metrics.json b/src/CCD_analysis/results/ccd/zone_metrics.json deleted file mode 100644 index be8deb4..0000000 --- a/src/CCD_analysis/results/ccd/zone_metrics.json +++ /dev/null @@ -1,282 +0,0 @@ -{ - "illusion_0.75L_dq_blk": { - "near_body": { - "n_points": 76800, - "mean_KE": 0.0007648248574696481, - "mean_enstrophy": 2.9027347636656486e-07, - "KE_fraction": 0.0023668697103857994 - }, - "body_wake": { - "n_points": 102400, - "mean_KE": 0.07021629065275192, - "mean_enstrophy": 0.0008679562015458941, - "KE_fraction": 0.28972700238227844 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.09573789685964584, - "mean_enstrophy": 0.001991209341213107, - "KE_fraction": 0.1382620632648468 - } - }, - "illusion_0.75L_dq_ctl": { - "near_body": { - "n_points": 76800, - "mean_KE": 3.614435627241619e-05, - "mean_enstrophy": 1.743321575986556e-08, - "KE_fraction": 0.000138708230224438 - }, - "body_wake": { - "n_points": 102400, - "mean_KE": 0.031560733914375305, - "mean_enstrophy": 0.001438321196474135, - "KE_fraction": 0.16149072349071503 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.04915999248623848, - "mean_enstrophy": 0.003602436976507306, - "KE_fraction": 0.0880400612950325 - } - }, - "illusion_1.0L_dq_blk": { - "near_body": { - "n_points": 76800, - "mean_KE": 0.0007648248574696481, - "mean_enstrophy": 2.9027347636656486e-07, - "KE_fraction": 0.0023668697103857994 - }, - "body_wake": { - "n_points": 102400, - "mean_KE": 0.07021629065275192, - "mean_enstrophy": 0.0008679562015458941, - "KE_fraction": 0.28972700238227844 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.09573789685964584, - "mean_enstrophy": 0.001991209341213107, - "KE_fraction": 0.1382620632648468 - } - }, - "illusion_1.0L_dq_ctl": { - "near_body": { - "n_points": 76800, - "mean_KE": 2.7065932954428717e-05, - "mean_enstrophy": 1.3172467561162193e-08, - "KE_fraction": 0.000290345138637349 - }, - "body_wake": { - "n_points": 102400, - "mean_KE": 0.0158767718821764, - "mean_enstrophy": 0.0004291802179068327, - "KE_fraction": 0.2270871102809906 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.015243785455822945, - "mean_enstrophy": 0.0009271366288885474, - "KE_fraction": 0.07631170749664307 - } - }, - "illusion_1.5L_dq_blk": { - "near_body": { - "n_points": 76800, - "mean_KE": 0.0007648248574696481, - "mean_enstrophy": 2.9027347636656486e-07, - "KE_fraction": 0.0023668697103857994 - }, - "body_wake": { - "n_points": 102400, - "mean_KE": 0.07021629065275192, - "mean_enstrophy": 0.0008679562015458941, - "KE_fraction": 0.28972700238227844 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.09573789685964584, - "mean_enstrophy": 0.001991209341213107, - "KE_fraction": 0.1382620632648468 - } - }, - "illusion_1.5L_dq_ctl": { - "near_body": { - "n_points": 76800, - "mean_KE": 2.4420158297289163e-05, - "mean_enstrophy": 1.2525031012344812e-09, - "KE_fraction": 8.383671229239553e-05 - }, - "body_wake": { - "n_points": 102400, - "mean_KE": 0.04845782741904259, - "mean_enstrophy": 0.00021500905859284103, - "KE_fraction": 0.22181373834609985 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.11757005751132965, - "mean_enstrophy": 0.0005185439949855208, - "KE_fraction": 0.18836025893688202 - } - }, - "steady_cloak_dq_blk": { - "near_body": { - "n_points": 76800, - "mean_KE": 0.0007648248574696481, - "mean_enstrophy": 2.9027347636656486e-07, - "KE_fraction": 0.0023668697103857994 - }, - "body_wake": { - "n_points": 102400, - "mean_KE": 0.07021629065275192, - "mean_enstrophy": 0.0008679562015458941, - "KE_fraction": 0.28972700238227844 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.09573789685964584, - "mean_enstrophy": 0.001991209341213107, - "KE_fraction": 0.1382620632648468 - } - }, - "steady_cloak_dq_ctl": { - "near_body": { - "n_points": 76800, - "mean_KE": 6.172616849653423e-05, - "mean_enstrophy": 3.953555349767157e-08, - "KE_fraction": 0.00015435306704603136 - }, - "body_wake": { - "n_points": 102400, - "mean_KE": 0.13577842712402344, - "mean_enstrophy": 0.008918941020965576, - "KE_fraction": 0.45270517468452454 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.3214779794216156, - "mean_enstrophy": 0.02501676417887211, - "KE_fraction": 0.375149130821228 - } - }, - "karman_re100_dq_blk": { - "near_body": { - "n_points": 71680, - "mean_KE": 0.14623497426509857, - "mean_enstrophy": 0.0006234294269233942, - "KE_fraction": 0.20486503839492798 - }, - "body_wake": { - "n_points": 66560, - "mean_KE": 0.10329263657331467, - "mean_enstrophy": 0.0001615467481315136, - "KE_fraction": 0.13436967134475708 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.0839567631483078, - "mean_enstrophy": 0.00010710477363318205, - "KE_fraction": 0.05880879983305931 - } - }, - "karman_re100_dq_ctl": { - "near_body": { - "n_points": 71680, - "mean_KE": 0.20531150698661804, - "mean_enstrophy": 0.008244173601269722, - "KE_fraction": 0.2548401653766632 - }, - "body_wake": { - "n_points": 66560, - "mean_KE": 0.12807133793830872, - "mean_enstrophy": 0.00012567009252961725, - "KE_fraction": 0.147612065076828 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.12139707058668137, - "mean_enstrophy": 8.339789928868413e-05, - "KE_fraction": 0.07534124702215195 - } - }, - "vortex_lamb_dq_blk": { - "near_body": { - "n_points": 71680, - "mean_KE": 0.13352914154529572, - "mean_enstrophy": 0.0010146809509024024, - "KE_fraction": 0.46505114436149597 - }, - "body_wake": { - "n_points": 66560, - "mean_KE": 0.11699999123811722, - "mean_enstrophy": 0.0002210606326116249, - "KE_fraction": 0.37837791442871094 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.09222870320081711, - "mean_enstrophy": 0.00014765505329705775, - "KE_fraction": 0.16060562431812286 - } - }, - "vortex_lamb_dq_ctl": { - "near_body": { - "n_points": 71680, - "mean_KE": 0.18387039005756378, - "mean_enstrophy": 0.008811434730887413, - "KE_fraction": 0.48393887281417847 - }, - "body_wake": { - "n_points": 66560, - "mean_KE": 0.13139627873897552, - "mean_enstrophy": 0.0002448623126838356, - "KE_fraction": 0.32112717628479004 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.12105172127485275, - "mean_enstrophy": 0.00019949178386013955, - "KE_fraction": 0.15930141508579254 - } - }, - "vortex_taylor_dq_blk": { - "near_body": { - "n_points": 71680, - "mean_KE": 0.1294081211090088, - "mean_enstrophy": 0.0011693575652316213, - "KE_fraction": 0.44358906149864197 - }, - "body_wake": { - "n_points": 66560, - "mean_KE": 0.10131462663412094, - "mean_enstrophy": 0.0001417396852048114, - "KE_fraction": 0.3224829137325287 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.07597015053033829, - "mean_enstrophy": 4.9815782404039055e-05, - "KE_fraction": 0.1302063763141632 - } - }, - "vortex_taylor_dq_ctl": { - "near_body": { - "n_points": 71680, - "mean_KE": 0.16766612231731415, - "mean_enstrophy": 0.008833244442939758, - "KE_fraction": 0.48838308453559875 - }, - "body_wake": { - "n_points": 66560, - "mean_KE": 0.08698907494544983, - "mean_enstrophy": 0.000124395388411358, - "KE_fraction": 0.23528558015823364 - }, - "sensor_zone": { - "n_points": 35840, - "mean_KE": 0.06907761096954346, - "mean_enstrophy": 3.919627488357946e-05, - "KE_fraction": 0.10060571134090424 - } - } -} \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/01_panorama_all_scenes.png b/src/CCD_analysis/results/figures/01_panorama_all_scenes.png deleted file mode 120000 index 786fb61..0000000 --- a/src/CCD_analysis/results/figures/01_panorama_all_scenes.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/corr_comparison_all_scenes.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/02_cloak_comparison.png b/src/CCD_analysis/results/figures/02_cloak_comparison.png deleted file mode 120000 index 3404cd5..0000000 --- a/src/CCD_analysis/results/figures/02_cloak_comparison.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/corr_cloak_comparison_dqctl.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/03_illusion_comparison.png b/src/CCD_analysis/results/figures/03_illusion_comparison.png deleted file mode 120000 index 04241f2..0000000 --- a/src/CCD_analysis/results/figures/03_illusion_comparison.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/corr_illusion_comparison_dqctl.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/04_steady_cloak_cancel.png b/src/CCD_analysis/results/figures/04_steady_cloak_cancel.png deleted file mode 120000 index efda685..0000000 --- a/src/CCD_analysis/results/figures/04_steady_cloak_cancel.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/steady_cloak_cancel_test.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/05_illusion_075L_ctl_vs_tar.png b/src/CCD_analysis/results/figures/05_illusion_075L_ctl_vs_tar.png deleted file mode 120000 index 6e3c646..0000000 --- a/src/CCD_analysis/results/figures/05_illusion_075L_ctl_vs_tar.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/corr_illusion_0.75L_ctl_vs_tar.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/06_illusion_10L_ctl_vs_tar.png b/src/CCD_analysis/results/figures/06_illusion_10L_ctl_vs_tar.png deleted file mode 120000 index 5510d44..0000000 --- a/src/CCD_analysis/results/figures/06_illusion_10L_ctl_vs_tar.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/corr_illusion_1.0L_ctl_vs_tar.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/07_illusion_15L_ctl_vs_tar.png b/src/CCD_analysis/results/figures/07_illusion_15L_ctl_vs_tar.png deleted file mode 120000 index 91e6e11..0000000 --- a/src/CCD_analysis/results/figures/07_illusion_15L_ctl_vs_tar.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/corr_illusion_1.5L_ctl_vs_tar.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/08_karman_ctl_vs_tar.png b/src/CCD_analysis/results/figures/08_karman_ctl_vs_tar.png deleted file mode 120000 index eaed7d2..0000000 --- a/src/CCD_analysis/results/figures/08_karman_ctl_vs_tar.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/corr_karman_re100_ctl_vs_tar.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/09_vortex_lamb_ctl_vs_tar.png b/src/CCD_analysis/results/figures/09_vortex_lamb_ctl_vs_tar.png deleted file mode 120000 index 11505ca..0000000 --- a/src/CCD_analysis/results/figures/09_vortex_lamb_ctl_vs_tar.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/corr_vortex_lamb_ctl_vs_tar.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/10_vortex_taylor_ctl_vs_tar.png b/src/CCD_analysis/results/figures/10_vortex_taylor_ctl_vs_tar.png deleted file mode 120000 index 4f5f9ad..0000000 --- a/src/CCD_analysis/results/figures/10_vortex_taylor_ctl_vs_tar.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/corr_vortex_taylor_ctl_vs_tar.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/11_vortex_lamb_diagnosis.png b/src/CCD_analysis/results/figures/11_vortex_lamb_diagnosis.png deleted file mode 120000 index 81d2a5b..0000000 --- a/src/CCD_analysis/results/figures/11_vortex_lamb_diagnosis.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/vortex_lamb_diagnosis.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/12_vortex_taylor_diagnosis.png b/src/CCD_analysis/results/figures/12_vortex_taylor_diagnosis.png deleted file mode 120000 index 1f8ad21..0000000 --- a/src/CCD_analysis/results/figures/12_vortex_taylor_diagnosis.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/vortex_taylor_diagnosis.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/13_vortex_lamb_vorticity.png b/src/CCD_analysis/results/figures/13_vortex_lamb_vorticity.png deleted file mode 120000 index c1dad85..0000000 --- a/src/CCD_analysis/results/figures/13_vortex_lamb_vorticity.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/vortex_lamb_vorticity.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/14_vortex_taylor_vorticity.png b/src/CCD_analysis/results/figures/14_vortex_taylor_vorticity.png deleted file mode 120000 index 9720ad2..0000000 --- a/src/CCD_analysis/results/figures/14_vortex_taylor_vorticity.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/vortex_taylor_vorticity.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/15_vortex_lamb_target_vorticity.png b/src/CCD_analysis/results/figures/15_vortex_lamb_target_vorticity.png deleted file mode 120000 index f018151..0000000 --- a/src/CCD_analysis/results/figures/15_vortex_lamb_target_vorticity.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/vortex_lamb_target_vorticity.png \ No newline at end of file diff --git a/src/CCD_analysis/results/figures/16_vortex_taylor_target_vorticity.png b/src/CCD_analysis/results/figures/16_vortex_taylor_target_vorticity.png deleted file mode 120000 index fd9c39e..0000000 --- a/src/CCD_analysis/results/figures/16_vortex_taylor_target_vorticity.png +++ /dev/null @@ -1 +0,0 @@ -../../data/figures/vortex_taylor_target_vorticity.png \ No newline at end of file diff --git a/src/CCD_analysis/scripts/collect_empty_channel.py b/src/CCD_analysis/scripts/collect_empty_channel.py deleted file mode 100644 index 619c9d0..0000000 --- a/src/CCD_analysis/scripts/collect_empty_channel.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Collect empty channel reference flow (parabolic profile, no bodies). - -This provides the "target" reference for steady cloak metrics — -the clean channel flow that the cloak should restore. - -Usage: - conda run -n pycuda_3_10 python scripts/collect_empty_channel.py --device 2 - -Output: data/target_channel/target_channel/ -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import get_scene, data_dir_for_scene, LEGACY_CFG_DIR -from CCD_analysis.utils.cfd_interface import ( - load_legacy_configs, get_velocity_field, save_vorticity_png, - vorticity_from_ddf, -) - - -def collect(): - ap = argparse.ArgumentParser() - ap.add_argument("--device", type=int, default=2) - args = ap.parse_args() - - cfg = get_scene("target_channel") - out_dir = data_dir_for_scene("target_channel") - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(cfg["nu"])) - - ff = FlowField(field_cfg, cuda_cfg, device_id=args.device) - from CCD_analysis.configs import L0, CENTER_Y - l0 = L0 - - # 3 dummy sensors (no cylinders — just empty channel) - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((40.0 * l0, CENTER_Y + y_off * l0, 0.0), l0 / 4.0) - - n_obj = 3 - stabilize = int(4 * 1280 / cfg["u0"]) - print(f"Stabilising empty channel ({stabilize} steps)...") - ff.run(stabilize, np.zeros(n_obj, dtype=np.float32)) - - n_steps = 100 # steady-state, 100 is more than enough - si = cfg["sample_interval"] - sens_list = [] - ux_list, uy_list = [], [] - - for step in range(n_steps): - ff.run(si, np.zeros(n_obj, dtype=np.float32)) - obs = ff.obs.copy() - sens_list.append(obs[0:6]) - ux, uy = get_velocity_field(ff, u0=cfg["u0"]) - ux_list.append(ux) - uy_list.append(uy) - - # Save - np.savez_compressed(os.path.join(out_dir, "fields.npz"), - ux=np.stack(ux_list), uy=np.stack(uy_list)) - np.savez(os.path.join(out_dir, "sensors.npz"), - sensors=np.array(sens_list, dtype=np.float32)) - - # Validate: should be nearly steady - sens_arr = np.array(sens_list, dtype=np.float32) - print(f" Sensor mean: {np.mean(sens_arr, axis=0)}") - print(f" Sensor std: {np.std(sens_arr, axis=0)}") - total_std = float(np.sqrt(np.mean(sens_arr ** 2))) - print(f" Total std (should be near 0 for steady channel): {total_std:.6f}") - - # Vorticity - omega = vorticity_from_ddf(ff, u0=cfg["u0"]) - save_vorticity_png(os.path.join(out_dir, "vorticity.png"), - omega, title="Empty channel Re=100") - - # Meta - meta = dict(cfg, n_steps=n_steps) - with open(os.path.join(out_dir, "meta.json"), "w") as f: - json.dump(meta, f, indent=2) - print(f"Empty channel data saved to {out_dir}") - - del ff - - -if __name__ == "__main__": - t0 = time.time() - collect() - print(f"Time: {time.time() - t0:.1f}s") diff --git a/src/CCD_analysis/scripts/collect_illusion.py b/src/CCD_analysis/scripts/collect_illusion.py deleted file mode 100644 index c213cfd..0000000 --- a/src/CCD_analysis/scripts/collect_illusion.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Illusion DRL inference (all S_DIM=14, regardless of model name). - -All illusion models use 14-D observation space -(sensors(6) + forces(6) + target_cd(1) + target_cl(1)), -with target forces reconstructed from harmonics. - -Usage: - conda run -n pycuda_3_10 python scripts/collect_illusion.py --device 2 --steps 500 - -Output: data/illusion/{scene_name}/ -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time -from collections import deque -from pathlib import Path - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import get_scene, get_scene_list, data_dir_for_scene, model_path_for_scene, LEGACY_CFG_DIR -from CCD_analysis.utils.cfd_interface import ( - load_legacy_configs, save_vorticity_png, vorticity_from_ddf, - load_ppo_model, scale_action, get_velocity_field, - calc_lag, calc_dtw_sim, -) -from CCD_analysis.utils.resampling import analyze_harmonics, gen_target_states_at -from CCD_analysis.utils.dual_clock import DualClockCollector, field_steps_from_interval - -DATA_TYPE = np.float32 -L0 = 20.0 -CENTER_Y = (512 - 1) / 2.0 -FIFO_LEN = 150 -CONV_LEN = 36 - - -def run_single( - scene_name: str, - device_id: int, - n_steps: int, - *, - field_interval: int | None = None, - output_dir: str | None = None, -) -> dict: - cfg = get_scene(scene_name) - out_dir = output_dir or data_dir_for_scene(scene_name) - Path(out_dir).mkdir(parents=True, exist_ok=True) - u0 = cfg["u0"] - si = cfg["sample_interval"] - ac_scale = cfg["action_scale"] - ac_bias = cfg["action_bias"] - n_obj = cfg["n_objects_env"] - s_dim = cfg["s_dim"] - - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(cfg["nu"]), velocity=float(u0)) - - with open(os.path.join(out_dir, "config.json"), "w") as f: - json.dump({k: str(v) if not isinstance(v, (int, float, list, bool)) else v - for k, v in cfg.items()}, f, indent=2) - - # === Target recording (separate FlowField) === - print("=== Target recording ===") - ff_tgt = FlowField(field_cfg, cuda_cfg, device_id=device_id) - tgt_radius = cfg["target_diameter"] * L0 - ff_tgt.add_cylinder((30.65 * L0, CENTER_Y, 0.0), tgt_radius) # UNIFIED: was 20.0 - print(f" target cylinder: diameter={cfg['target_diameter']}L, radius={tgt_radius}", flush=True) - for y_off in [2.0, 0.0, -2.0]: - ff_tgt.add_sensor((40.0 * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) # UNIFIED: was 30.0 - n_tgt = 4 - ff_tgt.run(int(4 * 1280 / u0), np.zeros(n_tgt, dtype=DATA_TYPE)) - - target_states = np.empty((0, 8), dtype=DATA_TYPE) - for _ in range(FIFO_LEN): - ff_tgt.run(si, np.zeros(n_tgt, dtype=DATA_TYPE)) - target_states = np.vstack((target_states, ff_tgt.obs.copy()[0:8])) - target_harmonics = analyze_harmonics(target_states, n_harmonics=5) - np.savez(os.path.join(out_dir, "target.npz"), target_states=target_states) - harm_save = [{k: v for k, v in h.items()} for h in target_harmonics] - with open(os.path.join(out_dir, "target_harmonics.json"), "w") as f: - json.dump(harm_save, f, indent=2) - - save_vorticity_png(os.path.join(out_dir, "vorticity_target.png"), - vorticity_from_ddf(ff_tgt, u0=u0), - title="Illusion target cylinder") - del ff_tgt - - # === Control env (6 objects) === - print("=== Pinball env + norm ===") - ff = FlowField(field_cfg, cuda_cfg, device_id=device_id) - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((40.0 * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) # UNIFIED: was 30.0 - ff.add_cylinder((30.0 * L0, CENTER_Y, 0.0), L0 / 2.0) # UNIFIED: was 19.0 - ff.add_cylinder((31.3 * L0, CENTER_Y + 0.75 * L0, 0.0), L0 / 2.0) # UNIFIED: was 20.3 - ff.add_cylinder((31.3 * L0, CENTER_Y - 0.75 * L0, 0.0), L0 / 2.0) # UNIFIED: was 20.3 - - n_env = 6 - ff.run(int(4 * 1280 / u0), np.zeros(n_env, dtype=DATA_TYPE)) - ff.get_ddf() - ff.save_ddf() - - # Norm - fifo = deque(maxlen=FIFO_LEN) - for _ in range(FIFO_LEN): - ff.run(si, np.zeros(n_env, dtype=DATA_TYPE)) - fifo.append(ff.obs.copy()[0:12]) - temp = np.array(fifo, dtype=DATA_TYPE) - force_norm_fact = 6.0 * float(np.max(np.abs(temp[:, 6:12]))) - sens_deviation = np.mean(temp[:, 0:6], axis=0).astype(DATA_TYPE) - sens_norm_fact = np.zeros(6, dtype=DATA_TYPE) - for i in range(6): - sens_norm_fact[i] = 5.0 * float(np.max(np.abs(temp[:, i] - sens_deviation[i]))) - - norm = {"force_norm_fact": force_norm_fact, - "sens_deviation": sens_deviation.tolist(), - "sens_norm_fact": sens_norm_fact.tolist()} - with open(os.path.join(out_dir, "norm.json"), "w") as f: - json.dump(norm, f, indent=2) - print(f" force_norm_fact={force_norm_fact:.6f}") - - # Preset-action FIFO init (matches legacy_env_imit: [0,0,0,0,-1*U0,1*U0]) - # NOTE: this is NOT the same as action_bias([0,-2,2]). action_bias controls DRL - # action scaling; preset_action is a fixed Omega array used to warm up the FIFO. - ff.apply_ddf() - bias = np.zeros(n_env, dtype=DATA_TYPE) - bias[4] = -1.0 * u0 - bias[5] = 1.0 * u0 - fifo.clear() - for _ in range(FIFO_LEN): - ff.run(si, bias) - fifo.append(ff.obs.copy()[0:12]) - save_states_arr = np.array(fifo, dtype=DATA_TYPE) - - # Save DDF+FIFO checkpoint for replay (state right after warmup, before step 0) - ff.get_ddf() - np.save(os.path.join(out_dir, "ddf_checkpoint.npy"), ff.ddf) - np.save(os.path.join(out_dir, "fifo_checkpoint.npy"), save_states_arr) - - ff.apply_ddf() - - # === PPO inference === - print("=== PPO inference ===") - model = load_ppo_model(model_path_for_scene(scene_name), - device=f"cuda:{device_id}", s_dim=s_dim, a_dim=3) - model.set_random_seed(19) - - fifo = deque(maxlen=FIFO_LEN) - for s in save_states_arr: - fifo.append(np.array(s, dtype=DATA_TYPE)) - - obs = np.zeros(s_dim, dtype=np.float32) - sens_c, forc_c, act_c, rew_c, sim_c = [], [], [], [], [] - collector = None - if field_interval is not None: - collector = DualClockCollector( - ff, - control_interval=si, - control_count=n_steps, - u0=u0, - field_steps=field_steps_from_interval(si * n_steps, field_interval), - ) - - for step in range(n_steps): - action, _ = model.predict(obs, deterministic=True) - action = action.astype(np.float32).flatten() - act_c.append(action.copy()) - - temp_a = np.zeros(n_env, dtype=DATA_TYPE) - omega = (action * ac_scale + np.array(ac_bias, dtype=np.float32)) * u0 - temp_a[3:6] = omega - - ff.context.push() - if collector is None: - ff.run(si, temp_a) - else: - collector.run_interval(step, temp_a) - ff.context.pop() - - obs_slice = ff.obs.copy()[0:12] - fifo.append(obs_slice) - sens_c.append(obs_slice[0:6]) - forc_c.append(obs_slice[6:12]) - - # obs dimension depends on model type: - # d1a3o12_* = 12-dim (forces + sens only) - # d1a3o14_* = 14-dim (forces + sens + target_cd + target_cl) - forces_norm = obs_slice[6:12] / force_norm_fact - sens_norm = (obs_slice[0:6] - sens_deviation) / sens_norm_fact - if s_dim == 14: - target_recon = gen_target_states_at(step, target_harmonics) - t_cd_n = float(target_recon[0]) / force_norm_fact - t_cl_n = float(target_recon[1]) / force_norm_fact - obs = np.clip(np.hstack([forces_norm, sens_norm, t_cd_n, t_cl_n]), -1.0, 1.0).astype(np.float32) - else: - obs = np.clip(np.hstack([forces_norm, sens_norm]), -1.0, 1.0).astype(np.float32) - - # Reward - sarr = np.array(fifo, dtype=np.float32) - if len(sarr) >= CONV_LEN: - f = sarr[-1, 6:12] / force_norm_fact - cd = float(f[0] + f[2] + f[4]) - cl = float(f[1] + f[3] + f[5]) - - # DTW - ref_seq = target_states[CONV_LEN:2*CONV_LEN, 3] - cur_seq = sarr[-CONV_LEN:, 1] - lag = calc_lag(ref_seq, cur_seq) - sim_sum = 0.0 - for i in range(6): - t_seq = np.roll(target_states[:, i+2], -lag)[CONV_LEN:2*CONV_LEN] - s_seq = sarr[-CONV_LEN:, i] - sim_sum += calc_dtw_sim(t_seq, s_seq) / 6.0 - similarities = float(sim_sum) - sim_c.append(similarities) - - t_recon = gen_target_states_at(step, target_harmonics) - t_cd = float(t_recon[0]) / force_norm_fact - t_cl = float(t_recon[1]) / force_norm_fact - r_cd = np.exp(-abs((cd - t_cd) * 10)) - r_cl = np.exp(-abs((cl - t_cl) * 10)) - r_sim = np.exp(-10 * abs(similarities - 1)) - reward = float(min(0.3*r_cd + 0.3*r_cl + 0.4*r_sim, 1.0)) - rew_c.append(reward) - - sens_arr = np.array(sens_c, dtype=np.float32) - forc_arr = np.array(forc_c, dtype=np.float32) - act_arr = np.array(act_c, dtype=np.float32) - - np.savez(os.path.join(out_dir, "controlled.npz"), - sensors=sens_arr, forces=forc_arr, actions=act_arr, - rewards=np.array(rew_c, dtype=np.float32)) - if collector is not None: - collector.save(Path(out_dir) / "fields.npz") - - save_vorticity_png(os.path.join(out_dir, "vorticity_controlled.png"), - vorticity_from_ddf(ff, u0=u0), - title=f"{scene_name} controlled") - - tail = min(100, len(rew_c)) - avg_reward = float(np.mean(rew_c[-tail:])) if tail > 0 else 0.0 - avg_sim = float(np.mean(sim_c[-tail:])) if sim_c else 0.0 - print(f" reward={avg_reward:.4f} similarity={avg_sim:.4f}") - - result = { - "scene": scene_name, - "similarity": avg_sim, - "avg_reward": avg_reward, - "control_interval": si, - "field_interval": field_interval, - } - with open(os.path.join(out_dir, "result.json"), "w") as f: - json.dump(result, f, indent=2) - - del ff, model - return result - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--scene", type=str, default="illusion_1.0L", - help="Scene name (illusion_0.75L, illusion_1.0L, illusion_1.5L)") - ap.add_argument("--diameter", type=float, default=None, - help="Diameter shortcut (0.75, 1.0, 1.5)") - ap.add_argument("--device", type=int, default=2) - ap.add_argument("--steps", type=int, default=200) - ap.add_argument("--field-interval", type=int, default=None, - help="Save fields every N absolute lattice steps") - ap.add_argument("--output-dir", type=str, default=None, - help="Optional isolated output directory") - args = ap.parse_args() - - if args.diameter is not None: - scene_name = f"illusion_{args.diameter}L" - else: - scene_name = args.scene - - if scene_name not in get_scene_list("illusion"): - print(f"Unknown scene: {scene_name}. Available: {get_scene_list('illusion')}") - return 1 - - t0 = time.time() - r = run_single( - scene_name, args.device, args.steps, - field_interval=args.field_interval, output_dir=args.output_dir, - ) - print(f"Done in {time.time()-t0:.1f}s: sim={r['similarity']:.4f}") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/scripts/collect_karman.py b/src/CCD_analysis/scripts/collect_karman.py deleted file mode 100644 index 9b42601..0000000 --- a/src/CCD_analysis/scripts/collect_karman.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Karman cloak DRL inference (Re=100). - -Uses utils/cfd_interface.py (copied and verified from SR_analysis). - -Usage: - conda run -n pycuda_3_10 python scripts/collect_karman.py --device 2 --steps 200 - -Output: data/karman/karman_re200/ -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time -from collections import deque -from pathlib import Path - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import get_scene, get_scene_list, data_dir_for_scene, model_path_for_scene, LEGACY_CFG_DIR -from CCD_analysis.utils.dual_clock import DualClockCollector, field_steps_from_interval -from CCD_analysis.utils.cfd_interface import ( - load_legacy_configs, - build_karman_cloak_env, add_pinball, build_observation, - scale_action, load_ppo_model, save_vorticity_png, - vorticity_from_ddf, compute_similarity, -) - -DATA_TYPE = np.float32 -L0 = 20.0 - - -def run_single( - scene_name: str, - device_id: int, - n_steps: int, - *, - field_interval: int | None = None, - output_dir: str | None = None, -) -> dict: - cfg = get_scene(scene_name) - out_dir = output_dir or data_dir_for_scene(scene_name) - Path(out_dir).mkdir(parents=True, exist_ok=True) - u0 = cfg["u0"] - si = cfg["sample_interval"] - ac_scale = cfg["action_scale"] - ac_bias = cfg["action_bias"] - n_obj = cfg["n_objects_env"] - - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(cfg["nu"])) - - # Save config - with open(os.path.join(out_dir, "config.json"), "w") as f: - json.dump({k: str(v) if not isinstance(v, (int, float, list, bool)) else v - for k, v in cfg.items()}, f, indent=2) - - # --- Target recording --- - print("=== Target recording ===") - ff = FlowField(field_cfg, cuda_cfg, device_id=device_id) - target_states, _ = build_karman_cloak_env( - ff, u0=u0, l0=L0, sample_interval=si, fifo_len=150, data_type=DATA_TYPE) - np.savez(os.path.join(out_dir, "target.npz"), target_states=target_states) - - # --- Add pinball + norm --- - print("=== Norm ===") - norm = add_pinball( - ff, l0=L0, u0=u0, sample_interval=si, fifo_len=150, data_type=DATA_TYPE, - action_bias=ac_bias, - pinball_front_x=cfg["pinball_front_x"], - pinball_rear_x=cfg["pinball_rear_x"], - obs_slice_start=cfg["obs_slice"][0], obs_slice_end=cfg["obs_slice"][1], - ) - # Save norm (without save_states array) - norm_json = {k: v for k, v in norm.items() if not isinstance(v, np.ndarray)} - with open(os.path.join(out_dir, "norm.json"), "w") as f: - json.dump(norm_json, f, indent=2) - - # --- Uncontrolled rollout --- - print("=== Uncontrolled ===") - ff.restore_ddf() - ff.apply_ddf() - sens_u, forc_u = [], [] - for _ in range(n_steps): - ff.run(si, np.zeros(n_obj, dtype=DATA_TYPE)) - obs_slice = ff.obs.copy()[2:14] - sens_u.append(obs_slice[0:6]) - forc_u.append(obs_slice[6:12]) - np.savez(os.path.join(out_dir, "uncontrolled.npz"), - sensors=np.array(sens_u, dtype=np.float32), - forces=np.array(forc_u, dtype=np.float32)) - save_vorticity_png(os.path.join(out_dir, "vorticity_uncontrolled.png"), - vorticity_from_ddf(ff, u0=u0), - title=f"{scene_name} uncontrolled") - - # --- Controlled rollout --- - print("=== Controlled ===") - model_path = model_path_for_scene(scene_name) - s_dim = cfg.get("s_dim", 12) - model = load_ppo_model(model_path, device=f"cuda:{device_id}", s_dim=s_dim) - model.set_random_seed(0) - - ff.restore_ddf() - ff.apply_ddf() - fifo = deque(maxlen=150) - bias_action = scale_action(np.zeros(3, dtype=np.float32), - scale=ac_scale, bias=ac_bias, u0=u0, n_total_bodies=n_obj) - for _ in range(150): - ff.context.push() - ff.run(si, bias_action) - ff.context.pop() - fifo.append(ff.obs.copy()[2:14]) - - # Save DDF+FIFO checkpoint for replay (state right after warmup, before step 0) - save_states_arr = np.array(fifo, dtype=DATA_TYPE) - ff.get_ddf() - np.save(os.path.join(out_dir, "ddf_checkpoint.npy"), ff.ddf) - np.save(os.path.join(out_dir, "fifo_checkpoint.npy"), save_states_arr) - - sens_c, forc_c, act_c, rew_c = [], [], [], [] - obs = np.zeros(s_dim, dtype=np.float32) - collector = None - if field_interval is not None: - collector = DualClockCollector( - ff, - control_interval=si, - control_count=n_steps, - u0=u0, - field_steps=field_steps_from_interval(si * n_steps, field_interval), - ) - - for step in range(n_steps): - action, _ = model.predict(obs, deterministic=True) - action = action.astype(np.float32).flatten() - act_c.append(action.copy()) - - action_arr = scale_action(action, scale=ac_scale, bias=ac_bias, - u0=u0, n_total_bodies=n_obj) - ff.context.push() - if collector is None: - ff.run(si, action_arr) - else: - collector.run_interval(step, action_arr) - ff.context.pop() - - obs_slice = ff.obs.copy()[2:14] - fifo.append(obs_slice) - sens_c.append(obs_slice[0:6]) - forc_c.append(obs_slice[6:12]) - obs = build_observation(obs_slice, norm) - - # Reward - sarr = np.array(fifo, dtype=np.float32) - if len(sarr) >= 30: - f = sarr[-1, 6:12] / norm["force_norm_fact"] - cd = float((f[0] + f[2] + f[4]) / 3.0) - cl = float((f[1] + f[3] + f[5]) / 3.0) - sim = compute_similarity(target_states, sarr[:, 0:6], 30) - r = min(0.3 * np.exp(-abs(cd * 20)) + 0.4 * np.exp(-abs(cl * 80)) - + 0.3 * np.exp(-10 * abs(sim - 1)), 1.0) - rew_c.append(float(r)) - - sens_arr = np.array(sens_c, dtype=np.float32) - forc_arr = np.array(forc_c, dtype=np.float32) - act_arr = np.array(act_c, dtype=np.float32) - rew_arr = np.array(rew_c, dtype=np.float32) - - np.savez(os.path.join(out_dir, "controlled.npz"), - sensors=sens_arr, forces=forc_arr, actions=act_arr, rewards=rew_arr) - if collector is not None: - collector.save(Path(out_dir) / "fields.npz") - - save_vorticity_png(os.path.join(out_dir, "vorticity_controlled.png"), - vorticity_from_ddf(ff, u0=u0), - title=f"{scene_name} controlled") - - avg_reward = float(np.mean(rew_arr[-100:])) if len(rew_arr) >= 100 else 0.0 - sim_score = compute_similarity(target_states, sens_arr, 30) - print(f" reward={avg_reward:.4f} similarity={sim_score:.4f}") - - result = { - "scene": scene_name, - "similarity": sim_score, - "avg_reward": avg_reward, - "control_interval": si, - "field_interval": field_interval, - } - with open(os.path.join(out_dir, "result.json"), "w") as f: - json.dump(result, f, indent=2) - - del ff, model - return result - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--scene", type=str, default="karman_re100", - help="Scene name (karman_re50, re100, re200, re400)") - ap.add_argument("--re", type=int, default=None, - help="Re number shortcut (50, 100, 200, 400)") - ap.add_argument("--device", type=int, default=2) - ap.add_argument("--steps", type=int, default=200) - ap.add_argument("--field-interval", type=int, default=None, - help="Save fields every N absolute lattice steps") - ap.add_argument("--output-dir", type=str, default=None, - help="Optional isolated output directory") - args = ap.parse_args() - - if args.re is not None: - scene_name = f"karman_re{args.re}" - else: - scene_name = args.scene - - if scene_name not in get_scene_list("karman"): - print(f"Unknown scene: {scene_name}. Available: {get_scene_list('karman')}") - return 1 - - t0 = time.time() - r = run_single( - scene_name, args.device, args.steps, - field_interval=args.field_interval, output_dir=args.output_dir, - ) - print(f"Done in {time.time()-t0:.1f}s: sim={r['similarity']:.4f}") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/scripts/collect_karman_q_blk.py b/src/CCD_analysis/scripts/collect_karman_q_blk.py deleted file mode 100644 index 581d4f1..0000000 --- a/src/CCD_analysis/scripts/collect_karman_q_blk.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Collect pinball-in-vortex-street baseline with zero control. - -Records the pinball interacting with the Karman vortex street, with NO control -(zero rotations). Saves both telemetry and full velocity field snapshots. - -Usage: - conda run -n pycuda_3_10 python scripts/collect_karman_q_blk.py --device 3 - -Output: data/karman_blocked/karman_q_blk/ -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import get_scene, data_dir_for_scene, LEGACY_CFG_DIR, L0, CENTER_Y -from CCD_analysis.utils.cfd_interface import ( - load_legacy_configs, get_velocity_field, -) - -DATA_TYPE = np.float32 - - -def collect(): - ap = argparse.ArgumentParser( - description="Collect pinball in Karman vortex street (zero control)") - ap.add_argument("--device", type=int, default=3, help="CUDA device ID") - ap.add_argument("--steps", type=int, default=500, - help="Number of recording steps after stabilisation") - ap.add_argument("--target-fifo", type=int, default=150, - help="Number of target recording steps") - args = ap.parse_args() - - scene_name = "karman_q_blk" - cfg = get_scene(scene_name) - out_dir = data_dir_for_scene(scene_name) - u0 = cfg["u0"] - si = cfg["sample_interval"] - print(f"Output: {out_dir}", flush=True) - - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(cfg["nu"])) - - ff = FlowField(field_cfg, cuda_cfg, device_id=args.device) - - # ---- Phase 1: Build karman cloak env (dist cylinder + 3 sensors) ---- - print("=== Target recording (disturbance + sensors) ===", flush=True) - ff.add_cylinder((10.0 * L0, CENTER_Y, 0.0), L0) - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((40.0 * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) - - n_obj_phase1 = 4 - stabilize = int(4 * 1280 / u0) - print(f"Stabilising ({stabilize} steps)...", flush=True) - ff.run(stabilize, np.zeros(n_obj_phase1, dtype=DATA_TYPE)) - print("Stabilisation done.", flush=True) - - # Record target states - target_states = np.empty((0, 6), dtype=DATA_TYPE) - for _ in range(args.target_fifo): - ff.run(si, np.zeros(n_obj_phase1, dtype=DATA_TYPE)) - target_states = np.vstack((target_states, ff.obs.copy()[2:8])) - np.savez(os.path.join(out_dir, "target.npz"), target_states=target_states) - print(f"Target recorded: {target_states.shape}", flush=True) - - # ---- Phase 2: Add pinball cylinders ---- - print("=== Adding pinball ===", flush=True) - ff.add_cylinder((30.0 * L0, CENTER_Y, 0.0), L0 / 2.0) - ff.add_cylinder((31.3 * L0, CENTER_Y + 0.75 * L0, 0.0), L0 / 2.0) - ff.add_cylinder((31.3 * L0, CENTER_Y - 0.75 * L0, 0.0), L0 / 2.0) - - n_obj = 7 - stabilize = int(4 * 1280 / u0) - print(f"Stabilising pinball ({stabilize} steps)...", flush=True) - ff.run(stabilize, np.zeros(n_obj, dtype=DATA_TYPE)) - print("Pinball stabilisation done.", flush=True) - - # ---- Phase 3: Record blocked flow (zero actions) ---- - print("=== Recording blocked flow ===", flush=True) - sens_list, forc_list = [], [] - ux_list, uy_list = [], [] - - for step in range(args.steps): - ff.run(si, np.zeros(n_obj, dtype=DATA_TYPE)) - obs = ff.obs.copy() - # obs[0:2] = dist cylinder forces - # obs[2:8] = 3 sensors - # obs[8:14] = 3 pinball forces - sens_list.append(obs[2:8]) - forc_list.append(np.concatenate([obs[0:2], obs[8:14]])) - ux, uy = get_velocity_field(ff, u0=u0) - ux_list.append(ux) - uy_list.append(uy) - - # Save - np.savez_compressed(os.path.join(out_dir, "fields.npz"), - ux=np.stack(ux_list), uy=np.stack(uy_list)) - np.savez(os.path.join(out_dir, "sensors.npz"), - sensors=np.array(sens_list, dtype=np.float32), - forces=np.array(forc_list, dtype=np.float32)) - - # Save meta - meta = dict(cfg, n_steps=args.steps) - with open(os.path.join(out_dir, "meta.json"), "w") as f: - json.dump({k: str(v) if not isinstance(v, (int, float, list, bool)) else v - for k, v in meta.items()}, f, indent=2) - - del ff - print(f"Done, saved {args.steps} steps to {out_dir}", flush=True) - - -if __name__ == "__main__": - t0 = time.time() - collect() - print(f"Time: {time.time() - t0:.1f}s", flush=True) diff --git a/src/CCD_analysis/scripts/collect_karman_q_in.py b/src/CCD_analysis/scripts/collect_karman_q_in.py deleted file mode 100644 index 9a64003..0000000 --- a/src/CCD_analysis/scripts/collect_karman_q_in.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Collect target/incoming vortex street from disturbance cylinder (no pinball). - -Records the Karman vortex street from the upstream disturbance cylinder alone, -saving both telemetry (sensors + forces) and full velocity field snapshots. - -Usage: - conda run -n pycuda_3_10 python scripts/collect_karman_q_in.py --device 3 - -Output: data/karman_target/karman_q_in/ -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import get_scene, data_dir_for_scene, LEGACY_CFG_DIR, L0, CENTER_Y -from CCD_analysis.utils.cfd_interface import ( - load_legacy_configs, get_velocity_field, -) - -DATA_TYPE = np.float32 - - -def collect(): - ap = argparse.ArgumentParser( - description="Collect incoming Karman vortex street (disturbance cylinder only)") - ap.add_argument("--device", type=int, default=3, help="CUDA device ID") - ap.add_argument("--steps", type=int, default=500, - help="Number of recording steps") - args = ap.parse_args() - - scene_name = "karman_q_in" - cfg = get_scene(scene_name) - out_dir = data_dir_for_scene(scene_name) - u0 = cfg["u0"] - si = cfg["sample_interval"] - print(f"Output: {out_dir}", flush=True) - - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=0.004) - - ff = FlowField(field_cfg, cuda_cfg, device_id=args.device) - - # Add disturbance cylinder - ff.add_cylinder((10.0 * L0, CENTER_Y, 0.0), L0) - # Add 3 sensors at x=40*L0 - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((40.0 * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) - - n_obj = 4 - stabilize = int(4 * 1280 / u0) - print(f"Stabilising ({stabilize} steps)...", flush=True) - ff.run(stabilize, np.zeros(n_obj, dtype=DATA_TYPE)) - print("Stabilisation done.", flush=True) - - # Record 500 steps - sens_list, forc_list = [], [] - ux_list, uy_list = [], [] - - for step in range(args.steps): - ff.run(si, np.zeros(n_obj, dtype=DATA_TYPE)) - obs = ff.obs.copy() - # obs[0:2] = dist cylinder forces, obs[2:8] = 3 sensors - sens_list.append(obs[2:8]) - forc_list.append(obs[0:2]) - ux, uy = get_velocity_field(ff, u0=u0) - ux_list.append(ux) - uy_list.append(uy) - - # Save - np.savez_compressed(os.path.join(out_dir, "fields.npz"), - ux=np.stack(ux_list), uy=np.stack(uy_list)) - np.savez(os.path.join(out_dir, "sensors.npz"), - sensors=np.array(sens_list, dtype=np.float32), - forces=np.array(forc_list, dtype=np.float32)) - - # Save meta - meta = dict(cfg, n_steps=args.steps) - with open(os.path.join(out_dir, "meta.json"), "w") as f: - json.dump({k: str(v) if not isinstance(v, (int, float, list, bool)) else v - for k, v in meta.items()}, f, indent=2) - - del ff - print(f"Done, saved {args.steps} steps to {out_dir}", flush=True) - - -if __name__ == "__main__": - t0 = time.time() - collect() - print(f"Time: {time.time() - t0:.1f}s", flush=True) diff --git a/src/CCD_analysis/scripts/collect_pinball.py b/src/CCD_analysis/scripts/collect_pinball.py deleted file mode 100644 index 84b7ee3..0000000 --- a/src/CCD_analysis/scripts/collect_pinball.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Collect pinball baseline data. - -Usage: - conda run -n pycuda_3_10 python scripts/collect_pinball.py --device 2 - -Output: data/pinball/pinball/ -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import get_scene, data_dir_for_scene, LEGACY_CFG_DIR -from CCD_analysis.utils.cfd_interface import ( - load_legacy_configs, get_velocity_field, save_vorticity_png, -) - - -def collect(): - ap = argparse.ArgumentParser() - ap.add_argument("--device", type=int, default=2) - args = ap.parse_args() - - cfg = get_scene("pinball") - out_dir = data_dir_for_scene("pinball") - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(cfg["nu"])) - - ff = FlowField(field_cfg, cuda_cfg, device_id=args.device) - for sc_y in [40.0, 40.0, 40.0]: - pass # sensor positions handled below - from CCD_analysis.configs import L0, CENTER_Y - l0 = L0 - # sensors - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((40.0 * l0, CENTER_Y + y_off * l0, 0.0), l0 / 4.0) - # pinball - ff.add_cylinder((30.0 * l0, CENTER_Y, 0.0), l0 / 2.0) - ff.add_cylinder((31.3 * l0, CENTER_Y + 0.75 * l0, 0.0), l0 / 2.0) - ff.add_cylinder((31.3 * l0, CENTER_Y - 0.75 * l0, 0.0), l0 / 2.0) - - n_obj = 6 - stabilize = int(4 * 1280 / cfg["u0"]) - print(f"Stabilising ({stabilize} steps)...") - ff.run(stabilize, np.zeros(n_obj, dtype=np.float32)) - - n_steps = 200 - si = cfg["sample_interval"] - sens_list, forc_list = [], [] - ux_list, uy_list = [], [] - - for step in range(n_steps): - ff.run(si, np.zeros(n_obj, dtype=np.float32)) - obs = ff.obs.copy() - sens_list.append(obs[0:6]) - forc_list.append(obs[6:12]) - ux, uy = get_velocity_field(ff, u0=cfg["u0"]) - ux_list.append(ux) - uy_list.append(uy) - - # Save - np.savez_compressed(os.path.join(out_dir, "fields.npz"), - ux=np.stack(ux_list), uy=np.stack(uy_list)) - np.savez(os.path.join(out_dir, "sensors.npz"), - sensors=np.array(sens_list, dtype=np.float32), - forces=np.array(forc_list, dtype=np.float32)) - - # Vorticity - from CCD_analysis.utils.cfd_interface import vorticity_from_ddf - omega = vorticity_from_ddf(ff, u0=cfg["u0"]) - save_vorticity_png(os.path.join(out_dir, "vorticity.png"), - omega, title="Pinball uncontrolled Re=100") - - # Meta - from CCD_analysis.utils.resampling import detect_dominant_frequency - signal = np.array(sens_list, dtype=np.float32)[:, 3] - f_dom, T_dom, _ = detect_dominant_frequency(signal, float(si)) - St = f_dom * 20.0 / cfg["u0"] - meta = dict(cfg, St=St, f_dom=f_dom, n_steps=n_steps) - with open(os.path.join(out_dir, "meta.json"), "w") as f: - json.dump(meta, f, indent=2) - print(f"St={St:.4f}, done") - - del ff - - -if __name__ == "__main__": - t0 = time.time() - collect() - print(f"Time: {time.time() - t0:.1f}s") diff --git a/src/CCD_analysis/scripts/collect_steady_cloak.py b/src/CCD_analysis/scripts/collect_steady_cloak.py deleted file mode 100644 index 0eaf8c7..0000000 --- a/src/CCD_analysis/scripts/collect_steady_cloak.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Collect steady cloak (open-loop constant rotation). - -Usage: - conda run -n pycuda_3_10 python scripts/collect_steady_cloak.py --device 2 - -Output: data/steady_cloak/steady_cloak/ -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import get_scene, data_dir_for_scene, LEGACY_CFG_DIR, L0, CENTER_Y -from CCD_analysis.utils.cfd_interface import ( - load_legacy_configs, get_velocity_field, save_vorticity_png, vorticity_from_ddf, -) - -print("Steady cloak collection starting...", flush=True) - -def collect(): - ap = argparse.ArgumentParser() - ap.add_argument("--device", type=int, default=2) - ap.add_argument("--tune", action="store_true", help="scan rear omega") - args = ap.parse_args() - - cfg = get_scene("steady_cloak") - out_dir = data_dir_for_scene("steady_cloak") - print(f"Output dir: {out_dir}", flush=True) - - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(cfg["nu"])) - print(f"Configs loaded. Viscosity={cfg['nu']}", flush=True) - - print(f"Creating FlowField on device {args.device}...", flush=True) - ff = FlowField(field_cfg, cuda_cfg, device_id=args.device) - print("FlowField created.", flush=True) - l0 = L0 - - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((40.0 * l0, CENTER_Y + y_off * l0, 0.0), l0 / 4.0) - ff.add_cylinder((30.0 * l0, CENTER_Y, 0.0), l0 / 2.0) - ff.add_cylinder((31.3 * l0, CENTER_Y + 0.75 * l0, 0.0), l0 / 2.0) - ff.add_cylinder((31.3 * l0, CENTER_Y - 0.75 * l0, 0.0), l0 / 2.0) - print("Objects added.", flush=True) - - n_obj = 6 - stabilize = int(4 * 1280 / cfg["u0"]) - print(f"Initial stabilization ({stabilize} steps)...", flush=True) - ff.run(stabilize, np.zeros(n_obj, dtype=np.float32)) - print("Initial stabilization done.", flush=True) - - rear_scale = cfg["omega_rear_scale"] - if args.tune: - candidates = [4.7, 4.9, 5.1, 5.3, 5.5] - else: - candidates = [rear_scale] - - for scale in candidates: - rear_val = scale * cfg["u0"] - temp = np.zeros(n_obj, dtype=np.float32) - temp[3] = cfg["omega_front"] - temp[4] = rear_val - temp[5] = -rear_val - print(f"Stabilizing with rear={scale:.1f}xU0 ({stabilize} steps)...", flush=True) - ff.run(stabilize, temp) - - sens_list = [] - for _ in range(30): - ff.run(cfg["sample_interval"], temp) - sens_list.append(ff.obs.copy()[0:6]) - std = float(np.std(np.array(sens_list), axis=0).mean()) - print(f" rear={scale:.1f}xU0 -> sensor std={std:.6f}", flush=True) - - # Save with best (or single) value - rear_val = candidates[-1] * cfg["u0"] - temp = np.zeros(n_obj, dtype=np.float32) - temp[3] = cfg["omega_front"] - temp[4] = rear_val - temp[5] = -rear_val - - print(f"Saving final data (rear={candidates[-1]:.1f}xU0)...", flush=True) - sens_list, forc_list, ux_list, uy_list = [], [], [], [] - for _ in range(30): - ff.run(cfg["sample_interval"], temp) - obs = ff.obs.copy() - sens_list.append(obs[0:6]) - forc_list.append(obs[6:12]) - ux, uy = get_velocity_field(ff, u0=cfg["u0"]) - ux_list.append(ux) - uy_list.append(uy) - - np.savez_compressed(os.path.join(out_dir, "fields.npz"), - ux=np.stack(ux_list), uy=np.stack(uy_list)) - np.savez(os.path.join(out_dir, "sensors.npz"), - sensors=np.array(sens_list, dtype=np.float32), - forces=np.array(forc_list, dtype=np.float32)) - - omega = vorticity_from_ddf(ff, u0=cfg["u0"]) - save_vorticity_png(os.path.join(out_dir, "vorticity.png"), - omega, title="Steady Cloak Re=100") - del ff - - meta = dict(cfg, rear_scale=candidates[-1], n_samples=30) - with open(os.path.join(out_dir, "meta.json"), "w") as f: - json.dump(meta, f, indent=2) - print(f"Done, saved to {out_dir}", flush=True) - - -if __name__ == "__main__": - t0 = time.time() - collect() - print(f"Time: {time.time() - t0:.1f}s", flush=True) diff --git a/src/CCD_analysis/scripts/collect_target_cylinder.py b/src/CCD_analysis/scripts/collect_target_cylinder.py deleted file mode 100644 index c354d94..0000000 --- a/src/CCD_analysis/scripts/collect_target_cylinder.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Collect target cylinder baseline data (2D cylinder wake, Re=100). - -Records the periodic vortex shedding from a single cylinder of specified diameter, -positioned at x=20*L0, with 3 sensors at x=30*L0. - -Usage: - conda run -n pycuda_3_10 python scripts/collect_target_cylinder.py --diameter 1.0 --device 2 - -Output: data/target_cylinder/target_cylinder_{diam}L/ -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import get_scene, get_scene_list, data_dir_for_scene, LEGACY_CFG_DIR, L0, CENTER_Y -from CCD_analysis.utils.cfd_interface import ( - load_legacy_configs, get_velocity_field, save_vorticity_png, - vorticity_from_ddf, -) - -DATA_TYPE = np.float32 - - -def collect(): - ap = argparse.ArgumentParser() - ap.add_argument("--diameter", type=float, default=1.0, - help="Target cylinder diameter in L0 units (0.75, 1.0, 1.5)") - ap.add_argument("--device", type=int, default=2) - ap.add_argument("--steps", type=int, default=500) - args = ap.parse_args() - - scene_name = f"target_cylinder_{args.diameter}L" - if scene_name not in get_scene_list("target_cylinder"): - print(f"Unknown scene: {scene_name}. Available: {get_scene_list('target_cylinder')}", flush=True) - return - - cfg = get_scene(scene_name) - out_dir = data_dir_for_scene(scene_name) - u0 = cfg["u0"] - si = cfg["sample_interval"] - diam = args.diameter - radius = diam * L0 - cyl_x = cfg["cylinder_x"] - sensor_x = cfg["sensor_x"] - print(f"Target cylinder: diameter={diam}L (radius={radius}), u0={u0}, si={si}", flush=True) - print(f"Output: {out_dir}", flush=True) - - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(cfg["nu"]), velocity=float(u0)) - - ff = FlowField(field_cfg, cuda_cfg, device_id=args.device) - - # Add cylinder with specified diameter - ff.add_cylinder((cyl_x * L0, CENTER_Y, 0.0), radius) - # Add 3 sensors at x=sensor_x*L0 - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((sensor_x * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) - - n_obj = 4 - stabilize = int(4 * 1280 / u0) - print(f"Stabilising target cylinder ({stabilize} steps)...", flush=True) - ff.run(stabilize, np.zeros(n_obj, dtype=DATA_TYPE)) - print("Stabilisation done.", flush=True) - - # Record - sens_list, forc_list = [], [] - ux_list, uy_list = [], [] - - for step in range(args.steps): - ff.run(si, np.zeros(n_obj, dtype=DATA_TYPE)) - obs = ff.obs.copy() - sens_list.append(obs[2:8]) # 3 sensors x 2 = 6 - forc_list.append(obs[0:2]) # cylinder force (fx, fy) - ux, uy = get_velocity_field(ff, u0=u0) - ux_list.append(ux) - uy_list.append(uy) - - # Save - np.savez_compressed(os.path.join(out_dir, "fields.npz"), - ux=np.stack(ux_list), uy=np.stack(uy_list)) - np.savez(os.path.join(out_dir, "sensors.npz"), - sensors=np.array(sens_list, dtype=np.float32), - forces=np.array(forc_list, dtype=np.float32)) - - omega = vorticity_from_ddf(ff, u0=u0) - save_vorticity_png(os.path.join(out_dir, "vorticity.png"), - omega, title=f"Target cylinder {diam}L, u0={u0}") - - # Strouhal number - from CCD_analysis.utils.resampling import detect_dominant_frequency - signal = np.array(sens_list, dtype=np.float32)[:, 3] - f_dom, T_dom, _ = detect_dominant_frequency(signal, float(si)) - St = f_dom * (diam * L0) / u0 - print(f"St={St:.4f} (f_dom={f_dom:.6f}, T={T_dom:.0f})", flush=True) - - meta = dict(cfg, St=St, f_dom=f_dom, n_steps=args.steps) - with open(os.path.join(out_dir, "meta.json"), "w") as f: - json.dump(meta, f, indent=2) - - del ff - print(f"Done, saved to {out_dir}", flush=True) - - -if __name__ == "__main__": - t0 = time.time() - collect() - print(f"Time: {time.time() - t0:.1f}s", flush=True) diff --git a/src/CCD_analysis/scripts/collect_vortex.py b/src/CCD_analysis/scripts/collect_vortex.py deleted file mode 100644 index 5e56cac..0000000 --- a/src/CCD_analysis/scripts/collect_vortex.py +++ /dev/null @@ -1,509 +0,0 @@ -"""Collect vortex cloak data for CCD correction-field analysis. - -Collects field snapshots for three scene types per vortex type: - - vortex_target_{type}: vortex only (no pinball), the "ideal" target flow - - vortex_uncontrolled_{type}: vortex + pinball, zero control (q_blk) - - vortex_{type}: vortex + pinball + PPO control (q_ctl) - -All use LegacyCelerisLab (matching existing CCD data convention). - -Usage: - conda run -n pycuda_3_10 python scripts/collect_vortex.py \\ - --type lamb --device 2 - conda run -n pycuda_3_10 python scripts/collect_vortex.py \\ - --type taylor --device 2 - conda run -n pycuda_3_10 python scripts/collect_vortex.py \\ - --type all --device 2 - ---- BUG HISTORY (2026-06-29) --- -Three independent bugs in this script were discovered and fixed after -Lamb dipole showed non-physical front cylinder oscillation: - -BUG 1 - Cylinder order swap (lines 166-168, 254-257): - Original training env adds cylinders: front(id3) -> TOP(+y, id4) -> BOTTOM(-y, id5). - action = [aF, aT(+bias), aB(-bias)] -> temp[3]=aF, temp[4]=aT(+bias), temp[5]=aB(-bias). - We had TOP and BOTTOM swapped, so bias -4 went to the wrong cylinder. - Consequence: rear cylinders rotated in opposite directions; front overcompensated. - -BUG 2 - Observation normalization swap (lines 368-371): - Training env produces obs = [forces/force_norm, sensors/sens_norm] (force-first). - We incorrectly fed [sensors/force_norm, forces/sens_norm] (channel-swapped + wrong norms). - Consequence: model received garbage feedback; on Lamb, similarity appeared 0.94 from - the similarity computation but cross-correlation was only 0.73. - -BUG 3 - Missing fade-in/out transitions (lines 314-357): - uni_test.ipynb uses 25-step fade-in from steady-cloak bias [-5.1U0, +5.1U0] to PPO action, - and 25-step fade-out back to steady-cloak. We applied PPO action immediately at full scale. - Consequence: flow instability from abrupt control changes. - -Fix summary: - - Cylinder order: TOP(+y) at id4, BOTTOM(-y) at id5. - - Bias: [-5, +5] U0 for both Lamb and Taylor (matching uni_test). - - Observation: forces_norm=obs[6:12]/force_norm_fact, sens_norm=(obs[0:6]-sens_dev)/sens_norm_fact. - Assembled as hstack([forces_norm, sens_norm]) — force first. - - Fade-in/out: 25 steps linear interpolation between steady_bias and PPO action. - -Comparison with other collect scripts: - collect_karman.py and collect_illusion.py use build_observation() from cfd_interface.py - which does force-first correctly. Only this script built obs manually. -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time -from collections import deque - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import ( - get_scene, data_dir_for_scene, model_path_for_scene, - LEGACY_CFG_DIR, L0, CENTER_Y, U0, -) -from CCD_analysis.utils.cfd_interface import ( - load_legacy_configs, get_velocity_field, - load_ppo_model, -) - -DATA_TYPE = np.float32 -FIFO_LEN = 150 - - -# --------------------------------------------------------------------------- -# Vortex configuration -# --------------------------------------------------------------------------- - -_VORTEX_CFG = { - "lamb": {"vortex_type": "lamb", "vortex_strength": 0.5 * U0}, - "taylor": {"vortex_type": "taylor", "vortex_strength": 0.03 * U0}, -} - - -# --------------------------------------------------------------------------- -# Target: vortex only (no pinball), save fields.npz -# --------------------------------------------------------------------------- - -def collect_target(vtype: str, device_id: int, out_dir: str, - n_steps: int = 150) -> dict: - """Collect field snapshots for vortex-only target flow. - - Records the vortex evolving through a sensor-only environment - at x=40*L0. Saves fields.npz and sensors.npz. - """ - cfg = _VORTEX_CFG[vtype] - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(0.004)) - - ff = FlowField(field_cfg, cuda_cfg, device_id=device_id) - ny = ff.FIELD_SHAPE[1] - n_sensors = 3 - - # Add 3 sensors at x=40*L0 - sensor_positions = [2.0, 0.0, -2.0] - for y_off in sensor_positions: - ff.add_sensor((40.0 * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) - - # Short stabilization (1xNX/U0 for vortex, not 4x) - stabilize_steps = int(1 * ff.FIELD_SHAPE[0] / U0) - ff.run(stabilize_steps, np.zeros(n_sensors, dtype=DATA_TYPE)) - - # Save clean flow DDF - ff.get_ddf() - ff.save_ddf() - - # Add vortex at x=10*L0 - vc = cfg["vortex_type"] - vs = cfg["vortex_strength"] - ff.add_vortex((10.0 * L0, CENTER_Y, 0.0), - 2.0 * L0, vs, 0, vc) - - # Record vortex evolution - sens_list, forc_list = [], [] - ux_list, uy_list = [], [] - - for step in range(n_steps): - ff.run(800, np.zeros(n_sensors, dtype=DATA_TYPE)) - obs = ff.obs.copy() # (n_sensors*2,) = 6 sensor channels - sens_list.append(obs) - forc_list.append(np.zeros(6, dtype=DATA_TYPE)) # placeholder - ux, uy = get_velocity_field(ff, u0=U0) - ux_list.append(ux) - uy_list.append(uy) - - # Save - np.savez_compressed(os.path.join(out_dir, "fields.npz"), - ux=np.stack(ux_list), uy=np.stack(uy_list)) - np.savez(os.path.join(out_dir, "sensors.npz"), - sensors=np.array(sens_list, dtype=np.float32), - forces=np.array(forc_list, dtype=np.float32)) - - del ff - return {"scene": f"vortex_target_{vtype}", "n_steps": n_steps} - - -# --------------------------------------------------------------------------- -# Uncontrolled: vortex + pinball, zero control (q_blk) -# --------------------------------------------------------------------------- - -def collect_uncontrolled(vtype: str, device_id: int, out_dir: str, - n_steps: int = 150) -> dict: - """Collect field snapshots for vortex + pinball with zero control. - - Records the transient interaction of vortex with the pinball. - Target phases are recorded alongside field snapshots. - """ - cfg = _VORTEX_CFG[vtype] - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(0.004)) - - ff = FlowField(field_cfg, cuda_cfg, device_id=device_id) - ny = ff.FIELD_SHAPE[1] - n_sensors = 3 - - # ---- Phase 1: Sensors + target recording ---- - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((40.0 * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) - - stabilize_steps = int(1 * ff.FIELD_SHAPE[0] / U0) - ff.run(stabilize_steps, np.zeros(n_sensors, dtype=DATA_TYPE)) - - # Save clean DDF (pre-pinball, pre-vortex) - ff.get_ddf() - ff.save_ddf() - - # Record target (vortex only, for similarity reference) - ff.add_vortex((10.0 * L0, CENTER_Y, 0.0), - 2.0 * L0, cfg["vortex_strength"], 0, cfg["vortex_type"]) - target_states = np.empty((0, 6), dtype=DATA_TYPE) - for _ in range(min(FIFO_LEN, n_steps)): - ff.run(800, np.zeros(n_sensors, dtype=DATA_TYPE)) - target_states = np.vstack((target_states, ff.obs.copy())) - np.savez(os.path.join(out_dir, "target.npz"), target_states=target_states) - - # ---- Phase 2: Add pinball, record uncontrolled flow ---- - ff.restore_ddf() - ff.apply_ddf() - - # BUG-FIX (2026-06-29): cylinder order MUST match training env. - # Original env adds: front(3) -> TOP(+y,id4) -> BOTTOM(-y,id5). - # Swapping TOP/BOTTOM causes rear cylinders to rotate wrong direction. - ff.add_cylinder((30.0 * L0, CENTER_Y, 0.0), L0 / 2.0) # id 3: front - ff.add_cylinder((31.3 * L0, CENTER_Y + 0.75 * L0, 0.0), L0 / 2.0) # id 4: TOP (+y) - ff.add_cylinder((31.3 * L0, CENTER_Y - 0.75 * L0, 0.0), L0 / 2.0) # id 5: BOTTOM (-y) - - n_obj = ff.obs.size // 2 - assert n_obj == 6, f"Expected 6, got {n_obj}" - - # Bias action stabilization (matching uni_test: [-5, +5] for both Lamb and Taylor) - ff.run(stabilize_steps, np.zeros(n_obj, dtype=DATA_TYPE)) - ff.run(stabilize_steps, np.array([0.0, 0.0, 0.0, 0.0, -5.0 * U0, 5.0 * U0], dtype=DATA_TYPE)) - - # Add vortex at x=15*L0 - ff.add_vortex((15.0 * L0, CENTER_Y, 0.0), - 2.0 * L0, cfg["vortex_strength"], 0, cfg["vortex_type"]) - - # Record uncontrolled flow (zero actions on pinball) - sens_list, forc_list = [], [] - ux_list, uy_list = [], [] - - for step in range(n_steps): - # Zero control on pinball - ff.run(800, np.zeros(n_obj, dtype=DATA_TYPE)) - obs = ff.obs.copy() - sens_list.append(obs[0:6]) - forc_list.append(obs[6:12]) - ux, uy = get_velocity_field(ff, u0=U0) - ux_list.append(ux) - uy_list.append(uy) - - np.savez_compressed(os.path.join(out_dir, "fields.npz"), - ux=np.stack(ux_list), uy=np.stack(uy_list)) - np.savez(os.path.join(out_dir, "sensors.npz"), - sensors=np.array(sens_list, dtype=np.float32), - forces=np.array(forc_list, dtype=np.float32)) - - del ff - return {"scene": f"vortex_uncontrolled_{vtype}", "n_steps": n_steps} - - -# --------------------------------------------------------------------------- -# Controlled: vortex + pinball + PPO (q_ctl) -# --------------------------------------------------------------------------- - -def collect_controlled(vtype: str, device_id: int, out_dir: str, - n_steps: int = 150) -> dict: - """Collect field snapshots for vortex + pinball with PPO control. - - Loads the trained PPO model, runs inference, saves fields and telemetry. - Also saves norm.json and ddf/fifo checkpoints for possible replay. - """ - scene_name = f"vortex_{vtype}" - cfg_src = get_scene(scene_name) - cfg_v = _VORTEX_CFG[vtype] - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(0.004)) - - ff = FlowField(field_cfg, cuda_cfg, device_id=device_id) - ny = ff.FIELD_SHAPE[1] - n_sensors = 3 - - # Save config - with open(os.path.join(out_dir, "config.json"), "w") as f: - json.dump({k: str(v) if not isinstance(v, (int, float, list, bool)) else v - for k, v in cfg_src.items()}, f, indent=2) - - # ---- Phase 1: Sensor-only target recording with vortex ---- - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((40.0 * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) - - stabilize_steps_short = int(1 * ff.FIELD_SHAPE[0] / U0) - ff.run(stabilize_steps_short, np.zeros(n_sensors, dtype=DATA_TYPE)) - - ff.get_ddf() - ff.save_ddf() - - ff.add_vortex((10.0 * L0, CENTER_Y, 0.0), - 2.0 * L0, cfg_v["vortex_strength"], 0, cfg_v["vortex_type"]) - - target_states = np.empty((0, 6), dtype=DATA_TYPE) - for _ in range(min(FIFO_LEN, n_steps)): - ff.run(800, np.zeros(n_sensors, dtype=DATA_TYPE)) - target_states = np.vstack((target_states, ff.obs.copy())) - np.savez(os.path.join(out_dir, "target.npz"), target_states=target_states) - - # ---- Phase 2: Add pinball, compute norm ---- - ff.restore_ddf() - ff.apply_ddf() - - # Object order MUST match training env: front(id3), TOP(+y,id4), BOTTOM(-y,id5) - ff.add_cylinder((30.0 * L0, CENTER_Y, 0.0), L0 / 2.0) # id 3: front - ff.add_cylinder((31.3 * L0, CENTER_Y + 0.75 * L0, 0.0), L0 / 2.0) # id 4: TOP (+y) - ff.add_cylinder((31.3 * L0, CENTER_Y - 0.75 * L0, 0.0), L0 / 2.0) # id 5: BOTTOM (-y) - - n_obj = ff.obs.size // 2 - assert n_obj == 6, f"Expected 6, got {n_obj}" - - # Stabilize with zero action, then bias action matching training env - ff.run(stabilize_steps_short, np.zeros(n_obj, dtype=DATA_TYPE)) - ff.run(stabilize_steps_short, np.array([0.0, 0.0, 0.0, 0.0, -5.0 * U0, 5.0 * U0], dtype=DATA_TYPE)) - - # Add vortex at x=15*L0 - ff.add_vortex((15.0 * L0, CENTER_Y, 0.0), - 2.0 * L0, cfg_v["vortex_strength"], 0, cfg_v["vortex_type"]) - - # Save DDF checkpoint (vortex at x=15, pinball stabilized with bias) - ff.get_ddf() - ff.save_ddf() - - # Norm collection (zero action on vortex+pinball) - fifo = deque(maxlen=FIFO_LEN) - for _ in range(FIFO_LEN): - ff.run(800, np.zeros(n_obj, dtype=DATA_TYPE)) - fifo.append(ff.obs.copy()) - - temp_states = np.array(fifo, dtype=DATA_TYPE) - force_norm_fact = 6.0 * float(np.max(np.abs(temp_states[:, 6:12]))) - sens_deviation = np.mean(temp_states[:, 0:6], axis=0).astype(DATA_TYPE) - sens_norm_fact = np.zeros(6, dtype=DATA_TYPE) - for i in range(6): - sens_norm_fact[i] = 5.0 * float(np.max(np.abs(temp_states[:, i] - sens_deviation[i]))) - - norm = { - "force_norm_fact": force_norm_fact, - "sens_deviation": sens_deviation.tolist(), - "sens_norm_fact": sens_norm_fact.tolist(), - "action_bias": [0.0, -4.0, 4.0], - } - with open(os.path.join(out_dir, "norm.json"), "w") as f: - json.dump(norm, f, indent=2) - - # Bias FIFO init (restore DDF so vortex starts from x=15) - ff.restore_ddf() - ff.apply_ddf() - fifo.clear() - # BUG-FIX (2026-06-29): use uni_test bias [-5,+5], not training env bias [-4,+4]. - # The model was tested with [-5,+5] in uni_test.ipynb. - bias_fifo_arr = np.array([0.0, 0.0, 0.0, 0.0, -5.0 * U0, 5.0 * U0], dtype=DATA_TYPE) - for _ in range(FIFO_LEN): - ff.run(800, bias_fifo_arr) - fifo.append(ff.obs.copy()) - - save_states = np.array(list(fifo), dtype=DATA_TYPE) - # Restore DDF back to vortex-at-x=15 checkpoint - ff.restore_ddf() - ff.apply_ddf() - - # Save checkpoints for replay (vortex at initial position) - np.save(os.path.join(out_dir, "ddf_checkpoint.npy"), ff.ddf) - np.save(os.path.join(out_dir, "fifo_checkpoint.npy"), save_states) - - # ---- Phase 3: Controlled PPO inference (fade-in/out matching uni_test) ---- - model_path = model_path_for_scene(scene_name) - if model_path is None: - raise FileNotFoundError(f"No model found for scene: {scene_name}") - - model = load_ppo_model(model_path, device=f"cuda:{device_id}", s_dim=12) - model.set_random_seed(0) - - # Restore DDF to vortex-at-x=15 checkpoint - ff.restore_ddf() - ff.apply_ddf() - - # Start with zeros observation (matching uni_test) - obs = np.zeros(12, dtype=np.float32) - - sens_list, forc_list, act_list, rew_list = [], [], [], [] - ux_list = [] - uy_list = [] - fifo = deque(maxlen=FIFO_LEN) - - # Steady-cloak bias for transition (matching uni_test: [-5.1, +5.1]) - # BUG-FIX (2026-06-29): uni_test uses [-5.1,+5.1], not training env [-4,+4]. - steady_bias = np.array([0.0, -5.1 * U0, 5.1 * U0], dtype=DATA_TYPE) - - for step in range(n_steps): - action, _ = model.predict(obs, deterministic=True) - action = action.astype(np.float32).flatten() - act_list.append(action.copy()) - - temp_action = np.array(action * 4.0 + np.array([0.0, -4.0, 4.0]), dtype=DATA_TYPE) - - # Fade-in (0-24), active (25-44), fade-out (45-69), steady-cloak (70+) - if step < 25: - w = step / 25.0 - temp_val = temp_action * w * U0 + steady_bias * (1.0 - w) - elif 45 <= step < 70: - w = (step - 45) / 25.0 - temp_val = temp_action * (1.0 - w) * U0 + steady_bias * w - elif step >= 70: - temp_val = steady_bias - else: - temp_val = temp_action * U0 - - temp = np.zeros(n_obj, dtype=DATA_TYPE) - temp[3:6] = temp_val - - ff.context.push() - ff.run(800, temp) - ff.context.pop() - - obs_slice = ff.obs.copy() - fifo.append(obs_slice) - sens_list.append(obs_slice[0:6]) - forc_list.append(obs_slice[6:12]) - - # Build observation for next step - # BUG-FIX (2026-06-29): force-first with CORRECT norms. - # OLD (broken): sens_raw = obs[0:6]/force_norm, force_raw = (obs[6:12]-sens_dev)/sens_norm - # obs = hstack([sens_raw, force_raw]) ← completely wrong! - # NEW (correct): forces_norm = obs[6:12]/force_norm, sens_norm = (obs[0:6]-sens_dev)/sens_norm - # obs = hstack([forces_norm, sens_norm]) ← force first - # Training env produces: obs = hstack([forces/force_norm, sensors/sens_norm]). - # collect_karman and collect_illusion use build_observation() which does this correctly. - forces_norm = obs_slice[6:12] / force_norm_fact - sens_norm = (obs_slice[0:6] - sens_deviation) / sens_norm_fact - obs = np.clip(np.hstack([forces_norm, sens_norm]), -1.0, 1.0).astype(np.float32) - - # Save field snapshot - ux, uy = get_velocity_field(ff, u0=U0) - ux_list.append(ux) - uy_list.append(uy) - - # Save field snapshots - np.savez_compressed(os.path.join(out_dir, "fields.npz"), - ux=np.stack(ux_list), uy=np.stack(uy_list)) - - # Save telemetry - np.savez(os.path.join(out_dir, "controlled.npz"), - sensors=np.array(sens_list, dtype=np.float32), - forces=np.array(forc_list, dtype=np.float32), - actions=np.array(act_list, dtype=np.float32)) - - # Compute similarity - from CCD_analysis.utils.cfd_interface import compute_similarity - states_arr = np.array(sens_list, dtype=np.float32) - n_align = min(states_arr.shape[0], target_states.shape[0]) - if n_align >= 30: - sim = compute_similarity(target_states, states_arr[:n_align], 30) - else: - sim = 0.0 - - result = {"scene": scene_name, "similarity": float(sim), "n_steps": n_steps} - with open(os.path.join(out_dir, "result.json"), "w") as f: - json.dump(result, f, indent=2) - - del ff, model - return result - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - ap = argparse.ArgumentParser(description="Collect vortex cloak fields for CCD") - ap.add_argument("--type", type=str, default="lamb", - help='Vortex type: lamb, taylor, or "all"') - ap.add_argument("--device", type=int, default=2, help="GPU device ID") - ap.add_argument("--steps", type=int, default=150, - help="Number of recording steps (max 150 for transient)") - ap.add_argument("--skip-target", action="store_true", - help="Skip vortex_target collection") - ap.add_argument("--skip-uncontrolled", action="store_true", - help="Skip uncontrolled collection") - ap.add_argument("--skip-controlled", action="store_true", - help="Skip controlled collection") - args = ap.parse_args() - - if args.type.lower() == "all": - vtypes = ["lamb", "taylor"] - else: - vtypes = [args.type.lower()] - - t_start = time.time() - for vtype in vtypes: - print(f"\n{'=' * 60}") - print(f"Vortex type: {vtype}") - print(f"{'=' * 60}") - - # --- Target: vortex only --- - if not args.skip_target: - scene_name = f"vortex_target_{vtype}" - print(f"\n--- Collecting target: {scene_name} ---") - out_dir = data_dir_for_scene(scene_name) - r = collect_target(vtype, args.device, out_dir, args.steps) - print(f" Done: {r['scene']} -> {out_dir}") - - # --- Uncontrolled: vortex + pinball, zero control --- - if not args.skip_uncontrolled: - scene_name = f"vortex_uncontrolled_{vtype}" - print(f"\n--- Collecting uncontrolled: {scene_name} ---") - out_dir = data_dir_for_scene(scene_name) - r = collect_uncontrolled(vtype, args.device, out_dir, args.steps) - print(f" Done: {r['scene']} -> {out_dir}") - - # --- Controlled: vortex + pinball + PPO --- - if not args.skip_controlled: - scene_name = f"vortex_{vtype}" - print(f"\n--- Collecting controlled: {scene_name} ---") - out_dir = data_dir_for_scene(scene_name) - r = collect_controlled(vtype, args.device, out_dir, args.steps) - print(f" Done: {r['scene']} -> {out_dir} sim={r['similarity']:.4f}") - - elapsed = time.time() - t_start - print(f"\nTotal time: {elapsed:.1f}s") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/CCD_analysis/scripts/detect_period.py b/src/CCD_analysis/scripts/detect_period.py deleted file mode 100644 index c23426e..0000000 --- a/src/CCD_analysis/scripts/detect_period.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Detect period and generate phase-aligned field extraction plan. - -For a periodic case, this script: -1. Loads sensor data from controlled.npz (or sensors.npz for open-loop) -2. Detects dominant frequency and cycle stability (CV_T) -3. Selects the best 4-cycle stable window -4. Computes which step indices correspond to N_PTS uniform phase points -5. Generates a phase_plan.json for replay_fields.py - -Usage: - python3 src/CCD_analysis/scripts/detect_period.py --scene pinball - python3 src/CCD_analysis/scripts/detect_period.py --scene illusion_1.0L - python3 src/CCD_analysis/scripts/detect_period.py --scene target_cylinder_1.0L - python3 src/CCD_analysis/scripts/detect_period.py --scene karman_re100 - -Output: data/resampled/{scene_name}/phase_plan.json -""" -from __future__ import annotations - -import argparse -import json -import os -import sys - -import numpy as np - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import SCENES, DATA_DIR -from CCD_analysis.utils.resampling import ( - detect_dominant_frequency, detect_cycle_stability, -) - -N_CYCLES = 4 -N_PTS = 24 -CV_T_STRICT = 0.10 -CV_T_RELAXED = 0.12 - - -def run_single(scene_name: str, n_cycles: int = N_CYCLES, n_pts: int = N_PTS) -> dict: - if scene_name not in SCENES: - raise KeyError(f"Unknown scene: {scene_name}") - - cfg = SCENES[scene_name] - scene_id = cfg["scene_id"] - data_dir = os.path.join(DATA_DIR, scene_id, scene_name) - si = cfg["sample_interval"] - - # Load sensor data - controlled_path = os.path.join(data_dir, "controlled.npz") - sensors_path = os.path.join(data_dir, "sensors.npz") - - if os.path.isfile(controlled_path): - d = np.load(controlled_path) - sensors = d.get("sensors") - d.close() - elif os.path.isfile(sensors_path): - d = np.load(sensors_path) - sensors = d.get("sensors") - d.close() - else: - raise FileNotFoundError(f"No sensor data found for {scene_name}") - - if sensors is None or len(sensors) < 30: - raise ValueError(f"Insufficient sensor data ({len(sensors) if sensors is not None else 0})") - - # Use centre sensor v-component for period detection - signal = sensors[:, 3] - - # Frequency and stability - f_case, T_case, _ = detect_dominant_frequency(signal, float(si)) - cv_T, mean_T, cy_lengths = detect_cycle_stability(signal, float(si)) - N_raw = mean_T / si if mean_T > 0 else 0 - rho = float(n_pts) / N_raw if N_raw > 0 else 99 - - # Gate check (note: delta_f is always <= 0.1 since we self-compare) - if cv_T <= CV_T_STRICT: - gate = "strict" - elif cv_T <= CV_T_RELAXED: - gate = "relaxed" - else: - gate = "auxiliary" - - print(f" f={f_case:.6f}, T={T_case:.0f}, CV_T={cv_T:.4f}, gate={gate}") - print(f" N_raw/cycle={N_raw:.1f}, rho_interp={rho:.2f}") - - if gate not in ("strict", "relaxed"): - print(f" WARNING: gate={gate}, case may be too unstable for clean CCD") - # Still generate plan but flag it - - # Find cycle boundaries via rising zero-crossings - y = signal - np.mean(signal) - crossings = np.where((np.sign(y[:-1]) < 0) & (np.sign(y[1:]) > 0))[0] - if len(crossings) < n_cycles + 1: - raise ValueError(f"Only {len(crossings)} crossings found, need {n_cycles + 1}") - - # Select the most uniform block of n_cycles - cycle_lens_steps = np.diff(crossings) - T_exp_steps = T_case / si if T_case > 0 else float(np.median(cycle_lens_steps)) - best_score, best_start = float("inf"), 0 - for i in range(len(cycle_lens_steps) - n_cycles + 1): - score = np.sum((cycle_lens_steps[i:i + n_cycles] - T_exp_steps) ** 2) - if score < best_score: - best_score, best_start = score, i - selected_crossings = [int(crossings[best_start + k]) for k in range(n_cycles + 1)] - - # Map each (cycle, phase) to an exact step index - # For cycle c (0..n_cycles-1), the range is [start, end) in step indices - # We place n_pts equally spaced within [start, end) - step_indices = [] - for c in range(n_cycles): - i_start = selected_crossings[c] - i_end = selected_crossings[c + 1] - seg_len = i_end - i_start - for p in range(n_pts): - # Phase fraction: p / n_pts - frac = p / float(n_pts) - idx = int(i_start + frac * seg_len) - step_indices.append(idx) - - # The last sample point (end of last cycle) should be included for completeness - # But we keep exactly n_cycles * n_pts samples; step_indices[p + c * n_pts] - - phase_plan = { - "scene": scene_name, - "scene_id": scene_id, - "n_cycles": n_cycles, - "n_pts": n_pts, - "total_steps": n_cycles * n_pts, - "selected_crossings": selected_crossings, - "step_indices": step_indices, - "f_dom": f_case, - "T_dom_steps": T_case, - "CV_T": cv_T, - "N_raw_per_cycle": float(N_raw), - "rho_interp": rho, - "gate": gate, - } - - return phase_plan - - -def main(): - ap = argparse.ArgumentParser(description="Detect period and generate phase plan") - ap.add_argument("--scene", type=str, default=None, help="Scene name") - ap.add_argument("--all-periodic", action="store_true", help="Run all periodic scenes") - ap.add_argument("--n-cycles", type=int, default=N_CYCLES) - ap.add_argument("--n-pts", type=int, default=N_PTS) - args = ap.parse_args() - - if args.all_periodic: - scenes = [name for name, cfg in SCENES.items() if cfg["target_type"] == "periodic"] - elif args.scene is not None: - scenes = [args.scene] - else: - ap.print_help() - return - - for scene_name in scenes: - print(f"\n=== {scene_name} ===", flush=True) - try: - plan = run_single(scene_name, args.n_cycles, args.n_pts) - except (FileNotFoundError, ValueError, KeyError) as e: - print(f" SKIP: {e}", flush=True) - continue - - out_dir = os.path.join(DATA_DIR, "resampled", scene_name) - os.makedirs(out_dir, exist_ok=True) - with open(os.path.join(out_dir, "phase_plan.json"), "w") as f: - json.dump(plan, f, indent=2) - print(f" Saved: {out_dir}/phase_plan.json", flush=True) - - print("\nDone.", flush=True) - - -if __name__ == "__main__": - main() diff --git a/src/CCD_analysis/scripts/replay_fields.py b/src/CCD_analysis/scripts/replay_fields.py deleted file mode 100644 index 4be0bd0..0000000 --- a/src/CCD_analysis/scripts/replay_fields.py +++ /dev/null @@ -1,280 +0,0 @@ -"""Replay PPO actions with DDF+FIFO checkpoint, save phase-aligned fields. - -For PPO cases (illusion, karman): - - Loads DDF+FIFO checkpoint saved by collect_*.py - - Rebuilds exact same FlowField geometry - - Restores DDF and FIFO - - Replays ALL saved actions step by step - - Saves raw (non-interpolated) field snapshots at step indices from phase_plan.json - - Verifies replay fidelity by comparing sensors/forces with original controlled.npz - -For open-loop cases (target_cylinder, pinball): - - No DDF checkpoint needed (no PPO actions) - - Uses fields.npz directly + phase_plan.json to extract aligned fields - -Usage: - conda run -n pycuda_3_10 python src/CCD_analysis/scripts/replay_fields.py --scene illusion_1.0L --device 2 - conda run -n pycuda_3_10 python src/CCD_analysis/scripts/replay_fields.py --scene target_cylinder_1.0L --device 2 - conda run -n pycuda_3_10 python src/CCD_analysis/scripts/replay_fields.py --scene pinball --device 2 - -Output: data/{scene_id}/{scene_name}/fields_aligned.npz -""" -from __future__ import annotations - -import argparse -import json -import os -import sys -import time - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField - -from CCD_analysis.configs import ( - get_scene, get_scene_list, data_dir_for_scene, LEGACY_CFG_DIR, L0, CENTER_Y, -) -from CCD_analysis.utils.cfd_interface import load_legacy_configs, get_velocity_field - -DATA_TYPE = np.float32 -FIFO_LEN = 150 - - -def build_env(cfg: dict, cuda_cfg, field_cfg, device_id: int) -> FlowField: - """Build the exact same FlowField geometry as the original collection script.""" - ff = FlowField(field_cfg, cuda_cfg, device_id=device_id) - u0 = cfg["u0"] - - if cfg.get("has_disturbance", False): - # Karman layout: dist_cyl + 3 sensors first - ff.add_cylinder((10.0 * L0, CENTER_Y, 0.0), L0) - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((cfg["sensor_x"] * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) - n_phase1 = 4 - ff.run(int(4 * 1280 / u0), np.zeros(n_phase1, dtype=DATA_TYPE)) - else: - # Illusion / plain layout: 3 sensors first - for y_off in [2.0, 0.0, -2.0]: - ff.add_sensor((cfg["sensor_x"] * L0, CENTER_Y + y_off * L0, 0.0), L0 / 4.0) - - # Add pinball - ff.add_cylinder((cfg["pinball_front_x"] * L0, CENTER_Y, 0.0), L0 / 2.0) - ff.add_cylinder((cfg["pinball_rear_x"] * L0, CENTER_Y + 0.75 * L0, 0.0), L0 / 2.0) - ff.add_cylinder((cfg["pinball_rear_x"] * L0, CENTER_Y - 0.75 * L0, 0.0), L0 / 2.0) - return ff - - -def replay_ppo(scene_name: str, device_id: int, verify_tol: float = 1e-4) -> int: - """Replay PPO inference using DDF+FIFO checkpoint, save aligned fields.""" - cfg = get_scene(scene_name) - out_dir = data_dir_for_scene(scene_name) - u0 = cfg["u0"] - si = cfg["sample_interval"] - ac_scale = cfg["action_scale"] - ac_bias = cfg["action_bias"] - n_obj = cfg["n_objects_env"] - obs_start, obs_end = cfg["obs_slice"] - - # Load phase plan - plan_path = os.path.join(os.path.join(os.path.dirname(out_dir), "..", "resampled", scene_name), "phase_plan.json") - # Try alternate path - if not os.path.isfile(plan_path): - plan_path = os.path.join(os.path.join(os.path.dirname(__file__), "..", "data", "resampled", scene_name), "phase_plan.json") - if not os.path.isfile(plan_path): - plan_path = os.path.join(cfg["DATA_DIR"] if "DATA_DIR" in cfg else os.path.join(os.path.dirname(__file__), "..", "data"), "resampled", scene_name, "phase_plan.json") - # Final fallback: data/resampled - from CCD_analysis.configs import DATA_DIR as CCD_DATA_DIR - plan_path = os.path.join(CCD_DATA_DIR, "resampled", scene_name, "phase_plan.json") - - if not os.path.isfile(plan_path): - raise FileNotFoundError(f"phase_plan.json not found for {scene_name}. Run detect_period.py first.") - - with open(plan_path) as f: - plan = json.load(f) - step_indices = set(plan["step_indices"]) - print(f" phase plan: {plan['n_cycles']} cycles x {plan['n_pts']} pts = {len(step_indices)} steps", flush=True) - - # Load actions and original data for verification - controlled = np.load(os.path.join(out_dir, "controlled.npz")) - actions = controlled["actions"] - orig_sensors = controlled["sensors"] - orig_forces = controlled["forces"] - n_steps = len(actions) - - # Load DDF+FIFO checkpoint - ddf_ckpt = np.load(os.path.join(out_dir, "ddf_checkpoint.npy")) - fifo_ckpt = np.load(os.path.join(out_dir, "fifo_checkpoint.npy")) - - # Build env and restore - cuda_cfg, field_cfg = load_legacy_configs(LEGACY_CFG_DIR) - field_cfg = field_cfg._replace(viscosity=float(cfg["nu"]), velocity=float(u0)) - - ff = build_env(cfg, cuda_cfg, field_cfg, device_id) - n_obj_built = ff.obs.size // 2 - assert n_obj_built == n_obj, f"Object count mismatch: {n_obj_built} vs expected {n_obj}" - - # Restore DDF - ff.ddf = ddf_ckpt.copy() - ff.apply_ddf() - print(f" DDF checkpoint restored ({len(ddf_ckpt)} floats)", flush=True) - - # Replay - from collections import deque - fifo = deque(maxlen=FIFO_LEN) - for s in fifo_ckpt: - fifo.append(s) - - sens_replay, forc_replay = [], [] - ux_list, uy_list = [], [] - max_diff_sens, max_diff_forc = 0.0, 0.0 - - for step in range(n_steps): - action = actions[step] - omega = (action * ac_scale + np.array(ac_bias, dtype=np.float32)) * u0 - temp = np.zeros(n_obj, dtype=DATA_TYPE) - temp[n_obj - 3:] = omega - - ff.context.push() - ff.run(si, temp) - ff.context.pop() - - obs_slice = ff.obs.copy()[obs_start:obs_end] - fifo.append(obs_slice) - sens_replay.append(obs_slice[0:6]) - forc_replay.append(obs_slice[6:12]) - - # Save field at selected step indices - if step in step_indices: - ux, uy = get_velocity_field(ff, u0=u0) - ux_list.append(ux) - uy_list.append(uy) - - # Verify replay fidelity - sens_replay = np.array(sens_replay, dtype=np.float32) - forc_replay = np.array(forc_replay, dtype=np.float32) - - diff_sens = np.max(np.abs(sens_replay - orig_sensors)) - diff_forc = np.max(np.abs(forc_replay - orig_forces)) - print(f" Replay max diff: sensors={diff_sens:.6e}, forces={diff_forc:.6e}", flush=True) - - if diff_sens > verify_tol or diff_forc > verify_tol: - print(f" WARNING: replay diff exceeds tolerance ({verify_tol}). " - f"CUDA atomic non-determinism may cause minor variation.", flush=True) - - # Save aligned fields - out_path = os.path.join(out_dir, "fields_aligned.npz") - np.savez_compressed(out_path, - ux=np.stack(ux_list), uy=np.stack(uy_list)) - print(f" Saved {len(ux_list)} field snapshots to {out_path}", flush=True) - - # Save verification report - verify = { - "scene": scene_name, - "n_steps": n_steps, - "n_fields": len(ux_list), - "max_diff_sensors": float(diff_sens), - "max_diff_forces": float(diff_forc), - "tolerance": verify_tol, - "passed": bool(diff_sens <= verify_tol and diff_forc <= verify_tol), - } - with open(os.path.join(out_dir, "replay_verify.json"), "w") as f: - json.dump(verify, f, indent=2) - - del ff - controlled.close() - return len(ux_list) - - -def replay_open_loop(scene_name: str, device_id: int) -> int: - """For open-loop cases: extract fields from phase plan indices directly. - - Open-loop cases (target_cylinder, pinball) have no actions to replay. - Their fields.npz already contains the raw time series. - We just need to pick the phase-aligned snapshots. - """ - cfg = get_scene(scene_name) - out_dir = data_dir_for_scene(scene_name) - - from CCD_analysis.configs import DATA_DIR as CCD_DATA_DIR - plan_path = os.path.join(CCD_DATA_DIR, "resampled", scene_name, "phase_plan.json") - if not os.path.isfile(plan_path): - raise FileNotFoundError(f"phase_plan.json not found for {scene_name}") - - with open(plan_path) as f: - plan = json.load(f) - step_indices = plan["step_indices"] - - # Load existing fields - fields_path = os.path.join(out_dir, "fields.npz") - if not os.path.isfile(fields_path): - raise FileNotFoundError(f"fields.npz not found for {scene_name}") - - fd = np.load(fields_path) - ux_all, uy_all = fd["ux"], fd["uy"] - - n_avail = len(ux_all) - valid_idx = [idx for idx in step_indices if idx < n_avail] - if len(valid_idx) < len(step_indices): - print(f" WARNING: {len(step_indices) - len(valid_idx)} step indices out of range " - f"(max={n_avail - 1}). Using {len(valid_idx)} valid snapshots.", flush=True) - - ux_list = [ux_all[idx] for idx in valid_idx] - uy_list = [uy_all[idx] for idx in valid_idx] - - out_path = os.path.join(out_dir, "fields_aligned.npz") - np.savez_compressed(out_path, ux=np.stack(ux_list), uy=np.stack(uy_list)) - print(f" Extracted {len(ux_list)} phase-aligned field snapshots to {out_path}", flush=True) - - fd.close() - return len(ux_list) - - -def replay_single(scene_name: str, device_id: int, verify_tol: float = 1e-4) -> int: - """Route to the appropriate replay method based on scene type.""" - cfg = get_scene(scene_name) - source = cfg.get("source", "open_loop") - - if source == "PPO_inference": - return replay_ppo(scene_name, device_id, verify_tol) - else: - return replay_open_loop(scene_name, device_id) - - -def main(): - ap = argparse.ArgumentParser(description="Replay and save phase-aligned fields") - ap.add_argument("--scene", type=str, default="karman_re100") - ap.add_argument("--device", type=int, default=2) - ap.add_argument("--tol", type=float, default=1e-4, - help="Replay verification tolerance") - ap.add_argument("--all-periodic", action="store_true", - help="Replay all periodic scenes") - args = ap.parse_args() - - if args.all_periodic: - from CCD_analysis.configs import SCENES - scenes = [name for name, cfg in SCENES.items() - if cfg.get("target_type") == "periodic"] - else: - scenes = [args.scene] - - for scene in scenes: - print(f"\n=== Replaying fields for {scene} ===", flush=True) - t0 = time.time() - try: - n = replay_single(scene, args.device, args.tol) - print(f" {n} fields in {time.time() - t0:.1f}s", flush=True) - except (FileNotFoundError, AssertionError, ValueError) as e: - print(f" FAILED: {e}", flush=True) - - print("\nDone.", flush=True) - - -if __name__ == "__main__": - main() diff --git a/src/CCD_analysis/scripts/resample.py b/src/CCD_analysis/scripts/resample.py deleted file mode 100644 index 618aed9..0000000 --- a/src/CCD_analysis/scripts/resample.py +++ /dev/null @@ -1,172 +0,0 @@ -"""DEPRECATED — replaced by detect_period.py + replay_fields.py - -This script interpolates field data to phase-aligned points, introducing artifacts. -The new pipeline saves RAW (non-interpolated) fields at exact step indices. - -Keep this file for reference only. Do NOT rely on its output for CCD analysis. - -Usage (old, DEPRECATED): - python scripts/resample.py # <-- DO NOT USE -""" -from __future__ import annotations - -import json -import os -import sys - -import numpy as np - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import SCENES, DATA_DIR -from CCD_analysis.utils.resampling import ( - detect_dominant_frequency, detect_cycle_stability, phase_resample, -) - -N_CYCLES = 4 -N_PTS = 24 - -CV_T_STRICT = 0.10 -CV_T_RELAXED = 0.12 -DELTA_F_STRICT = 0.10 -DELTA_F_RELAXED = 0.20 - -pr = lambda *a, **kw: print(*a, **kw, flush=True) - - -def run(): - periodic = [name for name, cfg in SCENES.items() - if cfg["target_type"] == "periodic"] - pr(f"Periodic cases: {periodic}") - - for name in periodic: - pr(f"\n=== {name} ===") - cfg = SCENES[name] - data_dir = os.path.join(DATA_DIR, cfg["scene_id"], name) - meta_path = os.path.join(data_dir, "meta.json") - sens_path = os.path.join(data_dir, "sensors.npz") - fields_path = os.path.join(data_dir, "fields.npz") - controlled_path = os.path.join(data_dir, "controlled.npz") - - print(f"\n=== {name} ===") - - # Load sensor data - if os.path.isfile(controlled_path): - d = np.load(controlled_path) - elif os.path.isfile(sens_path): - d = np.load(sens_path) - else: - print(f" SKIP: no sensor data") - continue - - sensors = d.get("sensors") - if sensors is None or len(sensors) < 30: - print(f" SKIP: insufficient sensor data ({len(sensors) if sensors is not None else 0})") - continue - - si = cfg["sample_interval"] - signal = sensors[:, 3] # centre sensor v - - # Frequency and stability - f_case, T_case, _ = detect_dominant_frequency(signal, float(si)) - cv_T, mean_T, cy_lengths = detect_cycle_stability(signal, float(si)) - print(f" f={f_case:.6f}, T={T_case:.0f}, CV_T={cv_T:.4f}") - - # Gate check - N_raw = mean_T / si if mean_T > 0 else 0 - rho = 24.0 / N_raw if N_raw > 0 else 99 - delta_f = abs(f_case - f_case) / (f_case + 1e-12) - if cv_T <= CV_T_STRICT and delta_f <= DELTA_F_STRICT: - gate = "strict" - elif cv_T <= CV_T_RELAXED and delta_f <= DELTA_F_RELAXED: - gate = "relaxed" - else: - gate = "auxiliary" - print(f" gate={gate}, N_raw/cycle={N_raw:.1f}, rho_interp={rho:.2f}") - - if gate not in ("strict", "relaxed"): - print(f" SKIP: does not pass period gate") - continue - - # Find cycles - y = signal - np.mean(signal) - crossings = np.where((np.sign(y[:-1]) < 0) & (np.sign(y[1:]) > 0))[0] - if len(crossings) < N_CYCLES + 1: - print(f" SKIP: only {len(crossings)} cycles") - continue - - # Select best N_CYCLES - cycle_lens = np.diff(crossings) - T_exp = T_case / si if T_case > 0 else N_raw - best_score, best_start = float("inf"), 0 - for i in range(len(cycle_lens) - N_CYCLES + 1): - score = np.sum((cycle_lens[i:i+N_CYCLES] - T_exp) ** 2) - if score < best_score: - best_score, best_start = score, i - selected = list(crossings[best_start:best_start + N_CYCLES + 1]) - - # Resample - rs_sensors = phase_resample(sensors, selected, n_pts=N_PTS) - - out = {"sensors": rs_sensors, "n_cycles": N_CYCLES, "n_pts": N_PTS, - "gate": gate, "selected_crossings": selected} - - # Forces - forces = d.get("forces") - if forces is not None and forces.ndim == 2: - out["forces"] = phase_resample(forces, selected, n_pts=N_PTS) - - # Actions - actions = d.get("actions") - if actions is not None and actions.ndim == 2: - out["actions"] = phase_resample(actions, selected, n_pts=N_PTS) - - fd = None - # Fields (from controlled.npz or fields.npz) - if os.path.isfile(controlled_path) and "ux" not in d: - ol_path = os.path.join(data_dir, "open_loop_fields.npz") - if os.path.isfile(ol_path): - fd = np.load(ol_path) - elif os.path.isfile(fields_path): - fd = np.load(fields_path) - - if fd is not None and "ux" in fd: - ux, uy = fd["ux"], fd["uy"] - if len(ux) >= selected[-1]: - nx, ny = ux.shape[2], ux.shape[1] - field_flat = np.column_stack([ux.reshape(len(ux), -1), - uy.reshape(len(uy), -1)]) - rs_fields = phase_resample(field_flat, selected, n_pts=N_PTS) - half = rs_fields.shape[-1] // 2 - out["ux"] = rs_fields[:, :, :half].reshape(N_CYCLES, N_PTS, ny, nx) - out["uy"] = rs_fields[:, :, half:].reshape(N_CYCLES, N_PTS, ny, nx) - - # Save - resample_dir = os.path.join(DATA_DIR, "resampled", name) - os.makedirs(resample_dir, exist_ok=True) - - save_dict = {"sensors": out["sensors"], "n_cycles": N_CYCLES, "n_pts": N_PTS} - for k in ["forces", "actions", "ux", "uy"]: - if k in out: - save_dict[k] = out[k] - np.savez_compressed(os.path.join(resample_dir, "resampled.npz"), **save_dict) - - meta = {"case": name, "gate": gate, "f_case": f_case, "CV_T": cv_T, - "N_raw_per_cycle": N_raw, "rho_interp": rho, - "n_cycles": N_CYCLES, "n_pts": N_PTS, - "has_fields": "ux" in out} - with open(os.path.join(resample_dir, "meta.json"), "w") as f: - json.dump(meta, f, indent=2) - pr(f" Saved: {resample_dir} ({'fields' if meta['has_fields'] else 'no fields'})") - - d.close() - if fd is not None: - fd.close() - - print("\nDone.") - - -if __name__ == "__main__": - run() diff --git a/src/CCD_analysis/scripts/sanity_check_force.py b/src/CCD_analysis/scripts/sanity_check_force.py deleted file mode 100644 index 41c96e3..0000000 --- a/src/CCD_analysis/scripts/sanity_check_force.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Sanity check: compare raw force signals between target and illusion. - -This script answers the most basic question: - "Does the controller actually match the target's force signature?" - -For each diameter (0.75L, 1.0L, 1.5L), loads the target cylinder's forces -and the illusion's total forces, plots them, and computes basic statistics. - -Usage: - python3 src/CCD_analysis/scripts/sanity_check_force.py - -Output: data/figures/sanity_force_{diam}L.png - data/figures/sanity_force_report.json -""" -from __future__ import annotations - -import json -import os -import sys - -import numpy as np - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import SCENES, DATA_DIR - -FIG_DIR = os.path.join(DATA_DIR, "figures") -os.makedirs(FIG_DIR, exist_ok=True) - -DIAMETERS = [0.75, 1.0, 1.5] - - -def compute_cd_cl(forces: np.ndarray, scene_name: str) -> tuple: - """Compute total Cd and Cl from raw forces. - - For target_cylinder: forces shape (N, 2), just [Fx, Fy] - For illusion: forces shape (N, 6), need [F0+F2+F4, F1+F3+F5] - """ - if "target_cylinder" in scene_name: - return forces[:, 0], forces[:, 1] # Fx, Fy - else: - return (forces[:, 0] + forces[:, 2] + forces[:, 4], - forces[:, 1] + forces[:, 3] + forces[:, 5]) - - -def run(): - print("=" * 60, flush=True) - print("Sanity Check: Force Comparison", flush=True) - print("=" * 60, flush=True) - - report = {} - - for diam in DIAMETERS: - tgt_name = f"target_cylinder_{diam}L" - ill_name = f"illusion_{diam}L" - - cfg_tgt = SCENES[tgt_name] - cfg_ill = SCENES[ill_name] - - # Load target forces - if cfg_tgt.get("source") == "open_loop": - tgt_path = os.path.join(DATA_DIR, "target_cylinder", tgt_name, "sensors.npz") - tgt_raw = np.load(tgt_path) - tgt_forces = tgt_raw["forces"] # (N, 2) - else: - tgt_path = os.path.join(DATA_DIR, "target_cylinder", tgt_name, "controlled.npz") - tgt_raw = np.load(tgt_path) - tgt_forces = tgt_raw["forces"] - - tgt_cd, tgt_cl = compute_cd_cl(tgt_forces, tgt_name) - - # Load illusion forces - ill_raw = np.load(os.path.join(DATA_DIR, "illusion", ill_name, "controlled.npz")) - ill_forces = ill_raw["forces"] # (N, 6) - ill_cd, ill_cl = compute_cd_cl(ill_forces, ill_name) - - # Align lengths - n = min(len(tgt_cd), len(ill_cd)) - tgt_cd, tgt_cl = tgt_cd[:n], tgt_cl[:n] - ill_cd, ill_cl = ill_cd[:n], ill_cl[:n] - - # Statistics - cd_corr = float(np.corrcoef(tgt_cd, ill_cd)[0, 1]) - cl_corr = float(np.corrcoef(tgt_cl, ill_cl)[0, 1]) - cd_rmse = float(np.sqrt(np.mean((tgt_cd - ill_cd) ** 2))) - cl_rmse = float(np.sqrt(np.mean((tgt_cl - ill_cl) ** 2))) - cd_mean_tgt = float(np.mean(tgt_cd)) - cd_mean_ill = float(np.mean(ill_cd)) - cl_std_tgt = float(np.std(tgt_cl)) - cl_std_ill = float(np.std(ill_cl)) - - print(f"\n--- {diam}L ---", flush=True) - print(f" Cd correlation: {cd_corr:.4f}") - print(f" Cl correlation: {cl_corr:.4f}") - print(f" Cd RMSE: {cd_rmse:.6f}") - print(f" Cl RMSE: {cl_rmse:.6f}") - print(f" Cd mean: target={cd_mean_tgt:.6f}, illusion={cd_mean_ill:.6f}") - print(f" Cl std: target={cl_std_tgt:.6f}, illusion={cl_std_ill:.6f}") - - report[diam] = { - "cd_correlation": cd_corr, - "cl_correlation": cl_corr, - "cd_rmse": cd_rmse, - "cl_rmse": cl_rmse, - "cd_mean_target": cd_mean_tgt, - "cd_mean_illusion": cd_mean_ill, - "cl_std_target": cl_std_tgt, - "cl_std_illusion": cl_std_ill, - "n_samples": n, - } - - # Plot - t = np.arange(n) - fig, axes = plt.subplots(2, 2, figsize=(14, 8)) - - # Cd time series - ax = axes[0, 0] - ax.plot(t, tgt_cd, "r-", alpha=0.7, label=f"Target {diam}L", linewidth=1) - ax.plot(t, ill_cd, "b-", alpha=0.7, label=f"Illusion {diam}L", linewidth=1) - ax.set_ylabel("Total Fx (lattice)") - ax.set_title(f"{diam}L: Total Fx (Cd proxy)") - ax.legend() - ax.grid(True, alpha=0.3) - - # Cl time series - ax = axes[0, 1] - ax.plot(t, tgt_cl, "r-", alpha=0.7, label=f"Target {diam}L", linewidth=1) - ax.plot(t, ill_cl, "b-", alpha=0.7, label=f"Illusion {diam}L", linewidth=1) - ax.set_ylabel("Total Fy (lattice)") - ax.set_title(f"{diam}L: Total Fy (Cl proxy)") - ax.legend() - ax.grid(True, alpha=0.3) - - # Cd scatter - ax = axes[1, 0] - ax.scatter(tgt_cd, ill_cd, s=2, alpha=0.5, c="purple") - lim = max(np.abs(ax.get_xlim()).max(), np.abs(ax.get_ylim()).max()) - ax.plot([-lim, lim], [-lim, lim], "k--", alpha=0.3) - ax.set_xlabel("Target Fx") - ax.set_ylabel("Illusion Fx") - ax.set_title(f"Cd scatter (r={cd_corr:.3f})") - ax.set_aspect("equal") - ax.grid(True, alpha=0.3) - - # Cl scatter - ax = axes[1, 1] - ax.scatter(tgt_cl, ill_cl, s=2, alpha=0.5, c="orange") - lim = max(np.abs(ax.get_xlim()).max(), np.abs(ax.get_ylim()).max()) - ax.plot([-lim, lim], [-lim, lim], "k--", alpha=0.3) - ax.set_xlabel("Target Fy") - ax.set_ylabel("Illusion Fy") - ax.set_title(f"Cl scatter (r={cl_corr:.3f})") - ax.set_aspect("equal") - ax.grid(True, alpha=0.3) - - plt.tight_layout() - path = os.path.join(FIG_DIR, f"sanity_force_{diam}L.png") - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - print(f" Saved: {path}", flush=True) - - tgt_raw.close() - ill_raw.close() - - # Save report - report_path = os.path.join(FIG_DIR, "sanity_force_report.json") - with open(report_path, "w") as f: - json.dump(report, f, indent=2) - print(f"\nReport saved to {report_path}", flush=True) - print("Done.", flush=True) - - -if __name__ == "__main__": - run() diff --git a/src/CCD_analysis/scripts/verify_cases.py b/src/CCD_analysis/scripts/verify_cases.py deleted file mode 100644 index 0d201b7..0000000 --- a/src/CCD_analysis/scripts/verify_cases.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Phase 3: Generate vorticity images and verification reports. - -For each case with fields_aligned.npz: - 1. Average the 96 snapshots (4 cycles x 24 phases) to get mean flow - 2. Compute vorticity from the mean flow - 3. Save vorticity image - 4. Also save the mean flow ux/uy fields for reference - -Usage: - python3 src/CCD_analysis/scripts/verify_cases.py - -Output: data/figures/vorticity_{scene_name}.png - data/figures/meanflow_{scene_name}.png -""" -from __future__ import annotations - -import json -import os -import sys - -import numpy as np - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import SCENES, DATA_DIR - -FIG_DIR = os.path.join(DATA_DIR, "figures") -os.makedirs(FIG_DIR, exist_ok=True) - -NX = 1280 -NY = 512 - -# Cases with stable sampling periods -PERIODIC_CASES = [ - "pinball", - "karman_re100", - "illusion_0.75L", - "illusion_1.0L", - "illusion_1.5L", - "target_cylinder_0.75L", - "target_cylinder_1.0L", - "target_cylinder_1.5L", -] - -STEADY_CASES = [ - "steady_cloak", - "target_channel", -] - - -def vorticity_from_uv(ux: np.ndarray, uy: np.ndarray) -> np.ndarray: - """Compute z-vorticity from velocity fields. - - ux, uy have shape (NX, NY) as returned by get_velocity_field. - Returns omega_z with shape (NY, NX) for direct imshow use. - """ - # Transpose to (NY, NX) for correct gradient axes - ux_t = ux.T # (NY, NX) - uy_t = uy.T # (NY, NX) - # omega_z = duy/dx - dux/dy - # gradient(uy_t, axis=1) = duy/dx (axis=1 is x-direction in NY,NX layout) - # gradient(ux_t, axis=0) = dux/dy (axis=0 is y-direction in NY,NX layout) - return np.gradient(uy_t, axis=1) - np.gradient(ux_t, axis=0) - - -def load_aligned_fields(scene_name: str, scene_id: str): - """Load fields_aligned.npz or standard fields.npz.""" - d = os.path.join(DATA_DIR, scene_id, scene_name) - paths = [ - os.path.join(d, "fields_aligned.npz"), - os.path.join(d, "fields.npz"), - ] - for p in paths: - if os.path.isfile(p): - return np.load(p) - return None - - -def plot_vorticity(omega: np.ndarray, title: str, path: str): - """Save vorticity image with symmetric colorbar.""" - abs_o = np.abs(omega[np.isfinite(omega)]) - vmax = float(np.percentile(abs_o, 99.5)) if abs_o.size > 0 else 1.0 - if vmax <= 0: - vmax = 1.0 - - fig, ax = plt.subplots(figsize=(16, 5)) - im = ax.imshow(omega, origin="lower", aspect="equal", cmap="RdBu_r", - vmin=-vmax, vmax=vmax) - ax.set_xlabel("x (lattice)") - ax.set_ylabel("y (lattice)") - ax.set_title(title) - fig.colorbar(im, ax=ax, fraction=0.046, pad=0.04, label=r"$\omega_z$") - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def plot_meanflow(ux_mean: np.ndarray, uy_mean: np.ndarray, title: str, path: str): - """Save mean flow magnitude image.""" - speed = np.sqrt(ux_mean**2 + uy_mean**2) - vmax = float(np.percentile(speed, 99.5)) if speed.size > 0 else 1.0 - - fig, axes = plt.subplots(1, 2, figsize=(20, 5)) - im0 = axes[0].imshow(ux_mean, origin="lower", aspect="equal", cmap="viridis") - axes[0].set_title(f"{title}: ux mean") - fig.colorbar(im0, ax=axes[0]) - - im1 = axes[1].imshow(uy_mean, origin="lower", aspect="equal", cmap="viridis") - axes[1].set_title(f"{title}: uy mean") - fig.colorbar(im1, ax=axes[1]) - - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def process_periodic(scene_name: str): - """Process a periodic case: average fields, compute vorticity, save images.""" - cfg = SCENES[scene_name] - scene_id = cfg["scene_id"] - - print(f"\n=== {scene_name} ===", flush=True) - - fd = load_aligned_fields(scene_name, scene_id) - if fd is None: - print(f" SKIP: no field data found", flush=True) - return - - ux = fd["ux"] - uy = fd["uy"] - print(f" fields shape: {ux.shape}", flush=True) - - # Time-average over all snapshots - ux_mean = np.mean(ux, axis=0) - uy_mean = np.mean(uy, axis=0) - - # Vorticity of mean flow - omega = vorticity_from_uv(ux_mean, uy_mean) - - # Save images - plot_vorticity(omega, f"{scene_name}: mean vorticity (4-cycle avg)", - os.path.join(FIG_DIR, f"vorticity_{scene_name}.png")) - plot_meanflow(ux_mean, uy_mean, scene_name, - os.path.join(FIG_DIR, f"meanflow_{scene_name}.png")) - print(f" Saved vorticity and meanflow images", flush=True) - - # Also compute: instantaneous vorticity of each snapshot - if ux.shape[0] >= 96: - # Last cycle's snapshots for phase-resolved comparison - cycle_start = 72 # third cycle start - fig, axes = plt.subplots(4, 6, figsize=(24, 10)) - for p in range(24): - row, col = p // 6, p % 6 - idx = cycle_start + p - om = vorticity_from_uv(ux[idx], uy[idx]) - abs_o = np.abs(om[np.isfinite(om)]) - vm = float(np.percentile(abs_o, 99.5)) if abs_o.size > 0 else 1.0 - axes[row, col].imshow(om, origin="lower", aspect="equal", - cmap="RdBu_r", vmin=-vm, vmax=vm) - axes[row, col].set_title(f"phase {p}") - axes[row, col].axis("off") - plt.suptitle(f"{scene_name}: phase-resolved vorticity (1 cycle)") - plt.tight_layout() - fig.savefig(os.path.join(FIG_DIR, f"phase_vorticity_{scene_name}.png"), - dpi=150, bbox_inches="tight") - plt.close(fig) - print(f" Saved phase-resolved vorticity", flush=True) - - fd.close() - - -def process_steady(scene_name: str): - """Process steady cases (no periodic averaging needed).""" - cfg = SCENES[scene_name] - scene_id = cfg["scene_id"] - - print(f"\n=== {scene_name} (steady) ===", flush=True) - - fd = load_aligned_fields(scene_name, scene_id) - if fd is None: - print(f" SKIP: no field data", flush=True) - return - - ux = fd["ux"] - uy = fd["uy"] - - ux_mean = np.mean(ux, axis=0) - uy_mean = np.mean(uy, axis=0) - omega = vorticity_from_uv(ux_mean, uy_mean) - - plot_vorticity(omega, f"{scene_name}: mean vorticity", - os.path.join(FIG_DIR, f"vorticity_{scene_name}.png")) - plot_meanflow(ux_mean, uy_mean, scene_name, - os.path.join(FIG_DIR, f"meanflow_{scene_name}.png")) - print(f" Saved vorticity and meanflow images", flush=True) - fd.close() - - -def main(): - print("=" * 60, flush=True) - print("Phase 3: Case Verification", flush=True) - print("=" * 60, flush=True) - - for name in PERIODIC_CASES: - process_periodic(name) - - for name in STEADY_CASES: - process_steady(name) - - # Generate summary report - report = { - "periodic_cases": PERIODIC_CASES, - "steady_cases": STEADY_CASES, - "figure_dir": FIG_DIR, - } - with open(os.path.join(DATA_DIR, "figures", "verification_report.json"), "w") as f: - json.dump(report, f, indent=2) - - print(f"\nAll figures saved to {FIG_DIR}", flush=True) - - -if __name__ == "__main__": - main() diff --git a/src/CCD_analysis/scripts/visualize_ccd.py b/src/CCD_analysis/scripts/visualize_ccd.py deleted file mode 100644 index e8df03f..0000000 --- a/src/CCD_analysis/scripts/visualize_ccd.py +++ /dev/null @@ -1,605 +0,0 @@ -"""Phase 4: Visualization — O_k heatmap, CCD modes, POD phase portraits, 1.5L special case. - -Integrates 1.5L special-mechanism branch (no separate analyze_15L.py). - -Usage: - conda run -n pycuda_3_10 python src/CCD_analysis/scripts/visualize_ccd.py -""" -from __future__ import annotations - -import json -import os -import sys -from collections import deque - -import numpy as np - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt - -_SRC = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from CCD_analysis.configs import DATA_DIR, SCENES -from CCD_analysis.utils.resampling import ( - compute_pod, compute_reduced_ccd, cumulative_energy, - load_aligned_fields, make_force_obs, - build_field_matrix, project_into_basis, - detect_dominant_frequency, detect_cycle_stability, -) - -FIG_DIR = os.path.join(DATA_DIR, "figures") -os.makedirs(FIG_DIR, exist_ok=True) - -CCD_Q = 6 -N_PTS = 24 -N_CYCLES = 4 -NX_ = 1280 -NY_ = 512 - -# -- helper: warp CCD directions back to physical space -- -def warp(W: np.ndarray, modes: np.ndarray) -> np.ndarray: - """Convert CCD weight vectors to physical modes.""" - return modes @ W - - -# ==================================================================== -# Task 1: O_k heatmap (force_fy primary, from ccd_results.json) -# ==================================================================== -def task_1(): - print("=== Task 1: O_k heatmap (force_fy) ===", flush=True) - results_path = os.path.join(DATA_DIR, "ccd", "ccd_results.json") - if not os.path.isfile(results_path): - print(" SKIP: ccd_results.json not found", flush=True) - return - - with open(results_path) as f: - all_results = json.load(f) - - for r_label, r in [("r6", 6), ("r10", 10)]: - diameters = [0.75, 1.0, 1.5] - ov_matrix = np.full((3, 3), np.nan) - - for col_idx, diam in enumerate(diameters): - tgt_name = f"target_cylinder_{diam}L" - ill_name = f"illusion_{diam}L" - - # Build target-only POD basis and recompute CCD for O_k - try: - tgt_d = load_aligned_fields(tgt_name) - ill_d = load_aligned_fields(ill_name) - pin_d = load_aligned_fields("pinball") - except FileNotFoundError: - continue - - Q_tgt = build_field_matrix(tgt_d["ux"], tgt_d["uy"]) - mf, modes, _, coeffs = compute_pod(Q_tgt) - modes_r = modes[:, :r] - - def get_ccd_w(name, data): - a = project_into_basis(data["ux"], data["uy"], modes_r, mf) - frc = data.get("forces") - if frc is None: - return None - y = make_force_obs(frc, name, mode="fy") - W, _, _, _, _, _ = compute_reduced_ccd(a, y, Q_delay=CCD_Q) - return W - - W_tgt = get_ccd_w(tgt_name, tgt_d) - W_ill = get_ccd_w(ill_name, ill_d) - W_pin = get_ccd_w("pinball", pin_d) - - def ov(Wa, Wb, k=0): - if Wa is None or Wb is None: - return np.nan - n = min(Wa.shape[1], Wb.shape[1]) - if k >= n: - return np.nan - return float(abs( - Wa[:, k] / (np.linalg.norm(Wa[:, k]) + 1e-12) @ - Wb[:, k] / (np.linalg.norm(Wb[:, k]) + 1e-12) - )) - - ov_matrix[0, col_idx] = ov(W_tgt, W_ill) - ov_matrix[1, col_idx] = ov(W_tgt, W_pin) - ov_matrix[2, col_idx] = ov(W_ill, W_pin) - - for d in [tgt_d, ill_d, pin_d]: - if d is not None: - pass # No explicit close needed, gc will handle - - fig, ax = plt.subplots(figsize=(8, 6)) - im = ax.imshow(ov_matrix, cmap="viridis", vmin=0, vmax=1, aspect="auto") - ax.set_xticks(range(3)) - ax.set_xticklabels(["0.75L", "1.0L", "1.5L"]) - ax.set_yticks(range(3)) - ax.set_yticklabels(["target-illusion", "target-pinball", "illusion-pinball"]) - - for i in range(3): - for j in range(3): - v = ov_matrix[i, j] - if not np.isnan(v): - ax.text(j, i, f"{v:.3f}", ha="center", va="center", - color="white" if v > 0.5 else "black", fontsize=12) - - # Annotate 1.5L as special mechanism - ax.annotate("special mechanism", xy=(2.0, -0.15), fontsize=9, - ha="center", va="center", color="orange", - xycoords="axes fraction") - - plt.colorbar(im, label="O_1 (modal overlap)") - plt.title(f"Force-CCD (SigmaFy) O_1 heatmap ({r_label})") - plt.tight_layout() - path = os.path.join(FIG_DIR, f"Ok_heatmap_fy_{r_label}.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - -# ==================================================================== -# Task 2: CCD mode 1 physical fields (target-only basis) -# ==================================================================== -def task_2(): - print("=== Task 2: CCD mode 1 physical fields ===", flush=True) - r = 6 - - for diam in [0.75, 1.0]: - tgt_name = f"target_cylinder_{diam}L" - ill_name = f"illusion_{diam}L" - - try: - tgt_d = load_aligned_fields(tgt_name) - ill_d = load_aligned_fields(ill_name) - except FileNotFoundError: - continue - - # Build target-only POD basis - Q_tgt = build_field_matrix(tgt_d["ux"], tgt_d["uy"]) - mf, modes, _, _ = compute_pod(Q_tgt) - modes_r = modes[:, :r] - - for name, d_obj, label in [(tgt_name, tgt_d, "target"), - (ill_name, ill_d, "illusion")]: - a = project_into_basis(d_obj["ux"], d_obj["uy"], modes_r, mf) - frc = d_obj.get("forces") - if frc is None: - continue - y = make_force_obs(frc, name, mode="fy") - W, _, _, _, _, _ = compute_reduced_ccd(a, y, Q_delay=CCD_Q) - - ccd_mode = warp(W[:, :1], modes_r) - half = NX_ * NY_ - ux_m = ccd_mode[:half, 0].reshape(NY_, NX_) - uy_m = ccd_mode[half:, 0].reshape(NY_, NX_) - - fig, axes = plt.subplots(1, 2, figsize=(14, 5)) - vmax = max(np.abs(ux_m).max(), np.abs(uy_m).max()) + 1e-12 - - im0 = axes[0].imshow(ux_m, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", - extent=(0, NX_ - 1, 0, NY_ - 1)) - axes[0].set_title(f"{diam}L {label}: CCD mode 1 ux") - plt.colorbar(im0, ax=axes[0]) - - im1 = axes[1].imshow(uy_m, cmap="RdBu_r", vmin=-vmax, vmax=vmax, - origin="lower", aspect="equal", - extent=(0, NX_ - 1, 0, NY_ - 1)) - axes[1].set_title(f"{diam}L {label}: CCD mode 1 uy") - plt.colorbar(im1, ax=axes[1]) - - plt.tight_layout() - path = os.path.join(FIG_DIR, f"ccd_mode1_fy_{diam}L_{label}.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - # Mark end for this diameter - del tgt_d, ill_d - - -# ==================================================================== -# Task 3: z_1(t) verification -# ==================================================================== -def task_3(): - print("=== Task 3: z_1(t) verification ===", flush=True) - - for diam in [0.75, 1.0]: - ill_name = f"illusion_{diam}L" - tgt_name = f"target_cylinder_{diam}L" - - try: - tgt_d = load_aligned_fields(tgt_name) - ill_d = load_aligned_fields(ill_name) - except FileNotFoundError: - continue - - # Target-only POD basis - Q_tgt = build_field_matrix(tgt_d["ux"], tgt_d["uy"]) - mf, modes, _, _ = compute_pod(Q_tgt) - modes_r = modes[:, :6] - - a = project_into_basis(ill_d["ux"], ill_d["uy"], modes_r, mf) - frc = ill_d.get("forces") - if frc is None: - continue - y = make_force_obs(frc, ill_name, mode="fy") - W, sig, _, z, _, _ = compute_reduced_ccd(a, y, Q_delay=CCD_Q) - - fig, axes = plt.subplots(2, 1, figsize=(12, 6)) - - ax = axes[0] - ax.plot(z[0, :], "b-", label="z_1(t)", alpha=0.8) - ax.set_ylabel("CCD temporal coeff") - ax.set_title(f"{diam}L illusion: Force-CCD (SigmaFy) z_1(t)") - ax.legend() - ax.grid(True, alpha=0.3) - - ax = axes[1] - Nv = z.shape[1] - y_norm = (y[0, :Nv] - np.mean(y[0, :Nv])) / (np.std(y[0, :Nv]) + 1e-12) - z_norm = (z[0, :] - np.mean(z[0, :])) / (np.std(z[0, :]) + 1e-12) - ax.plot(y_norm, "r-", label="norm SigmaFy", alpha=0.7) - ax.plot(z_norm, "b--", label="norm z_1", alpha=0.7) - ax.set_xlabel("Flat sample index") - ax.set_ylabel("Normalized amplitude") - ax.set_title(f"{diam}L: z_1 vs SigmaFy (normalized)") - ax.legend() - ax.grid(True, alpha=0.3) - - plt.tight_layout() - path = os.path.join(FIG_DIR, f"z1_verification_fy_{diam}L.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - -# ==================================================================== -# Task 4: POD phase portraits (target-only basis) -# ==================================================================== -def task_4(): - print("=== Task 4: POD phase portraits ===", flush=True) - - fig, axes = plt.subplots(1, 3, figsize=(15, 4)) - - for idx, diam in enumerate([0.75, 1.0, 1.5]): - if diam in [0.75, 1.0]: - main_only = False - else: - main_only = False # include 1.5L in phase portrait - - tgt_name = f"target_cylinder_{diam}L" - ill_name = f"illusion_{diam}L" - - try: - tgt_d = load_aligned_fields(tgt_name) - ill_d = load_aligned_fields(ill_name) - pin_d = load_aligned_fields("pinball") - except FileNotFoundError: - continue - - Q_tgt = build_field_matrix(tgt_d["ux"], tgt_d["uy"]) - mf, modes, _, _ = compute_pod(Q_tgt) - modes_r = modes[:, :6] - - ax = axes[idx] - colors = {"target": "red", "illusion": "blue", "pinball": "green"} - - for kind, d_obj, label in [("target", tgt_d, "target"), - ("illusion", ill_d, "illusion"), - ("pinball", pin_d, "pinball (unc)")]: - if d_obj is None: - continue - a = project_into_basis(d_obj["ux"], d_obj["uy"], modes_r, mf) - ax.plot(a[0, :], a[1, :], ".", color=colors[kind], markersize=3, - alpha=0.5, label=label if idx == 0 else "") - - ax.set_xlabel("a_1") - ax.set_ylabel("a_2") - title = f"{diam}L POD attractor" - if diam == 1.5: - title += " (special mechanism)" - ax.set_title(title) - ax.grid(True, alpha=0.3) - ax.set_aspect("equal") - if idx == 0: - ax.legend(fontsize=8) - - plt.tight_layout() - path = os.path.join(FIG_DIR, "pod_phase_portraits_target_basis.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - -# ==================================================================== -# Task 5: 1.5L special-mechanism diagnostics -# ==================================================================== -def task_5(): - """1.5L special-mechanism analysis — raw diagnostics, action compactness, phase drift.""" - print("=== Task 5: 1.5L special-mechanism diagnostics ===", flush=True) - - SI = 800 # 1.5L sample interval - - try: - ill_d = load_aligned_fields("illusion_1.5L") - tgt_d = load_aligned_fields("target_cylinder_1.5L") - pin_d = load_aligned_fields("pinball") - except FileNotFoundError as e: - print(f" SKIP: {e}", flush=True) - return - - sens_i = ill_d.get("sensors") - forc_i = ill_d.get("forces") - act_i = ill_d.get("actions") - sens_t = tgt_d.get("sensors") - forc_t = tgt_d.get("forces") - - # ---- Panel 5a: Raw diagnostics (sensors, forces, actions) ---- - print(" -- 5a: Raw time-series diagnostics", flush=True) - n_plot = min(400, len(sens_i) if sens_i is not None else 0) - t = np.arange(n_plot) * SI / 1000.0 - - fig, axes = plt.subplots(3, 1, figsize=(14, 10)) - - ax = axes[0] - if sens_i is not None: - for ch in range(6): - ax.plot(t, sens_i[:n_plot, ch], label=f"ill_s{ch}", alpha=0.7) - if sens_t is not None: - ax.plot(t, sens_t[:n_plot, 3], "k--", label="target_s1_v", linewidth=2) - ax.set_ylabel("Velocity (lattice)") - ax.set_title("1.5L Sensors: Illusion vs Target") - ax.legend(fontsize=7, ncol=3) - ax.grid(True, alpha=0.3) - - ax = axes[1] - if forc_i is not None: - for ch in range(6): - ax.plot(t, forc_i[:n_plot, ch], label=f"ill_F{ch}", alpha=0.7) - if forc_t is not None: - ax.plot(t, forc_t[:n_plot, 0], "k--", label="target_Fx", linewidth=2) - ax.plot(t, forc_t[:n_plot, 1], "k:", label="target_Fy", linewidth=2) - ax.set_ylabel("Force (lattice)") - ax.set_title("1.5L Forces") - ax.legend(fontsize=7, ncol=3) - ax.grid(True, alpha=0.3) - - ax = axes[2] - if act_i is not None: - for ch in range(3): - ax.plot(t, act_i[:n_plot, ch], label=f"Omega_{ch}") - ax.set_xlabel("Time (T0 units)") - ax.set_ylabel("Omega (normalised)") - ax.set_title("1.5L Actions (DRL output, [-1, 1])") - ax.legend() - ax.grid(True, alpha=0.3) - - plt.tight_layout() - path = os.path.join(FIG_DIR, "15L_raw_timeseries.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - # ---- Panel 5b: Force-CCD compactness ---- - print(" -- 5b: Force-CCD compactness", flush=True) - results_path = os.path.join(DATA_DIR, "ccd", "ccd_results.json") - if os.path.isfile(results_path): - with open(results_path) as f: - all_res = json.load(f) - # Report key action-CCD m80 - for r in [6, 8, 10]: - key = f"1.5L_illusion_1.5L_action_r{r}" - if key in all_res: - print(f" {key}: m80={all_res[key]['m80']}, " - f"sigma1={all_res[key]['sigma_top3'][0]:.4f}", flush=True) - # Force-fy compactness - key_f = f"1.5L_illusion_1.5L_force_fy_r{r}" - if key_f in all_res: - print(f" {key_f}: m80={all_res[key_f]['m80']}, " - f"sigma1={all_res[key_f]['sigma_top3'][0]:.4f}", flush=True) - - # ---- Panel 5c: Windowed periodicity (phase drift) ---- - # Use raw (non-aligned) sensor data for sufficient window length - print(" -- 5c: Windowed periodicity", flush=True) - raw_path = os.path.join(DATA_DIR, "illusion", "illusion_1.5L", "controlled.npz") - if os.path.isfile(raw_path): - raw_d = np.load(raw_path) - raw_sensors = raw_d["sensors"] - raw_d.close() - else: - raw_sensors = sens_i # fallback to aligned data - - if raw_sensors is not None and len(raw_sensors) > 200: - signal = raw_sensors[:, 1] # center sensor v - window = 200 - stride = 20 - n_windows = (len(signal) - window) // stride - - cv_vals, T_vals, f_vals, t_centers = [], [], [], [] - for w in range(n_windows): - seg = signal[w * stride:w * stride + window] - cv_T, mean_T, _ = detect_cycle_stability(seg, SI) - f_dom, T_dom, _ = detect_dominant_frequency(seg, SI) - cv_vals.append(cv_T) - T_vals.append(mean_T) - f_vals.append(f_dom) - t_centers.append((w * stride + window // 2) * SI / 1000) - - fig, axes = plt.subplots(3, 1, figsize=(14, 8), sharex=True) - - ax = axes[0] - ax.plot(t_centers, cv_vals, "o-", markersize=3) - ax.axhline(0.10, color="r", ls="--", label="strict gate") - ax.axhline(0.12, color="orange", ls="--", label="relaxed gate") - ax.set_ylabel("CV_T") - ax.set_title("1.5L Windowed cycle stability (window=200 steps)") - ax.legend() - ax.grid(True, alpha=0.3) - - ax = axes[1] - ax.plot(t_centers, T_vals, "o-", markersize=3, color="green") - ax.set_ylabel("Mean period (steps)") - ax.grid(True, alpha=0.3) - - ax = axes[2] - ax.plot(t_centers, f_vals, "o-", markersize=3, color="purple") - ax.set_xlabel("Time (T0 units)") - ax.set_ylabel("Freq (1/step)") - ax.grid(True, alpha=0.3) - - plt.tight_layout() - path = os.path.join(FIG_DIR, "15L_windowed_periodicity.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - # ---- Panel 5d: O(target, illusion) overlap bar ---- - print(" -- 5d: Force-CCD overlap summary", flush=True) - fig, ax = plt.subplots(figsize=(6, 4)) - diam_labels = ["0.75L", "1.0L", "1.5L"] - ov_vals = [] - results_path = os.path.join(DATA_DIR, "ccd", "ccd_results.json") - if os.path.isfile(results_path): - with open(results_path) as f: - all_res = json.load(f) - for diam in [0.75, 1.0, 1.5]: - tgt = f"target_cylinder_{diam}L" - ill = f"illusion_{diam}L" - k_tgt = f"{diam}L_{tgt}_force_fy_r6" - k_ill = f"{diam}L_{ill}_force_fy_r6" - # Need W from the saved results — but we don't store W in json. - # Instead, recompute overlap quickly from the raw data. - try: - td = load_aligned_fields(tgt) - id_ = load_aligned_fields(ill) - Qt = build_field_matrix(td["ux"], td["uy"]) - mf, modes, _, _ = compute_pod(Qt) - modes6 = modes[:, :6] - a_t = project_into_basis(td["ux"], td["uy"], modes6, mf) - a_i = project_into_basis(id_["ux"], id_["uy"], modes6, mf) - y_t = make_force_obs(td["forces"], tgt, mode="fy") - y_i = make_force_obs(id_["forces"], ill, mode="fy") - Wt, _, _, _, _, _ = compute_reduced_ccd(a_t, y_t, Q_delay=CCD_Q) - Wi, _, _, _, _, _ = compute_reduced_ccd(a_i, y_i, Q_delay=CCD_Q) - ov_val = float(abs( - Wt[:, 0] / (np.linalg.norm(Wt[:, 0]) + 1e-12) @ - Wi[:, 0] / (np.linalg.norm(Wi[:, 0]) + 1e-12) - )) - ov_vals.append(ov_val) - except Exception as e: - print(f" {diam}L overlap failed: {e}", flush=True) - ov_vals.append(0.0) - - bars = ax.bar(diam_labels, ov_vals, color=["blue", "green", "orange"], alpha=0.7) - for bar, v in zip(bars, ov_vals): - ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.02, - f"{v:.3f}", ha="center", fontsize=11) - ax.set_ylim(0, 1.1) - ax.set_ylabel("O_1 (target-illusion)") - ax.set_title("Force-CCD (SigmaFy) overlap comparison") - ax.grid(True, alpha=0.3, axis="y") - - # Annotate 1.5L - ax.annotate("special mechanism", xy=(2, 0.05), fontsize=9, - ha="center", color="orange", fontweight="bold") - - plt.tight_layout() - path = os.path.join(FIG_DIR, "15L_overlap_summary.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - -# ==================================================================== -# Task 6: Cross-diameter overlap (all illusions in 1.0L target basis) -# ==================================================================== -def task_6(): - print("=== Task 6: Cross-diameter overlap (1.0L target basis) ===", flush=True) - - try: - tgt_10 = load_aligned_fields("target_cylinder_1.0L") - except FileNotFoundError: - print(" SKIP: missing 1.0L target data", flush=True) - return - - Q_10 = build_field_matrix(tgt_10["ux"], tgt_10["uy"]) - mf_10, modes_10, _, _ = compute_pod(Q_10) - modes6 = modes_10[:, :6] - - W_cross = {} - for diam in [0.75, 1.0, 1.5]: - name = f"illusion_{diam}L" - try: - d = load_aligned_fields(name) - except FileNotFoundError: - continue - a = project_into_basis(d["ux"], d["uy"], modes6, mf_10) - frc = d.get("forces") - if frc is None: - continue - y = make_force_obs(frc, name, mode="fy") - W, _, _, _, _, _ = compute_reduced_ccd(a, y, Q_delay=CCD_Q) - W_cross[diam] = W - - if len(W_cross) < 2: - print(" SKIP: not enough illusions", flush=True) - return - - diam_list = sorted(W_cross.keys()) - ov_mat = np.ones((len(diam_list), len(diam_list))) - print(" Cross-diameter O_1 matrix (1.0L target-only basis, force_fy):") - for i, da in enumerate(diam_list): - for j, db in enumerate(diam_list): - if i >= j: - continue - Wa, Wb = W_cross[da], W_cross[db] - ov = float(abs( - Wa[:, 0] / (np.linalg.norm(Wa[:, 0]) + 1e-12) @ - Wb[:, 0] / (np.linalg.norm(Wb[:, 0]) + 1e-12) - )) - ov_mat[i, j] = ov - ov_mat[j, i] = ov - print(f" O({da}L, {db}L) = {ov:.4f}") - - fig, ax = plt.subplots(figsize=(6, 5)) - im = ax.imshow(ov_mat, cmap="viridis", vmin=0, vmax=1) - ax.set_xticks(range(len(diam_list))) - ax.set_xticklabels([f"{d}L" for d in diam_list]) - ax.set_yticks(range(len(diam_list))) - ax.set_yticklabels([f"{d}L" for d in diam_list]) - for i in range(len(diam_list)): - for j in range(len(diam_list)): - v = ov_mat[i, j] - ax.text(j, i, f"{v:.3f}", ha="center", va="center", - color="white" if v > 0.5 else "black") - plt.colorbar(im, label="O_1") - plt.title("Cross-diam force-CCD (1.0L target basis, SigmaFy)") - plt.tight_layout() - path = os.path.join(FIG_DIR, "cross_diameter_overlap_fy.png") - fig.savefig(path, dpi=150) - plt.close(fig) - print(f" Saved: {path}", flush=True) - - -# ==================================================================== -# Main -# ==================================================================== -def main(): - print("=" * 60, flush=True) - print("Phase 4: Visualization (Round 5)", flush=True) - print("=" * 60, flush=True) - - task_1() # O_k heatmap - task_2() # CCD physical modes (0.75L, 1.0L) - task_4() # POD phase portraits (all diameters) - task_3() # z_1 verification (0.75L, 1.0L) - task_5() # 1.5L special mechanism - task_6() # Cross-diameter overlap - - print(f"\nAll figures saved to {FIG_DIR}", flush=True) - - -if __name__ == "__main__": - main() diff --git a/src/CCD_analysis/tests/__init__.py b/src/CCD_analysis/tests/__init__.py new file mode 100644 index 0000000..4469cfe --- /dev/null +++ b/src/CCD_analysis/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the active three-part CCD analysis surface.""" diff --git a/src/CCD_analysis/tests/test_acquisition.py b/src/CCD_analysis/tests/test_acquisition.py new file mode 100644 index 0000000..b4754fa --- /dev/null +++ b/src/CCD_analysis/tests/test_acquisition.py @@ -0,0 +1,579 @@ +import json, os, subprocess, sys +from pathlib import Path +import numpy as np +import pytest + +from CCD_analysis.acquisition.artifacts import ArtifactTransaction +from CCD_analysis.acquisition.cli import main as cli_main, orchestrate +from CCD_analysis.acquisition.contracts import CASES, ROLES, canonical_coordinate_axis, canonical_coordinate_frame, case_snapshot, validate_coordinate_arrays +from CCD_analysis.acquisition.dual_clock import DualClockCollector, ExactTelemetry, field_steps, solver_fluid_mask +from CCD_analysis.acquisition.runtime import RoleRuntime, _load_policy, build_role_runtime, preflight, require_cfd_environment, role_spec, run_role_acquisition +from CCD_analysis.acquisition.solver_state import copy_ping_pong_ddf, d2q9_q_over_u0_xy, ema_step + +class FakeFlow: + def __init__(self,width=6): + self.absolute=100; self.completed=0; self.total=0; self.begin_count=0; self.end_count=0 + self.action=np.zeros(width,np.float32); self.flags=np.array([[1,2],[1,1]],np.uint8) + def begin_control_interval(self,total_steps,action_target): + self.total=total_steps; self.completed=0; self.begin_count+=1; self.target=np.asarray(action_target) + def run_control_segment(self,steps): + self.absolute+=steps; self.completed+=steps; self.action=(.9*self.action+.1*self.target).astype(np.float32) + def current_step_observation(self): return np.arange(12,dtype=np.float32)+self.absolute + def current_effective_action(self): return self.action.copy() + def completed_flags_xy(self): return self.flags.copy() + def active_step_clock_state(self): + if not self.completed: raise RuntimeError("active only") + return {"solver_absolute_lattice_clock":self.absolute,"solver_absolute_control_clock":self.end_count} + def solver_clock_state(self): + if self.completed and self.completed!=self.total: raise RuntimeError("boundary only") + return {"solver_absolute_lattice_clock":self.absolute,"solver_absolute_control_clock":self.end_count} + def end_control_interval(self): + assert self.completed==self.total; self.end_count+=1; return self.current_step_observation() + +def velocity(flow): + return np.full((2,2),flow.absolute,np.float32),np.full((2,2),-flow.absolute,np.float32) +def phase(flow): return np.asarray([flow.absolute/10],np.float32) +def telemetry(role="q_ctl"): + t=ExactTelemetry(role=role,observation_slices={"disturbance_force":None,"pinball_forces":(6,12),"sensors":(0,6)},phase_probe=phase,physical_width=6) + t.set_requested(np.ones(3,np.float32) if role=="q_ctl" else np.zeros(3,np.float32),np.ones(6,np.float32) if role=="q_ctl" else np.zeros(6,np.float32)); return t + +def valid_state(): + h="a"*64 + return {"current_ddf":np.zeros(9,np.float32),"temp_ddf":np.ones(9,np.float32),"current_raw_observation":np.zeros(12,np.float32),"fifo_history":np.zeros((2,12),np.float32),"persisted_effective_ema_action":np.zeros(6,np.float32),"target_harmonic_phase":np.asarray(0,np.int64),"absolute_control_clock":np.asarray(2,np.int64),"absolute_lattice_clock":np.asarray(16,np.int64),"normalization_hash":np.asarray(h),"model_hash":np.asarray(h),"config_hash":np.asarray(h),"geometry_hash":np.asarray(h)} + +def collected_arrays(): + ff=FakeFlow(); c=DualClockCollector(ff,control_interval=8,control_count=2,absolute_origin=100,field_steps_absolute=(103,109,116),telemetry=telemetry(),velocity_reader=velocity) + c.run_interval(0,np.ones(6,np.float32)); c.run_interval(1,np.ones(6,np.float32)); arrays=c.arrays() + arrays.update(x_D=np.asarray([0.,1.],np.float32),y_D=np.asarray([-1.,1.],np.float32),fluid_mask=solver_fluid_mask(ff)) + return arrays,ff + +def test_frozen_cases_geometry_roles_and_caveat(): + assert tuple(CASES)==("karman_re100","illusion_1.0L") and ROLES==("q_target","q_blk","q_ctl") + k,i=CASES.values(); assert (k.geometry.front_x_D,k.geometry.rear_x_D,k.geometry.sensor_x_D)==(30,31.3,40) + assert k.code_reynolds==100 and k.physical_re_D==50 + assert (i.geometry.front_x_D,i.geometry.rear_x_D,i.geometry.sensor_x_D,i.geometry.target_x_D)==(30,31.3,41,31) + assert i.geometry.rear_y_D==(.75,-.75) and i.geometry.sensor_y_D==(2,0,-2) + assert i.history_smoke_required and "differs" in i.deployment_warning + for case in CASES: + for role in ROLES: assert "not a same-checkpoint counterfactual" in case_snapshot(case,role)["case"]["q_target_caveat"] + +def test_exact_absolute_timeline_and_control_lifecycle(): + arrays,ff=collected_arrays(); assert field_steps(16,3,absolute_origin=100)==(103,106,109,112,115) + assert arrays["lattice_steps"].tolist()==[103,109,116] and arrays["sample_ids"].tolist()==[103,109,116] + assert arrays["control_indices"].tolist()==[0,1,1] and arrays["control_offsets"].tolist()==[3,1,8] + assert arrays["phase_reference"].reshape(-1).tolist()==pytest.approx([10.3,10.9,11.6]) + assert ff.begin_count==2 and ff.end_count==2 + +def test_same_step_telemetry_does_not_read_boundary_obs(): + ff=FakeFlow(); ff.begin_control_interval(8,np.ones(6,np.float32)); ff.run_control_segment(3) + snap=telemetry()(ff,103,3,0); assert snap["sensors"].tolist()==[103,104,105,106,107,108] + assert not np.array_equal(snap["requested_physical_action"],snap["effective_applied_action"]) + +def test_mask_must_come_from_solver_accessor(): + ff=FakeFlow(); assert solver_fluid_mask(ff).tolist()==[[True,False],[True,True]] + with pytest.raises(RuntimeError,match="fallback is forbidden"): solver_fluid_mask(object()) + +def test_roles_reject_nonzero_reference_action(): + for role in ("q_target","q_blk"): + t=ExactTelemetry(role=role,observation_slices={"disturbance_force":None,"pinball_forces":(6,12),"sensors":(0,6)},phase_probe=phase,physical_width=6) + with pytest.raises(ValueError,match="exactly zero"): t.set_requested(np.ones(3,np.float32),np.ones(6,np.float32)) + +def test_environment_guard_and_cpu_import_safety(monkeypatch): + monkeypatch.setenv("CONDA_DEFAULT_ENV","pinball_math") + with pytest.raises(RuntimeError,match="pycuda_3_10"): require_cfd_environment() + code="import sys; import CCD_analysis.acquisition; assert 'pycuda' not in sys.modules and 'LegacyCelerisLab' not in sys.modules" + env=dict(os.environ,CONDA_DEFAULT_ENV="pinball_math",PYTHONPATH=str(Path(__file__).parents[2])) + subprocess.run([sys.executable,"-c",code],check=True,env=env) + +def test_orchestration_is_sequential_fresh_process_plan(tmp_path): + commands=orchestrate("karman_re100",tmp_path/"future",control_count=450,field_interval=2000,run=False) + assert [c[c.index("--role")+1] for c in commands]==list(ROLES) + assert all(c[:3]==[sys.executable,"-m","CCD_analysis.acquisition"] for c in commands) + assert all(c[c.index("--control-count")+1]=="450" for c in commands) + assert all(c[c.index("--field-interval")+1]=="2000" for c in commands) + + +def test_orchestration_propagates_case_specific_schedule_identically(tmp_path, monkeypatch): + certificate=tmp_path/"certificate.json"; certificate.write_text("stub") + monkeypatch.setattr("CCD_analysis.acquisition.cli.validate_certificate", lambda path: {"status":"PASS"}) + commands=orchestrate("illusion_1.0L",tmp_path/"future",control_count=450,field_interval=1250,compatibility_certificate=certificate) + assert all(c[c.index("--control-count")+1]=="450" for c in commands) + assert all(c[c.index("--field-interval")+1]=="1250" for c in commands) + + +@pytest.mark.parametrize("missing",("--control-count","--field-interval")) +def test_orchestrate_parser_requires_explicit_schedule(tmp_path,missing): + argv=["orchestrate","--case","karman_re100","--output",str(tmp_path/"future"),"--control-count","450","--field-interval","2000","--execute"] + index=argv.index(missing); del argv[index:index+2] + with pytest.raises(SystemExit) as error: cli_main(argv) + assert error.value.code==2 + + +@pytest.mark.parametrize("control_count,field_interval",((0,2000),(450,0),(450,2001))) +def test_orchestration_rejects_invalid_schedule_before_child_launch(tmp_path,monkeypatch,control_count,field_interval): + launched=[] + monkeypatch.setattr("CCD_analysis.acquisition.cli.require_cfd_environment",lambda: None) + monkeypatch.setattr("CCD_analysis.acquisition.cli.subprocess.run",lambda *args,**kwargs: launched.append((args,kwargs))) + with pytest.raises(ValueError): + orchestrate("karman_re100",tmp_path/"future",control_count=control_count,field_interval=field_interval,run=True) + assert launched==[] + + +def test_legacy_q_over_u0_matches_archived_literal_formula_and_xy_axis(): + nx,ny,u0=2,3,0.01 + populations=np.zeros((nx,ny,9),np.float32) + populations[:,:,0]=np.array([[1,2,3],[4,5,6]],np.float32) + populations[:,:,1]=np.float32(.02); populations[:,:,2]=np.float32(.01) + flat=populations.transpose(2,1,0).reshape(-1); mask=np.ones((nx,ny),np.bool_) + ux,uy=d2q9_q_over_u0_xy(flat,nx,ny,mask,u0) + archived_ux=(populations[:,:,1]+populations[:,:,5]+populations[:,:,8]-populations[:,:,3]-populations[:,:,6]-populations[:,:,7])/u0 + archived_uy=(populations[:,:,2]+populations[:,:,5]+populations[:,:,6]-populations[:,:,4]-populations[:,:,7]-populations[:,:,8])/u0 + np.testing.assert_array_equal(ux,archived_ux.astype(np.float32)); np.testing.assert_array_equal(uy,archived_uy.astype(np.float32)) + assert np.max(np.abs(ux)) == pytest.approx(2.0) and np.max(np.abs(uy)) == pytest.approx(1.0) + assert ux.shape==(nx,ny) and ux.dtype==np.float32 and ux.flags.c_contiguous + for bad_u0 in (0.,-1.,np.inf,np.nan): + with pytest.raises(ValueError,match="u0 must be positive and finite"): d2q9_q_over_u0_xy(flat,nx,ny,mask,bad_u0) + + +def test_d2q9_solver_mask_ignores_nonfluid_garbage_and_zeros_nonfluid(): + nx,ny,u0=2,2,0.01 + populations=np.zeros((nx,ny,9),np.float32); populations[:,:,1]=np.float32(.02); populations[:,:,2]=np.float32(.01) + flags=np.asarray([[1,2],[1,2|8]],np.uint8); populations[flags & 1 == 0]=np.nan + ux,uy=d2q9_q_over_u0_xy(populations.transpose(2,1,0).reshape(-1),nx,ny,flags,u0) + np.testing.assert_array_equal(ux[flags & 1 == 0],np.zeros(2,np.float32)); np.testing.assert_array_equal(uy[flags & 1 == 0],np.zeros(2,np.float32)) + np.testing.assert_allclose(ux[flags & 1 != 0],2); np.testing.assert_allclose(uy[flags & 1 != 0],1) + + +def test_d2q9_rejects_invalid_fluid_and_bad_mask(): + populations=np.zeros((2,2,9),np.float32); mask=np.asarray([[True,False],[True,False]],np.bool_) + populations[~mask]=np.nan + d2q9_q_over_u0_xy(populations.transpose(2,1,0).reshape(-1),2,2,mask,.01) + populations[0,0,0]=np.nan + with pytest.raises(ValueError,match="fluid populations.*non-finite"): d2q9_q_over_u0_xy(populations.transpose(2,1,0).reshape(-1),2,2,mask,.01) + flat=np.ones(36,np.float32) + with pytest.raises(ValueError,match="exact .*grid"): d2q9_q_over_u0_xy(flat,2,2,np.ones((2,3),np.bool_),.01) + with pytest.raises(ValueError,match="bool.*uint8"): d2q9_q_over_u0_xy(flat,2,2,np.ones((2,2),np.int32),.01) + + +def test_checkpoint_copy_ping_pong_semantics_and_hashes(): + current=np.arange(9,dtype=np.float32); previous=np.arange(9,dtype=np.float32)+20 + calls=[] + def copy(label,source,dest): calls.append(label); dest[:]=source + result=copy_ping_pong_ddf(lambda d:copy("current",current,d),lambda d:copy("temp",previous,d),9) + assert calls==["current","temp"] + np.testing.assert_array_equal(result["current_ddf"],current) + np.testing.assert_array_equal(result["temp_ddf"],previous) + current[:]=99; assert result["current_ddf"][0]==0 + assert len(result["current_sha256"])==len(result["temp_sha256"])==64 + + +def test_role_specs_exact_order_geometry_and_controller_requirements(): + kt=role_spec("karman_re100","q_target"); kb=role_spec("karman_re100","q_blk") + assert [o.identity for o in kt.objects]==["disturbance","sensor_upper","sensor_center","sensor_lower"] + assert [o.identity for o in kb.objects][-3:]==["front","upper","lower"] and kb.controller=="zero" + it=role_spec("illusion_1.0L","q_target"); ic=role_spec("illusion_1.0L","q_ctl") + assert [(o.identity,o.center_D) for o in it.objects]==[("target",(31.0,0.0)),("sensor_upper",(41.0,2.0)),("sensor_center",(41.0,0.0)),("sensor_lower",(41.0,-2.0))] + assert [(o.identity,o.center_D) for o in ic.objects][-3:]==[("front",(30.0,0.0)),("upper",(31.3,.75)),("lower",(31.3,-.75))] + assert ic.fifo_len==150 and ic.harmonic_channels==2 and it.harmonic_channels==8 and role_spec("illusion_1.0L","q_blk").harmonic_channels==0 and ic.controller=="ppo_history" + + +def test_cpu_preflight_checks_models_configs_solver_and_reports_no_cfd(): + report=preflight() + assert report["ready_for_independent_pre_cfd_gate"] and not report["cfd_executed"] and not report["runtime_success_claimed"] + assert report["blockers"]==[] and report["production_blockers"]==["Illusion +11D replay/history smoke not run"] + names={item["name"] for item in report["checks"]} + assert {"configs/legacy_configs/config_cuda.json","configs/legacy_configs/config_flowfield.json","solver_read_only_apis","karman_re100.model","illusion_1.0L.model","runner_reachable"}<=names + + +def test_preflight_fails_closed_for_missing_model(tmp_path): + (tmp_path/"LegacyCelerisLab").mkdir(); (tmp_path/"configs/legacy_configs").mkdir(parents=True) + (tmp_path/"LegacyCelerisLab/driver.py").write_text((Path(__file__).parents[3]/"LegacyCelerisLab/driver.py").read_text()) + report=preflight("karman_re100",repo_root=tmp_path) + assert not report["ready_for_independent_pre_cfd_gate"] + assert "configs/legacy_configs/config_cuda.json" in report["blockers"] and "karman_re100.model" in report["blockers"] + + +def test_builder_is_injectable_and_preserves_object_order(monkeypatch): + class Config: + def _replace(self,**kwargs): self.values=kwargs; return self + class Flow: + FIELD_SHAPE=(1280,512,1) + def __init__(self,*args,**kwargs): self.added=[]; self.objects={}; self.action=np.zeros(0,np.float32) + def add_sensor(self,c,r): self.added.append(("sensor",c,r)); self.objects[len(self.objects)]={"type":"sensor","center":c,"radius":r}; self.action=np.zeros(len(self.objects),np.float32) + def add_cylinder(self,c,r): self.added.append(("cylinder",c,r)); self.objects[len(self.objects)]={"type":"cylinder","center":c,"radius":r}; self.action=np.zeros(len(self.objects),np.float32) + monkeypatch.setenv("CONDA_DEFAULT_ENV","pycuda_3_10") + runtime=build_role_runtime(case="illusion_1.0L",role="q_blk",flow_factory=Flow,config_loader=lambda:(object(),Config())) + assert [kind for kind,_,_ in runtime.flow.added]==["sensor","sensor","sensor","cylinder","cylinder","cylinder"] + assert [item[1][0] for item in runtime.flow.added]==[820,820,820,600,626,626] + + +def test_default_policy_loader_passes_literal_cpu_without_torch_cuda(monkeypatch): + import types + class Module: pass + torch = types.ModuleType("torch") + torch.sin = np.sin + torch_nn = types.ModuleType("torch.nn"); torch_nn.Module = Module + gym = types.ModuleType("gymnasium") + class Env: pass + gym.Env = Env + spaces = types.ModuleType("gymnasium.spaces") + class Box: + def __init__(self, low, high, shape, dtype): self.low, self.high, self.shape, self.dtype = low, high, shape, dtype + spaces.Box = Box; gym.spaces = spaces + sb3 = types.ModuleType("stable_baselines3") + class PPO: + @staticmethod + def load(path, *, env, device): + assert path == "policy.zip" and device == "cpu" + return {"path": path, "device": device, "env": env} + sb3.PPO = PPO + for name, module in {"torch": torch, "torch.nn": torch_nn, "gymnasium": gym, "gymnasium.spaces": spaces, "stable_baselines3": sb3}.items(): + monkeypatch.setitem(sys.modules, name, module) + loaded = _load_policy("policy.zip", policy_device="cpu", s_dim=12) + assert loaded["device"] == "cpu" and loaded["env"].observation_space.shape == (12,) + assert not hasattr(torch, "cuda") + with pytest.raises(ValueError, match="policy_device must be 'cpu'"): + _load_policy("policy.zip", policy_device="cuda:0", s_dim=12) + + +def test_controlled_builder_keeps_policy_cpu_and_cfd_logical_zero(monkeypatch): + class Config: + def _replace(self, **kwargs): return self + class Flow: + FIELD_SHAPE = (1280, 512, 1) + def __init__(self, *args, **kwargs): + assert kwargs == {"device_id": 0} + self.objects = {}; self.action = np.zeros(0, np.float32) + def add_sensor(self, center, radius): + self.objects[len(self.objects)] = {"type": "sensor", "center": center, "radius": radius}; self.action = np.zeros(len(self.objects), np.float32) + def add_cylinder(self, center, radius): + self.objects[len(self.objects)] = {"type": "cylinder", "center": center, "radius": radius}; self.action = np.zeros(len(self.objects), np.float32) + class Policy: + device = "cpu" + observation_space = type("S", (), {"shape": (12,), "dtype": np.dtype("float32"), "low": -np.ones(12), "high": np.ones(12)})() + action_space = type("S", (), {"shape": (3,), "dtype": np.dtype("float32"), "low": -np.ones(3), "high": np.ones(3)})() + def set_random_seed(self, seed): self.seed = seed + calls = [] + def loader(path, *, policy_device, s_dim): + calls.append((path, policy_device, s_dim)); return Policy() + monkeypatch.setenv("CONDA_DEFAULT_ENV", "pycuda_3_10") + runtime = build_role_runtime(case="karman_re100", role="q_ctl", device_id=0, flow_factory=Flow, config_loader=lambda: (object(), Config()), policy_loader=loader) + assert calls == [(runtime.spec.model_path, "cpu", 12)] + assert runtime.policy_device == "cpu" and runtime.cfd_device == 0 and runtime.policy.device == "cpu" + with pytest.raises(ValueError, match="CUDA policy inference is forbidden"): + build_role_runtime(case="karman_re100", role="q_ctl", device_id=0, policy_device="cuda:0", flow_factory=Flow, config_loader=lambda: (object(), Config()), policy_loader=loader) + + +def test_preflight_policy_check_is_cpu_only_and_does_not_import_torch(monkeypatch): + sys.modules.pop("torch", None) + report = preflight("karman_re100") + check = next(item for item in report["checks"] if item["name"] == "policy_device_contract") + assert check == {"name": "policy_device_contract", "ok": True, "detail": {"policy_device": "cpu", "cfd_device": "separate logical device selected at runtime", "cuda_initialized": False}} + assert "torch" not in sys.modules + + +def test_exact_coordinate_contract_accepts_realistic_float32_lattice_and_rejects_noncanonical(): + frame = canonical_coordinate_frame(1280, 512) + x = canonical_coordinate_axis(1280, origin_lattice=0.0, spacing_lattice=1.0, reference_length_lattice=20.0) + y = canonical_coordinate_axis(512, origin_lattice=-255.5, spacing_lattice=1.0, reference_length_lattice=20.0) + assert np.unique(np.diff(x)).size > 1 # Expected float32 ULP variation is not a contract violation. + validate_coordinate_arrays(x, y, frame) + mutations = { + "shifted": x + np.float32(1 / 20), + "noncanonical": np.linspace(0, np.float32(1279 / 20), 1280, dtype=np.float32), + "skipped": np.concatenate((x[:640], x[641:], x[-1:])), + "nonmonotonic": x.copy(), + } + mutations["nonmonotonic"][700] = mutations["nonmonotonic"][699] + for value in mutations.values(): + with pytest.raises(ValueError, match="exactly equal"): + validate_coordinate_arrays(value, y, frame) + + +def test_coordinate_contract_requires_strict_float32_1d_finite(): + frame = canonical_coordinate_frame(3, 2) + x = canonical_coordinate_axis(3, origin_lattice=0, spacing_lattice=1, reference_length_lattice=20) + y = canonical_coordinate_axis(2, origin_lattice=-.5, spacing_lattice=1, reference_length_lattice=20) + for bad in (x.astype(np.float64), x[:, None], np.asarray([0, np.nan, .1], np.float32)): + with pytest.raises(ValueError): validate_coordinate_arrays(bad, y, frame) + + +def test_artifact_field_axis_is_time_x_y(): + arrays,_=collected_arrays() + assert arrays["ux"].shape==(3,2,2) + arrays["ux"]=arrays["ux"].transpose(0,2,1) + # A square fixture cannot reveal transposition; contract coordinates bind axis 1 to x and axis 2 to y. + assert arrays["x_D"].shape==(arrays["ux"].shape[1],) and arrays["y_D"].shape==(arrays["ux"].shape[2],) + + +class RunnerFlow: + FIELD_SHAPE=(3,2,1) + def __init__(self,width=6): + self.action=np.zeros(width,np.float32); self.obs=np.zeros(width*2,np.float32); self.objects={}; self.flags=np.ones((3,2),np.uint8) + self.absolute=0; self.absolute_controls=0; self.active=False; self.target=None; self.started=[]; self._last_raw=None + def add_sensor(self,c,r): self.objects[len(self.objects)]={"type":"sensor","center":c,"radius":r}; self.action=np.zeros(len(self.objects),np.float32); self.obs=np.zeros(2*len(self.objects),np.float32) + def add_cylinder(self,c,r): self.objects[len(self.objects)]={"type":"cylinder","center":c,"radius":r}; self.action=np.zeros(len(self.objects),np.float32); self.obs=np.zeros(2*len(self.objects),np.float32) + def run(self,steps,target): + self.begin_control_interval(steps,target); self.run_control_segment(steps); return self.end_control_interval() + def begin_control_interval(self,total_steps,action_target): + assert not self.active; self.active=True; self.total=total_steps; self.done=0; self.target=np.asarray(action_target).copy(); self.started.append(self.action.copy()); self.samples=[] + def run_control_segment(self,steps): + for _ in range(steps): + self.action=ema_step(self.action,self.target,1); self.absolute+=1; self.done+=1 + raw=(np.arange(self.obs.size,dtype=np.float32)+1+self.absolute/1000).astype(np.float32); self._last_raw=raw; self.samples.append(raw) + def end_control_interval(self): + assert self.done==self.total; self.obs=np.mean(self.samples,axis=0,dtype=np.float32); self.active=False; self.absolute_controls+=1; return self.obs.copy() + def current_step_observation(self): return self._last_raw.copy() + def current_effective_action(self): return self.action.copy() + def current_step_velocity_probe(self,index): return np.asarray([self.absolute/100,0],np.float32) + def current_step_velocity_field(self): return np.full((3,2),self.absolute,np.float32),np.full((3,2),-self.absolute,np.float32) + def active_step_clock_state(self): + if not self.active or self.done<1: raise RuntimeError("active-step only") + return {"solver_absolute_lattice_clock":self.absolute,"solver_absolute_control_clock":self.absolute_controls} + def solver_clock_state(self): + if self.active: raise RuntimeError("boundary only") + return {"solver_absolute_lattice_clock":self.absolute,"solver_absolute_control_clock":self.absolute_controls} + def current_step_ddf_checkpoint(self): + return {"current_ddf":np.full(54,self.absolute,np.float32),"temp_ddf":np.full(54,self.absolute-1,np.float32),"current_sha256":"a"*64,"temp_sha256":"b"*64} + def full_state_checkpoint(self): + cp=self.current_step_ddf_checkpoint(); cp.update(action=self.action.copy(),last_effective_action=self.action.copy(),raw_observation=None if self._last_raw is None else self._last_raw.copy(),boundary_observation=self.obs.copy(),solver_absolute_lattice_clock=self.absolute,solver_absolute_control_clock=self.absolute_controls); return cp + def restore_full_state(self,cp): + self.absolute=cp["solver_absolute_lattice_clock"]; self.action=cp["action"].copy(); self._last_raw=None if cp["raw_observation"] is None else cp["raw_observation"].copy(); self.obs=cp["boundary_observation"].copy(); self.absolute_controls=cp["solver_absolute_control_clock"]; self.restore_events=getattr(self,"restore_events",[])+[(self.absolute,self.action.copy())] + def completed_flags_xy(self): return self.flags.copy() + + +def make_runner_runtime(role="q_blk"): + spec=role_spec("illusion_1.0L",role); flow=RunnerFlow() + cy=(flow.FIELD_SHAPE[1]-1)/2 + for obj in spec.objects: + center=(obj.center_D[0]*20,cy+obj.center_D[1]*20,0.) + (flow.add_sensor if obj.kind=="sensor" else flow.add_cylinder)(center,obj.radius_D*20) + class Policy: + observation_space=type("S",(),{"shape":(14,),"dtype":np.dtype("float32"),"low":-np.ones(14),"high":np.ones(14)})() + action_space=type("S",(),{"shape":(3,),"dtype":np.dtype("float32"),"low":-np.ones(3),"high":np.ones(3)})() + def predict(self,obs,deterministic=True): return np.zeros(3,np.float32),None + return RoleRuntime(flow,spec,CASES["illusion_1.0L"],Policy() if role=="q_ctl" else None) + + +def fake_initializer(runtime,**kwargs): + norm={"force_norm_fact":np.asarray(2,np.float32),"sens_deviation":np.zeros(6,np.float32),"sens_norm_fact":np.ones(6,np.float32)} + harmonics=[{"dc":0.,"amps":[],"freqs":[],"phases":[]} for _ in range(runtime.spec.harmonic_channels)] if runtime.spec.harmonic_channels else None + runtime.set_controller_state(fifo_history=np.ones((150,12),np.float32),normalization=norm,harmonics=harmonics) + + +def test_ema_continuity_across_control_boundaries(): + flow=RunnerFlow(2); target=np.ones(2,np.float32) + flow.run(3,target); final=flow.action.copy(); flow.begin_control_interval(2,target) + np.testing.assert_array_equal(flow.started[-1],final) + flow.run_control_segment(2); flow.end_control_interval() + np.testing.assert_allclose(flow.action,ema_step(final,target,2)) + + +def test_end_to_end_fake_runner_publishes_complete_artifact(tmp_path): + runtime=make_runner_runtime(); destination=tmp_path/"role" + result=run_role_acquisition(case="illusion_1.0L",role="q_blk",output=destination,control_count=2,field_interval=600,runtime=runtime,initializer=fake_initializer) + assert result==destination and (destination/"fields.npz").is_file() and (destination/"controller_state.npz").is_file() + manifest=json.loads((destination/"manifest.json").read_text()); assert manifest["complete"] and manifest["field_count"]==2 + config=json.loads((destination/"config.json").read_text()); assert [o["identity"] for o in config["runtime"]["solver_objects"]]==["sensor_upper","sensor_center","sensor_lower","front","upper","lower"] + assert config["runtime"]["policy_device"] == "cpu" and config["runtime"]["cfd_device"] == 0 + assert config["case"]["geometry"]["action_identities"] == ["front_ccw_positive", "upper_ccw_positive", "lower_ccw_positive"] + assert config["runtime"]["role_spec"]["model_path"] is None + with np.load(destination/"fields.npz") as fields: + assert fields["ux"].shape==(2,3,2) + assert fields["fluid_mask"].dtype==np.bool_ and fields["fluid_mask"].shape==(3,2) + assert np.isfinite(fields["ux"]).all() and np.isfinite(fields["uy"]).all() + assert fields["lattice_steps"].tolist()==[600,1200] + assert fields["sample_ids"].tolist()==[600,1200] and fields["acquisition_relative_lattice_steps"].tolist()==[600,1200] + assert fields["control_indices"].tolist()==[0,1] and fields["solver_absolute_control_indices"].tolist()==[0,1] + assert fields["requested_physical_action"].shape==(2,6) and not fields["requested_physical_action"].any() + with np.load(destination/"controller_state.npz") as state: + assert state["fifo_history"].shape==(150,12) and int(state["acquisition_relative_control_index"])==2 and int(state["acquisition_relative_lattice_clock"])==1200 + assert int(state["solver_absolute_control_clock"])==2 and int(state["solver_absolute_lattice_clock"])==1200 and int(state["policy_harmonic_phase_index"])==2 + assert state["current_ddf"].shape==state["temp_ddf"].shape==(54,) + + +def test_runner_failure_leaves_no_artifact(tmp_path): + destination=tmp_path/"bad"; runtime=make_runner_runtime() + def bad_initializer(runtime,**kwargs): raise ValueError("initialization failed") + with pytest.raises(ValueError,match="initialization failed"): + run_role_acquisition(case="illusion_1.0L",role="q_blk",output=destination,control_count=1,field_interval=600,runtime=runtime,initializer=bad_initializer) + assert not destination.exists() and not list(tmp_path.glob("*.partial.*")) + + +def test_cli_role_reaches_runner_not_builder(monkeypatch,tmp_path): + called={} + monkeypatch.setenv("CONDA_DEFAULT_ENV","pycuda_3_10") + monkeypatch.setattr("CCD_analysis.acquisition.cli.run_role_acquisition",lambda **kwargs: called.update(kwargs)) + assert cli_main(["role","--case","karman_re100","--role","q_blk","--output",str(tmp_path/"out"),"--control-count","2","--field-interval","800"])==0 + assert called["role"]=="q_blk" and called["control_count"]==2 + + +def test_artifact_validation_failure_cleans_partial(tmp_path): + runtime=make_runner_runtime(); runtime.flow.current_step_ddf_checkpoint=lambda:{"current_ddf":np.zeros(2,np.float32),"temp_ddf":np.zeros(2,np.float32)} + destination=tmp_path/"bad" + with pytest.raises(ValueError,match="D2Q9"): + run_role_acquisition(case="illusion_1.0L",role="q_blk",output=destination,control_count=1,field_interval=600,runtime=runtime,initializer=fake_initializer) + assert not destination.exists() and not list(tmp_path.glob("*.partial.*")) + + +def test_artifact_rejects_nonzero_velocity_at_saved_solid_cell(tmp_path): + runtime=make_runner_runtime(); runtime.flow.flags[0,0]=np.uint8(2) + destination=tmp_path/"bad-solid" + with pytest.raises(ValueError,match="solid-cell velocities.*zero"): + run_role_acquisition(case="illusion_1.0L",role="q_blk",output=destination,control_count=1,field_interval=600,runtime=runtime,initializer=fake_initializer) + assert not destination.exists() and not list(tmp_path.glob("*.partial.*")) + + +def test_real_initializer_restores_checkpoint_before_warmup(monkeypatch): + from CCD_analysis.acquisition.runtime import initialize_role + runtime=make_runner_runtime("q_ctl") + frozen_norm={"force_norm_fact":np.asarray(2,np.float32),"sens_deviation":np.zeros(6,np.float32),"sens_norm_fact":np.ones(6,np.float32)} + frozen_h=[{"dc":0.,"amps":[],"freqs":[],"phases":[]} for _ in range(2)] + monkeypatch.setattr("CCD_analysis.acquisition.runtime.load_illusion_training_reference",lambda:(frozen_norm,frozen_h,{"normalization_sha256":"9"*64,"harmonics_sha256":"f"*64})) + initialize_role(runtime,stabilization_steps=2) + assert runtime.flow.restore_events and runtime.flow.restore_events[0][0]==2 + assert runtime.solver_absolute_lattice_clock==2+150*600 and runtime.solver_absolute_control_clock==151 + assert runtime.acquisition_relative_lattice_clock==runtime.acquisition_relative_control_index==runtime.policy_harmonic_phase_index==0 + assert not runtime.flow.restore_events[0][1].any() + # warmup starts after exact restore and uses [0,-1,+1]*u0 command + first_after_restore=runtime.flow.started[1+150] + assert not first_after_restore.any() + np.testing.assert_allclose(runtime.flow.target[-3:],np.asarray([0,-.01,.01],np.float32)) + + +def test_policy_first_inputs_are_case_specific(monkeypatch): + class Capture: + def __init__(self): self.calls=[] + def predict(self,obs,deterministic=True): self.calls.append(obs.copy()); return np.zeros(3,np.float32),None + norm={"force_norm_fact":np.asarray(2,np.float32),"sens_deviation":np.zeros(6,np.float32),"sens_norm_fact":np.ones(6,np.float32)} + kspec=role_spec("karman_re100","q_ctl"); kp=Capture(); kr=RoleRuntime(object(),kspec,CASES["karman_re100"],kp); kr.set_controller_state(fifo_history=np.ones((150,12),np.float32),normalization=norm) + kr.requested_action(); assert np.array_equal(kp.calls[0],np.zeros(12,np.float32)) + kr.policy_harmonic_phase_index=1; kr.requested_action(); assert not np.array_equal(kp.calls[1],np.zeros(12,np.float32)) + ir=make_runner_runtime("q_ctl"); ip=Capture(); ir.policy=ip; harmonics=[{"dc":2.,"amps":[],"freqs":[],"phases":[]} for _ in range(2)] + ir.set_controller_state(fifo_history=np.ones((150,12),np.float32),normalization=norm,harmonics=harmonics) + ir.requested_action(); np.testing.assert_allclose(ip.calls[0][:6],.5); np.testing.assert_allclose(ip.calls[0][6:12],1.); np.testing.assert_allclose(ip.calls[0][12:],1.) + + +def test_schedule_rejected_before_runtime_builder(tmp_path): + called=[] + with pytest.raises(ValueError,match="divisible"): + run_role_acquisition(case="karman_re100",role="q_blk",output=tmp_path/"x",control_count=1,field_interval=300,runtime_builder=lambda **kw:called.append(kw)) + assert called==[] + + +def test_frozen_illusion_reference_exact_files_and_hashes(): + from CCD_analysis.acquisition.runtime import load_illusion_training_reference + norm,harmonics,identity=load_illusion_training_reference() + assert identity["normalization_sha256"]=="9ec5ddbe68fb441cdb660c17b51bf84ac348ca87d565081a21cb092a5245e4fc" + assert identity["harmonics_sha256"]=="f135660ee3533c81d7dd175a4530f83e728ba52ec9485738ec0ad953e4a6378d" + assert set(norm)=={"force_norm_fact","sens_deviation","sens_norm_fact"} and len(harmonics)==2 + + +def test_full_fake_checkpoint_restore_excludes_norm_contamination(): + flow=RunnerFlow(2); zero=np.zeros(2,np.float32) + flow.run(2,zero); checkpoint=flow.full_state_checkpoint(); stabilized=flow.absolute + flow.run(7,zero); assert flow.absolute==stabilized+7 + flow.restore_full_state(checkpoint) + assert flow.absolute==stabilized and not flow.action.any() + flow.run(1,np.ones(2,np.float32)) + np.testing.assert_allclose(flow.action,np.full(2,.1,np.float32)) + + +def test_runner_clock_domains_with_nonzero_solver_origin(tmp_path): + runtime=make_runner_runtime() + def initialized(runtime,**kwargs): + runtime.flow.absolute=900; runtime.flow.absolute_controls=7 + norm={"force_norm_fact":np.asarray(2,np.float32),"sens_deviation":np.zeros(6,np.float32),"sens_norm_fact":np.ones(6,np.float32)} + runtime.set_controller_state(fifo_history=np.ones((150,12),np.float32),normalization=norm,solver_absolute_lattice_clock=900,solver_absolute_control_clock=7,policy_harmonic_phase_index=0) + destination=tmp_path/"clocked" + run_role_acquisition(case="illusion_1.0L",role="q_blk",output=destination,control_count=2,field_interval=600,runtime=runtime,initializer=initialized) + with np.load(destination/"fields.npz") as fields: + assert fields["lattice_steps"].tolist()==[1500,2100] + assert fields["acquisition_relative_lattice_steps"].tolist()==[600,1200] + assert fields["control_indices"].tolist()==[0,1] + assert fields["solver_absolute_control_indices"].tolist()==[7,8] + with np.load(destination/"controller_state.npz") as state: + assert int(state["solver_absolute_lattice_clock"])==2100 and int(state["solver_absolute_control_clock"])==9 + assert int(state["acquisition_relative_lattice_clock"])==1200 and int(state["acquisition_relative_control_index"])==2 + assert int(state["policy_harmonic_phase_index"])==2 + + +def test_clock_accessors_enforce_split_and_boundary_lifecycle(): + flow=RunnerFlow(2); zero=np.zeros(2,np.float32) + assert flow.solver_clock_state()=={"solver_absolute_lattice_clock":0,"solver_absolute_control_clock":0} + with pytest.raises(RuntimeError,match="active-step"): flow.active_step_clock_state() + flow.begin_control_interval(2,zero) + with pytest.raises(RuntimeError,match="boundary"): flow.solver_clock_state() + with pytest.raises(RuntimeError,match="active-step"): flow.active_step_clock_state() + flow.run_control_segment(1) + assert flow.active_step_clock_state()=={"solver_absolute_lattice_clock":1,"solver_absolute_control_clock":0} + with pytest.raises(RuntimeError,match="boundary"): flow.solver_clock_state() + flow.run_control_segment(1); flow.end_control_interval() + assert flow.solver_clock_state()=={"solver_absolute_lattice_clock":2,"solver_absolute_control_clock":1} + with pytest.raises(RuntimeError,match="active-step"): flow.active_step_clock_state() + + +def test_control_lineage_retains_initial_row_reconstructs_fifo_and_ignores_field_cadence(tmp_path): + runtime = make_runner_runtime("q_ctl") + initial = np.arange(150 * 12, dtype=np.float32).reshape(150, 12) + class Capture: + def predict(self, obs, deterministic=True): return np.zeros(3, np.float32), None + runtime.policy = Capture() + def initialized(runtime, **kwargs): + norm = {"force_norm_fact": np.asarray(2, np.float32), "sens_deviation": np.zeros(6, np.float32), "sens_norm_fact": np.ones(6, np.float32)} + harmonics = [{"dc": 0., "amps": [], "freqs": [], "phases": []} for _ in range(2)] + runtime.set_controller_state(fifo_history=initial, normalization=norm, harmonics=harmonics) + destination = tmp_path / "lineage" + run_role_acquisition(case="illusion_1.0L", role="q_ctl", output=destination, control_count=3, field_interval=900, runtime=runtime, initializer=initialized) + with np.load(destination / "fields.npz") as fields, np.load(destination / "controller_state.npz") as state: + assert fields["ux"].shape[0] == 2 + np.testing.assert_array_equal(state["initial_fifo_history"], initial) + assert state["boundary_observation_history"].shape == (3, 12) + np.testing.assert_array_equal(state["policy_source_observation_history"][0], initial[-1]) + np.testing.assert_array_equal(state["policy_source_observation_history"][1:], state["boundary_observation_history"][:-1]) + expected = np.concatenate((initial, state["boundary_observation_history"]), axis=0)[-150:] + np.testing.assert_array_equal(state["fifo_history"], expected) + np.testing.assert_array_equal(state["fifo_history"][:-3], initial[3:]) + np.testing.assert_array_equal(state["fifo_history"][-3:], state["boundary_observation_history"]) + np.testing.assert_array_equal(state["policy_harmonic_phase_indices"], np.arange(3, dtype=np.int64)) + assert state["policy_input_observation_history"].shape == (3, 14) + + +def test_lineage_mismatch_fails_closed(tmp_path): + runtime = make_runner_runtime() + destination = tmp_path / "valid" + run_role_acquisition(case="illusion_1.0L", role="q_blk", output=destination, control_count=2, field_interval=600, runtime=runtime, initializer=fake_initializer) + with np.load(destination / "fields.npz") as archive: arrays = {key: archive[key].copy() for key in archive.files} + with np.load(destination / "controller_state.npz") as archive: state = {key: archive[key].copy() for key in archive.files} + config = json.loads((destination / "config.json").read_text()) + from CCD_analysis.acquisition.validation import validate_acquisition_semantics + for key, match in (("boundary_observation_history", "terminal FIFO"), ("policy_harmonic_phase_indices", "phase indices"), ("policy_source_observation_sha256", "source observation hashes")): + broken = {name: value.copy() for name, value in state.items()} + if key == "policy_source_observation_sha256": broken[key][0] = "0" * 64 + else: broken[key][0] += np.asarray(1, broken[key].dtype) + with pytest.raises(ValueError, match=match): + validate_acquisition_semantics(arrays=arrays, config=config, state=broken) + + + +def test_atomic_no_replace_preserves_existing_empty_destination(tmp_path): + from CCD_analysis.acquisition.artifacts import rename_noreplace + source = tmp_path / "source"; source.mkdir(); (source / "payload").write_text("new") + destination = tmp_path / "destination"; destination.mkdir() + with pytest.raises(FileExistsError): rename_noreplace(source, destination) + assert destination.is_dir() and list(destination.iterdir()) == [] + assert (source / "payload").read_text() == "new" + + +def test_illusion_orchestration_fails_closed_without_compatibility_certificate(tmp_path): + with pytest.raises(ValueError, match="compatibility-certificate"): + orchestrate("illusion_1.0L", tmp_path / "future", control_count=450, field_interval=1250) + +def test_compatibility_certificate_rejects_tamper(tmp_path): + from CCD_analysis.acquisition.compatibility import publish_certificate, validate_certificate + pilot = Path("src/CCD_analysis/evidence/smoke-20260804-q-over-u0-lineage-v3/illusion_1.0L/q_ctl-pilot150") + certificate = tmp_path / "certificate.json" + publish_certificate(pilot, certificate); assert validate_certificate(certificate)["status"] == "PASS" + value=json.loads(certificate.read_text()); value["metrics"]["field_q_over_u0_max_abs"] += 1 + certificate.write_text(json.dumps(value,sort_keys=True,separators=(",",":"))+"\n") + with pytest.raises(ValueError, match="invalid or stale"): validate_certificate(certificate) diff --git a/src/CCD_analysis/tests/test_direct_dq.py b/src/CCD_analysis/tests/test_direct_dq.py new file mode 100644 index 0000000..94a48d0 --- /dev/null +++ b/src/CCD_analysis/tests/test_direct_dq.py @@ -0,0 +1,507 @@ +import json +from dataclasses import asdict +from hashlib import sha256 +import multiprocessing as mp +from pathlib import Path + +import numpy as np +import pytest + +from CCD_analysis.acquisition.artifacts import ArtifactTransaction, file_sha256 +from CCD_analysis.acquisition.contracts import ACTION_FORMULA, ACTION_FORMULA_SHA256, CONFIG_BINDINGS, MODEL_BINDINGS, canonical_coordinate_axis, canonical_coordinate_frame, canonical_json, case_snapshot, expected_controller_identity, expected_source_bindings, role_spec, role_spec_identity, VELOCITY_DECODER_FORMULA, VELOCITY_DECODER_FORMULA_SHA256, VELOCITY_DECODER_SCHEMA_ID +from CCD_analysis.acquisition.runtime import EXPECTED_SHA256 +from CCD_analysis.direct_dq.analysis import ( + analysis_mask, compute_estimands, phase_conditioned_outputs, profile_metrics, + parse_station_token, segmented_trapezoid, station_index, vorticity, weighted_vector_rms, +) +from CCD_analysis.direct_dq.cli import main as direct_dq_main +from CCD_analysis.direct_dq.convergence import convergence_report, validate_window_sizes +from CCD_analysis.direct_dq.io import ResultTransaction, load_acquisition_artifact, load_matched_inputs, load_result, load_result_metadata_unverified + + +def _race_cli_worker(arguments, queue): + try: + direct_dq_main(arguments) + except FileExistsError: + queue.put("lost") + except Exception as exc: + queue.put(f"error:{type(exc).__name__}:{exc}") + else: + queue.put("won") + + +def _rehash_acquisition_partial(transaction, config, state): + state["config_hash"] = np.asarray(sha256(canonical_json(config)).hexdigest()) + np.savez_compressed(transaction.partial / "controller_state.npz", **state) + (transaction.partial / "config.json").write_bytes(canonical_json(config)) + manifest = json.loads((transaction.partial / "manifest.json").read_text()) + manifest["config_sha256"] = sha256(canonical_json(config)).hexdigest() + manifest["state_array_sha256"] = {key: sha256(np.ascontiguousarray(value).tobytes()).hexdigest() for key, value in state.items()} + manifest["files"]["config.json"] = file_sha256(transaction.partial / "config.json") + manifest["files"]["controller_state.npz"] = file_sha256(transaction.partial / "controller_state.npz") + (transaction.partial / "manifest.json").write_bytes(canonical_json(manifest)) + + +def _rehash_result_json(output, name, value): + (output / name).write_bytes(canonical_json(value)) + manifest = json.loads((output / "manifest.json").read_text()) + manifest["files"][name] = file_sha256(output / name) + (output / "manifest.json").write_bytes(canonical_json(manifest)) + + +def _fixture_coordinate_frame(x, y): + x = np.asarray(x, np.float32); y = np.asarray(y, np.float32) + frame = canonical_coordinate_frame(x.size, y.size) + frame["x"].update(origin_lattice=float(x[0] * 20), spacing_lattice=float((x[1] - x[0]) * 20)) + frame["y"].update(origin_lattice=float(y[0] * 20), spacing_lattice=float((y[1] - y[0]) * 20)) + return frame + + +def _active_config(role, *, origin, steps, count, width, coordinate_frame=None): + spec = role_spec("karman_re100", role) + if width != spec.physical_action_width: raise ValueError("fixture width mismatch") + normalization = {"force_norm_fact": 1.0, "sens_deviation": [0.] * 6, "sens_norm_fact": [1.] * 6} + harmonics = [] + config = case_snapshot("karman_re100", role) + config["runtime"] = { + "role_spec": asdict(spec), "role_spec_sha256": role_spec_identity(spec), + "physical_action_width": width, + "solver_objects": [{"id": index, "identity": obj.identity, "kind": obj.kind, "center_lattice": (obj.center_D[0] * 20.0, 100.0 + obj.center_D[1] * 20.0, 0.0), "radius_lattice": obj.radius_D * 20.0} for index, obj in enumerate(spec.objects)], + "coordinate_frame": coordinate_frame or canonical_coordinate_frame(3, 4, y_origin_lattice=-30.0, spacing_lattice=20.0), + "policy_device": "cpu", "cfd_device": 0, + "action_formula": ACTION_FORMULA, "action_formula_sha256": ACTION_FORMULA_SHA256, + "velocity_decoder": {"schema_id": VELOCITY_DECODER_SCHEMA_ID, "quantity": "nondimensional velocity q/U0", "u0": 0.01, "formula": VELOCITY_DECODER_FORMULA, "formula_sha256": VELOCITY_DECODER_FORMULA_SHA256}, + } + config["controller_sources"] = { + "normalization": normalization, + "normalization_content_sha256": sha256(canonical_json(normalization)).hexdigest(), + "controller_harmonics": harmonics, + "controller_harmonics_content_sha256": sha256(canonical_json(harmonics)).hexdigest(), + "identity": expected_controller_identity("karman_re100", role), + "measured_plus11_phase_harmonics": None, + "compatibility": "synthetic active-schema fixture", + } + config["source_sha256"] = expected_source_bindings("karman_re100", role) + config["clock_domains"] = { + "solver_absolute_lattice_origin": origin, "solver_absolute_control_origin": 0, + "solver_absolute_lattice_final": int(steps[-1]), "solver_absolute_control_final": count, + "acquisition_relative_lattice_final": int(steps[-1] - origin), "acquisition_relative_control_final": count, + "policy_harmonic_phase_final": count, + } + config["acquisition"] = {"field_interval": 800, "checkpoint_lifecycle": "synthetic terminal completed boundary", "control_history": "complete boundary-average lineage independent of field cadence", "policy_input_contract": "reconstruct_from_prior_boundary_history" if role == "q_ctl" else "not_applicable_explicit_zero"} + return config + + +def _state(nx, ny, width, *, config, effective): + clocks = config["clock_domains"] + zero_model = sha256(b"zero-controller").hexdigest() + return { + "current_ddf": np.zeros(9 * nx * ny, np.float32), "temp_ddf": np.ones(9 * nx * ny, np.float32), + "current_raw_observation": np.zeros(12, np.float32), "fifo_history": np.zeros((150, 12), np.float32), + "initial_fifo_history": np.zeros((150, 12), np.float32), + "boundary_observation_history": np.zeros((clocks["acquisition_relative_control_final"], 12), np.float32), + "policy_source_observation_history": np.zeros((clocks["acquisition_relative_control_final"], 12), np.float32), + "policy_source_observation_sha256": np.asarray([sha256(np.zeros(12, np.float32).tobytes()).hexdigest()] * clocks["acquisition_relative_control_final"]), + "policy_input_observation_history": np.zeros((clocks["acquisition_relative_control_final"], 12), np.float32), + "policy_harmonic_phase_indices": np.arange(clocks["acquisition_relative_control_final"], dtype=np.int64), + "requested_normalized_action_history": np.zeros((clocks["acquisition_relative_control_final"], 3), np.float32), + "requested_physical_action_history": np.zeros((clocks["acquisition_relative_control_final"], width), np.float32), + "persisted_effective_ema_action": effective[-1].copy(), + "policy_harmonic_phase_index": np.asarray(clocks["policy_harmonic_phase_final"], np.int64), + "solver_absolute_control_clock": np.asarray(clocks["solver_absolute_control_final"], np.int64), + "solver_absolute_lattice_clock": np.asarray(clocks["solver_absolute_lattice_final"], np.int64), + "acquisition_relative_control_index": np.asarray(clocks["acquisition_relative_control_final"], np.int64), + "acquisition_relative_lattice_clock": np.asarray(clocks["acquisition_relative_lattice_final"], np.int64), + "normalization_hash": np.asarray(config["controller_sources"]["normalization_content_sha256"]), + "harmonics_hash": np.asarray(config["controller_sources"]["controller_harmonics_content_sha256"]), + "model_hash": np.asarray(EXPECTED_SHA256["models/old/d1a3o12_re100.zip"] if config["role"] == "q_ctl" else zero_model), + "cuda_config_hash": np.asarray(EXPECTED_SHA256["configs/legacy_configs/config_cuda.json"]), + "flow_config_hash": np.asarray(EXPECTED_SHA256["configs/legacy_configs/config_flowfield.json"]), + "config_hash": np.asarray(config["runtime"]["role_spec_sha256"]), + "geometry_hash": np.asarray(config["geometry_sha256"]), "action_formula_hash": np.asarray(ACTION_FORMULA_SHA256), + "velocity_decoder_formula_hash": np.asarray(VELOCITY_DECODER_FORMULA_SHA256), + } + + +def write_artifact(root: Path, role: str, *, origin=0, x=None, relative=None, mask=None, offset=0.0): + x = canonical_coordinate_axis(3, origin_lattice=0.0, spacing_lattice=20.0, reference_length_lattice=20.0) if x is None else np.asarray(x, np.float32) + y = canonical_coordinate_axis(4, origin_lattice=-30.0, spacing_lattice=20.0, reference_length_lattice=20.0) + relative = np.asarray([800, 1600, 2400, 3200], np.int64) if relative is None else np.asarray(relative, np.int64) + count, nx, ny = relative.size, x.size, y.size; steps = relative + origin + time = np.arange(count, dtype=np.float32)[:, None, None]; xx = x[None, :, None]; yy = y[None, None, :] + ux = (offset + time + 2 * xx + .5 * yy).astype(np.float32); uy = (offset - time + xx - yy).astype(np.float32) + mask = np.ones((nx, ny), bool) if mask is None else np.asarray(mask, bool); controls = (relative - 1) // 800 + normalized = np.zeros((count, 3), np.float32); physical = np.zeros((count, 6), np.float32); effective = np.zeros((count, 6), np.float32) + if role == "q_ctl": + physical[:, -3:] = np.asarray([0., -.04, .04], np.float32) + effective[:] = physical + arrays = {"ux": ux, "uy": uy, "x_D": x, "y_D": y, "fluid_mask": mask, "lattice_steps": steps, "control_indices": controls, "control_offsets": (relative - 1) % 800 + 1, "sample_ids": steps, "acquisition_relative_lattice_steps": relative, "solver_absolute_control_indices": controls, "requested_normalized_action": normalized, "requested_physical_action": physical, "effective_applied_action": effective, "disturbance_force": np.zeros((count, 2), np.float32), "pinball_forces": np.zeros((count, 6), np.float32), "sensors": np.zeros((count, 6), np.float32), "phase_reference": np.arange(count, dtype=np.float32)[:, None]} + width = role_spec("karman_re100", role).physical_action_width + normalized = np.zeros((count, 3), np.float32); physical = np.zeros((count, width), np.float32); effective = np.zeros((count, width), np.float32) + if role == "q_ctl": + physical[:, -3:] = np.asarray([0., -.04, .04], np.float32); effective[:] = physical + arrays["requested_normalized_action"] = normalized; arrays["requested_physical_action"] = physical; arrays["effective_applied_action"] = effective + frame = _fixture_coordinate_frame(x, y) + config = _active_config(role, origin=origin, steps=steps, count=count, width=width, coordinate_frame=frame) + state = _state(nx, ny, width, config=config, effective=effective) + state["requested_normalized_action_history"] = normalized.copy() + state["requested_physical_action_history"] = physical.copy() + state["config_hash"] = np.asarray(sha256(canonical_json(config)).hexdigest()) + with ArtifactTransaction(root) as transaction: + transaction.write(arrays=arrays, config=config, state=state); transaction.publish() + return root + + +def write_triplet(tmp_path, *, masks=None, origins=(11, 22, 33)): + masks = masks or [None, None, None] + return {role: write_artifact(tmp_path / role, role, origin=origin, mask=mask, offset=offset) for role, origin, mask, offset in zip(("q_target", "q_blk", "q_ctl"), origins, masks, (1., 2., 4.))} + + +def test_estimand_algebra_and_pointwise_closure(): + shape = (3, 2, 2, 3) + blocked = np.ones(shape, np.float32) + target = blocked + np.float32(2) + controlled = blocked + np.float32(5) + result = compute_estimands(target, blocked, controlled) + np.testing.assert_array_equal(result["e_target"], np.full(shape, 3, np.float32)) + np.testing.assert_array_equal(result["dq_ctl"], np.full(shape, 5, np.float32)) + np.testing.assert_array_equal(result["dq_tar"], np.full(shape, 2, np.float32)) + assert not result["closure_residual"].any() + + +def test_solver_mask_intersection_preserves_roles_and_rejects_small(): + masks = [np.ones((3, 4), bool) for _ in range(3)] + masks[0][0, 0] = False; masks[1][1, 1] = False; masks[2][2, 2] = False + common = analysis_mask(masks) + np.testing.assert_array_equal(common, masks[0] & masks[1] & masks[2]) + assert masks[0][1, 1] and masks[1][2, 2] + with pytest.raises(ValueError, match="too small"): + analysis_mask([np.eye(2, dtype=bool)] * 3) + + +def test_active_writer_artifact_reload_hash_and_role_validation(tmp_path): + paths = write_triplet(tmp_path) + artifact = load_acquisition_artifact(paths["q_target"], expected_case="karman_re100", expected_role="q_target") + assert artifact.input_identity["manifest_sha256"] and artifact.fields["ux"].dtype == np.float32 + with pytest.raises(ValueError, match="case/role"): + load_acquisition_artifact(paths["q_target"], expected_case="karman_re100", expected_role="q_blk") + fields = paths["q_target"] / "fields.npz" + fields.write_bytes(fields.read_bytes() + b"tamper") + with pytest.raises(ValueError, match="hash mismatch"): + load_acquisition_artifact(paths["q_target"], expected_case="karman_re100", expected_role="q_target") + + +def test_exact_relative_timeline_allows_different_valid_absolute_origins(tmp_path): + paths = write_triplet(tmp_path, origins=(100, 900, 1700)) + loaded = load_matched_inputs(q_target=paths["q_target"], q_blk=paths["q_blk"], q_ctl=paths["q_ctl"], case_id="karman_re100") + assert loaded["q_target"].fields["lattice_steps"][0] != loaded["q_blk"].fields["lattice_steps"][0] + + +def test_exact_timeline_and_grid_rejection_without_trim(tmp_path): + paths = write_triplet(tmp_path / "timeline") + bad = write_artifact(tmp_path / "different" / "q_ctl", "q_ctl", relative=[800, 1600, 2500, 3200], offset=4) + with pytest.raises(ValueError, match="timeline exact-equality"): + load_matched_inputs(q_target=paths["q_target"], q_blk=paths["q_blk"], q_ctl=bad, case_id="karman_re100") + paths2 = write_triplet(tmp_path / "grid") + bad_grid = write_artifact(tmp_path / "bad-grid" / "q_ctl", "q_ctl", x=[0, 2, 4], offset=4) + with pytest.raises(ValueError, match="grid exact-equality"): + load_matched_inputs(q_target=paths2["q_target"], q_blk=paths2["q_blk"], q_ctl=bad_grid, case_id="karman_re100") + + +def test_no_nearest_station_and_phase_outputs_fail_closed(): + x = np.asarray([0., 1., 2.], np.float32) + assert station_index(x, "1.0") == 1 + with pytest.raises(ValueError, match="nearest/tolerance"): + station_index(x, "1.2") + assert station_index(x, "1.00000001") == 1 # canonical float32 token maps exactly, not by tolerance + x_decimal = np.asarray([31.2, 31.3, 31.4], np.float32) + assert station_index(x_decimal, "31.3") == 1 + with pytest.raises(ValueError, match="nearest/tolerance"): + station_index(np.asarray([31.2, 31.4], np.float32), "31.3") + with pytest.raises(NotImplementedError, match="phase-conditioned"): + phase_conditioned_outputs() + + +def test_segmented_quadrature_does_not_bridge_mask_gaps(): + y = np.asarray([0., 1., 10., 11.]) + value = np.ones(4) + mask = np.asarray([True, True, True, True]) + assert segmented_trapezoid(value, y, mask) == pytest.approx(11) + mask[1:3] = False + assert segmented_trapezoid(value, y, mask) == 0.0 + mask = np.asarray([True, True, False, True, True]) + assert segmented_trapezoid(np.ones(5), np.asarray([0., 1., 50., 100., 101.]), mask) == pytest.approx(2) + + +def test_profiles_integrals_wake_moments_and_weighted_rms(): + y = np.asarray([-1., 0., 1.]) + target = np.ones(3) + controlled = np.asarray([0., 1., 0.]) + metrics = profile_metrics(target, controlled, y, np.ones(3, bool)) + assert metrics["signed_target_relative_ux_deficit_integral"] == pytest.approx(1) + assert metrics["momentum_flux_proxy_incomplete"] == pytest.approx(1) + assert metrics["positive_deficit_area"] == pytest.approx(1) + assert metrics["positive_deficit_centroid_y_D"] == pytest.approx(0) + assert metrics["positive_deficit_width_D"] == pytest.approx(1) + vector = np.ones((2, 2, 3), np.float32) + assert weighted_vector_rms(vector, np.asarray([0., 2.]), y, np.ones((2, 3), bool)) == pytest.approx(np.sqrt(2)) + + +def test_nonuniform_vorticity_and_mask_gap_handling(): + x = np.asarray([0., .5, 2., 4.], np.float32) + y = np.asarray([-2., -.5, .25, 2.], np.float32) + xx, yy = np.meshgrid(x, y, indexing="ij") + ux = (-2 * yy).astype(np.float32); uy = (2 * xx).astype(np.float32) + mask = np.ones_like(ux, bool) + omega, valid = vorticity(ux, uy, x, y, mask) + np.testing.assert_allclose(omega[valid], 4, atol=1e-6) + mask[:, 2] = False + omega, valid = vorticity(ux, uy, x, y, mask) + assert not valid[:, 2].any() and np.isfinite(omega).all() + + +def test_declared_prefix_suffix_convergence_has_no_uncertainty_claim(): + x = np.asarray([0., 1.], np.float32); y = np.asarray([0., 1.], np.float32); mask = np.ones((2, 2), bool) + base = np.arange(4, dtype=np.float32)[:, None, None, None] * np.ones((4, 2, 2, 2), np.float32) + estimands = {"e_target": base, "dq_ctl": base + 1, "dq_tar": np.ones_like(base)} + report = convergence_report({"q_target": base}, estimands, x_D=x, y_D=y, mask=mask, window_sizes=[2, 4]) + assert report["window_sizes"] == [2, 4] and report["independent_realization_uncertainty"] is False + assert report["records"][-1]["prefix"]["e_target_mean_deviation_weighted_vector_rms"] == pytest.approx(0) + with pytest.raises(ValueError, match="ending at the full"): + validate_window_sizes([1, 2], 4) + + +def test_synthetic_cli_writes_full_result_and_input_hashes(tmp_path): + paths = write_triplet(tmp_path / "inputs") + output = tmp_path / "result" + argv = ["--case", "karman_re100", "--q-target", str(paths["q_target"]), "--q-blk", str(paths["q_blk"]), "--q-ctl", str(paths["q_ctl"]), "--output", str(output), "--station-x-D", "1", "--start-after-relative-step", "0", "--window-size", "2", "--window-size", "4"] + assert direct_dq_main(argv) == 0 + result = load_result(output) + assert result["summary"]["sample_count"] == 4 + assert result["summary"]["time_aggregation"].startswith("time mean only") + assert set(result["input_hashes"]) == {"q_target", "q_blk", "q_ctl"} + arrays = result["arrays"] + assert arrays["q_target_mean"].shape == (2, 3, 4) + assert arrays["q_target_instantaneous"].shape == arrays["q_blk_instantaneous"].shape == arrays["q_ctl_instantaneous"].shape == (4, 2, 3, 4) + np.testing.assert_array_equal(arrays["selected_timeline_indices"], np.arange(4, dtype=np.int64)) + assert arrays["e_target_instantaneous"].shape == (4, 2, 3, 4) + assert all(result["input_hashes"][role]["role_instantaneous_sha256"] for role in ("q_target","q_blk","q_ctl")) + assert arrays["e_target_mean_ux_profiles"].shape == (1, 4) + assert np.array_equal(arrays["analysis_fluid_mask"], arrays["q_target_solver_fluid_mask"] & arrays["q_blk_solver_fluid_mask"] & arrays["q_ctl_solver_fluid_mask"]) + + +def test_shared_writer_rejects_semantically_forged_reference_actions(tmp_path): + role = "q_blk"; root = tmp_path / "forged" + x = np.asarray([0., 1.], np.float32); y = np.asarray([0., 1.], np.float32); relative = np.asarray([800], np.int64); steps = relative.copy() + width = role_spec("karman_re100", role).physical_action_width + arrays = {"ux": np.zeros((1, 2, 2), np.float32), "uy": np.zeros((1, 2, 2), np.float32), "x_D": x, "y_D": y, "fluid_mask": np.ones((2, 2), bool), "lattice_steps": steps, "control_indices": np.asarray([0], np.int64), "control_offsets": np.asarray([800], np.int64), "sample_ids": steps, "acquisition_relative_lattice_steps": relative, "solver_absolute_control_indices": np.asarray([0], np.int64), "requested_normalized_action": np.ones((1, 3), np.float32), "requested_physical_action": np.ones((1, width), np.float32), "effective_applied_action": np.zeros((1, width), np.float32), "disturbance_force": np.zeros((1, 2), np.float32), "pinball_forces": np.zeros((1, 6), np.float32), "sensors": np.zeros((1, 6), np.float32), "phase_reference": np.zeros((1, 1), np.float32)} + config = _active_config(role, origin=0, steps=steps, count=1, width=width, coordinate_frame=_fixture_coordinate_frame(x, y)) + state = _state(2, 2, width, config=config, effective=arrays["effective_applied_action"]); state["config_hash"] = np.asarray(sha256(canonical_json(config)).hexdigest()) + with pytest.raises(ValueError, match="exactly zero"): + with ArtifactTransaction(root) as transaction: + transaction.write(arrays=arrays, config=config, state=state) + assert not root.exists() and not list(tmp_path.glob("*.partial.*")) + + +def test_result_reload_recomputes_closure_after_hash_consistent_tamper(tmp_path): + paths = write_triplet(tmp_path / "inputs"); output = tmp_path / "result" + argv = ["--case", "karman_re100", "--q-target", str(paths["q_target"]), "--q-blk", str(paths["q_blk"]), "--q-ctl", str(paths["q_ctl"]), "--output", str(output), "--station-x-D", "1", "--start-after-relative-step", "0", "--window-size", "4"] + direct_dq_main(argv) + with np.load(output / "arrays.npz", allow_pickle=False) as archive: arrays = {key: archive[key].copy() for key in archive.files} + arrays["e_target_instantaneous"][0, 0, 0, 0] += np.float32(1) + np.savez_compressed(output / "arrays.npz", **arrays) + manifest = json.loads((output / "manifest.json").read_text()); manifest["files"]["arrays.npz"] = file_sha256(output / "arrays.npz") + (output / "manifest.json").write_bytes(canonical_json(manifest)) + with pytest.raises(ValueError, match="absolute-role recomputation|closure"): + load_result(output) + + +def test_result_publication_multiprocess_race_one_wins_and_sentinel_untouched(tmp_path): + paths = write_triplet(tmp_path / "inputs"); output = tmp_path / "raced-result"; sentinel = tmp_path / "sentinel" + sentinel.write_bytes(b"do-not-touch") + argv = ["--case", "karman_re100", "--q-target", str(paths["q_target"]), "--q-blk", str(paths["q_blk"]), "--q-ctl", str(paths["q_ctl"]), "--output", str(output), "--station-x-D", "1", "--start-after-relative-step", "0", "--window-size", "4"] + context = mp.get_context("spawn"); queue = context.Queue() + processes = [context.Process(target=_race_cli_worker, args=(argv, queue)) for _ in range(2)] + for process in processes: process.start() + for process in processes: process.join(30); assert process.exitcode == 0 + outcomes = sorted(queue.get(timeout=5) for _ in processes) + assert outcomes == ["lost", "won"] and sentinel.read_bytes() == b"do-not-touch" + load_result(output) + assert not list(tmp_path.glob("*.partial.*")) + occupied = tmp_path / "occupied"; occupied.mkdir(); (occupied / "sentinel").write_bytes(b"existing") + with pytest.raises(FileExistsError): + with ResultTransaction(occupied): pass + assert (occupied / "sentinel").read_bytes() == b"existing" + + +@pytest.mark.parametrize("mutation", ["formula", "velocity_decoder", "observation_slices", "harmonic_channels", "model_path", "model_hash", "config_path_hash"]) +def test_forged_rehashed_frozen_authority_rejected_by_writer(tmp_path, mutation): + role="q_ctl"; x=np.asarray([0.,1.],np.float32); y=np.asarray([0.,1.],np.float32); relative=np.asarray([800],np.int64); steps=relative.copy(); width=role_spec("karman_re100",role).physical_action_width + normalized=np.zeros((1,3),np.float32); physical=np.zeros((1,width),np.float32); physical[:,-3:]=np.asarray([0.,-.04,.04],np.float32); effective=physical.copy() + arrays={"ux":np.zeros((1,2,2),np.float32),"uy":np.zeros((1,2,2),np.float32),"x_D":x,"y_D":y,"fluid_mask":np.ones((2,2),bool),"lattice_steps":steps,"control_indices":np.asarray([0],np.int64),"control_offsets":np.asarray([800],np.int64),"sample_ids":steps,"acquisition_relative_lattice_steps":relative,"solver_absolute_control_indices":np.asarray([0],np.int64),"requested_normalized_action":normalized,"requested_physical_action":physical,"effective_applied_action":effective,"disturbance_force":np.zeros((1,2),np.float32),"pinball_forces":np.zeros((1,6),np.float32),"sensors":np.zeros((1,6),np.float32),"phase_reference":np.zeros((1,1),np.float32)} + config=_active_config(role,origin=0,steps=steps,count=1,width=width,coordinate_frame=_fixture_coordinate_frame(x,y)); state=_state(2,2,width,config=config,effective=effective) + if mutation=="formula": config["runtime"]["action_formula"]="forged"; config["runtime"]["action_formula_sha256"]=sha256(b"forged").hexdigest(); state["action_formula_hash"]=np.asarray(config["runtime"]["action_formula_sha256"]) + elif mutation=="velocity_decoder": config["runtime"]["velocity_decoder"]["formula"]="momentum/density"; config["runtime"]["velocity_decoder"]["formula_sha256"]=sha256(b"momentum/density").hexdigest(); state["velocity_decoder_formula_hash"]=np.asarray(config["runtime"]["velocity_decoder"]["formula_sha256"]) + elif mutation=="observation_slices": config["runtime"]["role_spec"]["observation_slices"]["sensors"]=[1,7]; config["runtime"]["role_spec_sha256"]=sha256(canonical_json(config["runtime"]["role_spec"])).hexdigest() + elif mutation=="harmonic_channels": config["runtime"]["role_spec"]["harmonic_channels"]=9; config["runtime"]["role_spec_sha256"]=sha256(canonical_json(config["runtime"]["role_spec"])).hexdigest() + elif mutation=="model_path": config["runtime"]["role_spec"]["model_path"]="/forged/model.zip"; config["runtime"]["role_spec_sha256"]=sha256(canonical_json(config["runtime"]["role_spec"])).hexdigest() + elif mutation=="model_hash": state["model_hash"]=np.asarray("b"*64); config["source_sha256"][MODEL_BINDINGS["karman_re100"][0]]="b"*64 + else: config["source_sha256"]={"forged/config.json":"c"*64,MODEL_BINDINGS["karman_re100"][0]:MODEL_BINDINGS["karman_re100"][1]}; state["cuda_config_hash"]=np.asarray("c"*64); state["flow_config_hash"]=np.asarray("c"*64) + state["config_hash"]=np.asarray(sha256(canonical_json(config)).hexdigest()) + with pytest.raises(ValueError): + with ArtifactTransaction(tmp_path/mutation) as transaction: transaction.write(arrays=arrays,config=config,state=state) + + +@pytest.mark.parametrize("mutation", ["normalization_path", "normalization_sha256", "harmonics_path", "harmonics_sha256"]) +def test_forged_rehashed_illusion_training_refs_rejected_on_load(tmp_path, mutation): + from CCD_analysis.tests.test_acquisition import fake_initializer, make_runner_runtime + from CCD_analysis.acquisition.runtime import run_role_acquisition + root=tmp_path/mutation + run_role_acquisition(case="illusion_1.0L",role="q_ctl",output=root,control_count=1,field_interval=600,runtime=make_runner_runtime("q_ctl"),initializer=fake_initializer) + config=json.loads((root/"config.json").read_text()); identity=config["controller_sources"]["identity"] + identity[mutation] = ("forged/reference.json" if mutation.endswith("path") else "d"*64) + with np.load(root/"controller_state.npz",allow_pickle=False) as z: state={k:z[k].copy() for k in z.files} + if mutation=="normalization_sha256": state["normalization_hash"]=np.asarray("d"*64) + if mutation=="harmonics_sha256": state["harmonics_hash"]=np.asarray("d"*64) + state["config_hash"]=np.asarray(sha256(canonical_json(config)).hexdigest()) + (root/"config.json").write_bytes(canonical_json(config)); np.savez_compressed(root/"controller_state.npz",**state) + manifest=json.loads((root/"manifest.json").read_text()); manifest["config_sha256"]=sha256(canonical_json(config)).hexdigest(); manifest["state_array_sha256"]={k:sha256(np.ascontiguousarray(v).tobytes()).hexdigest() for k,v in state.items()}; manifest["files"]["config.json"]=file_sha256(root/"config.json"); manifest["files"]["controller_state.npz"]=file_sha256(root/"controller_state.npz"); (root/"manifest.json").write_bytes(canonical_json(manifest)) + with pytest.raises(ValueError,match="training reference"): + load_acquisition_artifact(root,expected_case="illusion_1.0L",expected_role="q_ctl") + + +def test_publish_and_load_reject_forged_rehashed_formula(tmp_path): + # Writer creates valid partial, then dependent files/hashes are forged before publish. + role="q_blk"; root=write_artifact(tmp_path/"valid",role) + config=json.loads((root/"config.json").read_text()); config["runtime"]["action_formula"]="forged"; config["runtime"]["action_formula_sha256"]=sha256(b"forged").hexdigest() + with np.load(root/"controller_state.npz",allow_pickle=False) as z: state={k:z[k].copy() for k in z.files} + state["action_formula_hash"]=np.asarray(config["runtime"]["action_formula_sha256"]); state["config_hash"]=np.asarray(sha256(canonical_json(config)).hexdigest()) + (root/"config.json").write_bytes(canonical_json(config)); np.savez_compressed(root/"controller_state.npz",**state) + manifest=json.loads((root/"manifest.json").read_text()); manifest["config_sha256"]=sha256(canonical_json(config)).hexdigest(); manifest["state_array_sha256"]={k:sha256(np.ascontiguousarray(v).tobytes()).hexdigest() for k,v in state.items()}; manifest["files"]["config.json"]=file_sha256(root/"config.json"); manifest["files"]["controller_state.npz"]=file_sha256(root/"controller_state.npz"); (root/"manifest.json").write_bytes(canonical_json(manifest)) + with pytest.raises(ValueError,match="frozen authority"): load_acquisition_artifact(root,expected_case="karman_re100",expected_role=role) + + +@pytest.mark.parametrize("tamper", ["prefix_metric","suffix_metric","window_semantics","window_sizes","flag","sample_count"]) +def test_hash_consistent_convergence_tamper_rejected(tmp_path,tamper): + paths=write_triplet(tmp_path/"inputs"); output=tmp_path/"result"; argv=["--case","karman_re100","--q-target",str(paths["q_target"]),"--q-blk",str(paths["q_blk"]),"--q-ctl",str(paths["q_ctl"]),"--output",str(output),"--station-x-D","1","--start-after-relative-step","0","--window-size","2","--window-size","4"] + direct_dq_main(argv); summary=json.loads((output/"summary.json").read_text()); config=json.loads((output/"config.json").read_text()) + if tamper=="prefix_metric": summary["convergence"]["records"][0]["prefix"]["e_target_mean_deviation_weighted_vector_rms"]+=1 + elif tamper=="suffix_metric": summary["convergence"]["records"][0]["suffix"]["dq_ctl_mean_deviation_weighted_vector_rms"]+=1 + elif tamper=="window_semantics": summary["convergence"]["window_semantics"]="forged" + elif tamper=="window_sizes": config["window_sizes"]=[1,4]; summary["convergence"]["window_sizes"]=[1,4]; summary["convergence"]["records"][0]["sample_count"]=1 + elif tamper=="flag": summary["convergence"]["independent_realization_uncertainty"]=True + else: summary["convergence"]["records"][0]["sample_count"]=3 + _rehash_result_json(output,"config.json",config); _rehash_result_json(output,"summary.json",summary) + with pytest.raises(ValueError,match="convergence"): load_result(output) + + +def test_publish_rejects_forged_rehashed_complete_role_spec(tmp_path): + role="q_blk"; source=write_artifact(tmp_path/"source",role) + with np.load(source/"fields.npz",allow_pickle=False) as z: arrays={k:z[k].copy() for k in z.files} + with np.load(source/"controller_state.npz",allow_pickle=False) as z: state={k:z[k].copy() for k in z.files} + config=json.loads((source/"config.json").read_text()); destination=tmp_path/"publish-forgery" + with ArtifactTransaction(destination) as transaction: + transaction.write(arrays=arrays,config=config,state=state) + forged=json.loads((transaction.partial/"config.json").read_text()); forged["runtime"]["role_spec"]["harmonic_channels"]=99; forged["runtime"]["role_spec_sha256"]=sha256(canonical_json(forged["runtime"]["role_spec"])).hexdigest() + with np.load(transaction.partial/"controller_state.npz",allow_pickle=False) as z: forged_state={k:z[k].copy() for k in z.files} + _rehash_acquisition_partial(transaction,forged,forged_state) + with pytest.raises(ValueError,match="authoritative role_spec"): transaction.publish() + assert not destination.exists() + + +def test_fully_rehashed_nonconstant_common_role_shift_rejected_by_input_binding(tmp_path): + paths=write_triplet(tmp_path/"inputs"); output=tmp_path/"result"; argv=["--case","karman_re100","--q-target",str(paths["q_target"]),"--q-blk",str(paths["q_blk"]),"--q-ctl",str(paths["q_ctl"]),"--output",str(output),"--station-x-D","1","--start-after-relative-step","0","--window-size","2","--window-size","4"] + direct_dq_main(argv) + with np.load(output/"arrays.npz",allow_pickle=False) as z: arrays={k:z[k].copy() for k in z.files} + x,y=arrays["x_D"],arrays["y_D"]; xx,yy=np.meshgrid(x,y,indexing="ij") + spatial=(np.float32(.125)*xx+np.float32(.0625)*yy).astype(np.float32) + temporal=np.arange(arrays["q_target_instantaneous"].shape[0],dtype=np.float32)[:,None,None] + shift=np.stack((spatial[None]+temporal*np.float32(.03125),-spatial[None]+temporal*np.float32(.015625)),axis=1).astype(np.float32) + indices=arrays["station_indices"] + for role in ("q_target","q_blk","q_ctl"): + arrays[f"{role}_instantaneous"] += shift + mean=arrays[f"{role}_instantaneous"].mean(axis=0,dtype=np.float64).astype(np.float32) + arrays[f"{role}_mean"]=mean + omega,valid=vorticity(mean[0],mean[1],x,y,arrays["analysis_fluid_mask"]) + arrays[f"{role}_mean_vorticity"]=omega; arrays[f"{role}_mean_vorticity_valid_mask"]=valid; arrays[f"{role}_mean_ux_profiles"]=mean[0,indices,:] + # Common role shift leaves all three estimands, convergence, and target-relative summary metrics unchanged. + summary=json.loads((output/"summary.json").read_text()) + for item,index in zip(summary["profiles"],indices): item.update(profile_metrics(arrays["q_target_mean"][0,index],arrays["q_ctl_mean"][0,index],y,arrays["analysis_fluid_mask"][index])) + (output/"summary.json").write_bytes(canonical_json(summary)) + np.savez_compressed(output/"arrays.npz",**arrays) + from CCD_analysis.direct_dq.schema import canonical_array_sha256 + input_hashes=json.loads((output/"input_hashes.json").read_text()) + for role in ("q_target","q_blk","q_ctl"): input_hashes[role]["role_instantaneous_sha256"]=canonical_array_sha256(arrays[f"{role}_instantaneous"]) + (output/"input_hashes.json").write_bytes(canonical_json(input_hashes)) + manifest=json.loads((output/"manifest.json").read_text()); manifest["files"]["arrays.npz"]=file_sha256(output/"arrays.npz"); manifest["files"]["input_hashes.json"]=file_sha256(output/"input_hashes.json"); manifest["files"]["summary.json"]=file_sha256(output/"summary.json"); (output/"manifest.json").write_bytes(canonical_json(manifest)) + assert load_result_metadata_unverified(output)["provenance_validation"].startswith("UNVERIFIED") + with pytest.raises(ValueError,match="identities changed|live validated acquisition"): + load_result(output) + + +def _make_result_for_live_input_failure(tmp_path): + paths=write_triplet(tmp_path/"inputs"); output=tmp_path/"result"; argv=["--case","karman_re100","--q-target",str(paths["q_target"]),"--q-blk",str(paths["q_blk"]),"--q-ctl",str(paths["q_ctl"]),"--output",str(output),"--station-x-D","1","--start-after-relative-step","0","--window-size","4"] + direct_dq_main(argv); return paths,output + + +def test_load_result_missing_acquisition_source_fails_closed(tmp_path): + paths,output=_make_result_for_live_input_failure(tmp_path); missing=paths["q_target"]; moved=tmp_path/"temporarily-moved"; missing.rename(moved) + with pytest.raises(ValueError,match="live acquisition input validation failed"): load_result(output) + assert load_result_metadata_unverified(output)["provenance_validation"].startswith("UNVERIFIED") + + +def test_load_result_moved_acquisition_source_fails_closed(tmp_path): + paths,output=_make_result_for_live_input_failure(tmp_path); paths["q_blk"].rename(tmp_path/"new-location") + with pytest.raises(ValueError,match="live acquisition input validation failed"): load_result(output) + + +def test_load_result_changed_acquisition_source_fails_closed(tmp_path): + paths,output=_make_result_for_live_input_failure(tmp_path); fields=paths["q_ctl"]/"fields.npz"; fields.write_bytes(fields.read_bytes()+b"changed") + with pytest.raises(ValueError,match="live acquisition input validation failed"): load_result(output) + + +def test_result_publish_pre_and_post_validation_require_live_inputs(tmp_path): + paths=write_triplet(tmp_path/"inputs"); artifacts=load_matched_inputs(q_target=paths["q_target"],q_blk=paths["q_blk"],q_ctl=paths["q_ctl"],case_id="karman_re100") + from CCD_analysis.direct_dq.analysis import analyze_loaded + arrays,summary=analyze_loaded(artifacts,station_tokens=["1"],window_sizes=[4],start_after_relative_step=0); config={"schema_id":"ccd-direct-dq-config/v2","case_id":"karman_re100","inputs":{role:str(artifacts[role].path.resolve()) for role in artifacts},"station_x_D_tokens":["1"],"window_sizes":[4],"selection":{"start_after_relative_step":0,"end_at_relative_step":None},"alignment":"exact full acquisition-relative lattice timeline matching followed by exact physical-step inequality selection and exact float32 coordinates; no index trim, nearest-time, phase guess, crop, or translation"}; identities={role:artifacts[role].input_identity for role in artifacts}; output=tmp_path/"result" + with ResultTransaction(output) as transaction: + transaction.write(arrays=arrays,summary=summary,config=config,input_hashes=identities) + paths["q_target"].rename(tmp_path/"gone-before-publish") + with pytest.raises(ValueError,match="live acquisition input validation failed"): transaction.publish() + assert not output.exists() + + +def test_burn_in_exclusive_nonmatching_boundary_and_exact_indices(tmp_path): + paths = write_triplet(tmp_path / "inputs") + output = tmp_path / "selected" + argv = ["--case", "karman_re100", "--q-target", str(paths["q_target"]), "--q-blk", str(paths["q_blk"]), "--q-ctl", str(paths["q_ctl"]), "--output", str(output), "--station-x-D", "1", "--start-after-relative-step", "801", "--end-at-relative-step", "3200", "--window-size", "3"] + assert direct_dq_main(argv) == 0 + result = load_result(output); arrays = result["arrays"] + np.testing.assert_array_equal(arrays["original_acquisition_relative_lattice_steps"], [800, 1600, 2400, 3200]) + np.testing.assert_array_equal(arrays["selected_timeline_indices"], [1, 2, 3]) + np.testing.assert_array_equal(arrays["selected_acquisition_relative_lattice_steps"], [1600, 2400, 3200]) + assert result["summary"]["selection"]["selected_count"] == 3 + live = load_acquisition_artifact(paths["q_target"], expected_case="karman_re100", expected_role="q_target") + expected = np.stack((live.fields["ux"][[1,2,3]], live.fields["uy"][[1,2,3]]), axis=1) + np.testing.assert_array_equal(arrays["q_target_instantaneous"], expected) + + +def test_selection_requires_nonempty_and_exact_terminal_end(tmp_path): + paths = write_triplet(tmp_path / "inputs") + base = ["--case", "karman_re100", "--q-target", str(paths["q_target"]), "--q-blk", str(paths["q_blk"]), "--q-ctl", str(paths["q_ctl"]), "--station-x-D", "1"] + with pytest.raises(ValueError, match="empty"): + direct_dq_main(base + ["--output", str(tmp_path/"empty"), "--start-after-relative-step", "3200", "--window-size", "1"]) + with pytest.raises(ValueError, match="terminal selected"): + direct_dq_main(base + ["--output", str(tmp_path/"bad-end"), "--start-after-relative-step", "801", "--end-at-relative-step", "3199", "--window-size", "2"]) + + +def test_hash_consistent_selection_tamper_fails(tmp_path): + paths = write_triplet(tmp_path / "inputs"); output = tmp_path / "result" + argv = ["--case", "karman_re100", "--q-target", str(paths["q_target"]), "--q-blk", str(paths["q_blk"]), "--q-ctl", str(paths["q_ctl"]), "--output", str(output), "--station-x-D", "1", "--start-after-relative-step", "800", "--window-size", "3"] + direct_dq_main(argv) + with np.load(output/"arrays.npz", allow_pickle=False) as z: arrays={k:z[k].copy() for k in z.files} + arrays["selected_timeline_indices"][0] = 0 + np.savez_compressed(output/"arrays.npz", **arrays) + manifest=json.loads((output/"manifest.json").read_text()); manifest["files"]["arrays.npz"]=file_sha256(output/"arrays.npz"); (output/"manifest.json").write_bytes(canonical_json(manifest)) + with pytest.raises(ValueError, match="selection|indices"): + load_result(output) diff --git a/src/CCD_analysis/tests/test_dual_clock.py b/src/CCD_analysis/tests/test_dual_clock.py deleted file mode 100644 index e28c70f..0000000 --- a/src/CCD_analysis/tests/test_dual_clock.py +++ /dev/null @@ -1,65 +0,0 @@ -import numpy as np -import pytest - -from CCD_analysis.utils.dual_clock import DualClockCollector, field_steps_from_interval - - -class FakeFlowField: - def __init__(self): - self.absolute_step = 0 - self.ddf = np.zeros(9, dtype=np.float32) - self.begin_calls = 0 - self.end_calls = 0 - self._completed = 0 - self._total = 0 - - def begin_control_interval(self, total_steps, action_target): - self.begin_calls += 1 - self._total = total_steps - self._completed = 0 - - def run_control_segment(self, steps): - self.absolute_step += steps - self._completed += steps - - def end_control_interval(self): - assert self._completed == self._total - self.end_calls += 1 - return np.array([self.absolute_step], dtype=np.float32) - - def get_ddf(self): - self.ddf[:] = self.absolute_step - - -def test_field_steps_from_nondivisible_interval(): - assert field_steps_from_interval(16, 3) == (3, 6, 9, 12, 15) - - -def test_irregular_field_reads_do_not_add_control_boundaries(monkeypatch): - def fake_velocity(ff, u0): - ff.get_ddf() - return ff.ddf[:3].copy(), ff.ddf[3:6].copy() - - monkeypatch.setattr("CCD_analysis.utils.dual_clock.get_velocity_field", fake_velocity) - ff = FakeFlowField() - collector = DualClockCollector( - ff, - control_interval=8, - control_count=2, - u0=0.01, - field_steps=[3, 6, 9, 12, 15, 16], - ) - collector.run_interval(0, np.zeros(1, dtype=np.float32)) - collector.run_interval(1, np.zeros(1, dtype=np.float32)) - assert collector.saved_steps == [3, 6, 9, 12, 15, 16] - assert ff.begin_calls == 2 - assert ff.end_calls == 2 - assert ff.absolute_step == 16 - - -def test_collector_rejects_invalid_field_timeline(): - with pytest.raises(ValueError): - DualClockCollector( - FakeFlowField(), control_interval=8, control_count=2, - u0=0.01, field_steps=[3, 3], - ) diff --git a/src/CCD_analysis/tests/test_karman_dynamic.py b/src/CCD_analysis/tests/test_karman_dynamic.py new file mode 100644 index 0000000..e5ffbf3 --- /dev/null +++ b/src/CCD_analysis/tests/test_karman_dynamic.py @@ -0,0 +1,97 @@ +import json,os,sys +from pathlib import Path +import numpy as np +import pytest +from CCD_analysis.karman_dynamic.contracts import * +from CCD_analysis.karman_dynamic.orchestration import CampaignSchedule,mark_campaign,orchestrate +from CCD_analysis.karman_dynamic.safety import enforce_cooldown,exclusive_lease,require_execution_environment,validate_optane_storage + +def test_contract_roles_claims_and_decomposition(): + s=contract_snapshot(); assert ROLES==("target","zero","drl","constant_mean") and s["contract"]["physical_re_D"]==50 + assert "passive" in s["claims"]["zero"] and "DRL minus" in s["claims"]["dynamic_comparison"] + d=np.arange(24,dtype=float).reshape(4,3,2); c=d*.2+3; assert verify_decomposition(d,c) + +def test_fresh_effective_mean_provenance_is_unsymmetrized_and_hash_bound(): + a=np.asarray([[0,1,2],[3,5,8],[6,11,14]],np.float32); p=constant_mean_provenance(drl_manifest_sha256="a"*64,effective=a,retained_start=1) + np.testing.assert_array_equal(p["constant_mean_physical_action"],np.asarray([4.5,8,11],np.float32)); assert not p["symmetrized"] and len(p["source_effective_action_sha256"])==64 + with pytest.raises(ValueError): effective_action_mean(a,len(a)) + +def test_environment_requires_exact_env_and_one_visible_gpu(): + assert require_execution_environment({"CONDA_DEFAULT_ENV":"pycuda_3_10","CUDA_VISIBLE_DEVICES":"2"})=="2" + for env in ({"CONDA_DEFAULT_ENV":"base","CUDA_VISIBLE_DEVICES":"0"},{"CONDA_DEFAULT_ENV":"pycuda_3_10","CUDA_VISIBLE_DEVICES":"0,1"},{"CONDA_DEFAULT_ENV":"pycuda_3_10"}): + with pytest.raises(RuntimeError): require_execution_environment(env) + +def test_exclusive_lease_never_steals(tmp_path): + lease=tmp_path/"lease" + with exclusive_lease(lease,campaign_id="c",role="drl"): + assert lease.exists() + with pytest.raises(RuntimeError,match="never steal"): + with exclusive_lease(lease,campaign_id="x",role="zero"): pass + assert not lease.exists() + +def test_cooldown_minimum_and_wait(tmp_path): + clock=iter([100.,120.]); slept=[]; path=tmp_path/"cool" + assert enforce_cooldown(path,delay_seconds=30,monotonic=lambda:next(clock),sleep=slept.append)==0 + path.write_text(json.dumps({"started_monotonic":100.})); clock=iter([110.,130.]); assert enforce_cooldown(path,delay_seconds=30,monotonic=lambda:next(clock),sleep=slept.append)==20 and slept[-1]==20 + with pytest.raises(ValueError,match="30"): enforce_cooldown(path,delay_seconds=29) + +def test_optane_requires_exact_symlink_and_device(tmp_path): + mount=tmp_path/"optane"; root=mount/"campaign"; root.mkdir(parents=True); mapping=tmp_path/"mapping"; mapping.symlink_to(root, target_is_directory=True) + fake=lambda p:type("S",(),{"st_dev":7})(); assert validate_optane_storage(repo_mapping=mapping,optane_root=root,mount=mount,is_mount=lambda p:True,stat=fake)==root.resolve() + mapping.unlink(); mapping.mkdir() + with pytest.raises(ValueError,match="symlink"): validate_optane_storage(repo_mapping=mapping,optane_root=root,mount=mount,is_mount=lambda p:True,stat=fake) + +def test_fresh_child_plan_order_delay_and_constant_parent(tmp_path): + s=CampaignSchedule("campaign-x",480,360,120); commands=orchestrate(root=tmp_path/"fresh",schedule=s) + assert [c[c.index("--role")+1] for c in commands]==list(EXECUTION_ORDER); assert all(c[:3]==[sys.executable,"-m","CCD_analysis.karman_dynamic"] for c in commands) + constant=commands[1]; assert "--drl-artifact" in constant and constant[constant.index("--drl-artifact")+1].endswith("/drl") + with pytest.raises(ValueError): CampaignSchedule("x",480,360,29) + +def test_campaign_quarantine_state(tmp_path): + state=mark_campaign(tmp_path,campaign_id="x",status="QUARANTINED",failed_role="drl",error="boom"); assert state["failed_role"]=="drl" and json.loads((tmp_path/"campaign_state.json").read_text())["status"]=="QUARANTINED" + + +def test_campaign_manifest_reload_validates_hash_phase_and_constant_provenance(tmp_path,monkeypatch): + from CCD_analysis.karman_dynamic import artifacts + from CCD_analysis.acquisition.artifacts import file_sha256 + root=tmp_path/"constant_mean"; payload=root/"payload"; payload.mkdir(parents=True); (payload/"manifest.json").write_text("{}\n") + arrays={"sensors":np.arange(24,dtype=np.float32).reshape(4,6)} + monkeypatch.setattr(artifacts,"_legacy",lambda path,role:arrays) + np.savez_compressed(root/"campaign_telemetry.npz",center_sensor_uy=arrays["sensors"][:,3]) + meta={"schema_id":ARTIFACT_SCHEMA_ID,"complete":True,"role":"constant_mean","retained_slice":[1,3],"collect_boundaries":2,"constant_mean_provenance":{"fresh":True},"acquisition_mode":"smoke"} + (root/"campaign.json").write_bytes(canonical_json(meta)) + files={name:file_sha256(root/name) for name in ("campaign.json","campaign_telemetry.npz")} + (root/"campaign_manifest.json").write_bytes(canonical_json({"schema_id":ARTIFACT_SCHEMA_ID,"complete":True,"files":files,"payload_manifest_sha256":file_sha256(payload/"manifest.json")})) + loaded=artifacts.load_role_artifact(root,expected_role="constant_mean"); np.testing.assert_array_equal(loaded["telemetry"]["center_sensor_uy"],arrays["sensors"][:,3]) + (root/"campaign.json").write_text("{}\n") + with pytest.raises(ValueError,match="schema|hash"): artifacts.load_role_artifact(root) + + +def test_execute_role_creates_missing_validated_optane_parents_before_staging(tmp_path,monkeypatch): + from contextlib import nullcontext + from CCD_analysis.karman_dynamic import runtime + backing=tmp_path/"optane"; backing.mkdir(); output=backing/"campaign"/"smoke"/"drl" + monkeypatch.setattr(runtime,"require_execution_environment",lambda: "0") + monkeypatch.setattr(runtime,"validate_optane_storage",lambda **kwargs:backing.resolve()) + monkeypatch.setattr(runtime,"exclusive_lease",lambda *args,**kwargs:nullcontext()) + monkeypatch.setattr(runtime,"enforce_cooldown",lambda *args,**kwargs:0.) + def stop_before_cfd(**kwargs): + assert output.parent.is_dir() + raise RuntimeError("stop before CFD") + monkeypatch.setattr(runtime,"run_role_acquisition",stop_before_cfd) + with pytest.raises(RuntimeError,match="stop before CFD"): + runtime.execute_role(role="drl",output=output,campaign_id="x",warmup_intervals=1,collect_boundaries=2,smoke=True) + assert output.parent.is_dir() and not output.exists() + assert not list(output.parent.glob(".drl.campaign-*")) + +def test_smoke_mode_is_forwarded_bound_and_persisted(tmp_path,monkeypatch): + from CCD_analysis.karman_dynamic import cli + called={} + monkeypatch.setattr(cli,"execute_role",lambda **kwargs:called.update(kwargs)) + assert cli.main(["role","--campaign-id","x","--role","drl","--output",str(tmp_path/"drl"),"--warmup-intervals","1","--collect-boundaries","2","--smoke"])==0 + assert called["smoke"] is True + from CCD_analysis.karman_dynamic.runtime import execute_role + monkeypatch.setattr("CCD_analysis.karman_dynamic.runtime.require_execution_environment",lambda:"0") + monkeypatch.setattr("CCD_analysis.karman_dynamic.runtime.validate_optane_storage",lambda **kwargs:tmp_path.resolve()) + with pytest.raises(ValueError,match="smoke requires exactly"): + execute_role(role="drl",output=tmp_path/"bad"/"drl",campaign_id="x",warmup_intervals=2,collect_boundaries=2,smoke=True) diff --git a/src/CCD_analysis/tests/test_karman_dynamic_increment.py b/src/CCD_analysis/tests/test_karman_dynamic_increment.py new file mode 100644 index 0000000..12ae0a9 --- /dev/null +++ b/src/CCD_analysis/tests/test_karman_dynamic_increment.py @@ -0,0 +1,27 @@ +import json +from pathlib import Path +import numpy as np +import pytest +from CCD_analysis.karman_dynamic.dynamic_increment import publish_dynamic_increment,load_dynamic_increment + +def test_synthetic_four_role_increment_closure_and_provenance(tmp_path,monkeypatch): + from CCD_analysis.karman_dynamic import dynamic_increment as mod + roles=('drl','constant_mean','target','zero'); nx,ny,n=4,3,12; mask=np.ones((nx,ny),bool); x=np.arange(nx,dtype=np.float32); y=np.arange(ny,dtype=np.float32) + offsets={'target':1.,'zero':4.,'constant_mean':3.,'drl':2.} + role_data={}; role_paths={}; phase_paths={} + for role in roles: + rp=tmp_path/role; pp=tmp_path/f'{role}-phase'; rp.mkdir(); pp.mkdir(); (rp/'campaign_manifest.json').write_text('{}'); (pp/'manifest.json').write_text('{}') + q=np.full((n,nx,ny),offsets[role],np.float32); q+=np.sin(np.arange(n)*2*np.pi/n).astype(np.float32)[:,None,None] + role_data[str(rp)]={'metadata':{'retained_slice':[0,n]},'legacy_arrays':{'ux':q,'uy':q*.5,'fluid_mask':mask,'x_D':x,'y_D':y},'telemetry':{'effective_applied_action':np.zeros((n,7),np.float32)}} + role_paths[role]=rp; phase_paths[role]=pp + monkeypatch.setattr(mod,'load_role_artifact',lambda path,expected_role=None:role_data[str(Path(path))]) + monkeypatch.setattr(mod,'file_sha256',lambda path:'a'*64) + def phase(path): + role=Path(path).name.replace('-phase',''); fields=np.full((10,2,mask.sum()),offsets[role],np.float32) + return {'summary':{'gate_passed':True},'fields':fields} + monkeypatch.setattr(mod,'load_phase_compact',lambda path:phase(path)) + monkeypatch.setattr(mod,'_phase_mean',lambda path,common:phase(path)['fields']) + out=tmp_path/'result'; result=publish_dynamic_increment(role_paths,phase_paths,out) + s=result['summary']; assert s['drl_constant_phase_differences_available']; assert s['benefits']['zero_to_constant_overall_mean_control_benefit_target_error_reduction']>0; assert s['benefits']['constant_to_drl_dynamic_increment_target_error_reduction']>0 + assert s['closure']['benefit_additivity_absolute_residual']<1e-12 and not s['dense_fields_deleted'] + with pytest.raises(FileExistsError): publish_dynamic_increment(role_paths,phase_paths,out) diff --git a/src/CCD_analysis/tests/test_karman_dynamic_phase.py b/src/CCD_analysis/tests/test_karman_dynamic_phase.py new file mode 100644 index 0000000..938b700 --- /dev/null +++ b/src/CCD_analysis/tests/test_karman_dynamic_phase.py @@ -0,0 +1,36 @@ +import json +from pathlib import Path +import numpy as np +import pytest +from CCD_analysis.karman_dynamic.phase import evaluate_gate,publish_phase_compact,load_phase_compact + +def signal(cycles=16,period=20,amplitude=1): + t=np.arange(cycles*period+1,dtype=np.float32); return (amplitude*np.sin(2*np.pi*(t-.25)/period)).astype(np.float32) +def fake_role(tmp_path,monkeypatch,y,cycles=16,period=20,nx=5,ny=4): + from CCD_analysis.karman_dynamic import phase + root=tmp_path/'drl'; (root/'payload').mkdir(parents=True); (root/'campaign_manifest.json').write_text('{}'); (root/'payload/manifest.json').write_text('{}'); n=len(y); rng=np.random.default_rng(4); mask=np.ones((nx,ny),bool); phase_grid=np.arange(n,dtype=np.float32)[:,None,None]; spatial=np.arange(nx*ny,dtype=np.float32).reshape(1,nx,ny)/20; arrays={'ux':(np.sin(2*np.pi*phase_grid/period)+spatial).astype(np.float32),'uy':(np.cos(2*np.pi*phase_grid/period)-spatial).astype(np.float32),'fluid_mask':mask,'x_D':np.arange(nx,dtype=np.float32),'y_D':np.arange(ny,dtype=np.float32)}; telemetry={'center_sensor_uy':y,'effective_applied_action':rng.normal(scale=.01,size=(n,7)).astype(np.float32)}; role={'metadata':{'retained_slice':[0,n]},'legacy_arrays':arrays,'telemetry':telemetry} + monkeypatch.setattr(phase,'load_role_artifact',lambda path,expected_role=None:role); monkeypatch.setattr(phase,'file_sha256',lambda path:'a'*64 if path.name=='campaign_manifest.json' else 'b'*64); return root,role +def test_stationary_phase_gate_passes_and_counts_complete_cycles(): + _,m,b=evaluate_gate(signal()); assert not b and m['complete_cycle_count']>=10 and m['period_cv']<1e-5 and m['double_crossing_cycle_count']==0 +def test_gate_fails_closed_for_too_few_and_low_amplitude_cycle(): + _,_,b=evaluate_gate(signal(cycles=7)); assert any('complete cycle' in x for x in b) + y=signal(); y[120:141]*=.01; _,_,b=evaluate_gate(y); assert any('amplitude' in x for x in b) +def test_compact_role_artifact_cycle_balances_and_freezes_mean(tmp_path,monkeypatch): + root,role=fake_role(tmp_path,monkeypatch,signal()); out=tmp_path/'derived'/'phase-v1'; result=publish_phase_compact(root,out); assert result['summary']['gate_passed'] and not result['summary']['dense_fields_deleted']; assert result['summary']['constant_mean_provenance']['symmetrized'] is False + with np.load(out/'compact.npz') as z: + assert z['cycle_bin_fields'].shape[1:3]==(10,2); assert np.all(z['cycle_bin_counts']>0); assert z['ensemble_bins8'].shape[0]==8 and z['ensemble_bins12'].shape[0]==12 + with pytest.raises(FileExistsError): publish_phase_compact(root,out) +def test_failed_gate_publishes_metrics_but_no_mean_or_fields(tmp_path,monkeypatch): + root,_=fake_role(tmp_path,monkeypatch,signal(cycles=7)); out=tmp_path/'derived'/'failed'; r=publish_phase_compact(root,out); assert not r['summary']['gate_passed'] and r['summary']['constant_mean_provenance'] is None; assert not (out/'compact.npz').exists() +def test_compact_loader_rejects_hash_tamper(tmp_path,monkeypatch): + root,_=fake_role(tmp_path,monkeypatch,signal()); out=tmp_path/'derived'/'phase'; publish_phase_compact(root,out); (out/'summary.json').write_text('{}') + with pytest.raises(ValueError,match='hash|schema'): load_phase_compact(out) + +def test_non_drl_phase_publication_has_independent_role_and_no_mean(tmp_path,monkeypatch): + root,_=fake_role(tmp_path,monkeypatch,signal()); out=tmp_path/'constant-phase' + from CCD_analysis.karman_dynamic import phase + original=phase.load_role_artifact + monkeypatch.setattr(phase,'load_role_artifact',lambda path,expected_role=None: original(path,expected_role='drl')) + result=publish_phase_compact(root,out,role='constant_mean') + assert result['summary']['role']=='constant_mean' and result['summary']['constant_mean_provenance'] is None + assert not (out/'constant_mean_provenance.json').exists() diff --git a/src/CCD_analysis/tests/test_karman_dynamic_phase_domain.py b/src/CCD_analysis/tests/test_karman_dynamic_phase_domain.py new file mode 100644 index 0000000..5a7f397 --- /dev/null +++ b/src/CCD_analysis/tests/test_karman_dynamic_phase_domain.py @@ -0,0 +1,40 @@ +import json +from pathlib import Path +import numpy as np +import pytest +from CCD_analysis.karman_dynamic.phase_domain_ccd import _decompose,_periodic_resample,decompose_phase_domain,PhaseDomainTransaction,load_phase_domain_result +from CCD_analysis.original_ccd import CCDConfig,decompose + +def test_literal_weighted_reference_and_harmonic_resample(): + rng=np.random.default_rng(2); u=rng.normal(size=(9,10)); p=rng.normal(size=(3,10)); w=np.linspace(.5,1.5,9); got=_decompose(u,p,w); ref=decompose(u,p,weight=w,config=CCDConfig(center_snapshots=True,center_observables=True)) + np.testing.assert_allclose(got['cross'],ref.cross_correlation,rtol=1e-14,atol=1e-14); np.testing.assert_allclose(got['singular'],ref.singular_values,rtol=1e-14,atol=1e-14) + theta=(np.arange(10)+.5)/10; curve=np.stack((np.cos(2*np.pi*theta),np.sin(4*np.pi*theta)),axis=1); np.testing.assert_allclose(_periodic_resample(curve,10,order=2),curve,atol=1e-14) + +def _synthetic(monkeypatch,tmp_path): + from CCD_analysis.karman_dynamic import phase_domain_ccd as mod + roots={k:tmp_path/k for k in ('drl','constant','dynamic')} + for root in roots.values(): root.mkdir(); (root/'manifest.json').write_text('{}') + mask=np.ones((2,2),bool); theta=(np.arange(10)+.5)*2*np.pi/10; spatial=np.arange(8).reshape(2,4)+1; delta=np.cos(theta)[:,None,None]*spatial[None] + drl=np.zeros((2,10,2,4)); const=np.zeros_like(drl); drl[:]=delta[None]; actions=np.stack((np.cos(theta),np.sin(theta),np.cos(2*theta)),axis=1) + def write_phase(root,role,fields): + vals={'fluid_mask':mask,'cycle_bin_fields':fields.astype(np.float32),'cycle_bin_effective_actions':np.broadcast_to(actions,(2,10,3)).astype(np.float32)} + for n in (8,12): vals[f'ensemble_bins{n}']=_periodic_resample(fields.mean(0),n).astype(np.float32) + vals['ensemble_bins10_half_shift']=_periodic_resample(fields.mean(0),10,.5).astype(np.float32); np.savez(root/'compact.npz',**vals) + write_phase(roots['drl'],'drl',drl); write_phase(roots['constant'],'constant_mean',const) + means=np.zeros((2,4),np.float32); np.savez(roots['dynamic']/'arrays.npz',four_role_fluid_mask=mask,quadrature_weights=np.ones(4),mean_drl=means,mean_constant_mean=means) + monkeypatch.setattr(mod,'load_phase_compact',lambda path:{'summary':{'gate_passed':True,'role':'drl' if Path(path)==roots['drl'] else 'constant_mean'}}); monkeypatch.setattr(mod,'load_dynamic_increment',lambda path:{'summary':{}}); monkeypatch.setattr(mod,'file_sha256',lambda path:'a'*64) + return roots + +def test_synthetic_estimand_sensitivity_and_reference(monkeypatch,tmp_path): + roots=_synthetic(monkeypatch,tmp_path); r=decompose_phase_domain(roots['drl'],roots['constant'],roots['dynamic']); assert r.config['field_estimand'].startswith('separately_centered'); assert r.config['phase_offset_semantics'].startswith('circular phase offsets'); assert r.summary['numerical_rank']>=1; assert len(r.summary['circular_phase_offsets'])==10 + ref=decompose(r.arrays['primary_modes'].T*0 if False else np.zeros((1,1)),np.zeros((1,1)),config=CCDConfig()) if False else None + np.testing.assert_allclose(r.arrays['primary_singular'],np.linalg.svd(r.arrays['primary_cross'],compute_uv=False)) + +def test_immutable_reload_recomputes_and_tamper_rejected(monkeypatch,tmp_path): + roots=_synthetic(monkeypatch,tmp_path); result=decompose_phase_domain(roots['drl'],roots['constant'],roots['dynamic']); out=tmp_path/'result' + with PhaseDomainTransaction(out) as tx: tx.write(result); tx.publish() + loaded=load_phase_domain_result(out,recompute=True); assert loaded['summary']['scientific_contract_explicit'] + with pytest.raises(FileExistsError): + with PhaseDomainTransaction(out): pass + (out/'summary.json').write_text('{}') + with pytest.raises(ValueError,match='hash|contract'): load_phase_domain_result(out) diff --git a/src/CCD_analysis/tests/test_karman_dynamic_publication.py b/src/CCD_analysis/tests/test_karman_dynamic_publication.py new file mode 100644 index 0000000..13f5abb --- /dev/null +++ b/src/CCD_analysis/tests/test_karman_dynamic_publication.py @@ -0,0 +1,10 @@ +import json +from pathlib import Path +import pytest +from CCD_analysis.karman_dynamic.publication import SCHEMA_ID,load_dynamic_publication +def test_publication_loader_hash_source_and_claim_contract(tmp_path,monkeypatch): + root=tmp_path/"publication"; root.mkdir(); source=tmp_path/"source"; source.mkdir(); (source/"manifest.json").write_text("source") + report={"schema_id":SCHEMA_ID,"phase_domain_decision":"DOWNGRADE","zero_phase_output":"PROHIBITED: failed gate","sources":{"source":{"path":str(source),"manifest_sha256":"source-hash"}}}; (root/"RESULTS.json").write_text(json.dumps(report)); (root/"manifest.json").write_text(json.dumps({"schema_id":SCHEMA_ID,"complete":True,"files":{"RESULTS.json":"result-hash"}})) + monkeypatch.setattr("CCD_analysis.karman_dynamic.publication.file_sha256",lambda path:"result-hash" if Path(path).name=="RESULTS.json" else "source-hash"); assert load_dynamic_publication(root)["report"]["phase_domain_decision"]=="DOWNGRADE" + report["zero_phase_output"]="available"; (root/"RESULTS.json").write_text(json.dumps(report)) + with pytest.raises(ValueError,match="claim"): load_dynamic_publication(root) diff --git a/src/CCD_analysis/tests/test_karman_dynamic_temporal.py b/src/CCD_analysis/tests/test_karman_dynamic_temporal.py new file mode 100644 index 0000000..c30aeef --- /dev/null +++ b/src/CCD_analysis/tests/test_karman_dynamic_temporal.py @@ -0,0 +1,39 @@ +import numpy as np +import pytest +from CCD_analysis.karman_dynamic.temporal_ccd import _admit, _compare, PRIMARY_LAGS, ROW_ORDER +from CCD_analysis.original_ccd import CCDConfig, decompose + +def test_block_local_negative_lags_endpoint_sign_and_no_crossing(): + ids=np.array([-1,0,0,0,1,1,1,-1]); fi,oi=_admit(ids,(-2,-1,0)) + np.testing.assert_array_equal(fi,[3,6]); np.testing.assert_array_equal(oi,[[1,2,3],[4,5,6]]) + assert np.all(oi<=fi[:,None]) and ROW_ORDER=='channel-major_delay-minor' + with pytest.raises(ValueError): _admit(ids,(0,1)) + +def test_common_support_and_row_order_dense_original_reference(): + rng=np.random.default_rng(4); ids=np.repeat(np.arange(3),6); lags=(-2,-1,0); fi,oi=_admit(ids,lags); u=rng.normal(size=(7,len(ids))); actions=rng.normal(size=(len(ids),3)); p=actions[oi].transpose(2,1,0).reshape(9,len(fi)); w=np.linspace(.5,1.5,7) + dense=decompose(u[:,fi],p.reshape(3,3,len(fi)),weight=w,config=CCDConfig(center_snapshots=True,center_observables=True)) + uc=u[:,fi]-u[:,fi].mean(1,keepdims=True); pc=p-p.mean(1,keepdims=True); expected=pc@(uc*np.sqrt(w)[:,None]).T/(len(fi)*np.sqrt(9)) + np.testing.assert_allclose(dense.cross_correlation,expected,rtol=1e-14,atol=1e-14) + fi2,_=_admit(ids,(-1,0),support=fi); np.testing.assert_array_equal(fi2,fi) + +def test_weighted_projector_comparison_is_identity_for_same_modes(): + rng=np.random.default_rng(1); w=np.linspace(.5,1.5,6); q,_=np.linalg.qr(rng.normal(size=(6,3))*np.sqrt(w)[:,None]); modes=q/np.sqrt(w)[:,None]; result={'singular_values':np.array([3.,2.,1.]),'physical_modes':modes} + got=_compare(result,result,w); np.testing.assert_allclose(got['leading_weighted_subspace_principal_cosines'],np.ones(3),rtol=1e-14,atol=1e-14) + + +def _synthetic_input(tmp_path): + from CCD_analysis.karman_dynamic.temporal_ccd import TemporalInput + rng=np.random.default_rng(8); nx,ny=4,3; n=57; actions=rng.normal(size=(n,3)).astype(np.float32); fields=rng.normal(size=(n,2,nx,ny)).astype(np.float32) + ids=np.repeat(np.arange(3),19).astype(np.int64); rel=np.arange(n,dtype=np.int64)*800 + return TemporalInput(tmp_path/'role',tmp_path/'phase','r','p',np.arange(nx,dtype=np.float32),np.arange(ny,dtype=np.float32),np.ones((nx,ny),bool),fields,actions,rel,rel+1000,ids) + +def test_streaming_chunk_invariance_memory_and_literal_dense(tmp_path): + from CCD_analysis.karman_dynamic.temporal_ccd import TemporalConfig,decompose_temporal + inp=_synthetic_input(tmp_path); r1=decompose_temporal(inp,streaming_config=TemporalConfig(1,10_000_000)); r2=decompose_temporal(inp,streaming_config=TemporalConfig(7,10_000_000)) + for key in ('primary_cross_correlation','primary_singular_values'): + np.testing.assert_allclose(r1.arrays[key],r2.arrays[key],rtol=1e-13,atol=1e-13) + identifiable=int(np.sum(r1.arrays['primary_singular_values']>1e-10*r1.arrays['primary_singular_values'][0])); w=r1.arrays['coordinate_weights']; v1=r1.arrays['primary_physical_modes'][:,:identifiable]*np.sqrt(w)[:,None]; v2=r2.arrays['primary_physical_modes'][:,:identifiable]*np.sqrt(w)[:,None] + np.testing.assert_allclose(v1@v1.T,v2@v2.T,rtol=1e-12,atol=1e-12); np.testing.assert_allclose(r1.arrays['primary_coefficients'][:identifiable],r2.arrays['primary_coefficients'][:identifiable],rtol=1e-11,atol=1e-11) + fi,oi=_admit(inp.cycle_ids,PRIMARY_LAGS); u=np.concatenate((inp.fields[:,0].reshape(len(inp.fields),-1),inp.fields[:,1].reshape(len(inp.fields),-1)),axis=1).T.astype(float); p=inp.actions[oi].transpose(2,1,0).reshape(54,len(fi)); w=r1.arrays['coordinate_weights']; dense=decompose(u[:,fi]-u.mean(1,keepdims=True),p,weight=w,config=CCDConfig(center_snapshots=False,center_observables=True)) + np.testing.assert_allclose(r1.arrays['primary_cross_correlation'],dense.cross_correlation,rtol=1e-13,atol=1e-13) + with pytest.raises(MemoryError): decompose_temporal(inp,streaming_config=TemporalConfig(2,100)) diff --git a/src/CCD_analysis/tests/test_original_ccd.py b/src/CCD_analysis/tests/test_original_ccd.py new file mode 100644 index 0000000..76bb3ea --- /dev/null +++ b/src/CCD_analysis/tests/test_original_ccd.py @@ -0,0 +1,347 @@ +import inspect + +import numpy as np +import pytest + +from CCD_analysis import original_ccd +from CCD_analysis.original_ccd import ( + CCDConfig, + LaggedObservables, + build_lagged_observables, + decompose, + fit, + flatten_fields, + unflatten_fields, +) +from CCD_analysis.original_ccd._reference import _build_lag_matrix, _reference_ccd + + +def _projector(columns): + return columns @ columns.conj().T + + +def _hpd(rng, size, complex_values=False): + values = rng.normal(size=(size, size)) + if complex_values: + values = values + 1j * rng.normal(size=(size, size)) + return values.conj().T @ values + np.eye(size) + + +def _assert_reference_equal(u, p, weight=None, center=False, chunk_size=None): + reference_weight = np.diag(weight) if weight is not None and np.asarray(weight).ndim == 1 else weight + reference = _reference_ccd(u, p, weight=reference_weight, center=center) + production = decompose( + u, + p, + weight=weight, + config=CCDConfig( + center_snapshots=center, + center_observables=center, + chunk_size=chunk_size, + phase_convention=False, + ), + ) + rank = min(np.asarray(p).reshape(-1, u.shape[1]).shape[0], u.shape[0]) + assert np.allclose(production.cross_correlation, reference.cross_correlation) + assert np.allclose(production.singular_values, reference.singular_values[:rank]) + assert np.allclose(_projector(production.weighted_modes), _projector(reference.weighted_modes[:, :rank])) + assert np.allclose(_projector(production.left_functions), _projector(reference.left_functions[:, :rank])) + assert np.allclose(production.physical_modes.conj().T @ (np.eye(u.shape[0]) if weight is None else (np.diag(weight) if np.asarray(weight).ndim == 1 else weight)) @ production.physical_modes, np.eye(rank)) + return production, reference + + +def test_public_api_is_available_and_private_reference_not_exported(): + assert original_ccd.IMPLEMENTATION_AVAILABLE is True + assert not hasattr(original_ccd, "reference_ccd") + assert not hasattr(original_ccd, "pod") + signature = inspect.signature(decompose) + assert "whiten" not in signature.parameters + assert "standardize" not in signature.parameters + result = decompose(np.eye(2), np.eye(2)) + assert result.rank == 2 + + +def test_hand_case_literal_normalization_and_singular_equations(): + u = np.array([[1.0, 2.0, 0.0], [0.0, 1.0, 3.0]]) + p = np.array([[2.0, 0.0, 1.0], [1.0, -1.0, 2.0]]) + result = decompose(u, p) + expected = np.array([[2.0, 3.0], [-1.0, 5.0]]) / (3 * np.sqrt(2)) + assert np.allclose(result.cross_correlation, expected) + for index, singular in enumerate(result.singular_values): + left = result.left_functions[:, index] + right = result.weighted_modes[:, index] + assert np.allclose(result.cross_correlation @ right, singular * left) + assert np.allclose(result.cross_correlation.conj().T @ left, singular * right) + + +@pytest.mark.parametrize("complex_values", [False, True]) +@pytest.mark.parametrize("dense_weight", [False, True]) +def test_random_weighted_real_complex_matches_private_reference(complex_values, dense_weight): + rng = np.random.default_rng(41 + 10 * complex_values + dense_weight) + u = rng.normal(size=(7, 31)) + p = rng.normal(size=(3, 4, 31)) + if complex_values: + u = u + 1j * rng.normal(size=u.shape) + p = p + 1j * rng.normal(size=p.shape) + weight = _hpd(rng, 7, complex_values=True) if dense_weight else rng.uniform(0.2, 2.0, size=7) + reference_weight = weight if dense_weight else np.diag(weight) + production, reference = _assert_reference_equal(u, p, weight, chunk_size=6) + if not dense_weight: + dense = decompose(u, p, weight=np.diag(weight), config=CCDConfig(chunk_size=6, phase_convention=False)) + assert np.allclose(production.cross_correlation, dense.cross_correlation) + assert np.allclose(production.singular_values, dense.singular_values) + assert np.allclose(production.coefficients, production.physical_modes.conj().T @ (reference_weight @ production.preprocessed_snapshots)) + assert np.allclose(_projector(production.physical_modes), _projector(reference.physical_modes[:, : production.rank])) + + +def test_chunk_invariance_multiobservable_and_independent_centering(): + rng = np.random.default_rng(90) + u = rng.normal(size=(5, 103)) + 1j * rng.normal(size=(5, 103)) + p = rng.normal(size=(2, 3, 103)) + 1j * rng.normal(size=(2, 3, 103)) + baseline = decompose(u, p, config=CCDConfig(center_snapshots=True, center_observables=False, phase_convention=False)) + assert np.allclose(baseline.preprocessed_snapshots.mean(axis=1), 0) + assert np.allclose(baseline.observable_mean, 0) + for chunk in (1, 7, 64, 1000): + actual = decompose(u, p, config=CCDConfig(center_snapshots=True, center_observables=False, chunk_size=chunk, phase_convention=False)) + assert np.allclose(actual.cross_correlation, baseline.cross_correlation, rtol=2e-13, atol=2e-13) + assert np.allclose(actual.singular_values, baseline.singular_values, rtol=2e-13, atol=2e-13) + both, reference = _assert_reference_equal(u, p, center=True, chunk_size=7) + assert np.allclose(both.snapshot_mean, u.mean(axis=1, keepdims=True)) + assert np.allclose(both.observable_mean, p.reshape(6, 103).mean(axis=1, keepdims=True)) + assert np.allclose(both.cross_correlation, reference.cross_correlation) + + +def test_full_basis_reconstruction_and_truncation_residual_monotonicity(): + rng = np.random.default_rng(12) + u = rng.normal(size=(6, 40)) + 1j * rng.normal(size=(6, 40)) + p = rng.normal(size=(8, 40)) + 1j * rng.normal(size=(8, 40)) + weight = _hpd(rng, 6, complex_values=True) + result = decompose(u, p, weight=weight) + assert np.allclose(result.reconstruct(), u) + assert np.allclose(result.residual(), 0, atol=2e-14) + norms = [] + for rank in range(1, result.rank + 1): + residual = result.residual(rank=rank) + norms.append(np.sqrt(np.real(np.sum(residual.conj() * (weight @ residual))))) + assert np.all(np.diff(norms) <= 1e-11) + for rank in range(1, result.rank + 1): + phi = result.physical_modes[:, :rank] + assert np.allclose(phi.conj().T @ weight @ result.residual(rank=rank), 0, atol=2e-12) + + +def test_rectangular_complete_basis_reconstructs_supported_projection_only(): + rng = np.random.default_rng(7) + result = decompose(rng.normal(size=(8, 20)), rng.normal(size=(3, 20))) + assert result.rank == 3 + assert np.linalg.norm(result.residual()) > 0 + assert np.allclose(result.physical_modes.conj().T @ result.residual(), 0) + + +def test_deterministic_phase_convention_is_transparent_and_degeneracy_is_subspace_only(): + u = 2 * np.eye(3) + p = np.eye(3) + result = decompose(u, p) + pivots = result.weighted_modes[np.argmax(np.abs(result.weighted_modes), axis=0), np.arange(3)] + assert np.allclose(pivots.imag, 0) + assert np.all(pivots.real >= 0) + rotation = np.array([[1, 1j, 0], [1j, 1, 0], [0, 0, np.sqrt(2)]]) / np.sqrt(2) + rotated = decompose(rotation.conj().T @ u, p) + assert np.allclose(_projector(result.weighted_modes), _projector(rotation @ rotated.weighted_modes)) + + +def test_exact_lag_builder_matches_reference_for_time_index_blocks_and_mapping(): + field_times = np.array([0, 0, 1, 1, 2, 2]) + blocks = np.array([0, 1, 0, 1, 0, 1]) + values = np.array([[10, 20, 11, 21, 12, 22], [100, 200, 101, 201, 102, 202]]) + for delay_kind, delays in (("index", [0, 1]), ("time", [0, 1])): + actual = build_lagged_observables(field_times, field_times, values, delays, field_blocks=blocks, observable_blocks=blocks, delay_kind=delay_kind) + expected, expected_indices = _build_lag_matrix(field_times, field_times, values, delays, field_blocks=blocks, observable_blocks=blocks, delay_kind=delay_kind) + assert np.array_equal(actual.matrix, expected) + assert np.array_equal(actual.field_indices, expected_indices) + u = np.arange(18).reshape(3, 6) + fitted = fit(u, actual) + direct = decompose(u[:, expected_indices], expected) + assert np.array_equal(fitted.selected_field_indices, expected_indices) + assert np.allclose(fitted.cross_correlation, direct.cross_correlation) + + +def test_exact_lag_disparate_cadence_duplicate_delays_and_q_one(): + field = np.array([0.0, 1.0, 2.0]) + observable = np.arange(0.0, 3.1, 0.5) + values = np.vstack([observable**2, observable + 4]) + lagged = build_lagged_observables(field, observable, values, [0.0, 0.5, 0.5, 1.0]) + assert lagged.matrix.shape == (8, 3) + assert np.array_equal(lagged.matrix[1:2], lagged.matrix[2:3]) + assert np.array_equal(lagged.matrix[5:6], lagged.matrix[6:7]) + q1 = build_lagged_observables(field, observable, values, [0.0]) + assert q1.matrix.shape == (2, 3) + + +def test_field_flatten_roundtrip(): + fields = np.arange(2 * 3 * 4 * 5).reshape(2, 3, 4, 5) + flattened, shape = flatten_fields(fields) + assert flattened.shape == (24, 5) + assert np.array_equal(unflatten_fields(flattened, shape), fields) + + +def test_validation_fails_closed(): + u = np.eye(3) + p = np.eye(3) + invalid_calls = [ + lambda: decompose(u[:, :0], p[:, :0]), + lambda: decompose(u, p[:, :2]), + lambda: decompose(u.astype(object), p), + lambda: decompose(np.where(u == 1, np.nan, u), p), + lambda: decompose(u, p, weight=[1, 0, 1]), + lambda: decompose(u, p, weight=np.ones((3, 2))), + lambda: decompose(u, p, weight=np.array([[1, 1j, 0], [1j, 1, 0], [0, 0, 1]])), + lambda: decompose(u, p, config=CCDConfig(rank=0)), + lambda: decompose(u, p, config=CCDConfig(rank=4)), + lambda: decompose(u, p, config=CCDConfig(chunk_size=0)), + ] + for call in invalid_calls: + with pytest.raises((TypeError, ValueError)): + call() + with pytest.raises(ValueError, match="duplicates"): + decompose(u, p).reconstruct(mode_indices=[0, 0]) + with pytest.raises(ValueError, match="duplicates"): + fit(u, LaggedObservables(p, np.array([0, 0, 2]), (0, 1, 2), 1, 3)) + malformed_lags = [ + lambda: LaggedObservables(p, np.arange(3), (0,), 99, 77), + lambda: LaggedObservables(p, np.arange(3), (0,), 1, 3), + lambda: LaggedObservables(p, np.arange(3), (0,), 0, 3), + lambda: LaggedObservables(p[:, :0], np.array([], dtype=int), (0, 1, 2), 1, 3), + lambda: LaggedObservables(p, np.arange(3), (0, 1, 2), 1, 3, row_order="delay-major"), + ] + for construct in malformed_lags: + with pytest.raises(ValueError): + construct() + + +def test_weight_conditioning_is_scale_invariant_and_dense_diagonal_consistent(): + u = np.array([[1.0, 2.0], [3.0, -1.0]]) + p = np.eye(2) + base = np.array([1.0, 0.25]) + baseline = decompose(u, p, weight=base) + for scale in (1e-20, 1e20): + for weight in (scale * base, np.diag(scale * base)): + result = decompose(u, p, weight=weight) + assert np.allclose(result.singular_values, np.sqrt(scale) * baseline.singular_values, rtol=2e-12) + assert np.allclose(result.reconstruct(), baseline.reconstruct(), rtol=2e-12, atol=2e-12) + eps = np.finfo(float).eps + for weight in (np.array([1.0, eps]), np.diag([1.0, eps])): + with pytest.raises(ValueError, match="min/max ratio"): + decompose(u, p, weight=weight) + + +def test_lag_validation_rejects_nearest_unsorted_duplicates_and_bad_offsets(): + with pytest.raises(NotImplementedError, match="exact"): + build_lagged_observables([0], [0], [1], [0], interpolation="nearest") + with pytest.raises(ValueError, match="strictly increasing"): + build_lagged_observables([0], [1, 0], [1, 2], [0]) + with pytest.raises(ValueError, match="unique"): + build_lagged_observables([0, 0], [0, 1], [1, 2], [0]) + with pytest.raises(ValueError, match="integers"): + build_lagged_observables([0], [0], [1], [0.5], delay_kind="index") + with pytest.raises(ValueError, match="no complete"): + build_lagged_observables([0.1], [0.0, 0.5], [1, 2], [0.0]) + + +def _lyu_field(times, x): + return (2 * np.cos(times[:, None] - x) + 1.5 * np.cos(2 * times)[:, None] * np.cos(2 * x) + np.cos(3 * times)[:, None] * np.cos(3 * x) + 0.5 * np.cos(4 * times)[:, None] * np.cos(4 * x) + np.cos(6 * times)[:, None] * np.cos(6 * x) * np.exp(-0.1 * (x - np.pi) ** 2)) + + +def test_smaller_lyu_production_output_matches_literal_private_reference(): + q = 32 + n = 32 * 8 + dt = 2 * np.pi / q + times = np.arange(n + q - 1) * dt + x = np.linspace(0, 2 * np.pi, 48, endpoint=False) + observable = np.cos(times - np.pi / 4) + np.sin(2 * times - np.pi / 3) + np.cos(4 * times) + np.cos(6 * times - np.pi / 12) + lagged = build_lagged_observables(times[:n], times, observable, np.arange(q), delay_kind="index") + u = _lyu_field(times[:n], x).T + production = fit(u, lagged, config=CCDConfig(chunk_size=17, phase_convention=False)) + reference = _reference_ccd(u, lagged.matrix) + assert np.allclose(production.cross_correlation, reference.cross_correlation, atol=2e-13) + assert np.allclose(production.singular_values, reference.singular_values) + targets = np.column_stack([np.cos(x), np.sin(x), np.cos(2 * x), np.cos(6 * x) * np.exp(-0.1 * (x - np.pi) ** 2), np.cos(4 * x)]) + assert np.linalg.norm((np.eye(x.size) - _projector(production.weighted_modes[:, :5])) @ targets) / np.linalg.norm(targets) < 1e-8 + assert production.singular_values[4] > 1e10 * production.singular_values[5] + + + +def test_left_functions_empirical_identity_complex_multiobservable_order(): + rng = np.random.default_rng(813) + u = rng.normal(size=(4, 23)) + 1j * rng.normal(size=(4, 23)) + p = rng.normal(size=(2, 3, 23)) + 1j * rng.normal(size=(2, 3, 23)) + result = decompose(u, p, config=CCDConfig(phase_convention=True)) + assert result.left_functions_lq().shape == (2, 3, result.rank) + assert np.allclose(result.left_functions_lq().reshape(6, result.rank), result.left_functions) + assert np.allclose(result.left_functions.conj().T @ result.left_functions, np.eye(result.rank)) + for k, sigma in enumerate(result.singular_values): + empirical = p.reshape(6, 23) @ result.coefficients[k].conj() / 23 + assert np.allclose(empirical, np.sqrt(6) * sigma * result.left_functions[:, k]) + + +def test_endpoint_drop_uses_n_valid_and_reports_selected_indices(): + u = np.array([[1.0, 2.0, 4.0, 1000.0], [2.0, -1.0, 3.0, -900.0]]) + field_times = np.arange(4) + observable_times = np.arange(4) + observable = np.array([2.0, 5.0, -1.0, 700.0]) + lagged = build_lagged_observables(field_times, observable_times, observable, [0, 1], delay_kind="index") + result = fit(u, lagged, config=CCDConfig(phase_convention=False)) + selected = np.array([0, 1, 2]) + p = np.stack([observable[:3], observable[1:4]]) + expected = p @ u[:, selected].T / (3 * np.sqrt(2)) + wrong_original_n = p @ u[:, selected].T / (4 * np.sqrt(2)) + assert result.valid_sample_count == 3 + assert np.array_equal(result.selected_field_indices, selected) + assert np.allclose(result.cross_correlation, expected) + assert not np.allclose(result.cross_correlation, wrong_original_n) + + +def test_centering_after_endpoint_selection_excludes_dropped_field_outlier(): + u = np.array([[1.0, 2.0, 3.0, 1e12], [-1.0, 4.0, 2.0, -1e12]]) + observable = np.array([[2.0, 3.0, 5.0, 7.0]]) + lagged = build_lagged_observables(np.arange(4), np.arange(4), observable, [0, 1], delay_kind="index") + result = fit(u, lagged, config=CCDConfig(center_snapshots=True, center_observables=True)) + selected_u = u[:, :3] + selected_p = np.vstack([observable[0, :3], observable[0, 1:4]]) + assert np.allclose(result.snapshot_mean, selected_u.mean(axis=1, keepdims=True)) + assert np.allclose(result.observable_mean, selected_p.mean(axis=1, keepdims=True)) + assert np.allclose(result.preprocessed_snapshots.mean(axis=1), 0) + + +def test_degenerate_blocks_fail_closed_and_weighted_projector_is_invariant(): + weight = np.array([2.0, 3.0, 5.0]) + root = np.sqrt(weight) + u = np.diag(1 / root) + p = 3 * np.sqrt(3) * np.diag([3.0, 3.0, 1.0]) + result = decompose(u, p, weight=weight, config=CCDConfig(singular_block_rtol=1e-12)) + assert result.singular_blocks == ((0, 2),) + assert result.identifiable_rank == 3 + with pytest.raises(ValueError, match="splits"): + result.reconstruct(rank=1) + basis_dependent = result.reconstruct(rank=1, allow_basis_dependent=True) + assert basis_dependent.shape == u.shape + with pytest.raises(ValueError, match="splits"): + decompose(u, p, weight=weight, config=CCDConfig(rank=1, singular_block_rtol=1e-12)) + rotation = np.array([[np.cos(.41), -np.sin(.41)], [np.sin(.41), np.cos(.41)]]) + phi = result.physical_modes[:, :2] + rotated_phi = phi @ rotation + projector = phi @ phi.conj().T @ np.diag(weight) + rotated_projector = rotated_phi @ rotated_phi.conj().T @ np.diag(weight) + assert np.allclose(projector, rotated_projector) + + +def test_rectangular_null_modes_excluded_from_default_reconstruction(): + u = np.eye(5) + p = np.array([[1.0, 0, 0, 0, 0], [0, 1.0, 0, 0, 0], [0, 0, 0, 0, 0]]) + result = decompose(u, p) + assert result.rank == 3 + assert result.identifiable_rank == 2 + assert np.array_equal(result.identifiable_mode_mask, [True, True, False]) + assert np.allclose(result.reconstruct(), result.reconstruct(rank=2)) + with pytest.raises(ValueError, match="null"): + result.reconstruct(mode_indices=[2]) + assert result.reconstruct(mode_indices=[2], allow_basis_dependent=True).shape == u.shape diff --git a/src/CCD_analysis/tests/test_original_ccd_derivation.py b/src/CCD_analysis/tests/test_original_ccd_derivation.py new file mode 100644 index 0000000..ad118af --- /dev/null +++ b/src/CCD_analysis/tests/test_original_ccd_derivation.py @@ -0,0 +1,303 @@ +import importlib + +import numpy as np +import pytest + +from CCD_analysis.original_ccd._reference import _build_lag_matrix, _literal_cross_correlation, _reference_ccd + + +def _projector(columns): + return columns @ columns.conj().T + + +def _hpd_root(weight): + values, vectors = np.linalg.eigh(weight) + return (vectors * np.sqrt(values)) @ vectors.conj().T + + +def _align_phase(actual, expected): + phase = np.vdot(expected, actual) + return actual if phase == 0 else actual * np.exp(-1j * np.angle(phase)) + + +def test_public_api_available_and_reference_private(): + public = importlib.import_module("CCD_analysis.original_ccd") + assert public.IMPLEMENTATION_AVAILABLE is True + assert not hasattr(public, "reference_ccd") + assert public.decompose(np.eye(2), np.eye(2)).rank == 2 + + +def test_literal_equation_complex_multiobservable_and_common_scalar(): + u = np.array([[1., 2., 0.], [0., 1., 3.]]) + p = np.array([[2., 0., 1.], [1., -1., 2.]]) + expected = np.array([[2., 3.], [-1., 5.]]) / (3 * np.sqrt(2)) + assert np.allclose(_literal_cross_correlation(u, p), expected) + stacked = np.stack([p + 1j * p[::-1], 2 * p - .5j * p]) + result = _reference_ccd(u, stacked) + assert np.allclose(result.cross_correlation, stacked.reshape(4, 3) @ u.T / 6) + scalar = 2 - 3j + scaled = _reference_ccd(u, scalar * stacked) + assert np.allclose(scaled.singular_values, abs(scalar) * result.singular_values) + assert np.allclose(_projector(scaled.weighted_modes), _projector(result.weighted_modes)) + + +def test_real_inputs_complex_hpd_weight_preserves_dtype_and_direct_formula(): + u = np.array([[1., 2., -1.], [3., .5, 2.]]) + p = np.array([[2., -1., 4.], [.5, 3., 1.]]) + w = np.array([[2., .35 + .4j], [.35 - .4j, 1.7]]) + root = _hpd_root(w) + result = _reference_ccd(u, p, weight=w) + direct = p @ (root @ u).conj().T / (3 * np.sqrt(2)) + assert np.iscomplexobj(result.cross_correlation) + assert np.allclose(result.cross_correlation, direct) + assert np.allclose(result.physical_modes.conj().T @ w @ result.physical_modes, np.eye(2)) + with pytest.raises(ValueError, match="exact shape"): + _reference_ccd(u, p, weight=np.eye(3)) + with pytest.raises(ValueError, match="Hermitian"): + _reference_ccd(u, p, weight=np.array([[1, 1j], [1j, 1]])) + with pytest.raises(ValueError, match="positive definite"): + _reference_ccd(u, p, weight=np.diag([1., 0.])) + + +def test_right_and_left_singular_relations_and_normalization(): + u = np.array([[1 + 1j, 2, -1j], [2 - 1j, .5j, 1]]) + p = np.array([[1j, 2, 1 - 1j], [2, -1j, .5]]) + result = _reference_ccd(u, p) + a = result.cross_correlation + for k, sigma in enumerate(result.singular_values): + v = result.weighted_modes[:, k]; r = result.left_functions[:, k] + assert np.allclose(a.conj().T @ a @ v, sigma**2 * v) + assert np.allclose(a @ a.conj().T @ r, sigma**2 * r) + assert np.allclose(a @ v, sigma * r) + assert np.allclose(result.left_functions.conj().T @ result.left_functions, np.eye(2)) + + +def test_weighted_full_rank_complex_pod_coordinate_equivalence_and_reconstruction(): + rng = np.random.default_rng(4) + u = rng.normal(size=(4, 20)) + 1j * rng.normal(size=(4, 20)) + p = rng.normal(size=(3, 20)) + 1j * rng.normal(size=(3, 20)) + b = rng.normal(size=(4, 4)) + 1j * rng.normal(size=(4, 4)); w = b.conj().T @ b + np.eye(4) + root = _hpd_root(w); x = root @ u + full = _reference_ccd(u, p, weight=w) + psi, _, _ = np.linalg.svd(x, full_matrices=False); c = psi.conj().T @ x + coordinate = _reference_ccd(c, p); lifted = psi @ coordinate.weighted_modes + assert np.allclose(full.singular_values, coordinate.singular_values) + for k in range(4): + assert np.allclose(_align_phase(lifted[:, k], full.weighted_modes[:, k]), full.weighted_modes[:, k]) + assert np.allclose(full.physical_amplitude_coefficients, full.physical_modes.conj().T @ w @ u) + assert np.allclose(full.physical_modes @ full.physical_amplitude_coefficients, u) + + +def test_degenerate_full_rank_equivalence_compares_projector(): + u = 2 * np.eye(2); p = np.eye(2) + full = _reference_ccd(u, p) + rotation = np.array([[1, 1j], [1j, 1]]) / np.sqrt(2) + coordinate = _reference_ccd(rotation.conj().T @ u, p) + lifted = rotation @ coordinate.weighted_modes + assert np.allclose(full.singular_values, coordinate.singular_values) + assert np.allclose(_projector(full.weighted_modes[:, :2]), _projector(lifted[:, :2])) + + +def _restricted_top(a, basis): + reduced = basis.conj().T @ a.conj().T @ a @ basis + values, vectors = np.linalg.eigh(reduced) + return values[-1], basis @ vectors[:, -1] + + +def test_truncated_simple_equality_when_top_vector_retained(): + a = np.diag([3., 2., 1.]); basis = np.eye(3)[:, :2] + value, vector = _restricted_top(a, basis) + assert np.isclose(value, 9.) + assert np.allclose(abs(vector), [1, 0, 0]) + + +def test_truncated_degenerate_intersection_preserves_value_not_projector(): + a = np.diag([3., 3., 1.]); basis = np.column_stack([np.array([1., 1., 0.]) / np.sqrt(2), [0., 0., 1.]]) + value, vector = _restricted_top(a, basis) + full_projector = np.diag([1., 1., 0.]) + assert np.isclose(value, 9.) and np.isclose(vector.conj() @ full_projector @ vector, 1.) + assert not np.allclose(_projector(vector[:, None]), full_projector) + + +def test_truncated_no_top_eigenspace_intersection_is_strict_and_loses_signal(): + a = np.diag([3., 2., 1.]); basis = np.eye(3)[:, 1:] + value, _ = _restricted_top(a, basis) + assert value < 9. and np.isclose(value, 4.) + n = 400; t = np.arange(n) * 2 * np.pi / n + u = np.vstack([20*np.cos(7*t), np.sin(3*t), 2*np.cos(11*t)]); p = np.sin(3*t)[None, :] + full = _reference_ccd(u, p); psi = np.linalg.svd(u, full_matrices=False)[0][:, :1] + restricted = _reference_ccd(psi.T @ u, p) + assert full.singular_values[0] > 1e10 * restricted.singular_values[0] + + +def test_unequal_scale_standardization_changes_metric(): + t = np.linspace(0, 2*np.pi, 400, endpoint=False) + c = np.vstack([10*np.cos(t), np.sin(t)]); p = (np.cos(t) + .4*np.sin(t))[None, :] + raw = _reference_ccd(c, p).physical_modes[:, 0] + standardized = _reference_ccd(c / c.std(axis=1, keepdims=True), p).physical_modes[:, 0] + assert not np.allclose(abs(raw), abs(standardized), atol=.1) + + +def test_exact_degenerate_isotropic_standardization_rotation_invariant(): + t = np.linspace(0, 2*np.pi, 400, endpoint=False) + c = np.sqrt(2) * np.vstack([np.cos(t), np.sin(t)]); p = (np.cos(t) + .4*np.sin(t))[None, :] + theta = .37; rotation = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]]) + first = _reference_ccd(c / c.std(axis=1, keepdims=True), p).physical_modes[:, 0] + rotated = rotation.T @ c + second = rotation @ _reference_ccd(rotated / rotated.std(axis=1, keepdims=True), p).physical_modes[:, 0] + assert np.allclose(abs(first), abs(second), atol=1e-12) + + +def test_lags_repeated_local_clocks_interleaved_storage_and_no_crossing(): + times = [0, 0, 1, 1, 2, 2]; blocks = [0, 1, 0, 1, 0, 1]; values = np.array([10, 20, 11, 21, 12, 22]) + p, indices = _build_lag_matrix(times, times, values, [0, 1], field_blocks=blocks, observable_blocks=blocks, delay_kind="index") + assert np.array_equal(indices, [0, 1, 2, 3]) + assert np.array_equal(p, [[10, 20, 11, 21], [11, 21, 12, 22]]) + + +def test_lag_unsorted_duplicate_pairs_negative_and_duplicate_delay_policy(): + with pytest.raises(ValueError, match="strictly increasing"): + _build_lag_matrix([0], [1, 0], [11, 10], [0], observable_blocks=[0, 0]) + with pytest.raises(ValueError, match="pairs must be unique"): + _build_lag_matrix([0], [0, 0], [1, 2], [0], observable_blocks=[0, 0]) + p, indices = _build_lag_matrix(np.arange(4), np.arange(4), 10 + np.arange(4), [-1, 0, 0, 1], delay_kind="index") + assert np.array_equal(indices, [1, 2]) and np.array_equal(p[:, 0], [10, 11, 11, 12]) + + +def test_q_one_odd_even_asymmetric_and_exact_disparate_cadence(): + times = np.arange(5) + q1, _ = _build_lag_matrix(times, times, times, [0]); assert q1.shape == (1, 5) + odd, _ = _build_lag_matrix(times, times, times, [-1, 0, 1]); assert odd.shape == (3, 3) + even, _ = _build_lag_matrix(times, times, times, [-1, 0, 2, 3]); assert even.shape == (4, 1) + field = np.array([0., 1., 2.]); observable = np.arange(0., 3.1, .5); values = observable**2 + exact, indices = _build_lag_matrix(field, observable, values, [0., .5, 1.]) + assert np.array_equal(indices, [0, 1, 2]) and np.allclose(exact[:, 1], [1, 2.25, 4]) + with pytest.raises(ValueError, match="no complete"): + _build_lag_matrix([.1], observable, values, [0.]) + + +def test_centering_zeroes_both_u_and_p_rows(): + u = np.array([[100., 101., 99., 100.], [2., -2., 2., -2.]]) + p = np.array([[50., 51., 49., 50.], [3., 4., 5., 6.]]) + centered = _reference_ccd(u, p, center=True) + assert np.allclose(centered.preprocessed_snapshots.mean(axis=1), 0) + expected_p = p - p.mean(axis=1, keepdims=True) + assert np.allclose(centered.cross_correlation, expected_p @ centered.preprocessed_snapshots.T / (4*np.sqrt(2))) + + +def _lyu_basis(times): + return np.vstack([np.cos(times), np.sin(times), np.cos(2*times), np.sin(2*times), np.cos(4*times), np.sin(4*times), np.cos(6*times), np.sin(6*times)]) + + +def _lyu_delay_map(delays): + # p(t+d) represented in the cos/sin rows returned by _lyu_basis(t). + terms = [(1, "cos", -np.pi/4), (2, "sin", -np.pi/3), (4, "cos", 0.), (6, "cos", -np.pi/12)] + rows = [] + for d in delays: + coefficients = [] + for harmonic, kind, phase in terms: + angle = harmonic*d + phase + coefficients.extend(([np.cos(angle), -np.sin(angle)] if kind == "cos" else [np.sin(angle), np.cos(angle)])) + rows.append(coefficients) + return np.asarray(rows) + + +def _lyu_field(times, x, noise_amplitude, rng): + deterministic = (2*np.cos(times[:,None]-x) + 1.5*np.cos(2*times)[:,None]*np.cos(2*x) + np.cos(3*times)[:,None]*np.cos(3*x) + .5*np.cos(4*times)[:,None]*np.cos(4*x) + np.cos(6*times)[:,None]*np.cos(6*x)*np.exp(-.1*(x-np.pi)**2)) + return deterministic + noise_amplitude*rng.uniform(-.5, .5, deterministic.shape) + + +def _lyu_cross_covariance(cycles=None, noise_amplitude=100., seed=0, chunk=4096, *, sample_count=None): + q = 128; dt = 2*np.pi/128 + n = cycles*128 if sample_count is None else int(sample_count) + if n <= 0 or (cycles is None and sample_count is None): + raise ValueError("a positive sample count is required") + x = np.linspace(0, 2*np.pi, 128, endpoint=False); delays = np.arange(q)*dt + delay_map = _lyu_delay_map(delays); accumulated = np.zeros((8, 128)); rng = np.random.default_rng(seed) + for start in range(0, n, chunk): + stop = min(start+chunk, n); t = np.arange(start, stop)*dt + accumulated += _lyu_basis(t) @ _lyu_field(t, x, noise_amplitude, rng) + a = delay_map @ accumulated / (n*np.sqrt(q)) + _, sigma, vh = np.linalg.svd(a, full_matrices=False) + return sigma, vh.T, x, a + + +def _lyu_targets(x): + return [np.cos(x), np.sin(x), np.cos(2*x), np.cos(6*x)*np.exp(-.1*(x-np.pi)**2), np.cos(4*x)] + + +def test_lyu_exact_noiseless_structures_order_and_ratio(): + sigma, modes, x, a = _lyu_cross_covariance(4, 0., 1) + targets = _lyu_targets(x); recovered = _projector(modes[:, :5]) + assert np.linalg.norm(recovered @ np.column_stack(targets) - np.column_stack(targets)) < 1e-9 + assert sigma[4] > 1e10*sigma[5] + ratio = sigma[:3]**2 / sigma[2]**2 + assert np.allclose(ratio, [4/2.25, 4/2.25, 1.], rtol=3e-5) + pair_projector = _projector(modes[:, :2]) + traveling = np.column_stack([np.cos(x), np.sin(x)]) + assert np.linalg.norm(pair_projector @ traveling - traveling) / np.linalg.norm(traveling) < 1e-3 + assert abs(np.vdot(modes[:, 2], np.cos(2*x)/8)) > .999 + assert abs(np.vdot(modes[:, 3], targets[3]/np.linalg.norm(targets[3]))) > .99 + assert abs(np.vdot(modes[:, 4], np.cos(4*x)/8)) > .99 + assert np.linalg.norm(a @ (np.cos(3*x)/np.linalg.norm(np.cos(3*x)))) < .05 + + +def test_lyu_published_parameters_n10000_q128_noise100(): + sigma, modes, x, a = _lyu_cross_covariance(10_000, 100., 20260804) + targets = _lyu_targets(x); recovered = _projector(modes[:, :5]) + relative = np.linalg.norm((np.eye(128)-recovered) @ np.column_stack(targets)) / np.linalg.norm(targets) + assert relative < .08 + traveling = np.column_stack(targets[:2]) + pair_residual = np.linalg.norm((np.eye(128)-_projector(modes[:, :2])) @ traveling) / np.linalg.norm(traveling) + assert pair_residual < .04 # observed 0.0265 for the frozen seed + assert abs(np.vdot(modes[:, 2], targets[2]/np.linalg.norm(targets[2]))) > .995 # observed 0.9994 + assert abs(np.vdot(modes[:, 3], targets[3]/np.linalg.norm(targets[3]))) > .993 # observed 0.9981 + assert abs(np.vdot(modes[:, 4], targets[4]/np.linalg.norm(targets[4]))) > .990 # observed 0.9956 + ratio = sigma[:3]**2 / sigma[2]**2 + assert np.allclose(ratio, [4/2.25, 4/2.25, 1.], rtol=.12) + assert sigma[4] > 2*sigma[5] + frequency3 = np.linalg.norm(a @ (np.cos(3*x)/8)) + assert frequency3 < sigma[4] and sigma[5] < .5*sigma[4] + assert np.isclose(np.pi/(2*np.pi/128), 64.) and np.isclose(2*np.pi/(128*(2*np.pi/128)), 1.) + + + +def test_factorized_chunked_accumulator_equals_literal_p_u_dagger(): + n = 128*3 + 37; q = 128; dt = 2*np.pi/128; seed = 91; noise = 17. + x = np.linspace(0, 2*np.pi, 128, endpoint=False); t = np.arange(n)*dt + u_rows = _lyu_field(t, x, noise, np.random.default_rng(seed)) + observable_times = np.arange(n+q-1)*dt + observable = np.cos(observable_times-np.pi/4)+np.sin(2*observable_times-np.pi/3)+np.cos(4*observable_times)+np.cos(6*observable_times-np.pi/12) + literal_p = np.stack([observable[j:j+n] for j in range(q)]) + literal_a = literal_p @ u_rows / (n*np.sqrt(q)) + for chunk in (1, 17, 128, 257, 4096): + _, _, _, factorized_a = _lyu_cross_covariance(noise_amplitude=noise, seed=seed, chunk=chunk, sample_count=n) + assert np.allclose(factorized_a, literal_a, rtol=2e-13, atol=2e-13) + + +def test_invalid_inputs_and_lag_policies_fail_closed(): + u = np.eye(2); p = np.eye(2) + for bad_u, bad_p in ((np.array([[np.nan, 0.], [0., 1.]]), p), (u, np.array([[1., np.inf], [0., 1.]]))): + with pytest.raises(ValueError, match="finite"): + _reference_ccd(bad_u, bad_p) + bad_w = np.array([[1., 0.], [0., np.inf]]) + with pytest.raises(ValueError, match="finite"): + _reference_ccd(u, p, weight=bad_w) + with pytest.raises(ValueError, match="field .*pairs must be unique"): + _build_lag_matrix([0, 0], [0, 1], [1, 2], [0]) + with pytest.raises(ValueError, match="delay_kind"): + _build_lag_matrix([0], [0], [1], [0], delay_kind="seconds") + with pytest.raises(ValueError, match="integers"): + _build_lag_matrix([0], [0], [1], [.5], delay_kind="index") + with pytest.raises(NotImplementedError, match="exact"): + _build_lag_matrix([0], [0], [1], [0], interpolation="nearest") + + +def test_lyu_noise_convergence_multiple_seed_statistics(): + errors = {8: [], 64: []} + for cycles in errors: + for seed in range(4): + _, modes, x, _ = _lyu_cross_covariance(cycles, 100., seed) + targets = np.column_stack(_lyu_targets(x)) + errors[cycles].append(np.linalg.norm((np.eye(128)-_projector(modes[:, :5])) @ targets)/np.linalg.norm(targets)) + assert np.mean(errors[64]) < .65*np.mean(errors[8]) diff --git a/src/CCD_analysis/tests/test_real_ccd.py b/src/CCD_analysis/tests/test_real_ccd.py new file mode 100644 index 0000000..314798b --- /dev/null +++ b/src/CCD_analysis/tests/test_real_ccd.py @@ -0,0 +1,258 @@ +import json +from pathlib import Path +import numpy as np +import pytest +from CCD_analysis.acquisition.artifacts import file_sha256 +from CCD_analysis.acquisition.contracts import ACTION_IDENTITIES,canonical_json +from CCD_analysis.direct_dq.cli import main as direct_main +from CCD_analysis.tests.test_direct_dq import write_triplet +from CCD_analysis.real_ccd import MemoryBudget,RealCCDInput,StreamingConfig,ResultTransaction,decompose_streaming,estimate_memory,load_result,load_validated_input,preflight_real_artifact,safe_host_budget +from CCD_analysis.real_ccd.cli import main as real_ccd_main + +def _direct(tmp_path): + paths=write_triplet(tmp_path/'inputs'); out=tmp_path/'direct' + direct_main(['--case','karman_re100','--q-target',str(paths['q_target']),'--q-blk',str(paths['q_blk']),'--q-ctl',str(paths['q_ctl']),'--output',str(out),'--station-x-D','1','--start-after-relative-step','0','--window-size','4']) + return paths,out + +def _fit(tmp_path,chunk=2): + paths,direct=_direct(tmp_path); cfg=StreamingConfig(chunk,MemoryBudget(100_000_000,0,1.25)); inp,mem=load_validated_input(direct,streaming_config=cfg) + return paths,direct,inp,decompose_streaming(inp,streaming_config=cfg,memory_estimate=mem) + +def test_streaming_matches_dense_literal_and_flatten_order(tmp_path): + _,_,inp,result=_fit(tmp_path); a=result.arrays; mask=inp.analysis_mask + u=np.concatenate((inp.dq_ctl[:,0][:,mask],inp.dq_ctl[:,1][:,mask]),axis=1).T.astype(float) + p=inp.actions.T.astype(float); w=a['coordinate_weights']; uc=u-u.mean(1,keepdims=True); pc=p-p.mean(1,keepdims=True) + dense=pc@(uc*np.sqrt(w)[:,None]).T/(u.shape[1]*np.sqrt(3)) + np.testing.assert_allclose(a['cross_correlation'],dense,rtol=1e-13,atol=1e-13) + np.testing.assert_allclose(a['coefficients'],(a['physical_modes']*np.sqrt(w)[:,None]).T@(uc*np.sqrt(w)[:,None]),rtol=1e-12,atol=1e-12) + np.testing.assert_array_equal(a['field_mean'],u.mean(1)) + np.testing.assert_array_equal(a['effective_actions'],inp.actions) + assert result.config['channel_names']==['front','upper','lower'] and result.config['Q']==1 and result.config['tau']==0 + assert result.config['full_reconstructions_persisted'] is False + +def test_chunk_invariance_subspaces_coefficients_and_residuals(tmp_path): + _,direct,inp,r1=_fit(tmp_path/'one',1); cfg=StreamingConfig(3,MemoryBudget(100_000_000,0)); _,mem=load_validated_input(direct,streaming_config=cfg); r2=decompose_streaming(inp,streaming_config=cfg,memory_estimate=mem) + np.testing.assert_allclose(r1.arrays['cross_correlation'],r2.arrays['cross_correlation'],rtol=1e-13,atol=1e-13) + np.testing.assert_allclose(r1.arrays['physical_modes']@r1.arrays['physical_modes'].T,r2.arrays['physical_modes']@r2.arrays['physical_modes'].T,rtol=1e-11,atol=1e-11) + np.testing.assert_allclose(r1.arrays['weighted_relative_residuals'],r2.arrays['weighted_relative_residuals'],rtol=1e-12,atol=1e-12) + +def test_memory_budget_fail_closed_and_has_no_quadratic_term(): + with pytest.raises(MemoryError): estimate_memory(m=100,n=20,nx=10,ny=5,full_count=20,chunk_size=4,budget=MemoryBudget(1000,0)) + value=estimate_memory(m=100,n=20,nx=10,ny=5,full_count=20,chunk_size=4,budget=MemoryBudget(10_000_000,0)) + assert value['decision']=='PASS' and 'MxM' in value['formula'] and value['estimated_scratch_bytes']==0 + assert all('m_squared' not in key.lower() for key in value['terms_bytes']) + +def test_atomic_no_replace_verified_reload_and_live_provenance(tmp_path): + paths,_,_,result=_fit(tmp_path/'case'); out=tmp_path/'ccd' + with ResultTransaction(out) as tx: tx.write(result); tx.publish() + loaded=load_result(out); assert loaded['provenance_validation'].startswith('VERIFIED') + assert set(loaded['manifest']['files'])=={'arrays.npz','config.json','summary.json','input_hashes.json'} + with pytest.raises(FileExistsError): + with ResultTransaction(out): pass + fields=paths['q_ctl']/'fields.npz'; fields.write_bytes(fields.read_bytes()+b'tamper') + with pytest.raises(ValueError,match='live acquisition|hash mismatch'): load_result(out) + +def test_hash_consistent_result_action_tamper_fails_essential_binding(tmp_path): + _,_,_,result=_fit(tmp_path/'case'); out=tmp_path/'ccd' + with ResultTransaction(out) as tx: tx.write(result); tx.publish() + with np.load(out/'arrays.npz',allow_pickle=False) as z: arrays={k:z[k].copy() for k in z.files} + arrays['effective_actions'][0,0]+=np.float32(.01) + from CCD_analysis.direct_dq.schema import canonical_array_sha256 + hashes=json.loads((out/'input_hashes.json').read_text()); hashes['canonical_arrays']['effective_actions']=canonical_array_sha256(arrays['effective_actions']) + np.savez_compressed(out/'arrays.npz',**arrays); (out/'input_hashes.json').write_bytes(canonical_json(hashes)) + manifest=json.loads((out/'manifest.json').read_text()); manifest['files']['arrays.npz']=file_sha256(out/'arrays.npz'); manifest['files']['input_hashes.json']=file_sha256(out/'input_hashes.json'); (out/'manifest.json').write_bytes(canonical_json(manifest)) + with pytest.raises(ValueError,match='effective-action provenance'): load_result(out) + + +def _active_schema_like_input(tmp_path): + paths, direct = _direct(tmp_path) + cfg = StreamingConfig(2, MemoryBudget(100_000_000, 0)) + base, memory = load_validated_input(direct, streaming_config=cfg) + n = base.selected_indices.size + t = np.linspace(-1.0, 1.0, n, dtype=np.float32) + actions = np.stack((t, t * t, np.sin(np.pi * t).astype(np.float32)), axis=1).astype(np.float32) + dq = base.dq_ctl.copy() + xx = base.x_D[:, None].astype(np.float32); yy = base.y_D[None, :].astype(np.float32) + for j in range(n): + dq[j, 0] += actions[j, 0] * (1 + xx) + actions[j, 1] * (2 - yy) + dq[j, 1] += actions[j, 2] * (1 + xx * yy) + actions[j, 1] * (1 + yy) + inp = RealCCDInput(base.case_id, base.direct_dq_root, base.direct_manifest_sha256, base.acquisition_identities, base.x_D, base.y_D, base.role_masks, base.analysis_mask, base.selected_indices, base.selected_relative_steps, base.q_ctl_absolute_steps, dq, actions, dq.mean(axis=0, dtype=np.float64).astype(np.float32)) + return paths, direct, inp, memory + +def _dense_original(inp, chunk): + from CCD_analysis.original_ccd import CCDConfig, decompose + mask = inp.analysis_mask + u = np.concatenate((inp.dq_ctl[:, 0][:, mask], inp.dq_ctl[:, 1][:, mask]), axis=1).T.astype(np.float64) + from CCD_analysis.direct_dq.analysis import coordinate_weights + point_w = (coordinate_weights(inp.x_D)[:, None] * coordinate_weights(inp.y_D)[None, :])[mask] + w = np.concatenate((point_w, point_w)) + result = decompose(u, inp.actions.T, weight=w, config=CCDConfig(center_snapshots=True, center_observables=True, chunk_size=chunk)) + return u, w, result + +@pytest.mark.parametrize("chunk", [1, 2, 3, 4, 7]) +def test_streaming_exactly_matches_original_ccd_dense_api(tmp_path, chunk): + _, _, inp, _ = _active_schema_like_input(tmp_path) + cfg = StreamingConfig(chunk, MemoryBudget(100_000_000, 0)) + memory = estimate_memory(m=2*int(inp.analysis_mask.sum()), n=inp.selected_indices.size, nx=inp.x_D.size, ny=inp.y_D.size, full_count=inp.dq_ctl.shape[0], chunk_size=chunk, budget=cfg.budget) + stream = decompose_streaming(inp, streaming_config=cfg, memory_estimate=memory) + u, w, dense = _dense_original(inp, chunk) + a = stream.arrays + np.testing.assert_allclose(a["cross_correlation"], dense.cross_correlation, rtol=2e-15, atol=2e-15) + np.testing.assert_allclose(a["singular_values"], dense.singular_values, rtol=2e-14, atol=2e-14) + np.testing.assert_allclose(a["physical_modes"], dense.physical_modes, rtol=2e-13, atol=2e-13) + np.testing.assert_allclose(a["left_functions"], dense.left_functions, rtol=2e-13, atol=2e-13) + np.testing.assert_allclose(a["coefficients"], dense.coefficients, rtol=2e-13, atol=2e-13) + np.testing.assert_array_equal(a["coordinate_weights"], w) + expected = [] + total = np.sum(((u-u.mean(1, keepdims=True))*np.sqrt(w)[:, None])**2) + for boundary in a["residual_block_boundaries"]: + expected.append(np.sqrt(max(total-np.sum(dense.coefficients[:boundary]**2), 0)/total)) + np.testing.assert_allclose(a["weighted_relative_residuals"], expected, rtol=3e-14, atol=3e-14) + +def test_mask_flatten_weights_actions_identity_order_and_units(tmp_path): + _, _, inp, _ = _active_schema_like_input(tmp_path) + mask = inp.analysis_mask.copy(); mask[0, 0] = False + roles = {key: value.copy() for key, value in inp.role_masks.items()} + for value in roles.values(): value[0, 0] = False + changed = RealCCDInput(inp.case_id, inp.direct_dq_root, inp.direct_manifest_sha256, inp.acquisition_identities, inp.x_D, inp.y_D, roles, mask, inp.selected_indices, inp.selected_relative_steps, inp.q_ctl_absolute_steps, inp.dq_ctl, inp.actions, inp.authoritative_mean_dq_ctl) + cfg = StreamingConfig(3, MemoryBudget(100_000_000, 0)); result = decompose_streaming(changed, streaming_config=cfg) + a = result.arrays; point_count = int(mask.sum()) + assert a["field_mean"].shape == (2*point_count,) + np.testing.assert_array_equal(a["field_mean"][:point_count], inp.dq_ctl[:, 0][:, mask].astype(np.float64).mean(0)) + np.testing.assert_array_equal(a["field_mean"][point_count:], inp.dq_ctl[:, 1][:, mask].astype(np.float64).mean(0)) + assert result.config["action_identities"] == list(ACTION_IDENTITIES) + assert result.config["channel_names"] == ["front", "upper", "lower"] + assert result.config["action_units"] == "native solver angular-velocity command units" + np.testing.assert_array_equal(a["effective_actions"], inp.actions) + +@pytest.mark.parametrize("key", ["selected_indices", "selected_relative_steps", "q_ctl_absolute_steps"] ) +def test_timestamp_and_index_order_mismatch_fails_closed(tmp_path, key): + _, _, inp, _ = _active_schema_like_input(tmp_path) + values = getattr(inp, key).copy(); values[1] = values[0] + kwargs = {name: getattr(inp, name) for name in inp.__dataclass_fields__} + kwargs[key] = values + bad = RealCCDInput(**kwargs) + with pytest.raises(ValueError, match="selected|timestamp|increasing"): + decompose_streaming(bad, streaming_config=StreamingConfig(2, MemoryBudget(100_000_000, 0))) + +def test_schema_action_order_units_and_hashes_fail_closed(tmp_path): + _, _, _, result = _fit(tmp_path/'case'); out = tmp_path/'ccd' + with ResultTransaction(out) as tx: tx.write(result); tx.publish() + for key, value in (("channel_names", ["upper", "front", "lower"]), ("action_units", "dimensionless")): + config = json.loads((out/'config.json').read_text()); config[key] = value + (out/'config.json').write_bytes(canonical_json(config)) + manifest = json.loads((out/'manifest.json').read_text()); manifest['files']['config.json'] = file_sha256(out/'config.json'); (out/'manifest.json').write_bytes(canonical_json(manifest)) + with pytest.raises(ValueError, match="estimand|config"): + load_result(out) + config[key] = result.config[key]; (out/'config.json').write_bytes(canonical_json(config)); manifest['files']['config.json'] = file_sha256(out/'config.json'); (out/'manifest.json').write_bytes(canonical_json(manifest)) + (out/'summary.json').write_bytes((out/'summary.json').read_bytes()+b' ') + with pytest.raises(ValueError, match="hash mismatch|canonical JSON"): + load_result(out) + +def test_missing_and_corrupted_live_roots_fail_closed(tmp_path): + paths, _, _, result = _fit(tmp_path/'case'); out = tmp_path/'ccd' + with ResultTransaction(out) as tx: tx.write(result); tx.publish() + moved = paths['q_ctl'].with_name('q_ctl-moved'); paths['q_ctl'].rename(moved) + with pytest.raises((FileNotFoundError, ValueError)): + load_result(out) + +def test_safe_budget_policy_and_preflight_is_read_only(tmp_path): + host = safe_host_budget(available_bytes=64*1024**3) + assert host == {"available_host_ram_bytes": 64*1024**3, "safety_reserve_bytes": 16*1024**3, "ram_budget_bytes": 48*1024**3} + _, direct = _direct(tmp_path/'input'); before = sorted(str(p.relative_to(tmp_path)) for p in tmp_path.rglob('*')) + report = preflight_real_artifact(direct, chunk_size=2, available_bytes=64*1024**3) + after = sorted(str(p.relative_to(tmp_path)) for p in tmp_path.rglob('*')) + assert before == after and report['read_only'] is True and report['decision'] == 'PASS' + assert report['dimensions']['Q'] == 1 and report['dimensions']['L'] == 3 + with pytest.raises(MemoryError): + preflight_real_artifact(direct, chunk_size=2, available_bytes=8*1024**3) + + +def _rewrite_result_arrays(out, arrays): + from CCD_analysis.direct_dq.schema import canonical_array_sha256 + hashes = json.loads((out/'input_hashes.json').read_text()) + for key, value in arrays.items(): hashes['canonical_arrays'][key] = canonical_array_sha256(value) + np.savez_compressed(out/'arrays.npz', **arrays); (out/'input_hashes.json').write_bytes(canonical_json(hashes)) + manifest = json.loads((out/'manifest.json').read_text()) + for name in ('arrays.npz', 'input_hashes.json'): manifest['files'][name] = file_sha256(out/name) + (out/'manifest.json').write_bytes(canonical_json(manifest)) + +def test_hash_consistent_absolute_timestamp_tamper_fails_live_binding(tmp_path): + _, _, _, result = _fit(tmp_path/'case'); out = tmp_path/'ccd' + with ResultTransaction(out) as tx: tx.write(result); tx.publish() + with np.load(out/'arrays.npz', allow_pickle=False) as z: arrays = {k:z[k].copy() for k in z.files} + arrays['selected_q_ctl_absolute_lattice_steps'] += np.int64(1) + _rewrite_result_arrays(out, arrays) + with pytest.raises(ValueError, match='absolute timestamp provenance'): + load_result(out) + +@pytest.mark.parametrize(('key','value'), [ + ('spectrum_label', 'explained variance'), + ('claim_boundary', 'causal response-time claim allowed'), + ('provenance_status', 'VERIFIED_WITHOUT_LIVE_INPUTS'), +]) +def test_hash_consistent_scientific_summary_relabel_fails(tmp_path, key, value): + _, _, _, result = _fit(tmp_path/'case'); out = tmp_path/'ccd' + with ResultTransaction(out) as tx: tx.write(result); tx.publish() + summary = json.loads((out/'summary.json').read_text()); summary[key] = value + (out/'summary.json').write_bytes(canonical_json(summary)) + manifest = json.loads((out/'manifest.json').read_text()); manifest['files']['summary.json'] = file_sha256(out/'summary.json'); (out/'manifest.json').write_bytes(canonical_json(manifest)) + with pytest.raises(ValueError, match='summary schema/claims'): + load_result(out) + +def test_preflight_corrupted_live_root_fails_without_outputs(tmp_path): + paths, direct = _direct(tmp_path/'input'); fields = paths['q_ctl']/'fields.npz'; original = fields.read_bytes() + before = sorted(str(p.relative_to(tmp_path)) for p in tmp_path.rglob('*')) + fields.write_bytes(original+b'corrupt') + try: + with pytest.raises(ValueError, match='hash mismatch|live acquisition'): + preflight_real_artifact(direct, chunk_size=2, available_bytes=64*1024**3) + finally: + fields.write_bytes(original) + after = sorted(str(p.relative_to(tmp_path)) for p in tmp_path.rglob('*')) + assert before == after + + +def test_cli_preflight_and_run_use_explicit_inputs_and_no_clobber(tmp_path, capsys): + _, direct = _direct(tmp_path/'input') + available = str(64*1024**3) + assert real_ccd_main(['preflight', '--direct-dq-root', str(direct), '--chunk-size', '2', '--available-host-bytes', available]) == 0 + preflight = json.loads(capsys.readouterr().out.splitlines()[-1]) + assert preflight['decision'] == 'PASS' + assert preflight['direct_dq_root'] == str(direct.resolve()) + assert preflight['dimensions']['Q'] == 1 + output = tmp_path/'fresh-real-ccd' + assert real_ccd_main(['run', '--direct-dq-root', str(direct), '--output', str(output), '--chunk-size', '2', '--available-host-bytes', available]) == 0 + run = json.loads(capsys.readouterr().out.splitlines()[-1]) + assert run['result'] == str(output.resolve()) + assert run['chunk_size'] == 2 + assert load_result(output)['provenance_validation'].startswith('VERIFIED') + with pytest.raises(FileExistsError): + real_ccd_main(['run', '--direct-dq-root', str(direct), '--output', str(output), '--chunk-size', '2', '--available-host-bytes', available]) + + +def test_karman_figures_are_artifact_only_full_resolution_labeled_and_no_clobber(tmp_path, monkeypatch): + import matplotlib.pyplot as plt + from CCD_analysis.real_ccd import figures + nx, ny, n = 7, 5, 6 + mask = np.ones((nx, ny), dtype=bool); mask[0, 0] = False; m = 2*int(mask.sum()) + rng = np.random.default_rng(4); modes = rng.normal(size=(m, 3)); coefficients = rng.normal(size=(3, n)) + steps = np.arange(100, 100*(n+1), 100, dtype=np.int64); snapshots = modes@coefficients + .1*rng.normal(size=(m, n)) + arrays = {"x_D":np.linspace(0, 6, nx), "y_D":np.linspace(-2, 2, ny), "analysis_fluid_mask":mask, "physical_modes":modes, "coefficients":coefficients, "selected_acquisition_relative_lattice_steps":steps, "singular_values":np.array([.3,.2,.1]), "authoritative_mean_dq_ctl":rng.normal(size=(2,nx,ny)).astype(np.float32), "left_functions":np.eye(3), "residual_block_boundaries":np.arange(1,4,dtype=np.int64), "weighted_relative_residuals":np.array([.8,.6,.5]), "coordinate_weights":np.ones(m), "effective_actions":rng.normal(size=(n,3)).astype(np.float32), "action_mean":np.zeros(3)} + loaded = {"arrays":arrays, "config":{"case_id":"karman_re100","channel_names":["front","upper","lower"]}, "summary":{"claim_boundary":"no causal or CCD>POD claim"}, "provenance_validation":"VERIFIED synthetic public loader", "centered_snapshots":snapshots} + calls=[] + def public_loader(path, *, include_centered_snapshots=False): + calls.append((path,include_centered_snapshots)); return loaded + monkeypatch.setattr(figures,"load_result",public_loader) + labels=[]; original=plt.Axes.set_ylabel + def capture(self,label,*args,**kwargs): labels.append(str(label)); return original(self,label,*args,**kwargs) + monkeypatch.setattr(plt.Axes,"set_ylabel",capture) + out=figures.publish_karman_figures(tmp_path/'verified-result',tmp_path/'figures',snapshot_indices=(0,3,5)) + report=json.loads((out/'KARMAN_INTERPRETATION.json').read_text()) + assert calls == [(tmp_path/'verified-result',True)] + assert report['grid_dimensions']=={"nx":nx,"ny":ny,"mask_points":int(mask.sum()),"spatial_dof":m,"downsampling":"none"} + assert report['snapshot_acquisition_relative_lattice_steps']==[100,400,600] + assert len(report['figure_files'])==16 and all((out/name).stat().st_size>0 for name in report['figure_files']) + assert any('cross-correlation strength' in label for label in labels) and any('squared cross-correlation strength' in label for label in labels) + assert not any(p.suffix in {'.npy','.npz'} for p in out.iterdir()) + with pytest.raises(FileExistsError): figures.publish_karman_figures(tmp_path/'verified-result',out) diff --git a/src/CCD_analysis/utils/__init__.py b/src/CCD_analysis/utils/__init__.py deleted file mode 100644 index 1d650b4..0000000 --- a/src/CCD_analysis/utils/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""CCD_analysis utilities — non-pycuda exports. - -cfd_interface.py requires pycuda_3_10 and is NOT exported here. -Import it directly: from CCD_analysis.utils.cfd_interface import ... -""" -from .resampling import ( - detect_dominant_frequency, detect_cycle_stability, - phase_resample, compute_pod, cumulative_energy, - e95_index, compute_reduced_ccd, - stack_velocity_fields, unstack_velocity_modes, - analyze_harmonics, gen_target_states_at, - load_aligned_fields, make_force_obs, - build_field_matrix, project_into_basis, -) diff --git a/src/CCD_analysis/utils/cfd_interface.py b/src/CCD_analysis/utils/cfd_interface.py deleted file mode 100644 index 7e0dc57..0000000 --- a/src/CCD_analysis/utils/cfd_interface.py +++ /dev/null @@ -1,341 +0,0 @@ -"""CFD interface for LegacyCelerisLab (pycuda_3_10 env). - -Copied and adapted from SR_analysis/utils/cfd_interface.py (verified working). -All functions use the LegacyCelerisLab (old) CFD API via: - from LegacyCelerisLab import FlowField - -Must be run inside: conda run -n pycuda_3_10 - -NOTE: This module should be imported directly, not through utils/__init__.py, -because it requires pycuda. Other utils (resampling) do NOT require pycuda. -""" -from __future__ import annotations - -import json -import os -import sys -from collections import deque -from typing import Any, Dict, List, Optional, Tuple - -import numpy as np - -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if _REPO not in sys.path: - sys.path.insert(0, _REPO) -_SRC = os.path.join(_REPO, "src") -if _SRC not in sys.path: - sys.path.insert(0, _SRC) - -from LegacyCelerisLab import FlowField # noqa: E402 -from LegacyCelerisLab import utils as legacy_utils # noqa: E402 - -ACTION_SMOOTH_WEIGHT = 0.1 # used by FlowField.run() internally - - -def load_legacy_configs(config_dir: str) -> Tuple[Any, Any]: - """Load and return legacy (cuda_config, field_config).""" - cuda_cfg = legacy_utils.load_cuda_config( - os.path.join(config_dir, "config_cuda.json")) - field_cfg = legacy_utils.load_flow_field_config( - os.path.join(config_dir, "config_flowfield.json")) - return cuda_cfg, field_cfg - - -# --------------------------------------------------------------------------- -# Karman cloak env builder (disturbance cylinder + 3 sensors) -# --------------------------------------------------------------------------- - -def build_karman_cloak_env( - flow_field: FlowField, - *, - u0: float, - l0: float, - sample_interval: int, - fifo_len: int, - data_type: type, -) -> Tuple[np.ndarray, dict]: - """Add dist-cylinder & 3 sensors, stabilize, record target. - - Steps (mirrors env_karman_cloak_standard.__init__): - 1. add dist_cylinder (id=0) - 2. add 3 sensors (id=1,2,3) - 3. stabilize run(4*NX/U0, zero-action[4]) - 4. record FIFO_LEN x run(SAMPLE_INTERVAL, zero[4]), collect obs[2:8] - - Returns (target_states, info_dict). - """ - cy = (flow_field.FIELD_SHAPE[1] - 1) / 2.0 - flow_field.add_cylinder((10.0 * l0, cy, 0.0), l0) - for y_off in [2.0, 0.0, -2.0]: - flow_field.add_sensor((40.0 * l0, cy + y_off * l0, 0.0), l0 / 4.0) - - n_obj = flow_field.obs.size // 2 - stabilize_steps = int(4 * flow_field.FIELD_SHAPE[0] / u0) - print(f" stabilising ({stabilize_steps} steps)...") - flow_field.run(stabilize_steps, np.zeros(n_obj, dtype=data_type)) - - target_states = np.empty((0, 6), dtype=data_type) - for _ in range(fifo_len): - flow_field.run(sample_interval, np.zeros(n_obj, dtype=data_type)) - target_states = np.vstack((target_states, flow_field.obs.copy()[2:8])) - - print(f" target recorded: {target_states.shape}") - return target_states, {"n_objects": n_obj, "NX": flow_field.FIELD_SHAPE[0], - "NY": flow_field.FIELD_SHAPE[1]} - - -# --------------------------------------------------------------------------- -# Pinball adder + norm computation (configurable for Karman / Illusion) -# --------------------------------------------------------------------------- - -def add_pinball( - flow_field: FlowField, - *, - l0: float, - u0: float, - sample_interval: int, - fifo_len: int, - data_type: type, - action_bias: Optional[Tuple[float, float, float]] = None, - pinball_front_x: float = 30.0, - pinball_rear_x: float = 31.3, - obs_slice_start: int = 2, - obs_slice_end: int = 14, - n_objects_total: Optional[int] = None, -) -> dict: - """Add pinball cylinders, stabilize, compute norm, preset-action FIFO init. - - Parameters - ---------- - pinball_front_x, pinball_rear_x : pinball geometry (L0 units). - obs_slice_start, obs_slice_end : slice of obs for norm. - - Returns dict with norm values and save_states. - """ - if action_bias is None: - action_bias = (0.0, -4.0, 4.0) - - u0_float = float(u0) - ny = flow_field.FIELD_SHAPE[1] - centers = [ - (pinball_front_x * l0, (ny - 1) / 2, 0.0), - (pinball_rear_x * l0, (ny - 1) / 2 + 0.75 * l0, 0.0), - (pinball_rear_x * l0, (ny - 1) / 2 - 0.75 * l0, 0.0), - ] - for c in centers: - flow_field.add_cylinder(c, l0 / 2.0) - - n_obj = flow_field.obs.size // 2 if n_objects_total is None else n_objects_total - print(f" bodies after pinball: {n_obj}") - stabilize_steps = int(4 * flow_field.FIELD_SHAPE[0] / u0_float) - print(f" stabilising pinball ({stabilize_steps} steps)...") - flow_field.run(stabilize_steps, np.zeros(n_obj, dtype=data_type)) - flow_field.get_ddf() - flow_field.save_ddf() - - # ---- norm phase (zero-action) ---- - fifo = deque(maxlen=fifo_len) - for _ in range(fifo_len): - flow_field.run(sample_interval, np.zeros(n_obj, dtype=data_type)) - fifo.append(flow_field.obs.copy()[obs_slice_start:obs_slice_end]) - - temp_states = np.array(fifo, dtype=data_type) - # forces are at the last 6 positions of the slice - force_start = obs_slice_end - obs_slice_start - 6 - force_end = force_start + 6 - force_norm_fact = 6.0 * float(np.max(np.abs(temp_states[:, force_start:force_end]))) - sens_deviation = np.mean(temp_states[:, 0:6], axis=0).astype(data_type) - sens_norm_fact = np.zeros(6, dtype=data_type) - for i in range(6): - sens_norm_fact[i] = 5.0 * float(np.max(np.abs(temp_states[:, i] - sens_deviation[i]))) - - print(f" norm: force_norm_fact={force_norm_fact:.6f}") - - # ---- preset-action FIFO init ---- - flow_field.apply_ddf() - bias = np.zeros(n_obj, dtype=data_type) - bias[n_obj - 3] = float(action_bias[0] * u0_float) - bias[n_obj - 2] = float(action_bias[1] * u0_float) - bias[n_obj - 1] = float(action_bias[2] * u0_float) - - fifo.clear() - for _ in range(fifo_len): - flow_field.run(sample_interval, bias) - fifo.append(flow_field.obs.copy()[obs_slice_start:obs_slice_end]) - - save_states = np.array(list(fifo), dtype=data_type) - flow_field.apply_ddf() - - return { - "force_norm_fact": force_norm_fact, - "sens_deviation": sens_deviation.tolist(), - "sens_norm_fact": sens_norm_fact.tolist(), - "action_bias": list(action_bias), - "save_states": save_states, - } - - -# --------------------------------------------------------------------------- -# Observation builder and action helpers -# --------------------------------------------------------------------------- - -def build_observation(obs_slice: np.ndarray, norm: dict) -> np.ndarray: - """Assemble normalised DRL observation (12-dim) from obs slice. - - obs_slice is 12-element: sensor[0:6] + force[6:12]. - Returns clipped 12-dim array in [-1, 1]. - """ - forces = obs_slice[6:12] / norm["force_norm_fact"] - sens = (obs_slice[0:6] - norm["sens_deviation"]) / norm["sens_norm_fact"] - obs = np.clip(np.hstack([forces, sens]), -1.0, 1.0).astype(np.float32) - return obs - - -def scale_action( - action_norm: np.ndarray, - *, - scale: float = 8.0, - bias: Tuple[float, float, float] = (0.0, -4.0, 4.0), - u0: float = 0.01, - n_total_bodies: int = 7, -) -> np.ndarray: - """Convert normalised action ([-1,1]^3) to legacy CFD action array. - - Returns array of length n_total_bodies with cylinders' omegas at the last 3 slots. - """ - a = np.zeros(n_total_bodies, dtype=np.float32) - omega = (np.array(action_norm, dtype=np.float32) * scale - + np.array(bias, dtype=np.float32)) * u0 - a[n_total_bodies - 3:] = omega - return a - - -# --------------------------------------------------------------------------- -# Vorticity & field export -# --------------------------------------------------------------------------- - -def get_velocity_field(flow_field: FlowField, u0: float = 0.01): - """Extract ux, uy fields from DDF on host. Returns (ux, uy) each (NX, NY).""" - flow_field.get_ddf() - NX = flow_field.FIELD_SHAPE[0] - NY = flow_field.FIELD_SHAPE[1] - ddf = flow_field.ddf.copy().reshape((9, NY, NX)).transpose(2, 1, 0) - ux = (ddf[:, :, 1] + ddf[:, :, 5] + ddf[:, :, 8] - - ddf[:, :, 3] - ddf[:, :, 6] - ddf[:, :, 7]) / u0 - uy = (ddf[:, :, 2] + ddf[:, :, 5] + ddf[:, :, 6] - - ddf[:, :, 4] - ddf[:, :, 7] - ddf[:, :, 8]) / u0 - return ux.astype(np.float32), uy.astype(np.float32) - - -def vorticity_from_ddf(flow_field: FlowField, u0: float) -> np.ndarray: - """Compute z-vorticity from current DDF on host. - - Returns omega_z with shape (NY, NX) for direct imshow use. - """ - ux, uy = get_velocity_field(flow_field, u0) - # ux, uy have shape (NX, NY), transpose to (NY, NX) for correct axes - ux_t = ux.T.astype(np.float64) - uy_t = uy.T.astype(np.float64) - # omega_z = duy/dx - dux/dy - omega = np.gradient(uy_t, axis=1) - np.gradient(ux_t, axis=0) - return omega - - -def save_vorticity_png(path: str, omega: np.ndarray, title: str = ""): - """Save vorticity field as a PNG with symmetric colour bar.""" - import matplotlib - matplotlib.use("Agg") - import matplotlib.pyplot as plt - - abs_o = np.abs(omega[np.isfinite(omega)]) - vmax = float(np.percentile(abs_o, 99.5)) if abs_o.size > 0 else 1.0 - if vmax <= 0: - vmax = 1.0 - - ny, nx = omega.shape - fig, ax = plt.subplots(figsize=(min(18, max(8, nx / 60)), min(10, max(3, ny / 40)))) - im = ax.imshow(omega, origin="lower", aspect="equal", cmap="RdBu_r", - vmin=-vmax, vmax=vmax, extent=(0, nx - 1, 0, ny - 1)) - ax.set_xlabel("x (lattice)") - ax.set_ylabel("y (lattice)") - if title: - ax.set_title(title) - fig.colorbar(im, ax=ax, fraction=0.046, pad=0.04, label=r"$\omega_z$") - fig.tight_layout() - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -# --------------------------------------------------------------------------- -# DTW similarity -# --------------------------------------------------------------------------- - -def calc_lag(target: np.ndarray, state: np.ndarray) -> int: - t = target - np.mean(target) - s = state - np.mean(state) - corr = np.correlate(t, s, mode="full") - lags = np.arange(-len(target) + 1, len(target)) - return int(lags[np.argmax(corr)]) - - -def calc_dtw_sim(target: np.ndarray, state: np.ndarray) -> float: - n, m = len(target), len(state) - dtw = np.full((n + 1, m + 1), np.inf) - dtw[0, 0] = 0.0 - for i in range(1, n + 1): - for j in range(1, m + 1): - cost = abs(float(target[i - 1]) - float(state[j - 1])) - dtw[i, j] = cost + min(dtw[i - 1, j], dtw[i, j - 1], dtw[i - 1, j - 1]) - return float(1.0 - dtw[n, m] / n) - - -def compute_similarity(target_states: np.ndarray, state_series: np.ndarray, conv_len: int) -> float: - """Lag-compensated DTW similarity over conv_len window.""" - ref = target_states[conv_len:2 * conv_len, 1] - cur = state_series[-conv_len:, 1] - lag = calc_lag(ref, cur) - - sim_sum = 0.0 - for i in range(6): - target_seq = np.roll(target_states[:, i], -lag)[conv_len:2 * conv_len] - state_seq = state_series[-conv_len:, i] - sim_sum += calc_dtw_sim(target_seq, state_seq) / 6.0 - return float(sim_sum) - - -# --------------------------------------------------------------------------- -# PPO model loading (DummyEnv + Sin activation) -# --------------------------------------------------------------------------- - -def create_dummy_env(s_dim: int = 12, a_dim: int = 3): - """Return a gym.Env with correct observation/action spaces for model loading.""" - import gymnasium as gym - from gymnasium import spaces - - class DummyEnv(gym.Env): - def __init__(self): - super().__init__() - self.observation_space = spaces.Box(low=-1, high=1, shape=(s_dim,), dtype=np.float32) - self.action_space = spaces.Box(low=-1, high=1, shape=(a_dim,), dtype=np.float32) - def reset(self, seed=None): - return np.zeros(s_dim, dtype=np.float32), {} - def step(self, action): - return np.zeros(s_dim, dtype=np.float32), 0.0, False, False, {} - def render(self): - pass - return DummyEnv() - - -def load_ppo_model(model_path: str, device: str = "cuda:0", s_dim: int = 12, a_dim: int = 3): - """Load a PPO model with Sin activation.""" - import torch - from torch.nn import Module - from stable_baselines3 import PPO - - class Sin(Module): - def forward(self, x): - return torch.sin(x) - - dummy_env = create_dummy_env(s_dim, a_dim) - model = PPO.load(model_path, env=dummy_env, device=device) - return model diff --git a/src/CCD_analysis/utils/dual_clock.py b/src/CCD_analysis/utils/dual_clock.py deleted file mode 100644 index 74d35c4..0000000 --- a/src/CCD_analysis/utils/dual_clock.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Minimal dual-clock field sampling for legacy CelerisLab rollouts.""" -from __future__ import annotations - -from collections.abc import Iterable -from pathlib import Path - -import numpy as np - -from CCD_analysis.utils.cfd_interface import get_velocity_field - - -def field_steps_from_interval(total_steps: int, field_interval: int) -> tuple[int, ...]: - """Return absolute field-save steps within a finite rollout.""" - if type(total_steps) is not int or total_steps < 1: - raise ValueError("total_steps must be a positive integer") - if type(field_interval) is not int or field_interval < 1: - raise ValueError("field_interval must be a positive integer") - return tuple(range(field_interval, total_steps + 1, field_interval)) - - -class DualClockCollector: - """Keep policy decisions fixed while reading fields at independent steps.""" - - def __init__( - self, - flow_field, - *, - control_interval: int, - control_count: int, - u0: float, - field_steps: Iterable[int], - ): - if type(control_interval) is not int or control_interval < 1: - raise ValueError("control_interval must be a positive integer") - if type(control_count) is not int or control_count < 1: - raise ValueError("control_count must be a positive integer") - self.flow_field = flow_field - self.control_interval = control_interval - self.control_count = control_count - self.u0 = u0 - self.field_steps = tuple(field_steps) - horizon = control_interval * control_count - if any(type(step) is not int for step in self.field_steps): - raise ValueError("field steps must be integers") - if tuple(sorted(set(self.field_steps))) != self.field_steps: - raise ValueError("field steps must be strictly increasing and unique") - if any(step < 1 or step > horizon for step in self.field_steps): - raise ValueError("field steps must lie within the rollout") - self._next_field = 0 - self._next_control = 0 - self.ux: list[np.ndarray] = [] - self.uy: list[np.ndarray] = [] - self.saved_steps: list[int] = [] - - def run_interval(self, control_index: int, action_target: np.ndarray) -> np.ndarray: - """Advance exactly one policy interval, pausing only at due field steps.""" - if control_index != self._next_control: - raise ValueError("control intervals must run sequentially") - start = control_index * self.control_interval - stop = start + self.control_interval - due = [] - while self._next_field < len(self.field_steps): - step = self.field_steps[self._next_field] - if step > stop: - break - if step > start: - due.append(step) - self._next_field += 1 - - ff = self.flow_field - ff.begin_control_interval(self.control_interval, action_target) - current = start - boundaries = due if due and due[-1] == stop else [*due, stop] - for step in boundaries: - if step > current: - ff.run_control_segment(step - current) - current = step - if step in due: - ux, uy = get_velocity_field(ff, u0=self.u0) - self.ux.append(ux) - self.uy.append(uy) - self.saved_steps.append(step) - obs = ff.end_control_interval().copy() - self._next_control += 1 - return obs - - def save(self, path: str | Path) -> None: - """Save fields and their absolute lattice-step timeline.""" - if self._next_control != self.control_count: - raise RuntimeError("cannot save an incomplete rollout") - destination = Path(path) - destination.parent.mkdir(parents=True, exist_ok=True) - np.savez_compressed( - destination, - ux=np.asarray(self.ux, dtype=np.float32), - uy=np.asarray(self.uy, dtype=np.float32), - lattice_steps=np.asarray(self.saved_steps, dtype=np.int64), - control_indices=(np.asarray(self.saved_steps, dtype=np.int64) - 1) // self.control_interval, - control_offsets=(np.asarray(self.saved_steps, dtype=np.int64) - 1) % self.control_interval + 1, - control_interval=np.asarray(self.control_interval, dtype=np.int64), - ) diff --git a/src/CCD_analysis/utils/field_translate.py b/src/CCD_analysis/utils/field_translate.py deleted file mode 100644 index f716435..0000000 --- a/src/CCD_analysis/utils/field_translate.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Field translation (spatial shifting) utilities for CCD analysis. - -When computing correction fields across scenes with different object positions -(e.g., illusion pinball at x=393 vs cloak pinball at x=613), fields must be -translated so that reference points align before subtraction. - -All functions operate on field arrays with shape (NY, NX) or (N, NY, NX). -NX = 1280, NY = 512 in the standard grid. -""" -from __future__ import annotations - -import numpy as np - - -def translate_field_x(field: np.ndarray, shift_x: int, - fill_edge: bool = True) -> np.ndarray: - """Horizontally shift a field by a given number of pixels. - - Parameters - ---------- - field : (NY, NX) or (N, NY, NX) ndarray - Velocity field(s) in legacy (NY, NX) order. - shift_x : int - Positive = shift right, negative = shift left. - fill_edge : bool - If True, fill vacated columns with edge values (smooth). - If False, fill with zeros (creates boundary artifacts). - - Returns - ------- - shifted : ndarray with same shape as input. - """ - if shift_x == 0: - return field.copy() - - if field.ndim == 2: - ny, nx = field.shape - result = np.zeros_like(field) - shift = shift_x - if shift > 0: - result[:, shift:] = field[:, :-shift] - if fill_edge: - # Fill left vacated columns with leftmost column value - result[:, :shift] = field[:, :1] - else: - s = -shift - result[:, :-s] = field[:, s:] - if fill_edge: - # Fill right vacated columns with rightmost column value - result[:, -s:] = field[:, -1:] - return result - elif field.ndim == 3: - n, ny, nx = field.shape - result = np.zeros_like(field) - shift = shift_x - if shift > 0: - result[:, :, shift:] = field[:, :, :-shift] - if fill_edge: - # Broadcast leftmost column across vacated columns - result[:, :, :shift] = field[:, :, :1] - else: - s = -shift - result[:, :, :-s] = field[:, :, s:] - if fill_edge: - result[:, :, -s:] = field[:, :, -1:] - return result - else: - raise ValueError(f"Unsupported field ndim: {field.ndim}") - - -def translate_fields_dict(q: dict, shift_x: int) -> dict: - """Translate ux and uy fields in a data dict, leaving telemetry unchanged.""" - if shift_x == 0: - return q - return { - "ux": translate_field_x(q["ux"], shift_x), - "uy": translate_field_x(q["uy"], shift_x), - "forces": q.get("forces"), - "sensors": q.get("sensors"), - "actions": q.get("actions"), - "meta": {**q.get("meta", {}), "translated_by": shift_x}, - "step_indices": q.get("step_indices"), - } - - -# --------------------------------------------------------------------------- -# Reference positions (pixel coordinates) -# --------------------------------------------------------------------------- -# All scenes now use UNIFIED geometry: -# pinball at (30, 31.3) x L0 -> center = 613 px -# sensors at 40 x L0 -# target cylinder at 30.65 x L0 = 613 px (same as pinball center) -# field_translate kept for optional cross-comparison or future use. - -ILLUSION_REF_X = 613 # UNIFIED: was 393 -CLOAK_REF_X = 613 # unchanged -TARGET_CYL_REF_X = 613 # UNIFIED: was 400 (now same as pinball center) - - -def get_scene_ref_x(scene_name: str) -> int | None: - """Get the reference x-position (pinball/cylinder center) for a scene. - - Returns pixel coordinate, or None if no reference (e.g. target_channel). - """ - if "illusion" in scene_name: - return ILLUSION_REF_X - if "target_cylinder" in scene_name: - return TARGET_CYL_REF_X - if scene_name in ("pinball", "steady_cloak", "karman_re100", "karman_q_blk", - "vortex_lamb", "vortex_taylor", - "vortex_uncontrolled_lamb", "vortex_uncontrolled_taylor"): - return CLOAK_REF_X - return None diff --git a/src/CCD_analysis/utils/load_vortex_fields.py b/src/CCD_analysis/utils/load_vortex_fields.py deleted file mode 100644 index bbf1bb6..0000000 --- a/src/CCD_analysis/utils/load_vortex_fields.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Load vortex scene fields.npz format for transient CCD analysis. - -Vortex scenes (vortex_lamb, vortex_taylor, vortex_target_*, vortex_uncontrolled_*) -have field snapshots saved as raw fields.npz with shape (N, NX, NY) and no phase -plan (transient, not periodic). - -Converts to the same convention as load_aligned_fields(): - - Transposes fields from (N, NX, NY) -> (N, NY, NX) - - Loads telemetry from controlled.npz or sensors.npz - - Returns dict with identical key structure -""" -from __future__ import annotations - -import json -import os -from typing import Any - -import numpy as np - -from CCD_analysis.configs import DATA_DIR, NX, NY, SCENES - - -def load_vortex_fields(scene_name: str) -> dict: - """Load vortex scene field data from raw fields.npz. - - Parameters - ---------- - scene_name : str — one of the vortex scene names registered in configs. - - Returns - ------- - dict with same keys as load_aligned_fields(): - ux, uy : (N, NY, NX) ndarray — field snapshots (transposed) - forces : (N, 6) ndarray or None - sensors : (N, 6) ndarray or None - actions : (N, 3) ndarray or None - meta : dict with scene info - step_indices : list of int (sequential 0..N-1 for transient) - """ - if scene_name not in SCENES: - raise KeyError(f"Unknown scene: {scene_name}") - - cfg = SCENES[scene_name] - scene_id = cfg["scene_id"] - data_dir = os.path.join(DATA_DIR, scene_id, scene_name) - - if not os.path.isdir(data_dir): - raise FileNotFoundError(f"Vortex scene directory not found: {data_dir}") - - # -- fields.npz (native simulation order: NX first) -- - fields_path = os.path.join(data_dir, "fields.npz") - if not os.path.isfile(fields_path): - raise FileNotFoundError(f"{fields_path} not found") - - fd = np.load(fields_path) - ux_raw = fd["ux"] # (N, NX, NY) - uy_raw = fd["uy"] - N = ux_raw.shape[0] - fd.close() - - # Transpose (N, NX, NY) -> (N, NY, NX) to match load_aligned_fields convention - ux = np.ascontiguousarray(ux_raw.transpose(0, 2, 1)) - uy = np.ascontiguousarray(uy_raw.transpose(0, 2, 1)) - - # -- Telemetry (controlled.npz or sensors.npz) -- - tele_path = None - for p in [ - os.path.join(data_dir, "controlled.npz"), - os.path.join(data_dir, "sensors.npz"), - ]: - if os.path.isfile(p): - tele_path = p - break - - sensors, forces, actions = None, None, None - if tele_path is not None: - td = np.load(tele_path) - if "sensors" in td: - sensors = td["sensors"] # (N, 6) - if "forces" in td: - forces = td["forces"] # (N, 6) - if "actions" in td: - actions = td["actions"] # (N, 3) - td.close() - - # Verify N matches - if sensors is not None and sensors.shape[0] != N: - raise ValueError( - f"sensors ({sensors.shape[0]}) != fields ({N})" - ) - if forces is not None and forces.shape[0] != N: - raise ValueError( - f"forces ({forces.shape[0]}) != fields ({N})" - ) - - meta = { - "scene": scene_name, - "scene_id": scene_id, - "source": "vortex_transient", - "target_type": "transient", - "n_frames": N, - } - - result: dict[str, Any] = { - "ux": ux, - "uy": uy, - "forces": forces, - "actions": actions, - "sensors": sensors, - "meta": meta, - "step_indices": list(range(N)), # sequential — no phase plan - } - - return result diff --git a/src/CCD_analysis/utils/resampling.py b/src/CCD_analysis/utils/resampling.py deleted file mode 100644 index cd3131b..0000000 --- a/src/CCD_analysis/utils/resampling.py +++ /dev/null @@ -1,433 +0,0 @@ -"""CPU-only analysis utilities for CCD pipeline. - -No pycuda or LegacyCelerisLab dependency — can run with plain python3. -""" -from __future__ import annotations - -from typing import Any, Dict, List, Optional, Tuple - -import numpy as np - - -# --------------------------------------------------------------------------- -# Period detection helpers -# --------------------------------------------------------------------------- - -def detect_dominant_frequency(signal: np.ndarray, sample_dt: float) -> Tuple[float, float, float]: - """Detect dominant frequency via FFT. Returns (f_dom, period, peak_power).""" - n = len(signal) - if n < 16: - return 0.0, 0.0, 0.0 - y = signal - np.mean(signal) - window = np.hanning(n) - spec = np.abs(np.fft.rfft(y * window)) ** 2 - freqs = np.fft.rfftfreq(n, d=sample_dt) - idx = 1 + np.argmax(spec[1:]) - f_dom = float(freqs[idx]) - period = 1.0 / f_dom if f_dom > 0 else 0.0 - return f_dom, period, float(spec[idx]) - - -def detect_cycle_stability(signal: np.ndarray, sample_dt: float) -> Tuple[float, float, list]: - """Detect cycle lengths using rising zero-crossings. Returns (CV_T, mean_T, lengths).""" - y = signal - np.mean(signal) - crossings = np.where((np.sign(y[:-1]) < 0) & (np.sign(y[1:]) > 0))[0] - if len(crossings) < 2: - return 0.0, 0.0, [] - cycle_lengths = np.diff(crossings).astype(float) * sample_dt - if len(cycle_lengths) < 2: - return 0.0, float(cycle_lengths[0]) if len(cycle_lengths) > 0 else 0.0, cycle_lengths.tolist() - mean_T = float(np.mean(cycle_lengths)) - std_T = float(np.std(cycle_lengths)) - cv_T = std_T / mean_T if mean_T > 0 else 0.0 - return cv_T, mean_T, cycle_lengths.tolist() - - -# --------------------------------------------------------------------------- -# Phase resampling -# --------------------------------------------------------------------------- - -def phase_resample(data: np.ndarray, cycle_starts: List[int], n_pts: int = 24) -> np.ndarray: - """Resample a multi-channel signal to uniform phase points per cycle. - - Uses piecewise linear interpolation (no scipy dependency). - - Parameters - ---------- - data : (T, C) ndarray — multi-channel time series. - cycle_starts : list of int — indices where each cycle starts. - n_pts : int — number of phase points per cycle. - - Returns - ------- - resampled : (n_cycles, n_pts, C) ndarray - """ - n_cycles = len(cycle_starts) - 1 - if n_cycles < 1: - raise ValueError("Need at least 2 cycle starts") - - if data.ndim == 1: - data = data[:, None] - C = data.shape[1] - out = np.zeros((n_cycles, n_pts, C), dtype=np.float64) - - for c in range(n_cycles): - i_start = cycle_starts[c] - i_end = cycle_starts[c + 1] - segment = data[i_start:i_end + 1] - seg_len = len(segment) - if seg_len < 2: - continue - - old_idx = np.linspace(0, 1, seg_len) - new_idx = np.linspace(0, 1, n_pts, endpoint=False) - for ch in range(C): - out[c, :, ch] = np.interp(new_idx, old_idx, segment[:, ch]) - - return out - - -# --------------------------------------------------------------------------- -# POD -# --------------------------------------------------------------------------- - -def compute_pod(snapshot_matrix: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - """Compute POD from snapshot matrix using method of snapshots. - - For N << M (e.g. N=96 snapshots, M=2*NX*NY=1.3M), uses temporal - correlation matrix (N x N) instead of full SVD on (M x N). - This is ~100x faster and numerically identical. - - Returns (mean_field, modes, singular_values, coefficients). - """ - mean_field = np.mean(snapshot_matrix, axis=1) - Q = snapshot_matrix - mean_field[:, None] - M, N = Q.shape - - if N < M // 10: # method of snapshots: N << M - # Temporal correlation: C = Q^T @ Q (N x N) - C = Q.T.astype(np.float64) @ Q.astype(np.float64) - s2, V = np.linalg.eigh(C) - # Sort descending - idx = np.argsort(s2)[::-1] - s = np.sqrt(np.maximum(s2[idx], 0.0)) - V = V[:, idx] - # Spatial modes: U = Q @ V * diag(1/s) - U = Q.astype(np.float64) @ V - U[:, s > 1e-12] /= s[s > 1e-12] - # Coefficients: diag(s) @ V^T - coefficients = (V * s).T - else: # fallback to full SVD for small matrices - U, s, Vt = np.linalg.svd(Q, full_matrices=False) - coefficients = np.diag(s) @ Vt - - return mean_field, U, s, coefficients - - -def cumulative_energy(singular_values: np.ndarray) -> np.ndarray: - e = singular_values ** 2 - return np.cumsum(e) / np.sum(e) - - -def e95_index(cumulative_energy: np.ndarray) -> int: - return int(np.searchsorted(cumulative_energy, 0.95) + 1) - - -# --------------------------------------------------------------------------- -# CCD (reduced, Lyu23-inspired) -# --------------------------------------------------------------------------- - -def compute_reduced_ccd(pod_coeffs: np.ndarray, observable: np.ndarray, Q_delay: int = 12) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, int, int]: - """Compute reduced CCD in POD coefficient space, with clean boundary handling. - - Replaces np.roll-based delay with explicit lagged matrix construction, - dropping wrapped-around samples instead of zero-padding them. - - Parameters - ---------- - pod_coeffs : (r, N) ndarray — standardized POD coefficients. - observable : (m, N) ndarray — standardized observable. - Q_delay : int — number of delay steps (must be odd for symmetric window). - - Returns - ------- - W : (r, min(r, m*Q_delay)) — CCD directions. - sigma : (min_dim,) — singular values. - R : (m*Q_delay, min_dim) — CCD response modes. - z : (min_dim, N) — CCD temporal coefficients. - N_original : int — original sample count. - N_valid : int — sample count after boundary trimming. - """ - N = pod_coeffs.shape[1] - N_original = N - m = observable.shape[0] - - half = Q_delay // 2 - # Keep only columns where ALL shifts have valid (non-wrapped) indices - valid_start = half - valid_end = N - half - if Q_delay % 2 == 0: - valid_end = N - half # symmetric even: one extra on each side - valid_end = N - (Q_delay - 1) // 2 # general formula - # For symmetric Q_delay: valid = [half, N - half) - valid_start = half - valid_end = N - half - N_valid = valid_end - valid_start - - if N_valid < Q_delay: - raise ValueError(f"Too few valid samples ({N_valid}) for Q_delay={Q_delay}. Need at least {Q_delay}.") - - rows = [] - for shift in range(-half, half + 1): - shifted = observable[:, valid_start + shift:valid_end + shift] - rows.append(shifted) - P = np.vstack(rows) # (m*Q_delay, N_valid) - - A_valid = pod_coeffs[:, valid_start:valid_end] - - # Standardize - P_mean = np.mean(P, axis=1, keepdims=True) - P_std = np.std(P, axis=1, keepdims=True) + 1e-12 - P_z = (P - P_mean) / P_std - - A_mean = np.mean(A_valid, axis=1, keepdims=True) - A_std = np.std(A_valid, axis=1, keepdims=True) + 1e-12 - A_z = (A_valid - A_mean) / A_std - - C = P_z @ A_z.T / (N_valid * np.sqrt(float(Q_delay))) - R, s, Wt = np.linalg.svd(C, full_matrices=False) - W = Wt.T - z = W.T @ A_z - return W, s, R, z, N_original, N_valid - - -# --------------------------------------------------------------------------- -# Field stacking helpers -# --------------------------------------------------------------------------- - -def stack_velocity_fields(ux_fields: List[np.ndarray], uy_fields: List[np.ndarray]) -> np.ndarray: - """Stack list of (ux, uy) field pairs into snapshot matrix (2*nx*ny, N).""" - snapshots = [] - for ux, uy in zip(ux_fields, uy_fields): - q = np.concatenate([ux.ravel(), uy.ravel()]) - snapshots.append(q) - return np.column_stack(snapshots) - - -def unstack_velocity_modes(modes: np.ndarray, ny: int, nx: int, n_modes: int = 6) -> Tuple[List[np.ndarray], List[np.ndarray]]: - """Unstack POD/CCD modes back into ux, uy fields.""" - ux_list, uy_list = [], [] - half = nx * ny - for i in range(min(n_modes, modes.shape[1])): - mode = modes[:, i] - ux_list.append(mode[:half].reshape(ny, nx)) - uy_list.append(mode[half:].reshape(ny, nx)) - return ux_list, uy_list - - -# --------------------------------------------------------------------------- -# Aligned field data loader (new format: fields_aligned.npz + phase_plan.json) -# --------------------------------------------------------------------------- - -def load_aligned_fields(scene_name: str) -> dict: - """Load fields_aligned.npz + telemetry indexed by phase_plan step_indices. - - Transposes fields from (NX, NY) to (NY, NX) to match legacy convention - so that ravel() produces (y*NX + x) ordering for POD/CCD. - - Parameters - ---------- - scene_name : str — scene name (e.g. 'target_cylinder_1.0L', 'illusion_1.0L') - - Returns - ------- - dict with: - ux, uy : (N_snap, NY, NX) field snapshots - forces : (N_snap, n_force) or None - actions : (N_snap, 3) or None - sensors : (N_snap, 6) or None - meta : dict with gate, CV_T, f_dom, N_raw_per_cycle - step_indices : list of int — matching phase_plan - """ - import json - import os - - from CCD_analysis.configs import DATA_DIR, SCENES - - if scene_name not in SCENES: - raise KeyError(f"Unknown scene: {scene_name}") - - cfg = SCENES[scene_name] - scene_id = cfg["scene_id"] - data_dir = os.path.join(DATA_DIR, scene_id, scene_name) - - # -- fields_aligned.npz -- - fa_path = os.path.join(data_dir, "fields_aligned.npz") - if not os.path.isfile(fa_path): - raise FileNotFoundError(f"{fa_path} not found") - - fd = np.load(fa_path) - ux_raw = fd["ux"] # (N, NX, NY) - uy_raw = fd["uy"] - fd.close() - - N, NX, NY = ux_raw.shape - # Transpose to (N, NY, NX) matching legacy convention - ux = np.ascontiguousarray(ux_raw.transpose(0, 2, 1)) - uy = np.ascontiguousarray(uy_raw.transpose(0, 2, 1)) - - # -- phase_plan.json -- - plan_path = os.path.join(DATA_DIR, "resampled", scene_name, "phase_plan.json") - if not os.path.isfile(plan_path): - raise FileNotFoundError(f"{plan_path} not found") - - with open(plan_path) as f: - plan = json.load(f) - - step_indices = list(plan["step_indices"]) - assert len(step_indices) == N, ( - f"{scene_name}: step_indices has {len(step_indices)} entries but " - f"fields has {N} snapshots" - ) - - # -- telemetry (controlled.npz or sensors.npz) -- - tele_path = None - for p in [os.path.join(data_dir, "controlled.npz"), os.path.join(data_dir, "sensors.npz")]: - if os.path.isfile(p): - tele_path = p - break - if tele_path is None: - raise FileNotFoundError(f"No telemetry (*.npz) found in {data_dir}") - - td = np.load(tele_path) - - result = { - "ux": ux, - "uy": uy, - "forces": None, - "actions": None, - "sensors": None, - "meta": { - "scene": scene_name, - "scene_id": scene_id, - "gate": plan.get("gate", "unknown"), - "CV_T": plan.get("CV_T"), - "f_dom": plan.get("f_dom"), - "N_raw_per_cycle": plan.get("N_raw_per_cycle"), - "rho_interp": plan.get("rho_interp"), - "sample_interval": cfg.get("sample_interval"), - }, - "step_indices": step_indices, - } - - for key in ["forces", "actions", "sensors"]: - if key in td: - full = td[key] - result[key] = full[step_indices] # (N_snap, n_channels) - - td.close() - return result - - -def make_force_obs(forces: np.ndarray, scene_name: str, mode: str = "fy") -> np.ndarray: - """Construct force observable from raw forces. - - Parameters - ---------- - forces : (N, n_channels) ndarray - target_cylinder: (N, 2) = [Fx, Fy] - illusion/pinball: (N, 6) = [Fx1,Fy1,Fx2,Fy2,Fx3,Fy3] - scene_name : str — used to detect target_cylinder vs illusion - mode : str - 'fy' (primary) -> SigmaFy (1 channel) - 'fx' (secondary) -> SigmaFx (1 channel) - 'joint' (supplementary) -> [SigmaFx; SigmaFy] (2 channels) - - Returns - ------- - obs : (C, N) ndarray - """ - if "target_cylinder" in scene_name: - fx = forces[:, 0] - fy = forces[:, 1] - else: - fx = forces[:, 0] + forces[:, 2] + forces[:, 4] - fy = forces[:, 1] + forces[:, 3] + forces[:, 5] - - N = len(fx) - if mode == "fy": - return fy.reshape(1, N) - elif mode == "fx": - return fx.reshape(1, N) - elif mode == "joint": - return np.vstack([fx, fy]) - else: - raise ValueError(f"Unknown force observable mode: {mode}") - - -def build_field_matrix(ux: np.ndarray, uy: np.ndarray) -> np.ndarray: - """Stack velocity field snapshots into a snapshot matrix. - - Parameters - ---------- - ux, uy : (N, NY, NX) ndarray — field snapshots in legacy (NY, NX) order. - - Returns - ------- - Q : (2 * NX * NY, N) ndarray — snapshot matrix for POD. - """ - N = ux.shape[0] - q_list = [] - for s in range(N): - q_list.append(np.concatenate([ux[s].ravel(), uy[s].ravel()])) - return np.column_stack(q_list).astype(np.float64) - - -def project_into_basis(ux: np.ndarray, uy: np.ndarray, - modes: np.ndarray, mean_f: np.ndarray) -> np.ndarray: - """Project velocity fields onto a POD basis. - - Returns POD coefficients array of shape (r, N). - """ - Q = build_field_matrix(ux, uy) - return modes.T @ (Q - mean_f[:, None]).astype(np.float64) - - -# --------------------------------------------------------------------------- -# Harmonics analysis for illusion -# --------------------------------------------------------------------------- - -def analyze_harmonics(states: np.ndarray, n_harmonics: int = 5) -> list: - """FFT harmonics analysis. Returns list of dicts per channel.""" - N, D = states.shape - result = [] - for d in range(D): - y = states[:, d] - fft_coef = np.fft.rfft(y) - freqs = np.fft.rfftfreq(N, d=1) - amps = 2.0 * np.abs(fft_coef) / N - phases = np.angle(fft_coef) - idx = np.argsort(amps[1:])[::-1][:n_harmonics] + 1 - harmonics = { - 'dc': float(np.real(fft_coef[0]) / N), - 'amps': amps[idx].tolist(), - 'freqs': freqs[idx].tolist(), - 'phases': phases[idx].tolist(), - } - result.append(harmonics) - return result - - -def gen_target_states_at(t, harmonics): - """Reconstruct target observable at step index t from harmonics.""" - t = np.asarray(t) - D = len(harmonics) - result = np.zeros((t.size, D), dtype=np.float32) - for d, h in enumerate(harmonics): - val = np.full(t.shape, h['dc'], dtype=np.float32) - for amp, freq, phase in zip(h['amps'], h['freqs'], h['phases']): - val += amp * np.cos(2 * np.pi * freq * t + phase) - result[:, d] = val - if result.shape[0] == 1: - return result[0] - return result diff --git a/src/drl_pinball/acquisition.py b/src/drl_pinball/acquisition.py new file mode 100644 index 0000000..5476f4b --- /dev/null +++ b/src/drl_pinball/acquisition.py @@ -0,0 +1,575 @@ +"""CPU-only shared acquisition contracts; imports no solver/GPU dependencies.""" +from __future__ import annotations + +import csv +import os +import shutil +import tempfile +import uuid +from pathlib import Path + +import numpy as np + +from .legacy_test.core.dtw_metrics import calc_dtw_sim, calc_lag + +TAU = 2.0 * np.pi +EIGHT_PHASES = np.arange(8, dtype=np.float64) * TAU / 8.0 +EXPECTED_OPTANE_MOUNT = Path("/home/frank14f/optane") +EXPECTED_REPRODUCTION_ROOT = ( + EXPECTED_OPTANE_MOUNT / "DynamisLab" / "drl_pinball" / "reproduction" +) + + +def default_reproduction_mapping(repo_root=None): + """Return the repository-side default without creating its symlink.""" + root = Path(repo_root) if repo_root is not None else Path(__file__).resolve().parents[2] + return root / "src" / "drl_pinball" / "data" / "reproduction" + + +def _existing_ancestor(path): + candidate = Path(path) + while not candidate.exists(): + if candidate.parent == candidate: + raise FileNotFoundError(path) + candidate = candidate.parent + return candidate + + +def validate_output_storage( + *, + repo_mapping=None, + output_root=None, + expected_root=EXPECTED_REPRODUCTION_ROOT, + optane_mount=EXPECTED_OPTANE_MOUNT, + min_free_bytes=0, + is_mount=os.path.ismount, + disk_usage=shutil.disk_usage, + stat=os.stat, +): + """Validate the Optane mapping and output filesystem without creating paths.""" + mount = Path(optane_mount).resolve() + expected = Path(expected_root).resolve() + mapping = default_reproduction_mapping() if repo_mapping is None else Path(repo_mapping) + output = mapping if output_root is None else Path(output_root) + if type(min_free_bytes) is not int or min_free_bytes < 0: + raise ValueError("min_free_bytes must be a non-negative integer") + if not is_mount(mount): + raise ValueError(f"Optane backing path is not a mount: {mount}") + if expected != mount and mount not in expected.parents: + raise ValueError("expected reproduction root must be under the Optane mount") + if not mapping.is_symlink(): + raise ValueError(f"repository mapping must be a symlink: {mapping}") + try: + resolved_mapping = mapping.resolve(strict=True) + except FileNotFoundError as exc: + raise ValueError(f"repository mapping is broken: {mapping}") from exc + if resolved_mapping != expected: + raise ValueError( + f"repository mapping must resolve exactly to {expected}, got {resolved_mapping}" + ) + free_bytes = disk_usage(mount).free + if free_bytes < min_free_bytes: + raise OSError( + f"Optane free space {free_bytes} is below required {min_free_bytes} bytes" + ) + mount_device = stat(mount).st_dev + if stat(expected).st_dev != mount_device: + raise ValueError("expected reproduction root is not on the Optane filesystem") + output_device = stat(_existing_ancestor(output.resolve(strict=False))).st_dev + if output_device != mount_device: + raise ValueError("output root is not on the Optane filesystem") + return { + "repo_mapping": mapping, + "output_root": output, + "resolved_output_root": output.resolve(strict=False), + "expected_root": expected, + "optane_mount": mount, + "device": mount_device, + "free_bytes": free_bytes, + } + + +def _path_component(value, name): + text = os.fspath(value) + if not text or text in {".", ".."} or Path(text).name != text: + raise ValueError(f"{name} must be one non-empty path component") + return text + + +def prepare_role_output(storage, pipeline, case, role, *, overwrite=False): + """Create a hidden same-filesystem staging directory for one role transaction.""" + root = Path(storage["resolved_output_root"]) + pipeline = _path_component(pipeline, "pipeline") + case = _path_component(case, "case") + role = _path_component(role, "role") + pipeline_dir = root / pipeline + case_dir = pipeline_dir / case + final_role_dir = case_dir / role + root.mkdir(parents=True, exist_ok=True) + if os.stat(root).st_dev != storage["device"]: + raise ValueError("output root is no longer on the validated Optane filesystem") + for directory, label in ((pipeline_dir, "pipeline"), (case_dir, "case")): + if directory.is_symlink(): + raise ValueError(f"{label} directory must not be a symlink") + directory.mkdir(exist_ok=True) + if final_role_dir.is_symlink(): + raise ValueError("role directory must not be a symlink") + if final_role_dir.exists(): + if not overwrite: + raise FileExistsError(final_role_dir) + if not final_role_dir.is_dir(): + raise ValueError("role output must be a directory") + staging_dir = Path(tempfile.mkdtemp(prefix=f".{role}.staging-", dir=case_dir)) + scratch = staging_dir / "scratch" + scratch.mkdir() + if os.stat(staging_dir).st_dev != storage["device"]: + shutil.rmtree(staging_dir) + raise ValueError("role staging is not on the validated Optane filesystem") + return { + "role_dir": staging_dir, + "staging_dir": staging_dir, + "final_role_dir": final_role_dir, + "scratch_root": scratch, + "overwrite": bool(overwrite), + } + + +def publish_role_output(prepared): + """Atomically expose a complete staged role, restoring prior output on failure.""" + staging = Path(prepared["staging_dir"]) + final = Path(prepared["final_role_dir"]) + overwrite = bool(prepared["overwrite"]) + if staging.is_symlink() or not staging.is_dir(): + raise ValueError("role staging must be an owned directory") + backup = final.with_name(f".{final.name}.backup-{uuid.uuid4().hex}") + moved_old = False + try: + if final.exists(): + if not overwrite: + raise FileExistsError(final) + if final.is_symlink() or not final.is_dir(): + raise ValueError("existing role output must be a non-symlink directory") + os.rename(final, backup) + moved_old = True + os.rename(staging, final) + except Exception: + if moved_old and backup.exists() and not final.exists(): + os.rename(backup, final) + raise + if moved_old: + shutil.rmtree(backup) + directory_fd = os.open(final.parent, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + return final + + +def _times(values, name): + out = np.asarray(values, dtype=np.float64) + if out.ndim != 1 or len(out) < 2 or not np.all(np.isfinite(out)): + raise ValueError(f"{name} must be a finite one-dimensional array") + if np.any(np.diff(out) <= 0): + raise ValueError(f"{name} must be strictly increasing") + return out + + +def assign_phase(times, sensors): + """Use interpolated ``uy[i] <= 0 < uy[i+1]`` rising crossings.""" + time, sensor = _times(times, "times"), np.asarray(sensors) + if sensor.ndim != 2 or sensor.shape[0] != len(time) or sensor.shape[1] < 4: + raise ValueError("sensors must have shape (time, >=4)") + uy = np.asarray(sensor[:, 3], dtype=np.float64) + if not np.all(np.isfinite(uy)): + raise ValueError("center uy must be finite") + left = np.flatnonzero((uy[:-1] <= 0.0) & (uy[1:] > 0.0)) + if len(left) < 2: + raise ValueError("at least two rising crossings are required") + weight = -uy[left] / (uy[left + 1] - uy[left]) + crossings = time[left] + weight * (time[left + 1] - time[left]) + cycle = np.searchsorted(crossings, time, side="right") - 1 + valid = (cycle >= 0) & (cycle < len(crossings) - 1) & (time < crossings[-1]) + ids = np.full(len(time), -1, dtype=np.int64) + phase = np.full(len(time), np.nan, dtype=np.float64) + ids[valid] = cycle[valid] + start = crossings[cycle[valid]] + phase[valid] = TAU * (time[valid] - start) / (crossings[cycle[valid] + 1] - start) + return {"phase": phase, "cycle_id": ids, "crossing_times": crossings} + + +def select_phase_fields(field_times, crossing_times, phases=EIGHT_PHASES): + """Select one boundary globally for each target phase across complete cycles.""" + times = _times(field_times, "field_times") + crossings = _times(crossing_times, "crossing_times") + targets = np.asarray(phases, dtype=np.float64) + if targets.shape != (8,) or not np.all(np.isfinite(targets)): + raise ValueError("phases must contain exactly eight finite targets") + targets = np.mod(targets, TAU) + + candidate_index = [] + candidate_cycle = [] + candidate_phase = [] + for cycle_id, (start, stop) in enumerate(zip(crossings[:-1], crossings[1:])): + indices = np.flatnonzero((times >= start) & (times < stop)) + candidate_index.extend(indices.tolist()) + candidate_cycle.extend([cycle_id] * len(indices)) + candidate_phase.extend((TAU * (times[indices] - start) / (stop - start)).tolist()) + if not candidate_index: + raise ValueError("no field boundaries fall within complete cycles") + candidate_index = np.asarray(candidate_index, dtype=np.int64) + candidate_cycle = np.asarray(candidate_cycle, dtype=np.int64) + candidate_phase = np.asarray(candidate_phase, dtype=np.float64) + + selected = np.empty(8, dtype=np.int64) + cycle_ids = np.empty(8, dtype=np.int64) + actual = np.empty(8, dtype=np.float64) + errors = np.empty(8, dtype=np.float64) + for slot, target in enumerate(targets): + signed = (candidate_phase - target + np.pi) % TAU - np.pi + winner = int(np.lexsort((candidate_index, np.abs(signed)))[0]) + selected[slot] = candidate_index[winner] + cycle_ids[slot] = candidate_cycle[winner] + actual[slot] = candidate_phase[winner] + errors[slot] = signed[winner] + return { + "field_indices": selected, + "cycle_id": cycle_ids, + "target_phase": targets, + "actual_phase": actual, + "phase_error": errors, + } + + +def pooled_phase_bins(phase, values, *, bins=32): + """Pool named variables by phase and retain their source names.""" + phi = np.asarray(phase, dtype=np.float64) + if phi.ndim != 1 or type(bins) is not int or bins < 1: + raise ValueError("phase must be one-dimensional and bins must be positive") + if not isinstance(values, dict) or not values: + raise ValueError("values must be a non-empty named mapping") + data = {name: np.asarray(value) for name, value in values.items()} + if any(not name or value.ndim < 1 or value.shape[0] != len(phi) or + value.dtype.kind not in "iuf" for name, value in data.items()): + raise ValueError("named numeric values must share the phase dimension") + valid = np.isfinite(phi) + index = np.flatnonzero(valid)[np.argsort(np.mod(phi[valid], TAU), kind="stable")] + sorted_phase = np.mod(phi[index], TAU) + slot = np.minimum((sorted_phase / TAU * bins).astype(np.int64), bins - 1) + count = np.bincount(slot, minlength=bins).astype(np.int64) + pooled, mean, std = {}, {}, {} + for name, value in data.items(): + pooled[name] = value[index] + flat = np.asarray(pooled[name], dtype=np.float64).reshape(len(index), -1) + current_mean = np.full((bins, flat.shape[1]), np.nan) + current_std = np.full_like(current_mean, np.nan) + for bin_id in range(bins): + rows = flat[slot == bin_id] + for column in range(flat.shape[1]): + finite = rows[np.isfinite(rows[:, column]), column] + if len(finite): + current_mean[bin_id, column] = finite.mean() + current_std[bin_id, column] = finite.std() + shape = (bins,) + value.shape[1:] + mean[name], std[name] = current_mean.reshape(shape), current_std.reshape(shape) + return { + "phase_center": (np.arange(bins) + 0.5) * TAU / bins, + "phase_bin_count": count, + "sorted_index": index, + "sorted_phase": sorted_phase, + "pooled": pooled, + "mean": mean, + "std": std, + } + + +def _validate_dtw( + target, state, cycle_length, target_channels, state_channels, lag_channel, epsilon +): + target = np.asarray(target, dtype=np.float64) + state = np.asarray(state, dtype=np.float64) + target_channels, state_channels = tuple(target_channels), tuple(state_channels) + if target.ndim != 2 or state.ndim != 2 or cycle_length < 1: + raise ValueError("invalid DTW dimensions") + if len(target_channels) != 6 or len(state_channels) != 6 or lag_channel not in range(6): + raise ValueError("exactly six channels and lag_channel in 0..5 are required") + if len(target) < 2 * cycle_length: + raise ValueError("target must contain two cycle windows") + if epsilon <= 0 or not np.isfinite(epsilon): + raise ValueError("epsilon must be finite and positive") + target6, state6 = target[:, target_channels], state[:, state_channels] + if not np.all(np.isfinite(target6)) or not np.all(np.isfinite(state6)): + raise ValueError("DTW channels must be finite") + scale = np.maximum(np.max(np.abs(target6), axis=0), epsilon) + return target6 / scale, state6 / scale, scale + + +def dual_cycle_dtw( + target, + state, + native_reward_dtw=None, + *, + cycle_length, + target_channels=tuple(range(6)), + state_channels=tuple(range(6)), + lag_channel=1, + epsilon=1e-12, +): + """Preserve native reward DTW and compute a separate rolling normalized series.""" + target6, state6, scale = _validate_dtw( + target, + state, + cycle_length, + target_channels, + state_channels, + lag_channel, + epsilon, + ) + similarity = np.full(len(state6), np.nan, dtype=np.float64) + per_channel = np.full((len(state6), 6), np.nan, dtype=np.float64) + lag = np.full(len(state6), -1, dtype=np.int64) + target_reference = target6[cycle_length : 2 * cycle_length] + for boundary in range(cycle_length - 1, len(state6)): + state_window = state6[boundary - cycle_length + 1 : boundary + 1] + current_lag = calc_lag(target_reference[:, lag_channel], state_window[:, lag_channel]) + scores = [ + calc_dtw_sim( + np.roll(target6[:, channel], -current_lag)[cycle_length : 2 * cycle_length], + state_window[:, channel], + ) + for channel in range(6) + ] + lag[boundary] = current_lag + per_channel[boundary] = scores + similarity[boundary] = np.mean(scores) + + if native_reward_dtw is None: + native = np.full(len(state6), np.nan, dtype=np.float64) + else: + native = np.asarray(native_reward_dtw) + if native.ndim != 1 or len(native) != len(state6) or native.dtype.kind not in "iuf": + raise ValueError("native_reward_dtw must be a numeric per-boundary series") + return { + "native_reward_dtw": native, + "target_normalized_dtw": similarity, + "target_normalized_dtw_per_channel": per_channel, + "target_normalized_dtw_lag": lag, + "metadata": { + "definition": "target-channel max-abs normalized rolling six-channel cycle DTW", + "cycle_length": int(cycle_length), + "target_channels": np.asarray(target_channels, dtype=np.int64), + "state_channels": np.asarray(state_channels, dtype=np.int64), + "lag_channel": int(lag_channel), + "epsilon": float(epsilon), + "scale": scale, + }, + } + + +def _arrays(arrays): + data = {name: np.asarray(value) for name, value in arrays.items()} + if not data or any( + not name or value.ndim < 1 or value.dtype.kind not in "iuf" for name, value in data.items() + ): + raise ValueError("named numeric non-scalar arrays required") + rows = {value.shape[0] for value in data.values()} + if len(rows) != 1: + raise ValueError("equal first dimensions required") + return data, rows.pop() + + +def write_npz_csv(npz_path, csv_path, arrays): + data, row_count = _arrays(arrays) + npz, csv_path = Path(npz_path), Path(csv_path) + npz.parent.mkdir(parents=True, exist_ok=True) + csv_path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(npz, **data) + columns = [] + for name, value in data.items(): + flat = value.reshape(row_count, -1) + labels = [name] if flat.shape[1] == 1 else [f"{name}_{i}" for i in range(flat.shape[1])] + columns.extend(zip(labels, flat.T)) + with csv_path.open("w", newline="", encoding="ascii") as stream: + writer = csv.writer(stream) + writer.writerow([name for name, _ in columns]) + writer.writerows([[column[row] for _, column in columns] for row in range(row_count)]) + return npz, csv_path + + +def write_boundary_artifacts(npz_path, csv_path, arrays): + return write_npz_csv(npz_path, csv_path, arrays) + + +def write_phase_cycle_artifacts(npz_path, csv_path, folded): + """Write named pooled samples and named 32-bin statistics.""" + required = {"sorted_index", "sorted_phase", "phase_center", "phase_bin_count", + "pooled", "mean", "std"} + if not required.issubset(folded): + raise ValueError(f"phase-cycle artifact requires {sorted(required)}") + names = tuple(folded["pooled"]) + if not names or set(names) != set(folded["mean"]) or set(names) != set(folded["std"]): + raise ValueError("phase-cycle variables must be named consistently") + index = np.asarray(folded["sorted_index"]) + phase = np.asarray(folded["sorted_phase"]) + centers = np.asarray(folded["phase_center"]) + count = np.asarray(folded["phase_bin_count"]) + if len(centers) != 32 or len(count) != 32: + raise ValueError("phase-cycle artifact requires exactly 32 bins") + arrays = {"sorted_index": index, "sorted_phase": phase, + "phase_center": centers, "phase_bin_count": count} + widths = {} + for name in names: + pooled = np.asarray(folded["pooled"][name]) + mean = np.asarray(folded["mean"][name]) + std = np.asarray(folded["std"][name]) + if pooled.shape[0] != len(index) or mean.shape[0] != 32 or std.shape != mean.shape: + raise ValueError(f"invalid phase-cycle shapes for {name}") + arrays[f"{name}_pooled"] = pooled + arrays[f"{name}_mean"] = mean + arrays[f"{name}_std"] = std + widths[name] = pooled.reshape(len(index), -1).shape[1] + npz, csv_path = Path(npz_path), Path(csv_path) + npz.parent.mkdir(parents=True, exist_ok=True) + csv_path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(npz, **arrays) + labels = lambda name, suffix: [f"{name}_{i}_{suffix}" for i in range(widths[name])] + header = ["representation", "row_index", "original_index", "phase", "phase_bin_count"] + for name in names: + header += labels(name, "pooled") + labels(name, "mean") + labels(name, "std") + with csv_path.open("w", newline="", encoding="ascii") as stream: + writer = csv.writer(stream) + writer.writerow(header) + for row, (original, current_phase) in enumerate(zip(index, phase)): + values = ["pooled", row, original, current_phase, np.nan] + for name in names: + pooled = arrays[f"{name}_pooled"].reshape(len(index), -1)[row] + values += [*pooled, *([np.nan] * (2 * widths[name]))] + writer.writerow(values) + for row in range(32): + values = ["bin", row, np.nan, centers[row], count[row]] + for name in names: + mean = arrays[f"{name}_mean"].reshape(32, -1)[row] + std = arrays[f"{name}_std"].reshape(32, -1)[row] + values += [*([np.nan] * widths[name]), *mean, *std] + writer.writerow(values) + return npz, csv_path + + +def write_folded_artifacts(npz_path, csv_path, folded): + """Compatibility name for the complete phase-cycle artifact writer.""" + return write_phase_cycle_artifacts(npz_path, csv_path, folded) + + +def publish_selected_fields(destination, arrays): + """Atomic no-replace NPZ publication at a caller-supplied path.""" + data, _ = _arrays(arrays) + target = Path(destination) + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + raise FileExistsError(target) + fd, name = tempfile.mkstemp(prefix=f".{target.name}.", suffix=".tmp", dir=target.parent) + temporary = Path(name) + try: + with os.fdopen(fd, "wb") as stream: + np.savez_compressed(stream, **data) + stream.flush() + os.fsync(stream.fileno()) + os.link(temporary, target) + directory_fd = os.open(target.parent, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + return target + finally: + temporary.unlink(missing_ok=True) + + +def create_scratch(root, *, prefix="acquisition-"): + base = Path(root) + base.mkdir(parents=True, exist_ok=True) + return Path(tempfile.mkdtemp(prefix=prefix, dir=base)) + + +def cleanup_scratch(path, *, root): + base, candidate = Path(root).resolve(), Path(path) + if candidate.is_symlink(): + raise ValueError("scratch path must not be a symlink") + resolved = candidate.resolve() + if resolved == base or base not in resolved.parents: + raise ValueError("scratch path must be a child of root") + if resolved.exists(): + shutil.rmtree(resolved) + + +LEGACY_FLUID_BIT = np.uint8(0b00000001) + + +def decode_legacy_physical_velocity(ddf, *, flags, nx, ny, rho_ref=1.0, + include_pressure=False): + """Decode Legacy D2Q9 ``q/RHO_ref`` in canonical ``(ny, nx)`` order. + + Legacy uses the documented constant ``RHO_ref = 1`` by default; this is not + the local population sum. Fluid membership is exactly flag bit 0, including + sensor cells carrying that bit. Nonfluid populations are never inspected. + """ + if type(nx) is not int or type(ny) is not int or nx < 1 or ny < 1: + raise ValueError("nx and ny must be positive integers") + if type(rho_ref) not in (int, float) or not np.isfinite(rho_ref) or rho_ref <= 0: + raise ValueError("rho_ref must be positive and finite") + value = np.asarray(ddf) + if value.ndim != 1 or value.size != 9 * nx * ny or value.dtype.kind not in "iuf": + raise ValueError("expected flat 9*ny*nx numeric D2Q9 storage") + solver_flags = np.asarray(flags) + if solver_flags.shape != (nx, ny): + raise ValueError("exact Legacy solver flags/mask in (nx, ny) order required") + if solver_flags.dtype == np.uint8: + fluid_xy = (solver_flags & LEGACY_FLUID_BIT) != 0 + elif solver_flags.dtype == np.bool_: + fluid_xy = solver_flags + else: + raise ValueError("Legacy solver flags must be uint8 or exact fluid mask must be bool") + fluid = fluid_xy.T + f = np.asarray(value, dtype=np.float64).reshape(9, ny, nx) + if not np.isfinite(f[:, fluid]).all(): + raise ValueError("Legacy fluid populations contain non-finite values") + selected = f[:, fluid] + ux = np.zeros((ny, nx), dtype=np.float64) + uy = np.zeros((ny, nx), dtype=np.float64) + ux[fluid] = (selected[1] + selected[5] + selected[8] + - selected[3] - selected[6] - selected[7]) / rho_ref + uy[fluid] = (selected[2] + selected[5] + selected[6] + - selected[4] - selected[7] - selected[8]) / rho_ref + result = {"ux": ux, "uy": uy} + if include_pressure: + pressure = np.zeros((ny, nx), dtype=np.float64) + pressure[fluid] = selected.sum(axis=0) / 3.0 + result["pressure"] = pressure + return result + + +def decode_legacy_ddf(ddf, *, flags, nx, ny, rho_ref=1.0, + include_pressure=False): + """Deprecated compatibility alias for :func:`decode_legacy_physical_velocity`.""" + import warnings + warnings.warn("decode_legacy_ddf is deprecated; use decode_legacy_physical_velocity", + DeprecationWarning, stacklevel=2) + return decode_legacy_physical_velocity( + ddf, flags=flags, nx=nx, ny=ny, rho_ref=rho_ref, + include_pressure=include_pressure, + ) + + +def validate_modern_fields(fields, *, nx=None, ny=None): + if set(fields) != {"rho", "ux", "uy"}: + raise ValueError("exactly rho, ux, uy required") + out = {name: np.asarray(fields[name]) for name in ("rho", "ux", "uy")} + shapes = {value.shape for value in out.values()} + if len(shapes) != 1 or any(value.ndim != 2 for value in out.values()): + raise ValueError("shared 2-D shape required") + shape = next(iter(shapes)) + if (ny is not None and shape[0] != ny) or (nx is not None and shape[1] != nx): + raise ValueError(f"canonical shape ({ny}, {nx}) required") + if any(value.dtype.kind != "f" or not np.all(np.isfinite(value)) for value in out.values()): + raise ValueError("finite floating arrays required") + return out diff --git a/src/drl_pinball/legacy_test/core/dtw_metrics.py b/src/drl_pinball/legacy_test/core/dtw_metrics.py index 424bbef..546e838 100644 --- a/src/drl_pinball/legacy_test/core/dtw_metrics.py +++ b/src/drl_pinball/legacy_test/core/dtw_metrics.py @@ -1,23 +1,261 @@ -# legacy_test/core/dtw_metrics.py -"""DTW-based similarity metrics — imported from reproduce/core/ for consistency.""" +"""Canonical legacy DTW and harmonic utilities. -import os -import sys +The formulas mirror the active non-archive ``legacy_env`` implementations and +remain local so legacy evaluation has no dependency on deleted reproduce code. +Preserved historical scene semantics: +- Karman cloak: lag from sensor1 Uy, then DTW on 6 sensor channels +- Illusion: lag from target[:,3] vs state[:,1], then DTW on 6 sensor channels (offset +2) +- Vortex: no lag, roll by current_step+1 +- Erase: lag from force channels, uses enhanced calc_sim +""" +from __future__ import annotations -_REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) -_SRC = os.path.join(_REPO, "src") -for p in [_REPO, _SRC]: - if p not in sys.path: - sys.path.insert(0, p) +from typing import Optional -# Re-export from the verified reproduce/core/dtw_metrics module. -from drl_pinball.reproduce.core.dtw_metrics import ( # noqa: E402, F401 - calc_lag, - calc_dtw_sim, - calc_dtw_sim_enhanced, - compute_similarity_karman_cloak, - compute_similarity_vortex, - compute_similarity_illusion, - analyze_harmonics, - gen_target_states_at, -) +import numpy as np + + +def calc_lag(target: np.ndarray, state: np.ndarray) -> int: + """Cross-correlation lag between target and state sequences. + + Args: + target: shape ``(N,)`` reference signal. + state: shape ``(M,)`` observed signal. + + Returns: + Integer lag (positive = state is ahead of target). + """ + t_mean = np.mean(target) + s_mean = np.mean(state) + correlation = np.correlate(target - t_mean, state - s_mean, mode="full") + lags = np.arange(-len(target) + 1, len(target)) + return int(lags[np.argmax(correlation)]) + + +def calc_dtw_sim(target: np.ndarray, state: np.ndarray) -> float: + """Standard DTW similarity (used by cloak, illusion, vortex, reduce_obs). + + Args: + target: shape ``(N,)`` reference. + state: shape ``(M,)`` observed. + + Returns: + Similarity in [0, 1], where 1 = perfect match. + """ + n, m = len(target), len(state) + dtw = np.full((n + 1, m + 1), np.inf) + dtw[0, 0] = 0.0 + for i in range(1, n + 1): + for j in range(1, m + 1): + cost = abs(float(target[i - 1]) - float(state[j - 1])) + last_min = min(dtw[i - 1, j], dtw[i, j - 1], dtw[i - 1, j - 1]) + dtw[i, j] = cost + last_min + return float(1.0 - dtw[n, m] / float(n)) + + +def calc_dtw_sim_enhanced(target: np.ndarray, state: np.ndarray) -> float: + """Enhanced DTW similarity with amplitude-ratio and mean components. + + Used by the legacy erase env. Combines: + - 80% standard DTW (max-cost normalised) + - 10% amplitude ratio (min_std/max_std) + - 10% mean similarity (1/(1 + diff/scale*10)) + + Args: + target: shape ``(N,)`` reference. + state: shape ``(M,)`` observed. + + Returns: + Combined similarity in [0, 1]. + """ + target_arr = np.asarray(target, dtype=np.float64) + state_arr = np.asarray(state, dtype=np.float64) + n, m = len(target_arr), len(state_arr) + + # Amplitude ratio component + t_std = max(np.std(target_arr), 1e-8) + s_std = max(np.std(state_arr), 1e-8) + amplitude_ratio = float(min(t_std, s_std) / max(t_std, s_std)) + + # Mean similarity component + mean_diff = abs(np.mean(target_arr) - np.mean(state_arr)) + max_scale = max(abs(np.mean(target_arr)), abs(np.mean(state_arr)), 1e-8) + mean_similarity = 1.0 / (1.0 + mean_diff / max_scale * 10.0) + + # DTW with max-possible-cost normalisation + dtw = np.full((n + 1, m + 1), np.inf) + dtw[0, 0] = 0.0 + for i in range(1, n + 1): + for j in range(1, m + 1): + cost = abs(target_arr[i - 1] - state_arr[j - 1]) + last_min = min(dtw[i - 1, j], dtw[i, j - 1], dtw[i - 1, j - 1]) + dtw[i, j] = cost + last_min + + max_possible_cost = max(np.max(np.abs(target_arr)), np.max(np.abs(state_arr)), 1e-8) + dtw_distance = dtw[n, m] / (n * max_possible_cost) + dtw_sim = max(0.0, 1.0 - dtw_distance) + + return float(0.8 * dtw_sim + 0.1 * amplitude_ratio + 0.1 * mean_similarity) + + +def compute_similarity_karman_cloak( + target_states: np.ndarray, + fifo_states: np.ndarray, + conv_len: int = 30, +) -> float: + """Compute DTW similarity for Karman cloak (standard pattern). + + Matches legacy code: + 1. Compute lag from middle sensor (index 1) Uy component + 2. For all 6 sensor channels, roll target by lag, compute DTW, average + + Args: + target_states: shape ``(FIFO_LEN, 6)`` target sensor data. + fifo_states: shape ``(FIFO_LEN, 6)`` current FIFO sensor data. + conv_len: Convergence window length (default 30). + + Returns: + Average similarity over 6 channels in [0, 1]. + """ + target = np.asarray(target_states, dtype=np.float64) + state = np.asarray(fifo_states, dtype=np.float64) + + id_sens = 1 # middle sensor + target_seq = target[conv_len:2 * conv_len, id_sens] + state_seq = state[-conv_len:, id_sens] + lag = calc_lag(target_seq, state_seq) + + similarities = 0.0 + for i in range(6): + t_seq = np.roll(target[:, i], -lag)[conv_len:2 * conv_len] + s_seq = state[-conv_len:, i] + similarities += calc_dtw_sim(t_seq, s_seq) + return float(similarities / 6.0) + + +def compute_similarity_vortex( + target_states: np.ndarray, + fifo_states: np.ndarray, + current_step: int, + conv_len: int = 30, +) -> float: + """Compute DTW similarity for vortex (no lag, roll by current_step+1). + + Matches legacy vortex env. + + Args: + target_states: shape ``(FIFO_LEN, 6)`` target sensor data. + fifo_states: shape ``(FIFO_LEN, 6)`` current FIFO data. + current_step: The current simulation step index. + conv_len: Convergence window length (default 30). + + Returns: + Average similarity over 6 channels. + """ + target = np.asarray(target_states, dtype=np.float64) + state = np.asarray(fifo_states, dtype=np.float64) + + similarities = 0.0 + for i in range(6): + t_seq = np.roll(target[-conv_len:, i], -current_step - 1) + s_seq = state[-conv_len:, i] + similarities += calc_dtw_sim(t_seq, s_seq) + return float(similarities / 6.0) + + +def compute_similarity_illusion( + target_states: np.ndarray, + fifo_states: np.ndarray, + conv_len: int = 36, +) -> float: + """Compute DTW similarity for illusion. + + Matches legacy imit env: + 1. lag from target[:, id_sens+2] vs state[:, id_sens] (offset by 2) + 2. For 6 channels, target uses [:, i+2] offset + + Args: + target_states: shape ``(FIFO_LEN, 8)`` (2 force + 6 sensor channels). + fifo_states: shape ``(FIFO_LEN, 6)`` current FIFO (6 sensors only). + conv_len: Convergence window length (default 36). + + Returns: + Average similarity over 6 channels. + """ + target = np.asarray(target_states, dtype=np.float64) + state = np.asarray(fifo_states, dtype=np.float64) + + id_sens = 1 + t_seq_ref = target[conv_len:2 * conv_len, id_sens + 2] + s_seq_ref = state[-conv_len:, id_sens] + lag = calc_lag(t_seq_ref, s_seq_ref) + + similarities = 0.0 + for i in range(6): + t_seq = np.roll(target[:, i + 2], -lag)[conv_len:2 * conv_len] + s_seq = state[-conv_len:, i] + similarities += calc_dtw_sim(t_seq, s_seq) + return float(similarities / 6.0) + + +# --------------------------------------------------------------------------- +# Harmonics analysis (used by illusion) +# --------------------------------------------------------------------------- +def analyze_harmonics( + states: np.ndarray, + n_harmonics: int = 5, +) -> list: + """FFT-based harmonic analysis of multi-channel time series. + + Matches legacy ``analyze_harmonics()``. + + Args: + states: shape ``(N, D)`` time-series data. + n_harmonics: Number of harmonics to extract per channel. + + Returns: + List of D dicts, each with keys: + dc: float (DC component) + amps: (n_harmonics,) array + freqs: (n_harmonics,) array + phases: (n_harmonics,) array + """ + N, D = states.shape + result = [] + for d in range(D): + y = states[:, d] + fft_coef = np.fft.rfft(y) + freqs = np.fft.rfftfreq(N, d=1) + amps = 2.0 * np.abs(fft_coef) / N + phases = np.angle(fft_coef) + idx = np.argsort(amps[1:])[::-1][:n_harmonics] + 1 + harmonics = { + "dc": float(np.real(fft_coef[0]) / N), + "amps": np.array(amps[idx], dtype=np.float32), + "freqs": np.array(freqs[idx], dtype=np.float32), + "phases": np.array(phases[idx], dtype=np.float32), + } + result.append(harmonics) + return result + + +def gen_target_states_at(t, harmonics) -> np.ndarray: + """Reconstruct target state at time step t from harmonics. + + Matches legacy ``gen_target_states_at()``. + + Args: + t: Integer step index. + harmonics: Output from ``analyze_harmonics()``. + + Returns: + shape ``(D,)`` reconstructed state vector. + """ + D = len(harmonics) + result = np.zeros(D, dtype=np.float32) + for d, h in enumerate(harmonics): + val = float(h["dc"]) + for amp, freq, phase in zip(h["amps"], h["freqs"], h["phases"]): + val += amp * np.cos(2.0 * np.pi * freq * t + phase) + result[d] = val + return result diff --git a/tests/test_drl_pinball_acquisition.py b/tests/test_drl_pinball_acquisition.py new file mode 100644 index 0000000..d036eca --- /dev/null +++ b/tests/test_drl_pinball_acquisition.py @@ -0,0 +1,345 @@ +import csv +import os +import subprocess +import sys +from collections import namedtuple +from pathlib import Path + +import numpy as np +import pytest + +from drl_pinball.acquisition import ( + assign_phase, + cleanup_scratch, + create_scratch, + default_reproduction_mapping, + decode_legacy_physical_velocity, + dual_cycle_dtw, + pooled_phase_bins, + prepare_role_output, + publish_role_output, + publish_selected_fields, + select_phase_fields, + validate_output_storage, + validate_modern_fields, + write_boundary_artifacts, + write_phase_cycle_artifacts, +) + +Usage = namedtuple("Usage", "total used free") + + +def _storage_tree(tmp_path): + mount = tmp_path / "optane" + expected = mount / "DynamisLab" / "drl_pinball" / "reproduction" + expected.mkdir(parents=True) + mapping = tmp_path / "repo" / "src" / "drl_pinball" / "data" / "reproduction" + mapping.parent.mkdir(parents=True) + mapping.symlink_to(expected, target_is_directory=True) + return mount, expected, mapping + + +def _validate_tmp_storage(tmp_path, **overrides): + mount, expected, mapping = _storage_tree(tmp_path) + kwargs = { + "repo_mapping": mapping, "expected_root": expected, "optane_mount": mount, + "min_free_bytes": 100, "is_mount": lambda path: path == mount, + "disk_usage": lambda path: Usage(1000, 100, 900), + } + kwargs.update(overrides) + return validate_output_storage(**kwargs), mount, expected, mapping + + +def test_default_reproduction_mapping_is_derivable(tmp_path): + assert default_reproduction_mapping(tmp_path) == ( + tmp_path / "src" / "drl_pinball" / "data" / "reproduction" + ) + + +def test_storage_validation_defaults_output_to_repo_mapping(tmp_path): + storage, _, expected, mapping = _validate_tmp_storage(tmp_path) + assert storage["output_root"] == mapping + assert storage["resolved_output_root"] == expected +def test_storage_validation_accepts_exact_mapping_and_configurable_same_device(tmp_path): + + mount, expected, mapping = _storage_tree(tmp_path) + output = mount / "alternate" / "campaign" + result = validate_output_storage( + repo_mapping=mapping, output_root=output, expected_root=expected, + optane_mount=mount, min_free_bytes=500, is_mount=lambda path: path == mount, + disk_usage=lambda path: Usage(1000, 100, 900), + ) + assert result["resolved_output_root"] == output + assert result["free_bytes"] == 900 + + +def test_storage_validation_rejects_mount_space_mapping_and_device_failures(tmp_path): + mount, expected, mapping = _storage_tree(tmp_path) + common = dict(repo_mapping=mapping, expected_root=expected, optane_mount=mount) + with pytest.raises(ValueError, match="not a mount"): + validate_output_storage(**common, is_mount=lambda path: False) + with pytest.raises(OSError, match="free space"): + validate_output_storage( + **common, min_free_bytes=901, is_mount=lambda path: True, + disk_usage=lambda path: Usage(1000, 100, 900), + ) + wrong = tmp_path / "wrong" + wrong.mkdir() + mapping.unlink() + mapping.symlink_to(wrong, target_is_directory=True) + with pytest.raises(ValueError, match="resolve exactly"): + validate_output_storage(**common, is_mount=lambda path: True) + + mapping.unlink() + mapping.symlink_to(expected, target_is_directory=True) + real_stat = os.stat + outside = tmp_path / "outside" + outside.mkdir() + + def different_device(path): + value = real_stat(path) + if Path(path) == outside: + return type("Stat", (), {"st_dev": value.st_dev + 1})() + return value + + with pytest.raises(ValueError, match="output root"): + validate_output_storage( + **common, output_root=outside / "new", is_mount=lambda path: True, + stat=different_device, + ) + + +def test_prepare_role_output_conflict_and_scoped_overwrite(tmp_path): + storage, _, _, _ = _validate_tmp_storage(tmp_path) + prepared = prepare_role_output(storage, "v5", "karman_re100", "controlled") + (prepared["role_dir"] / "old.txt").write_text("old") + published = publish_role_output(prepared) + sentinel = published / "old.txt" + sibling = published.parent / "target" + sibling.mkdir() + (sibling / "keep.txt").write_text("keep") + with pytest.raises(FileExistsError): + prepare_role_output(storage, "v5", "karman_re100", "controlled") + replaced = prepare_role_output(storage, "v5", "karman_re100", "controlled", overwrite=True) + assert sentinel.exists() + (replaced["role_dir"] / "new.txt").write_text("new") + final = publish_role_output(replaced) + assert not sentinel.exists() and (final / "new.txt").exists() + assert (sibling / "keep.txt").read_text() == "keep" + with pytest.raises(ValueError, match="path component"): + prepare_role_output(storage, "v5", "karman_re100", "../target", overwrite=True) + + +def test_prepare_rejects_symlink_parent_without_following_it(tmp_path): + storage, _, _, _ = _validate_tmp_storage(tmp_path) + target = tmp_path / "external-pipeline" + target.mkdir() + pipeline = storage["resolved_output_root"] / "v5" + pipeline.symlink_to(target, target_is_directory=True) + with pytest.raises(ValueError, match="pipeline directory"): + prepare_role_output(storage, "v5", "karman_re100", "controlled", overwrite=True) + assert list(target.iterdir()) == [] +def test_prepare_rejects_symlink_role_without_following_it(tmp_path): + + storage, _, _, _ = _validate_tmp_storage(tmp_path) + target = tmp_path / "do-not-delete" + target.mkdir() + sentinel = target / "keep.txt" + sentinel.write_text("keep") + role = storage["resolved_output_root"] / "legacy" / "karman_re100" / "zero" + role.parent.mkdir(parents=True) + role.symlink_to(target, target_is_directory=True) + with pytest.raises(ValueError, match="must not be a symlink"): + prepare_role_output(storage, "legacy", "karman_re100", "zero", overwrite=True) + assert sentinel.read_text() == "keep" + + +def test_phase_complete_interpolated_half_open(): + times = np.arange(13, dtype=float) + sensors = np.zeros((13, 6)) + sensors[:, 3] = np.sin(2 * np.pi * (times - 0.25) / 4) + result = assign_phase(times, sensors) + assert np.allclose(result["crossing_times"], [0.29289322, 4.29289322, 8.29289322]) + assert result["cycle_id"].tolist() == [-1] + [0] * 4 + [1] * 4 + [-1] * 4 + assert np.isnan(result["phase"][9]) + + +def test_crossing_exact_zero_and_plateau_are_single_crossings(): + times = np.arange(10, dtype=float) + sensors = np.zeros((10, 6)) + sensors[:, 3] = [-1, 0, 0, 1, 1, -1, 0, 0, 1, 1] + result = assign_phase(times, sensors) + assert np.array_equal(result["crossing_times"], [2.0, 7.0]) + assert result["cycle_id"].tolist() == [-1, -1, 0, 0, 0, 0, 0, -1, -1, -1] + assert result["phase"][2] == 0.0 + + +def test_selection_is_global_eight_with_metadata_and_index_tie(): + field_times = np.concatenate((np.arange(0.5, 4.0, 0.5), np.arange(4.0, 8.0, 0.5))) + result = select_phase_fields(field_times, np.array([0.0, 4.0, 8.0])) + assert result["field_indices"].shape == (8,) + assert np.array_equal(result["field_indices"], [7, 0, 1, 2, 3, 4, 5, 6]) + assert np.array_equal(result["cycle_id"], [1, 0, 0, 0, 0, 0, 0, 0]) + assert np.allclose(result["actual_phase"], np.arange(8) * np.pi / 4, atol=1e-15) + assert np.allclose(result["phase_error"], [0.0] * 8, atol=1e-15) + + +def test_pooled_32_bins_population_std_and_original_indices(): + result = pooled_phase_bins( + np.array([np.pi / 32, 3 * np.pi / 32, np.pi / 32, np.nan]), + {"sensors": np.array([[1.0, 10.0], [4.0, 40.0], [3.0, 30.0], [9.0, 90.0]])}, + ) + assert result["phase_bin_count"][:2].tolist() == [2, 1] + assert np.allclose(result["mean"]["sensors"][0], [2, 20]) + assert np.allclose(result["std"]["sensors"][0], [1, 10]) + assert result["sorted_index"].tolist() == [0, 2, 1] + assert np.array_equal(result["pooled"]["sensors"], [[1, 10], [3, 30], [4, 40]]) + + +def test_named_phase_pooling_preserves_rows_with_unavailable_nan_variables(): + phase = np.array([0.1, 0.2, 0.3]) + result = pooled_phase_bins(phase, { + "sensors": np.array([[1.0], [2.0], [3.0]]), + "reward_raw": np.full(3, np.nan), + }) + assert result["sorted_index"].tolist() == [0, 1, 2] + assert result["phase_bin_count"].sum() == 3 + assert np.all(np.isnan(result["mean"]["reward_raw"])) + assert np.any(np.isfinite(result["mean"]["sensors"])) + + +def test_dual_dtw_preserves_native_series_and_separates_rolling_normalized(): + step = np.arange(90) + target = np.column_stack( + [(channel + 1) * np.sin(2 * np.pi * step / 30 + channel) for channel in range(6)] + ) + native = np.linspace(0.125, 0.875, 90, dtype=np.float32) + native_bytes = native.tobytes() + result = dual_cycle_dtw(target, target, native, cycle_length=30, epsilon=1e-9) + assert result["native_reward_dtw"] is native + assert result["native_reward_dtw"].tobytes() == native_bytes + assert np.all(np.isnan(result["target_normalized_dtw"][:29])) + assert np.all(np.isfinite(result["target_normalized_dtw"][29:])) + assert np.allclose(result["target_normalized_dtw"][[29, 59, 89]], 1.0) + assert result["metadata"]["lag_channel"] == 1 + assert result["metadata"]["epsilon"] == 1e-9 + assert np.array_equal(result["metadata"]["target_channels"], np.arange(6)) + assert np.array_equal(result["metadata"]["state_channels"], np.arange(6)) + + +def test_dual_dtw_unavailable_native_is_nan_series(): + step = np.arange(60) + target = np.column_stack([np.sin(2 * np.pi * step / 20 + channel) for channel in range(6)]) + result = dual_cycle_dtw(target, target, cycle_length=20) + assert result["native_reward_dtw"].shape == (60,) + assert np.all(np.isnan(result["native_reward_dtw"])) + assert result["target_normalized_dtw"].shape == (60,) + + +def test_boundary_and_complete_phase_cycle_artifact_schemas(tmp_path): + boundary = { + "time": np.array([0.0, 1.0]), + "sensor": np.array([[1.0, np.nan], [2.0, 3.0]]), + "field": np.array([4.0, np.nan]), + } + npz_path, csv_path = write_boundary_artifacts( + tmp_path / "raw.npz", tmp_path / "raw.csv", boundary + ) + with np.load(npz_path, allow_pickle=False) as saved: + assert np.isnan(saved["sensor"][0, 1]) + with csv_path.open(newline="") as stream: + rows = list(csv.DictReader(stream)) + assert list(rows[0]) == ["time", "sensor_0", "sensor_1", "field"] + assert rows[0]["sensor_1"] == "nan" + + folded = pooled_phase_bins( + np.array([0.1, 0.2]), {"sensors": np.array([[1.0, 10.0], [3.0, 30.0]])} + ) + npz_path, csv_path = write_phase_cycle_artifacts( + tmp_path / "phase_cycle.npz", tmp_path / "phase_cycle.csv", folded + ) + with np.load(npz_path, allow_pickle=False) as saved: + assert set(saved.files) == {"sorted_index", "sorted_phase", "phase_center", + "phase_bin_count", "sensors_pooled", "sensors_mean", + "sensors_std"} + assert np.array_equal(saved["sorted_index"], [0, 1]) + assert np.array_equal(saved["sensors_pooled"], [[1.0, 10.0], [3.0, 30.0]]) + assert len(saved["phase_bin_count"]) == 32 + with csv_path.open(newline="") as stream: + rows = list(csv.DictReader(stream)) + assert {"sensors_0_pooled", "sensors_1_mean", "sensors_1_std", "phase_bin_count"}.issubset(rows[0]) + assert [row["representation"] for row in rows[:2]] == ["pooled", "pooled"] + assert all(row["representation"] == "bin" for row in rows[2:]) + assert len(rows[2:]) == 32 + assert [row["original_index"] for row in rows[:2]] == ["0", "1"] + + +def test_publication_and_cleanup(tmp_path): + root = tmp_path / "scratch" + scratch = create_scratch(root) + (scratch / "x").write_text("x") + destination = tmp_path / "pub" / "fields.npz" + publish_selected_fields(destination, {"ux": np.ones((2, 3, 4))}) + with pytest.raises(FileExistsError): + publish_selected_fields(destination, {"ux": np.ones((1, 1, 1))}) + cleanup_scratch(scratch, root=root) + assert not scratch.exists() + with pytest.raises(ValueError): + cleanup_scratch(root, root=root) + + +def test_legacy_physical_decoder_is_mask_aware_and_pressure_invariant(): + ny, nx = 2, 3 + populations = np.zeros((9, ny, nx), dtype=np.float32) + populations[1] = 0.01 + populations[2] = -0.02 + flags = np.ones((nx, ny), dtype=np.uint8) + flags[1, 0] = np.uint8(0b00010001) # sensor plus FLUID remains fluid + flags[2, 1] = np.uint8(0b00000010) + populations[:, 1, 2] = np.nan # ignored nonfluid garbage + + first = decode_legacy_physical_velocity( + populations.reshape(-1), flags=flags, nx=nx, ny=ny, + include_pressure=True, + ) + populations[0, :, :] = 7.0 # pressure/f0 changes, velocity does not + populations[0, 1, 2] = np.nan + second = decode_legacy_physical_velocity( + populations.reshape(-1), flags=flags, nx=nx, ny=ny, + include_pressure=True, + ) + + fluid = ((flags & 1) != 0).T + assert first["ux"].shape == (ny, nx) + assert np.allclose(first["ux"][fluid], 0.01) + assert np.allclose(first["uy"][fluid], -0.02) + assert np.array_equal(first["ux"], second["ux"]) + assert np.array_equal(first["uy"], second["uy"]) + assert first["ux"][1, 2] == first["uy"][1, 2] == first["pressure"][1, 2] == 0.0 + assert not np.array_equal(first["pressure"][fluid], second["pressure"][fluid]) + + +def test_legacy_physical_decoder_accepts_zero_sum_nonzero_momentum(): + populations = np.zeros((9, 1, 1), dtype=np.float32) + populations[1, 0, 0] = 0.01 + populations[0, 0, 0] = -0.01 + decoded = decode_legacy_physical_velocity( + populations.reshape(-1), flags=np.ones((1, 1), np.uint8), nx=1, ny=1, + ) + assert decoded["ux"][0, 0] == pytest.approx(0.01) + + +def test_legacy_physical_decoder_rejects_nonfinite_fluid_populations(): + populations = np.zeros((9, 1, 1), dtype=np.float32) + populations[0, 0, 0] = np.nan + with pytest.raises(ValueError, match="fluid populations.*non-finite"): + decode_legacy_physical_velocity( + populations.reshape(-1), flags=np.ones((1, 1), np.uint8), nx=1, ny=1, + ) + +def test_module_import_does_not_load_gpu_dependencies(): + code = ( + "import sys; import drl_pinball.acquisition; " + "assert 'pycuda' not in sys.modules; assert 'torch' not in sys.modules" + ) + subprocess.run([sys.executable, "-c", code], check=True)