"""Authoritative semantic validator for active acquisition artifacts.""" from __future__ import annotations from dataclasses import asdict from hashlib import sha256 from pathlib import Path from typing import Any, Mapping import json import numpy as np from .contracts import (ACTION_FORMULA, ACTION_FORMULA_SHA256, ARTIFACT_SCHEMA_ID, CASES, CONFIG_BINDINGS, ILLUSION_TRAINING_BINDINGS, MODEL_BINDINGS, ROLES, SCHEMA_ID, canonical_json, case_snapshot, expected_controller_identity, expected_source_bindings, role_spec, role_spec_identity, validate_coordinate_arrays, VELOCITY_DECODER_FORMULA, VELOCITY_DECODER_FORMULA_SHA256, VELOCITY_DECODER_SCHEMA_ID) from .dual_clock import TELEMETRY_KEYS def _reconstruct_harmonics(t: int, harmonics: list[dict]) -> np.ndarray: result = np.zeros(len(harmonics), np.float32) for index, harmonic in enumerate(harmonics): value = float(harmonic["dc"]) for amp, freq, phase in zip(harmonic["amps"], harmonic["freqs"], harmonic["phases"]): value += float(amp) * np.cos(2 * np.pi * float(freq) * t + float(phase)) result[index] = value return result FIELD_KEYS = {"ux", "uy", "x_D", "y_D", "fluid_mask", "lattice_steps", "control_indices", "control_offsets", *TELEMETRY_KEYS} STATE_KEYS = {"current_ddf", "temp_ddf", "current_raw_observation", "fifo_history", "initial_fifo_history", "boundary_observation_history", "policy_source_observation_history", "policy_source_observation_sha256", "policy_input_observation_history", "policy_harmonic_phase_indices", "requested_normalized_action_history", "requested_physical_action_history", "persisted_effective_ema_action", "policy_harmonic_phase_index", "solver_absolute_control_clock", "solver_absolute_lattice_clock", "acquisition_relative_control_index", "acquisition_relative_lattice_clock", "normalization_hash", "harmonics_hash", "model_hash", "cuda_config_hash", "flow_config_hash", "config_hash", "geometry_hash", "action_formula_hash", "velocity_decoder_formula_hash"} CONFIG_KEYS = {"schema_id", "case", "role", "role_semantics", "timeline_semantics", "phase_reference_semantics", "action_semantics", "velocity_decoder", "geometry_sha256", "config_sha256", "runtime", "controller_sources", "source_sha256", "clock_domains", "acquisition"} LEGACY_ZERO_ROLE_RUNTIME_KEYS = {"role_spec", "role_spec_sha256", "physical_action_width", "solver_objects", "coordinate_frame", "action_formula", "action_formula_sha256", "velocity_decoder"} RUNTIME_KEYS = LEGACY_ZERO_ROLE_RUNTIME_KEYS | {"policy_device", "cfd_device"} ROLE_SPEC_KEYS = {"case_id", "role", "objects", "control_interval", "physical_action_width", "observation_slices", "controller", "fifo_len", "harmonic_channels", "model_path"} CLOCK_KEYS = {"solver_absolute_lattice_origin", "solver_absolute_control_origin", "solver_absolute_lattice_final", "solver_absolute_control_final", "acquisition_relative_lattice_final", "acquisition_relative_control_final", "policy_harmonic_phase_final"} CONTROLLER_KEYS = {"normalization", "normalization_content_sha256", "controller_harmonics", "controller_harmonics_content_sha256", "identity", "measured_plus11_phase_harmonics", "compatibility"} ACQUISITION_KEYS = {"field_interval", "checkpoint_lifecycle", "control_history", "policy_input_contract"} MANIFEST_KEYS = {"schema_id", "complete", "files", "state_array_sha256", "config_sha256", "field_count"} def array_sha256(value: np.ndarray) -> str: return sha256(np.ascontiguousarray(value).tobytes()).hexdigest() def require_sha256(value: Any, label: str) -> str: if not isinstance(value, str) or len(value) != 64: raise ValueError(f"{label} must be SHA256") try: int(value, 16) except ValueError as exc: raise ValueError(f"{label} must be SHA256") from exc return value def _state_sha(value: np.ndarray, label: str) -> str: if value.ndim != 0 or value.dtype.kind not in "SU": raise ValueError(f"{label} must be a scalar string") return require_sha256(str(value.item()), label) def _finite_json(value: Any, label: str) -> None: try: canonical_json(value) except (TypeError, ValueError) as exc: raise ValueError(f"{label} must be finite canonical-JSON compatible") from exc def validate_acquisition_semantics(*, arrays: Mapping[str, Any], config: Mapping[str, Any], state: Mapping[str, Any], manifest: Mapping[str, Any] | None = None, expected_case: str | None = None, expected_role: str | None = None) -> tuple[dict[str, np.ndarray], dict[str, np.ndarray]]: """Validate content semantics, not merely internal artifact hashes.""" if not isinstance(config, Mapping) or set(config) != CONFIG_KEYS: raise ValueError("acquisition config top-level schema is not exact") case_id, role = config.get("case", {}).get("case_id"), config.get("role") if case_id not in CASES or role not in ROLES or (expected_case is not None and case_id != expected_case) or (expected_role is not None and role != expected_role): raise ValueError("artifact exact case/role contract mismatch") config_json = json.loads(canonical_json(config)) frozen = json.loads(canonical_json(case_snapshot(case_id, role))) for key in ("schema_id", "case", "role", "role_semantics", "timeline_semantics", "phase_reference_semantics", "action_semantics", "velocity_decoder", "geometry_sha256", "config_sha256"): if config_json.get(key) != frozen[key]: raise ValueError(f"artifact exact case/role contract mismatch: {key}") if config["schema_id"] != SCHEMA_ID: raise ValueError("acquisition contract schema mismatch") runtime, clocks, acquisition, controller = config["runtime"], config["clock_domains"], config["acquisition"], config["controller_sources"] runtime_keys = set(runtime) if isinstance(runtime, Mapping) else set() allowed_runtime_schema = runtime_keys == RUNTIME_KEYS or (role in {"q_target", "q_blk"} and runtime_keys == LEGACY_ZERO_ROLE_RUNTIME_KEYS) if not allowed_runtime_schema or not isinstance(runtime["role_spec"], Mapping) or set(runtime["role_spec"]) != ROLE_SPEC_KEYS: raise ValueError("runtime/role-spec config schema is not exact") spec = runtime["role_spec"] authoritative_spec = json.loads(canonical_json(asdict(role_spec(case_id, role)))) if json.loads(canonical_json(spec)) != authoritative_spec or runtime["role_spec_sha256"] != role_spec_identity(role_spec(case_id, role)): raise ValueError("runtime role spec differs from frozen authoritative role_spec") width = runtime["physical_action_width"] objects, persisted_objects = spec["objects"], runtime["solver_objects"] if type(width) is not int or width < 1 or not isinstance(objects, (list, tuple)) or len(objects) != width or not isinstance(persisted_objects, list) or len(persisted_objects) != width: raise ValueError("runtime physical action/object width is invalid") inferred_center_y = None for index, (declared, persisted) in enumerate(zip(objects, persisted_objects)): if not isinstance(declared, Mapping) or set(declared) != {"kind", "identity", "center_D", "radius_D"} or not isinstance(persisted, Mapping) or set(persisted) != {"id", "identity", "kind", "center_lattice", "radius_lattice"}: raise ValueError("solver object schema is not exact") center_D, center_lattice = declared["center_D"], persisted["center_lattice"] if not isinstance(center_D, (list, tuple)) or len(center_D) != 2 or not isinstance(center_lattice, (list, tuple)) or len(center_lattice) != 3 or not all(type(v) in (int, float) and np.isfinite(v) for v in (*center_D, *center_lattice)): raise ValueError("solver object coordinates are invalid") candidate_center_y = float(center_lattice[1]) - float(center_D[1]) * 20.0 inferred_center_y = candidate_center_y if inferred_center_y is None else inferred_center_y if persisted["id"] != index or persisted["identity"] != declared["identity"] or persisted["kind"] != declared["kind"] or float(center_lattice[0]) != float(center_D[0]) * 20.0 or float(center_lattice[2]) != 0.0 or candidate_center_y != inferred_center_y or float(persisted["radius_lattice"]) != float(declared["radius_D"]) * 20.0: raise ValueError("solver object semantics contradict role spec") if runtime_keys == RUNTIME_KEYS and (runtime["policy_device"] != "cpu" or type(runtime["cfd_device"]) is not int or runtime["cfd_device"] < 0): raise ValueError("runtime compute-device roles are invalid") if runtime["action_formula"] != ACTION_FORMULA or runtime["action_formula_sha256"] != ACTION_FORMULA_SHA256: raise ValueError("runtime action formula differs from frozen authority") decoder = runtime["velocity_decoder"] if (not isinstance(decoder, Mapping) or set(decoder) != {"schema_id", "quantity", "u0", "formula", "formula_sha256"} or decoder["schema_id"] != VELOCITY_DECODER_SCHEMA_ID or decoder["quantity"] != "nondimensional velocity q/U0" or type(decoder["u0"]) not in (int, float) or not np.isfinite(decoder["u0"]) or decoder["u0"] <= 0 or float(decoder["u0"]) != float(config["case"]["u0"]) or decoder["formula"] != VELOCITY_DECODER_FORMULA or decoder["formula_sha256"] != VELOCITY_DECODER_FORMULA_SHA256): raise ValueError("runtime velocity decoder differs from frozen q/U0 authority") if not isinstance(clocks, Mapping) or set(clocks) != CLOCK_KEYS or any(type(clocks[key]) is not int or clocks[key] < 0 for key in CLOCK_KEYS): raise ValueError("clock-domain config schema is not exact") if not isinstance(acquisition, Mapping) or set(acquisition) != ACQUISITION_KEYS or type(acquisition["field_interval"]) is not int or acquisition["field_interval"] < 1 or not isinstance(acquisition["checkpoint_lifecycle"], str) or acquisition["control_history"] != "complete boundary-average lineage independent of field cadence" or acquisition["policy_input_contract"] != ("reconstruct_from_prior_boundary_history" if role == "q_ctl" else "not_applicable_explicit_zero"): raise ValueError("acquisition config schema is not exact") if not isinstance(controller, Mapping) or set(controller) != CONTROLLER_KEYS: raise ValueError("controller-source config schema is not exact") normalization = controller["normalization"] if not isinstance(normalization, Mapping) or set(normalization) != {"force_norm_fact", "sens_deviation", "sens_norm_fact"}: raise ValueError("controller normalization schema is not exact") force = np.asarray(normalization["force_norm_fact"]); deviation = np.asarray(normalization["sens_deviation"]); scale = np.asarray(normalization["sens_norm_fact"]) if force.ndim != 0 or not np.isfinite(force) or float(force) <= 0 or deviation.shape != (6,) or scale.shape != (6,) or not np.isfinite(deviation).all() or not np.isfinite(scale).all() or np.any(scale <= 0): raise ValueError("controller normalization semantics are invalid") if sha256(canonical_json(normalization)).hexdigest() != require_sha256(controller["normalization_content_sha256"], "normalization_content_sha256"): raise ValueError("normalization content hash mismatch") harmonics = controller["controller_harmonics"] if not isinstance(harmonics, list) or sha256(canonical_json(harmonics)).hexdigest() != require_sha256(controller["controller_harmonics_content_sha256"], "controller_harmonics_content_sha256"): raise ValueError("controller harmonics identity mismatch") if not isinstance(controller["identity"], Mapping) or not isinstance(controller["compatibility"], str): raise ValueError("controller source identity/compatibility is invalid") expected_identity = expected_controller_identity(case_id, role) if case_id == "illusion_1.0L" and role == "q_ctl": actual_identity = dict(controller["identity"]); actual_identity["normalization_path"] = str(Path(actual_identity.get("normalization_path", "")).resolve().relative_to(Path(__file__).resolve().parents[3])) if Path(actual_identity.get("normalization_path", "")).is_absolute() else actual_identity.get("normalization_path") actual_identity["harmonics_path"] = str(Path(actual_identity.get("harmonics_path", "")).resolve().relative_to(Path(__file__).resolve().parents[3])) if Path(actual_identity.get("harmonics_path", "")).is_absolute() else actual_identity.get("harmonics_path") if actual_identity != expected_identity: raise ValueError("Illusion training reference paths/hashes differ from frozen authority") elif controller["identity"] != expected_identity: raise ValueError("controller source identity differs from frozen authority") _finite_json(controller["measured_plus11_phase_harmonics"], "measured phase harmonics") if config["source_sha256"] != expected_source_bindings(case_id, role): raise ValueError("source paths/hashes differ from frozen case/role authority") if set(arrays) != FIELD_KEYS: raise ValueError("artifact arrays do not match unified schema") data = {key: np.asarray(value) for key, value in arrays.items()} ux, uy, x, y, mask = data["ux"], data["uy"], data["x_D"], data["y_D"], data["fluid_mask"] if ux.dtype != np.float32 or uy.dtype != np.float32 or ux.ndim != 3 or ux.shape != uy.shape or not np.isfinite(ux).all() or not np.isfinite(uy).all(): raise ValueError("fields must be finite matching float32 (time,x,y)") count, nx, ny = ux.shape if count < 1: raise ValueError("field time axis must be nonempty") validate_coordinate_arrays(x, y, runtime["coordinate_frame"]) if x.shape != (nx,) or y.shape != (ny,): raise ValueError("declared coordinates must match field axes") if mask.dtype != np.bool_ or mask.shape != (nx, ny) or not mask.any(): raise ValueError("saved solver fluid mask must match nonempty (x,y) grid") if np.any(ux[:, ~mask] != np.float32(0)) or np.any(uy[:, ~mask] != np.float32(0)): raise ValueError("solid-cell velocities must be exact float32 zero") integer_keys = ("lattice_steps", "control_indices", "control_offsets", "sample_ids", "acquisition_relative_lattice_steps", "solver_absolute_control_indices") for key in integer_keys: if data[key].dtype != np.int64 or data[key].shape != (count,): raise ValueError(f"{key} must be int64 length time") steps, relative = data["lattice_steps"], data["acquisition_relative_lattice_steps"] interval, origin, control_origin = config["case"]["sample_interval"], clocks["solver_absolute_lattice_origin"], clocks["solver_absolute_control_origin"] expected_controls = (relative - 1) // interval if np.any(np.diff(steps) <= 0) or np.any(relative <= 0) or not np.array_equal(data["sample_ids"], steps) or not np.array_equal(relative, steps - origin) or not np.array_equal(data["control_indices"], expected_controls) or not np.array_equal(data["control_offsets"], (relative - 1) % interval + 1) or not np.array_equal(data["solver_absolute_control_indices"], control_origin + expected_controls): raise ValueError("solver/acquisition timeline relations are invalid") if relative[-1] != clocks["acquisition_relative_lattice_final"] or steps[-1] != clocks["solver_absolute_lattice_final"] or clocks["solver_absolute_control_final"] - control_origin != clocks["acquisition_relative_control_final"] or clocks["policy_harmonic_phase_final"] != clocks["acquisition_relative_control_final"]: raise ValueError("final clock domains contradict timeline/control lineage") shapes = {"requested_normalized_action": (count, 3), "requested_physical_action": (count, width), "effective_applied_action": (count, width), "disturbance_force": (count, 2), "pinball_forces": (count, 6), "sensors": (count, 6), "phase_reference": (count, 1)} for key, shape in shapes.items(): value = data[key] if value.dtype != np.float32 or value.shape != shape or not np.isfinite(value).all(): raise ValueError(f"{key} has invalid dtype/shape/finiteness") normalized, physical, effective = data["requested_normalized_action"], data["requested_physical_action"], data["effective_applied_action"] if np.any(normalized < -1) or np.any(normalized > 1): raise ValueError("normalized actions outside bounds") if role != "q_ctl": if np.any(normalized) or np.any(physical) or np.any(effective): raise ValueError("q_target/q_blk requested and effective actions must be exactly zero") else: bias = np.asarray((0., -4., 4.) if case_id == "karman_re100" else (0., -2., 2.), np.float32) u0 = np.float32(config["case"]["u0"]); expected_physical = np.zeros_like(physical); expected_physical[:, -3:] = (normalized * np.float32(8) + bias) * u0 lower = (np.asarray([-8., -8., -8.], np.float32) + bias) * u0; upper = (np.asarray([8., 8., 8.], np.float32) + bias) * u0 if not np.array_equal(physical, expected_physical): raise ValueError("q_ctl requested physical actions contradict frozen formula") if np.any(physical[:, :-3]) or np.any(effective[:, :-3]) or np.any(effective[:, -3:] < lower) or np.any(effective[:, -3:] > upper): raise ValueError("q_ctl effective EMA actions violate actuated-channel semantics/bounds") if set(state) != STATE_KEYS: raise ValueError("controller state identity inventory is not exact") state_data = {key: np.asarray(value) for key, value in state.items()} ddf_size = 9 * nx * ny for key in ("current_ddf", "temp_ddf"): if state_data[key].dtype != np.float32 or state_data[key].shape != (ddf_size,) or not np.isfinite(state_data[key]).all(): raise ValueError(f"{key} has invalid D2Q9 storage") if state_data["current_raw_observation"].dtype != np.float32 or state_data["current_raw_observation"].ndim != 1 or state_data["current_raw_observation"].size < 1 or not np.isfinite(state_data["current_raw_observation"]).all(): raise ValueError("raw observation invalid") fifo = state_data["fifo_history"] initial_fifo = state_data["initial_fifo_history"] boundaries = state_data["boundary_observation_history"] sources = state_data["policy_source_observation_history"] source_hashes = state_data["policy_source_observation_sha256"] policy_inputs = state_data["policy_input_observation_history"] phase_indices = state_data["policy_harmonic_phase_indices"] normalized_controls = state_data["requested_normalized_action_history"] physical_controls = state_data["requested_physical_action_history"] control_count = clocks["acquisition_relative_control_final"] s_dim = 12 if case_id == "karman_re100" else 14 for label, value, shape in (("FIFO/history", fifo, (150, 12)), ("initial FIFO/history", initial_fifo, (150, 12)), ("boundary observation history", boundaries, (control_count, 12)), ("policy source observation history", sources, (control_count, 12)), ("policy input observation history", policy_inputs, (control_count, s_dim)), ("requested normalized control history", normalized_controls, (control_count, 3)), ("requested physical control history", physical_controls, (control_count, width))): if value.dtype != np.float32 or value.shape != shape or not np.isfinite(value).all(): raise ValueError(f"{label} has invalid dtype/shape/finiteness") if phase_indices.dtype != np.int64 or phase_indices.shape != (control_count,) or not np.array_equal(phase_indices, np.arange(control_count, dtype=np.int64)): raise ValueError("policy harmonic phase indices must be exact zero-origin control lineage") if source_hashes.dtype.kind not in "SU" or source_hashes.shape != (control_count,) or any(str(value) != array_sha256(sources[index]) for index, value in enumerate(source_hashes.tolist())): raise ValueError("policy source observation hashes mismatch") expected_fifo = np.concatenate((initial_fifo, boundaries), axis=0)[-150:] if not np.array_equal(fifo, expected_fifo): raise ValueError("terminal FIFO must equal rolling append of initial FIFO and all boundaries") expected_sources = np.zeros_like(sources) if role == "q_ctl": expected_sources[0] = initial_fifo[-1] if control_count > 1: expected_sources[1:] = boundaries[:-1] if not np.array_equal(sources, expected_sources): raise ValueError("policy source observations must use the appropriate prior FIFO/boundary history or explicit not-applicable zeros") expected_inputs = np.zeros_like(policy_inputs) if role == "q_ctl": for index, raw in enumerate(expected_sources): if not (case_id == "karman_re100" and index == 0): force_values = raw[6:12] / np.float32(force) sensor_values = (raw[:6] - deviation.astype(np.float32)) / scale.astype(np.float32) values = [*force_values, *sensor_values] if case_id == "illusion_1.0L": target = _reconstruct_harmonics(index, harmonics)[:2] / np.float32(force) values.extend(target.tolist()) expected_inputs[index] = np.clip(np.asarray(values, np.float32), -1, 1) if not np.array_equal(policy_inputs, expected_inputs): raise ValueError("policy inputs do not reconstruct exactly from prior history, normalization, harmonics, and initial semantics") if not np.array_equal(normalized, normalized_controls[data["control_indices"]]) or not np.array_equal(physical, physical_controls[data["control_indices"]]): raise ValueError("field-time requested actions contradict complete control action histories") expected_control_physical = np.zeros_like(physical_controls) if role == "q_ctl": bias = np.asarray((0., -4., 4.) if case_id == "karman_re100" else (0., -2., 2.), np.float32) expected_control_physical[:, -3:] = (normalized_controls * np.float32(8) + bias) * np.float32(config["case"]["u0"]) if not np.array_equal(physical_controls, expected_control_physical): raise ValueError("control action histories contradict frozen role/action formula") if state_data["persisted_effective_ema_action"].dtype != np.float32 or state_data["persisted_effective_ema_action"].shape != (width,) or not np.isfinite(state_data["persisted_effective_ema_action"]).all() or not np.array_equal(state_data["persisted_effective_ema_action"], effective[-1]): raise ValueError("persisted EMA action must equal terminal sampled effective action") clock_pairs = (("solver_absolute_lattice_clock", "solver_absolute_lattice_final"), ("solver_absolute_control_clock", "solver_absolute_control_final"), ("acquisition_relative_lattice_clock", "acquisition_relative_lattice_final"), ("acquisition_relative_control_index", "acquisition_relative_control_final"), ("policy_harmonic_phase_index", "policy_harmonic_phase_final")) for state_key, config_key in clock_pairs: value = state_data[state_key] if value.dtype != np.int64 or value.ndim != 0 or int(value) != clocks[config_key]: raise ValueError(f"{state_key} does not exactly match final clock domain") state_hash_values = {key: _state_sha(state_data[key], key) for key in STATE_KEYS if key.endswith("_hash")} if state_hash_values["geometry_hash"] != config["geometry_sha256"] or state_hash_values["action_formula_hash"] != runtime["action_formula_sha256"] or state_hash_values["velocity_decoder_formula_hash"] != runtime["velocity_decoder"]["formula_sha256"] or state_hash_values["config_hash"] != sha256(canonical_json(config)).hexdigest(): raise ValueError("state semantic identity hashes contradict config") identity_values = {value for key, value in controller["identity"].items() if key.endswith("_sha256") and isinstance(value, str)} if state_hash_values["normalization_hash"] not in {controller["normalization_content_sha256"], *identity_values} or state_hash_values["harmonics_hash"] not in {controller["controller_harmonics_content_sha256"], *identity_values}: raise ValueError("state controller-source hashes contradict config") if state_hash_values["cuda_config_hash"] != CONFIG_BINDINGS["configs/legacy_configs/config_cuda.json"] or state_hash_values["flow_config_hash"] != CONFIG_BINDINGS["configs/legacy_configs/config_flowfield.json"]: raise ValueError("state solver config hashes differ from frozen authority") expected_model_hash = MODEL_BINDINGS[case_id][1] if role == "q_ctl" else sha256(b"zero-controller").hexdigest() if state_hash_values["model_hash"] != expected_model_hash: raise ValueError("state model hash differs from frozen authority") if case_id == "illusion_1.0L" and role == "q_ctl" and (state_hash_values["normalization_hash"] != ILLUSION_TRAINING_BINDINGS["normalization_sha256"] or state_hash_values["harmonics_hash"] != ILLUSION_TRAINING_BINDINGS["harmonics_sha256"]): raise ValueError("state Illusion training reference hashes differ from frozen authority") if manifest is not None: if not isinstance(manifest, Mapping) or set(manifest) != MANIFEST_KEYS or manifest["schema_id"] != ARTIFACT_SCHEMA_ID or manifest["complete"] is not True or manifest["field_count"] != count: raise ValueError("artifact manifest semantic schema is not exact") if set(manifest["state_array_sha256"]) != STATE_KEYS or any(array_sha256(state_data[key]) != manifest["state_array_sha256"][key] for key in STATE_KEYS): raise ValueError("manifest state array hash inventory is invalid") if manifest["config_sha256"] != sha256(canonical_json(config)).hexdigest(): raise ValueError("manifest config hash mismatch") return data, state_data