"""Fail-closed CPU Phase 3--5 paired products and realization-held-out LR/LE-OID."""
from __future__ import annotations

import hashlib
import json
import resource
import time
from pathlib import Path
from typing import Any, Mapping, Sequence

import numpy as np

from .analysis import PODModel, fit_pod, fit_schlegel_oid, modal_assurance, principal_angles, projector_similarity
from .artifacts import atomic_save_npz, atomic_write_json, validate_field_artifact
from CCD_analysis.utils.run_contract import deterministic_json_bytes, sha256_file
from .contracts import telemetry_hashes

CANONICAL_SCHEMA = "oid-canonical-paired/v1"
ANALYSIS_SCHEMA = "oid-phase3-5-loro/v2"


def _hash(array: np.ndarray) -> str:
    value = np.ascontiguousarray(array)
    return hashlib.sha256(value.dtype.str.encode() + str(value.shape).encode() + value.tobytes()).hexdigest()


def _read_telemetry(path: Path, steps: np.ndarray) -> dict[str, np.ndarray]:
    if not path.is_file():
        raise FileNotFoundError(f"field-level telemetry is required: {path}")
    with np.load(path, allow_pickle=False) as data:
        required = {"sensors", "forces", "lattice_steps", "sample_ids", "times"}
        missing = required.difference(data.files)
        if missing:
            raise ValueError(f"field-level telemetry missing {sorted(missing)}; control-end telemetry is not valid")
        result = {key: np.asarray(data[key]).copy() for key in required}
    n = len(steps)
    if any(value.ndim == 0 or len(value) != n for value in result.values()):
        raise ValueError("field-level telemetry must have one row per field snapshot")
    if not np.array_equal(result["lattice_steps"], steps):
        raise ValueError("field and telemetry lattice_steps differ")
    if len(np.unique(result["sample_ids"])) != n or np.any(np.diff(result["times"]) <= 0):
        raise ValueError("sample_ids must be unique and times strictly increasing")
    if not all(np.isfinite(value).all() for value in result.values() if value.dtype.kind in "fc"):
        raise ValueError("field-level telemetry contains non-finite values")
    return result



def _field_content_sha256(path: Path) -> str:
    """Recompute online.py's ux/uy array-content digest exactly."""
    digest = hashlib.sha256()
    with np.load(path, allow_pickle=False) as archive:
        for key in ("ux", "uy"):
            value = np.ascontiguousarray(archive[key])
            digest.update(str(value.dtype).encode())
            digest.update(np.asarray(value.shape, np.int64).tobytes())
            digest.update(value.tobytes())
    return digest.hexdigest()


def _telemetry_content_sha256(path: Path) -> str:
    """Recompute online.py's sorted composite telemetry-array digest exactly."""
    with np.load(path, allow_pickle=False) as archive:
        hashes = telemetry_hashes(archive)
    return hashlib.sha256("".join(hashes[key] for key in sorted(hashes)).encode()).hexdigest()


def _artifact_provenance(role: str, record: Mapping[str, Any]) -> dict[str, Any]:
    source = Path(str(record.get("path", ""))).resolve()
    if not source.is_file():
        raise ValueError(f"artifact {role!r} requires an existing path")
    result: dict[str, Any] = {
        "path": str(source), "file_sha256": sha256_file(source),
        "file_hash_algorithm": "SHA-256 over exact file bytes", "size_bytes": source.stat().st_size,
    }
    semantics = record.get("content_hash_semantics")
    declared = record.get("declared_content_sha256")
    if semantics is not None or declared is not None:
        if semantics == "online.py fields_sha256: SHA-256 over ux then uy; each contributes str(dtype), int64 shape bytes, C-order data bytes":
            computed = _field_content_sha256(source)
        elif semantics == "online.py telemetry_sha256: SHA-256 over concatenated sorted telemetry array_sha256 hex digests":
            computed = _telemetry_content_sha256(source)
        else:
            raise ValueError(f"unsupported content hash semantics for {role}: {semantics!r}")
        if not isinstance(declared, str) or computed != declared:
            raise ValueError(f"declared content hash mismatch for {role}: {source}")
        result.update({"declared_content_sha256": declared, "computed_content_sha256": computed,
                       "content_hash_semantics": semantics, "content_hash_validated": True})
    return result

def build_canonical_paired_product(
    controlled_fields: str | Path,
    baseline_fields: str | Path,
    controlled_telemetry: str | Path,
    baseline_telemetry: str | Path,
    output: str | Path,
    *,
    realization_id: str,
    fluid_mask: np.ndarray,
    cell_metric: np.ndarray | None = None,
    metric_metadata: Mapping[str, Any] | None = None,
    crop: tuple[slice, slice] | None = None,
    stride: int = 1,
    source_artifacts: Mapping[str, Mapping[str, Any]] | None = None,
    geometry_declaration: Mapping[str, Any] | None = None,
    evidence_artifacts: Mapping[str, Mapping[str, Any]] | None = None,
) -> dict[str, Any]:
    """Publish one exact-time field/telemetry difference product.

    Telemetry must already be sampled at every field time.  This deliberately has
    no fallback from field indices to old control-end telemetry.
    """
    if not realization_id:
        raise ValueError("realization_id is required")
    if type(stride) is not int or stride < 1:
        raise ValueError("stride must be a positive integer")
    destination = Path(output)
    product_path = destination / "paired_product.npz"
    metadata_path = destination / "paired_product.json"
    if product_path.exists() or metadata_path.exists():
        raise FileExistsError(f"refusing to overwrite canonical product in {destination}")
    ctl_path, blk_path = Path(controlled_fields), Path(baseline_fields)
    with np.load(ctl_path, allow_pickle=False) as ctl, np.load(blk_path, allow_pickle=False) as blk:
        ctl_info, blk_info = validate_field_artifact(ctl), validate_field_artifact(blk)
        if ctl_info["shape"] != blk_info["shape"] or not np.array_equal(ctl["lattice_steps"], blk["lattice_steps"]):
            raise ValueError("controlled and baseline fields must have identical shape and lattice_steps")
        steps = np.asarray(ctl["lattice_steps"], dtype=np.int64).copy()
        region = crop or (slice(None), slice(None))
        sampled_region = tuple(slice(part.start, part.stop, stride) for part in region)
        selection = (slice(None), sampled_region[0], sampled_region[1])
        ux = ctl["ux"][selection] - blk["ux"][selection]
        uy = ctl["uy"][selection] - blk["uy"][selection]
        dq = np.concatenate((ux.reshape(len(steps), -1), uy.reshape(len(steps), -1)), axis=1).astype(np.float64)
    ct = _read_telemetry(Path(controlled_telemetry), steps)
    bt = _read_telemetry(Path(baseline_telemetry), steps)
    if not np.array_equal(ct["sample_ids"], bt["sample_ids"]) or not np.array_equal(ct["times"], bt["times"]):
        raise ValueError("controlled and baseline telemetry IDs/times differ")
    full_mask = np.asarray(fluid_mask, dtype=bool)
    full_shape = tuple(ctl_info["shape"][1:])
    if full_mask.shape != full_shape:
        raise ValueError(f"fluid_mask shape {full_mask.shape} differs from field shape {full_shape}")
    mask = full_mask[region][::stride, ::stride]
    spatial_shape = mask.shape
    cell = mask.astype(float) if cell_metric is None else np.asarray(cell_metric, dtype=float)
    if cell.shape != mask.shape or np.any(cell < 0) or np.any(cell[~mask] != 0) or not np.any(cell > 0):
        raise ValueError("cell_metric must be nonnegative, positive on some fluid cells, and zero on solids")
    weights = np.concatenate((cell.reshape(-1), cell.reshape(-1)))
    arrays = {
        "dq_ctl": dq, "delta_forces": ct["forces"] - bt["forces"],
        "delta_sensors": ct["sensors"] - bt["sensors"], "sample_ids": ct["sample_ids"],
        "lattice_steps": steps, "times": ct["times"], "fluid_mask": mask,
        "cell_metric": cell, "state_weights": weights,
    }
    if any(not np.isfinite(value).all() for value in (arrays["dq_ctl"], arrays["delta_forces"], arrays["delta_sensors"])):
        raise ValueError("canonical differences must be finite")
    sources = {role: _artifact_provenance(role, record)
               for role, record in dict(source_artifacts or {}).items()}
    evidence = {role: _artifact_provenance(role, record)
                for role, record in dict(evidence_artifacts or {}).items()}

    geometry = dict(geometry_declaration or {})
    geometry_hash = hashlib.sha256(deterministic_json_bytes(geometry)).hexdigest() if geometry else None
    atomic_save_npz(product_path, **arrays)
    metadata = {
        "schema_id": CANONICAL_SCHEMA, "realization_id": realization_id, "samples": len(steps),
        "state_features": dq.shape[1], "field_shape": list(spatial_shape),
        "pairing": "exact field-level IDs, lattice_steps, and times",
        "fluid_mask_sha256": _hash(mask), "full_fluid_mask_sha256": _hash(full_mask),
        "cell_metric_sha256": _hash(cell),
        "geometry_declaration": geometry, "geometry_declaration_sha256": geometry_hash,
        "canonical_pair_sources": sources, "reproducibility_gate_evidence": evidence,
        "layout_contract": {"field_axes": ["x", "y"], "component_order": ["ux", "uy"],
                            "flattening": "C order per component, ux then uy"},
        "timeline_contract": {"sample_ids_sha256": _hash(ct["sample_ids"]),
                              "lattice_steps_sha256": _hash(steps), "times_sha256": _hash(ct["times"]),
                              "pairing": "controlled and baseline arrays exactly equal"},
        "delta_contract": {"forces_sha256": _hash(arrays["delta_forces"]),
                           "sensors_sha256": _hash(arrays["delta_sensors"]),
                           "definition": "controlled minus baseline at exact field timeline"},
        "metric_metadata": {**dict(metric_metadata or {}), "crop": str(region), "stride": stride},
        "arrays": {key: _hash(value) for key, value in arrays.items()},
        "product": product_path.name,
    }
    atomic_write_json(metadata_path, metadata)
    return metadata


def load_canonical_product(directory: str | Path) -> dict[str, Any]:
    directory = Path(directory)
    metadata = json.loads((directory / "paired_product.json").read_text(encoding="utf-8"))
    if metadata.get("schema_id") != CANONICAL_SCHEMA:
        raise ValueError("not a canonical paired product")
    with np.load(directory / metadata["product"], allow_pickle=False) as archive:
        required = {"dq_ctl", "delta_forces", "delta_sensors", "sample_ids", "lattice_steps", "times", "fluid_mask", "cell_metric", "state_weights"}
        if required.difference(archive.files):
            raise ValueError("canonical paired product is incomplete")
        arrays = {key: np.asarray(archive[key]).copy() for key in required}
    if any(_hash(value) != metadata["arrays"].get(key) for key, value in arrays.items()):
        raise ValueError("canonical paired product hash mismatch")
    n = len(arrays["dq_ctl"])
    if any(len(arrays[key]) != n for key in ("delta_forces", "delta_sensors", "sample_ids", "lattice_steps", "times")):
        raise ValueError("canonical arrays are not sample aligned")
    return {"metadata": metadata, **arrays}


def _lagged(product: Mapping[str, Any], lag: int, observable: str) -> tuple[np.ndarray, np.ndarray]:
    q = product["dq_ctl"]
    y = product[observable]
    if lag < 0 or lag >= len(q) - 2:
        raise ValueError("lag leaves too few samples")
    return (q if lag == 0 else q[:-lag]), (y if lag == 0 else y[lag:])


def _r2(actual: np.ndarray, predicted: np.ndarray, center: np.ndarray) -> float:
    denominator = float(np.sum((actual - center) ** 2))
    return float(1 - np.sum((actual - predicted) ** 2) / denominator) if denominator else 0.0


def _scores(y, prediction, center, observable_metric):
    y, prediction, center = np.asarray(y, float), np.asarray(prediction, float), np.asarray(center, float)
    metric = np.asarray(observable_metric, float)
    channels = [_r2(y[:, [i]], prediction[:, [i]], center[[i]]) for i in range(y.shape[1])]
    residual, centered = y - prediction, y - center
    if metric.ndim == 1:
        squared_error = float(np.sum(residual**2 * metric))
        centered_sum_squares = float(np.sum(centered**2 * metric))
    elif metric.ndim == 2:
        squared_error = float(np.einsum("ti,ij,tj->", residual, metric, residual))
        centered_sum_squares = float(np.einsum("ti,ij,tj->", centered, metric, centered))
    else:
        raise ValueError("observable_metric must be a diagonal vector or dense matrix")
    metric_weighted_r2 = (float(1 - squared_error / centered_sum_squares)
                          if centered_sum_squares else 0.0)
    return {
        "metric_weighted_r2": metric_weighted_r2,
        "per_channel_r2": channels,
        "unweighted_mean_channel_r2": float(np.mean(channels)),
        "metric_weighted_squared_error": squared_error,
        "metric_weighted_centered_sum_squares": centered_sum_squares,
    }


def _seed(seed, *parts):
    raw = json.dumps([int(seed), *parts], separators=(",", ":")).encode()
    return int.from_bytes(hashlib.sha256(raw).digest()[:8], "little")


def _slice_pod(pod, rank):
    if rank < 1 or rank > pod.modes.shape[1]:
        raise ValueError("rank outside cached POD range")
    return PODModel(pod.mean, pod.modes[:, :rank], pod.singular_values[:rank], pod.coefficients[:, :rank],
                    pod.energy[:rank], pod.cumulative_energy[:rank], pod.total_energy, pod.metric)


class _PODCache:
    def __init__(self, max_rank, weights, products):
        self.max_rank, self.weights = int(max_rank), np.asarray(weights, float)
        self.products = tuple(products)
        self.models = {}
        self.coefficients = {}
        self.field_transform_count = 0
        self.physical_mode_materialization_count = 0

    def _key(self, train, lag):
        ids = tuple(sorted(x["metadata"]["realization_id"] for x in train))
        return (ids, int(lag), _hash(self.weights))

    def get(self, train, lag, rank):
        key = self._key(train, lag)
        if key not in self.models:
            fields = [_lagged(x, lag, "delta_forces")[0] for x in train]
            model = fit_pod(np.concatenate(fields), self.max_rank, self.weights)
            self.models[key] = model
            start = 0
            for item, field in zip(train, fields):
                stop = start + len(field)
                realization_id = item["metadata"]["realization_id"]
                self.coefficients[(key, realization_id)] = model.coefficients[start:stop]
                start = stop
            for item in self.products:
                realization_id = item["metadata"]["realization_id"]
                coefficient_key = (key, realization_id)
                if coefficient_key not in self.coefficients:
                    q = _lagged(item, lag, "delta_forces")[0]
                    self.coefficients[coefficient_key] = model.transform(q)
                    self.field_transform_count += 1
        return _slice_pod(self.models[key], rank)

    def project(self, train, item, lag, rank):
        key = self._key(train, lag)
        pod = self.get(train, lag, self.max_rank)
        realization_id = item["metadata"]["realization_id"]
        coefficient_key = (key, realization_id)
        if coefficient_key not in self.coefficients:
            q = _lagged(item, lag, "delta_forces")[0]
            self.coefficients[coefficient_key] = pod.transform(q)
            self.field_transform_count += 1
        return self.coefficients[coefficient_key][:, :rank]


def _fit_fold(train, test, *, rank, lag, observable, observable_metric, cache, rcond=1e-12,
              materialize_models=False):
    pairs = [_lagged(x, lag, observable) for x in train]
    ty = np.concatenate([x[1] for x in pairs])
    _, vy = _lagged(test, lag, observable)
    pod = cache.get(train, lag, rank)
    ta = np.concatenate([cache.project(train, x, lag, rank) for x in train])
    va = cache.project(train, test, lag, rank)
    oid = fit_schlegel_oid(ta, ty, observable_metric=observable_metric,
                           observable_rank=min(ty.shape[1], rank),
                           state_modes=pod.modes if materialize_models else None,
                           rcond=rcond, truncate=True)
    if materialize_models:
        cache.physical_mode_materialization_count += 1
    observed_generated = oid.observable_pod.transform(vy) @ oid.lr.observable_basis
    predicted_state = observed_generated @ oid.lr.inverse.T
    predicted_generated = va @ oid.lr.propagator.T
    prediction = oid.observable_pod.inverse_transform(predicted_generated @ oid.lr.observable_basis.T)
    metric = _scores(vy, prediction, ty.mean(0), observable_metric)
    s = oid.lr.singular_values
    retained = oid.lr.singular_values
    report = {"train_samples": len(ty), "test_samples": len(vy), "rank": rank, "sample_lag": lag,
              "observable": observable, "tested_statistic": "metric_weighted_r2", **metric,
              "lr_state_coefficient_r2": _r2(va, predicted_state, pod.coefficients.mean(0)),
              "numerical_status": "truncated", "regularization": None,
              "rcond": float(oid.rcond),
              "state_rank": int(pod.modes.shape[1]),
              "state_retained_energy": float(pod.cumulative_energy[-1]),
              "pod_energy": float(pod.cumulative_energy[-1]),
              "requested_observable_rank": int(oid.requested_observable_rank),
              "effective_observable_rank": int(oid.effective_observable_rank),
              "generated_rank": oid.lr.rank, "propagator_effective_rank": oid.lr.rank,
              "propagator_threshold": float(oid.lr.rank_tolerance),
              "propagator_smallest_retained_singular": float(retained[-1]),
              "propagator_condition_number": float(retained[0] / retained[-1]),
              "state_covariance_rank": int(oid.state_covariance_rank),
              "state_covariance_threshold": float(oid.state_covariance_threshold),
              "state_covariance_smallest_retained_singular":
                  float(oid.state_covariance_smallest_retained_singular),
              "lr_identity_defect": oid.lr.identity_defect, "le_identity_defect": oid.le.identity_defect,
              "lr_projection_defect": oid.lr.projection_defect, "le_projection_defect": oid.le.projection_defect,
              "train_discarded_pair_count": int(lag * len(train)), "test_discarded_pair_count": int(lag),
              "total_discarded_pair_count": int(lag * (len(train) + 1))}
    if not materialize_models:
        return report, {}
    return report, {"lr_modes": oid.lr_modes, "le_modes": oid.le_modes,
                    "state_pod_mean": pod.mean, "state_pod_modes": pod.modes,
                    "observable_pod_mean": oid.observable_pod.mean, "observable_pod_modes": oid.observable_pod.modes,
                    "observable_basis": oid.lr.observable_basis, "observable_metric": observable_metric,
                    "propagator": oid.lr.propagator,
                    "lr_inverse": oid.lr.inverse, "le_inverse": oid.le.inverse}


def _choose(train, ranks, lags, observable, observable_metric, cache, rcond=1e-12):
    table = []
    for rank in ranks:
        for lag in lags:
            values = []
            for i in range(len(train)):
                try:
                    values.append(_fit_fold([x for j, x in enumerate(train) if j != i], train[i], rank=rank,
                                            lag=lag, observable=observable, observable_metric=observable_metric,
                                            cache=cache, rcond=rcond)[0]["metric_weighted_r2"])
                except (ValueError, np.linalg.LinAlgError):
                    values = []; break
            table.append({"rank": rank, "sample_lag": lag, "tested_statistic": "metric_weighted_r2",
                          "train_only_metric_weighted_r2": float(np.mean(values)) if values else None})
    valid = [x for x in table if x["train_only_metric_weighted_r2"] is not None]
    if not valid:
        raise ValueError("no rank/lag candidate is estimable using training realizations only")
    best = max(valid, key=lambda x: (x["train_only_metric_weighted_r2"], -x["rank"], -x["sample_lag"]))
    return best["rank"], best["sample_lag"], table


def _dominant_period(observables: np.ndarray) -> dict[str, Any]:
    """Estimate one period from a complete multichannel training observable."""
    values = np.asarray(observables, dtype=float)
    if values.ndim != 2 or values.shape[0] < 4 or values.shape[1] < 1:
        raise ValueError("dominant-period estimation requires a complete multichannel record")
    centered = values - values.mean(axis=0, keepdims=True)
    power = np.abs(np.fft.rfft(centered, axis=0)) ** 2
    nonzero = power[1:]
    channel_energy = nonzero.sum(axis=0)
    if (not np.isfinite(nonzero).all() or np.any(channel_energy <= 0)):
        raise ValueError("dominant period is unavailable for a zero-energy observable channel")
    normalized = nonzero / channel_energy
    aggregate = normalized.mean(axis=1)
    peak_index = int(np.argmax(aggregate))
    dominant_bin = peak_index + 1
    peak = float(aggregate[peak_index])
    if not np.isfinite(peak) or peak <= 0:
        raise ValueError("dominant period is unavailable")
    return {
        "estimator": "deterministic normalized multichannel rFFT power spectrum",
        "preprocessing": "remove each channel mean; normalize each nonzero-bin power spectrum by its channel spectral energy; average channels",
        "dominant_nonzero_bin": dominant_bin,
        "period_samples": float(len(values) / dominant_bin),
        "peak_concentration": peak,
        "record_samples": int(len(values)),
        "channel_count": int(values.shape[1]),
    }


def _admissible_shift_record(
    item: Mapping[str, Any], observable: str, exclusion_width: int, physical_time_spacing: float
) -> dict[str, Any]:
    estimate = _dominant_period(np.asarray(item[observable]))
    n, period = estimate["record_samples"], estimate["period_samples"]
    offsets = np.arange(n, dtype=int)
    zero_excluded = (offsets <= exclusion_width) | (offsets >= n - exclusion_width)
    periodic_excluded = np.zeros(n, dtype=bool)
    multiples = []
    multiplier = 1
    while multiplier * period <= (n - 1) + exclusion_width:
        location = multiplier * period
        periodic_excluded |= np.abs(offsets - location) <= exclusion_width
        multiples.append(float(location))
        multiplier += 1
    excluded = offsets[zero_excluded | periodic_excluded]
    admissible = offsets[~(zero_excluded | periodic_excluded)]
    if len(admissible) < 2:
        raise ValueError(
            f"circular time-shift null has too few admissible offsets for "
            f"{item['metadata']['realization_id']}"
        )
    return {
        "realization_id": item["metadata"]["realization_id"],
        **estimate,
        "exclusion_width_samples": int(exclusion_width),
        "exclusion_width_physical_time": float(exclusion_width * physical_time_spacing),
        "period_physical_time": float(period * physical_time_spacing),
        "integer_period_multiple_locations_samples": multiples,
        "excluded_offset_count": int(len(excluded)),
        "admissible_offset_count": int(len(admissible)),
        "excluded_offsets": excluded.tolist(),
        "admissible_offsets": admissible.tolist(),
    }


def _shift(train, observable, rng, admissibility):
    """Draw uniformly from precomputed realization-specific admissible offsets."""
    result, offsets = [], []
    if len(train) != len(admissibility):
        raise ValueError("shift admissibility does not match training realizations")
    for item, contract in zip(train, admissibility):
        copy = dict(item); y = np.asarray(item[observable])
        if contract["realization_id"] != item["metadata"]["realization_id"]:
            raise ValueError("shift admissibility realization mismatch")
        candidates = np.asarray(contract["admissible_offsets"], dtype=int)
        offset = int(rng.choice(candidates))
        copy[observable] = np.roll(y, offset, axis=0); result.append(copy); offsets.append(offset)
    return result, offsets


def _derange(train, observable):
    result = []
    for i, receiver in enumerate(train):
        donor = np.asarray(train[(i + 1) % len(train)][observable]); n = len(receiver[observable])
        copy = dict(receiver); copy[observable] = donor[np.minimum(np.arange(n) * len(donor) // n, len(donor) - 1)]
        result.append(copy)
    return result


def _lag_metadata(products, lags):
    ds, dt = [], []
    for x in products:
        a, b = np.diff(x["lattice_steps"]), np.diff(x["times"])
        if not len(a) or not np.allclose(a, a[0]) or not np.allclose(b, b[0]):
            raise ValueError("physical lag metadata requires uniform sampling")
        ds.append(float(a[0])); dt.append(float(b[0]))
    if not np.allclose(ds, ds[0]) or not np.allclose(dt, dt[0]):
        raise ValueError("realizations use different sampling")
    return {"candidate_sample_lags": list(lags), "lattice_step_spacing": ds[0],
            "candidate_lattice_step_lags": [float(x * ds[0]) for x in lags],
            "physical_time_spacing": dt[0], "candidate_physical_time_lags": [float(x * dt[0]) for x in lags]}


def run_phase_3_5(product_dirs, output, *, ranks=(4, 8, 12), lags=(0, 1, 2), observable="delta_forces",
                  surrogate_count=99, seed=0, negative_control_margin=0.0,
                  rcond=1e-12,
                  observable_metric: np.ndarray | None = None,
                  observable_coordinate_units: Sequence[str] | None = None,
                  execution_metadata: Mapping[str, Any] | None = None,
                  convective_scale: Mapping[str, Any] | None = None):
    started = time.perf_counter()
    if len(product_dirs) != 3:
        raise ValueError("Phase 3-5 requires exactly three canonical realizations")
    product_dirs = tuple(Path(x).resolve() for x in product_dirs)
    products = sorted((load_canonical_product(x) for x in product_dirs), key=lambda x: x["metadata"]["realization_id"])
    input_products = [{"directory": str(path),
                       "metadata_sha256": sha256_file(path / "paired_product.json"),
                       "product_sha256": sha256_file(path / "paired_product.npz")}
                      for path in product_dirs]
    ids = [x["metadata"]["realization_id"] for x in products]
    if len(set(ids)) != 3: raise ValueError("canonical realization IDs must be unique")
    if observable not in {"delta_forces", "delta_sensors"}: raise ValueError("observable must be an explicit canonical delta")
    ranks, lags = tuple(sorted(set(map(int, ranks)))), tuple(sorted(set(map(int, lags))))
    if not ranks or ranks[0] < 1 or not lags or lags[0] < 0: raise ValueError("invalid ranks or lags")
    channel_count = int(products[0][observable].shape[1])
    observable_metric = (np.ones(channel_count, dtype=float) if observable_metric is None
                         else np.asarray(observable_metric, dtype=float))
    expected_metric_shapes = {(channel_count,), (channel_count, channel_count)}
    if observable_metric.shape not in expected_metric_shapes or not np.isfinite(observable_metric).all():
        raise ValueError("observable_metric must be a finite diagonal vector or square channel metric")
    if observable_metric.ndim == 1:
        if np.any(observable_metric <= 0):
            raise ValueError("diagonal observable_metric must be positive")
    elif not np.allclose(observable_metric, observable_metric.T) or np.linalg.eigvalsh(observable_metric)[0] <= 0:
        raise ValueError("dense observable_metric must be symmetric positive definite")
    coordinate_units = (["native incremental observable units"] * channel_count
                        if observable_coordinate_units is None else list(observable_coordinate_units))
    if len(coordinate_units) != channel_count or any(not isinstance(unit, str) or not unit for unit in coordinate_units):
        raise ValueError("observable_coordinate_units must name every physical observable coordinate")
    weights = np.asarray(products[0]["state_weights"], float)
    if any(not np.array_equal(weights, x["state_weights"]) for x in products[1:]): raise ValueError("realizations use different fluid masks/metrics")
    if surrogate_count < 1: raise ValueError("at least one surrogate is required")
    if not isinstance(rcond, (int, float)) or not np.isfinite(rcond) or rcond < 0:
        raise ValueError("rcond must be a finite nonnegative scalar")
    rcond = float(rcond)
    lag_meta = _lag_metadata(products, lags)
    cache, folds, modes = _PODCache(max(ranks), weights, products), [], []
    for candidate_lag in lags:
        for held in range(len(products)):
            cache.get([x for index, x in enumerate(products) if index != held],
                      candidate_lag, max(ranks))
        for product in products:
            cache.get([product], candidate_lag, max(ranks))
    minimum_shift_offset = max(2, max(lags) + 1)
    for held, test in enumerate(products):
        train = [x for i, x in enumerate(products) if i != held]
        admissibility = [
            _admissible_shift_record(
                item, observable, minimum_shift_offset, lag_meta["physical_time_spacing"]
            )
            for item in train
        ]
        rank, lag, table = _choose(
            train, ranks, lags, observable, observable_metric, cache, rcond
        )
        fold, fold_modes = _fit_fold(train, test, rank=rank, lag=lag, observable=observable,
                                     observable_metric=observable_metric, cache=cache,
                                     rcond=rcond, materialize_models=True)
        null, selections = [], []
        for n in range(surrogate_count):
            task_seed = _seed(seed, ids[held], "surrogate", n)
            shifted, offsets = _shift(
                train, observable, np.random.default_rng(task_seed), admissibility
            )
            nr, nl, nt = _choose(
                shifted, ranks, lags, observable, observable_metric, cache, rcond
            )
            null.append(_fit_fold(shifted, test, rank=nr, lag=nl, observable=observable,
                                  observable_metric=observable_metric, cache=cache,
                                  rcond=rcond)[0]["metric_weighted_r2"])
            selections.append({"surrogate_index": n, "seed": task_seed, "rank": nr, "sample_lag": nl, "offsets": offsets, "train_only_sensitivity": nt})
        deranged = _derange(train, observable)
        control_rank, control_lag, control_sensitivity = _choose(
            deranged, ranks, lags, observable, observable_metric, cache, rcond
        )
        control = _fit_fold(
            deranged, test, rank=control_rank, lag=control_lag, observable=observable,
            observable_metric=observable_metric, cache=cache, rcond=rcond
        )[0]["metric_weighted_r2"]
        observed = fold["metric_weighted_r2"]
        fold.update({"held_out_realization": ids[held], "training_realizations": sorted(x["metadata"]["realization_id"] for x in train),
                     "selection": "nested realization holdout on training realizations only", "rank_lag_sensitivity": table,
                     "lattice_step_lag": float(lag * lag_meta["lattice_step_spacing"]),
                     "physical_time_lag": float(lag * lag_meta["physical_time_spacing"]),
                     "circular_time_shift_surrogate_scores": null,
                     "circular_time_shift_surrogate_statistic": "metric_weighted_r2",
                     "circular_time_shift_surrogate_selections": selections,
                     "circular_time_shift_admissibility": admissibility,
                     "selection_adjusted_periodic_excluded_empirical_p_value":
                         float((1 + np.sum(np.asarray(null) >= observed)) / (surrogate_count + 1)),
                     "realization_derangement_negative_control_metric_weighted_r2": control,
                     "realization_derangement_selected_rank": control_rank,
                     "realization_derangement_selected_sample_lag": control_lag,
                     "realization_derangement_rank_lag_sensitivity": control_sensitivity,
                     "realization_derangement_selection":
                         "complete training-only rank/sample-lag reselection on deranged training observables",
                     "negative_control_margin": float(negative_control_margin),
                     "negative_control_pass": bool(observed > control + negative_control_margin)})
        folds.append(fold); modes.append(fold_modes)
    comparisons = []
    for i in range(3):
        for j in range(i + 1, 3):
            k = min(modes[i]["lr_modes"].shape[1], modes[j]["lr_modes"].shape[1]); entry = {"folds": [ids[i], ids[j]], "rank_compared": k}
            for kind in ("lr_modes", "le_modes"):
                a, b = modes[i][kind][:, :k], modes[j][kind][:, :k]
                entry[kind] = {"weighted_mac": [modal_assurance(a[:, m], b[:, m], weights) for m in range(k)],
                               "principal_angles_deg": np.degrees(principal_angles(a, b, weights)).tolist(),
                               "projector_similarity": projector_similarity(a, b, weights)}
            comparisons.append(entry)
    report = {"schema_id": ANALYSIS_SCHEMA, "realization_ids": ids, "fold_count": 3,
              "preprocessing": "one max-rank state POD per training-set/sample-lag/metric; all selection training-only",
              "surrogates": "training circular time shifts with complete rank/lag reselection and cached state POD",
              "circular_time_shift_null_contract": {
                  "tested_statistic": "metric_weighted_r2", "upper_tail": True,
                  "period_estimator": "deterministic normalized multichannel rFFT power spectrum",
                  "period_estimation_data":
                      "each training realization's complete multichannel observable only; held-out realization excluded",
                  "period_estimator_preprocessing":
                      "remove each channel mean; normalize nonzero-bin power by per-channel spectral energy; average channels; select first maximum nonzero bin; period=N/k",
                  "peak_concentration_definition":
                      "dominant value of the channel-averaged normalized nonzero-bin power spectrum",
                  "exclusion_width_samples": minimum_shift_offset,
                  "exclusion_width_physical_time":
                      float(minimum_shift_offset * lag_meta["physical_time_spacing"]),
                  "exclusion_width_rule": "max(2, max(candidate_sample_lags) + 1)",
                  "excluded_offsets":
                      "circular-zero neighborhood and offsets within the exclusion width of every integer multiple of each realization-specific estimated period",
                  "periodic_equivalent_exclusions_applied": True,
                  "admissible_offsets_fixed_before_null_scoring": True,
                  "draw": "independent uniform draw from each training realization's admissible integer offsets",
                  "minimum_admissible_offset_count": 2,
                  "fail_closed": "unavailable period or fewer than two admissible shifts",
                  "selection_repeated": "all training-only rank/sample-lag selection"},
              "metrics": {
                  "primary": "metric_weighted_r2",
                  "primary_formula": "1 - pooled sum_t e_t^T H e_t / pooled sum_t (y_t - training_mean)^T H (y_t - training_mean)",
                  "supplementary": ["per_channel_r2", "unweighted_mean_channel_r2"]},
              "lag_metadata": lag_meta,
              "metric_contract": {
                  "state_metric_sha256": _hash(weights),
                  "observable_metric": observable_metric.tolist(),
                  "observable_metric_sha256": _hash(observable_metric),
                  "observable_metric_coordinates": "physical incremental observable channel coordinates",
                  "observable_coordinate_units": coordinate_units,
                  "observable_metric_declaration": "H=I" if np.array_equal(observable_metric, np.ones(channel_count)) else "explicit H",
                  "sensitivity_kind": "global metric scaling is invariant"},
              "state_pod_fit_count": len(cache.models), "state_pod_fit_bound": len(lags) * 6,
              "state_field_transform_count": cache.field_transform_count,
              "state_field_transform_bound": len(cache.models) * (len(products) - 1),
              "physical_mode_materialization_count": cache.physical_mode_materialization_count,
              "physical_mode_materialization_bound": len(products),
              "negative_control_gate": {"tested_statistic": "metric_weighted_r2",
                                        "rule": "observed metric_weighted_r2 > deranged metric_weighted_r2 + margin",
                                        "margin": float(negative_control_margin),
                                        "deranged_selection_repeated":
                                            "all training-only rank/sample-lag selection",
                                        "all_folds_pass": all(x["negative_control_pass"] for x in folds)},
              "numerical_contract": {
                  "status": "truncated", "regularization": None, "rcond": rcond,
                  "rank_rule": "retain singular values strictly greater than rcond times the largest singular value",
                  "applies_to":
                      ["state covariance pseudoinverse", "propagator generated rank",
                       "LR Gram rank and pseudoinverse", "LE Gram rank and pseudoinverse"]},
              "folds": folds, "fold_mode_comparisons": comparisons,
              "input_products": input_products,
              "execution_metadata": dict(execution_metadata or {}),
              "convective_lag_contract": dict(convective_scale or {})}
    model_arrays = {}
    for index, model in enumerate(modes):
        for name, value in model.items():
            model_arrays[f"fold_{index}_{name}"] = np.asarray(value)
    destination = Path(output)
    model_path = destination / "phase_3_5_models.npz"
    atomic_save_npz(model_path, **model_arrays)
    file_sha256 = hashlib.sha256(model_path.read_bytes()).hexdigest()
    array_hashes = {name: _hash(value) for name, value in model_arrays.items()}
    with np.load(model_path, allow_pickle=False) as persisted:
        persisted_hashes = {name: _hash(persisted[name]) for name in persisted.files}
    if persisted_hashes != array_hashes:
        raise RuntimeError("persisted Phase 3-5 model verification failed")
    report["model_artifact"] = {"path": model_path.name, "sha256": file_sha256,
                                "array_sha256": array_hashes, "verified_after_write": True,
                                "binding": "fold_N arrays correspond to report folds[N] in canonical realization order"}
    usage = resource.getrusage(resource.RUSAGE_SELF)
    report["execution_metadata"].update({
        "analysis_wall_time_seconds": float(time.perf_counter() - started),
        "peak_rss_kib": int(usage.ru_maxrss),
        "peak_rss_semantics": "ru_maxrss for this process; KiB on Linux",
    })
    atomic_write_json(destination / "phase_3_5_report.json", report)
    return report
