"""Second descriptive delivery using only the authorised frozen Greif arrays.""" import hashlib from io import BytesIO import json import math import os from pathlib import Path import resource from concurrent.futures import ProcessPoolExecutor import numpy as np from scipy.sparse import coo_matrix from scipy.sparse.csgraph import connected_components, dijkstra import phase3_h2_shared_miss_attributes as first from s2_tail_diagnostic_math import ALPHA, tail_membership, weights ROOT = first.BASE / "artifacts/phase3-h2-shared-miss-true-tail-2026-11-19" FROZEN_ROSTER = "artifacts/s2-main-preparation-2026-11-15/final-roster.json" FROZEN_ROSTER_SHA = "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e" FIRST_ROWS_SHA = "fd18ac43986d5ecbf60adaaa96fab3c55f8f55da19bccfc3032deae9d76a8f17" FIRST_RESULT_SHA = "c6596097021a5d6eb59925aaa261c93e1c51959609ee9a90665ce81505237750" UNRESOLVED = {k: first.UNAVAILABLE[k] for k in ( "physical_branch_count_and_length", "physical_flow_length_to_wall_thickness", "semantic_family_type")} NEW_METRICS = { "late_tail_component_count": "count", "late_tail_major_component_count": "count", "late_tail_component_depth": "depth over Rg, valid tail mass weighted", "late_tail_component_depth_max": "depth over Rg", "late_tail_component_depth_mass_coverage": "fraction of fixed tail mass", "near_tie": "proportion", "major_component_peak_gap": "T / t_set", "L_tail_raw": "raw coordinate, Greif mm convention", "L_tail_over_Rg": "distance / Rg", "tail_unreachable_mass_fraction": "fraction of fixed tail mass", "L_tail_to_h_g": "dimensionless", "surface_area_raw_squared": "raw coordinate squared", "sqrt_lambda2_over_lambda1": "dimensionless", "sqrt_lambda3_over_lambda1": "dimensionless", } def checked_array(path, sha): return np.load(BytesIO(first.checked_bytes(path, sha)), allow_pickle=False) def graph(u, v, lengths, n): return coo_matrix((np.tile(lengths, 2), (np.r_[u, v], np.r_[v, u])), shape=(n, n)).tocsr() def tail_graph_values(x, edge, mass, unit_time, membership, rg, gates): u, v = edge.T if not (np.all(u < v) and np.all(np.diff(u.astype(np.int64) * len(x) + v) > 0)): raise ValueError("Original tet edge identities must be sorted and unique") lengths = np.linalg.norm(x[u] - x[v], axis=1) if not np.isfinite(lengths).all() or np.any(lengths <= 0): raise ValueError("Invalid physical edge length") selected = membership > 0 tail = np.flatnonzero(selected) ix = np.full(len(x), -1, dtype=np.int64) ix[tail] = np.arange(len(tail)) inner = selected[u] & selected[v] local = graph(ix[u[inner]], ix[v[inner]], lengths[inner], len(tail)) count, labels = connected_components(local, directed=False) tail_mass = mass[tail] * membership[tail] total_tail_mass = ALPHA * math.fsum(mass) fractions = np.bincount(labels, weights=tail_mass, minlength=count) / total_tail_mass major = fractions >= .05 maxima = np.full(count, -np.inf) np.maximum.at(maxima, labels, unit_time[tail]) peak_order = np.sort(maxima[major]) gap = float(peak_order[-1] - peak_order[-2]) if len(peak_order) >= 2 else None near = None if np.ptp(unit_time) == 0 else int(gap is not None and gap <= .01) entering_u, entering_v = selected[u] & ~selected[v], selected[v] & ~selected[u] entry_nodes = np.unique(np.r_[ix[u[entering_u]], ix[v[entering_v]]]) depths = np.zeros(count) valid_depth = np.zeros(count, dtype=bool) if len(entry_nodes): dist = np.asarray(dijkstra(local, directed=False, indices=entry_nodes, min_only=True)) finite = np.isfinite(dist) np.maximum.at(depths, labels[finite], dist[finite] / rg) valid_depth[labels[finite]] = True if not np.array_equal(finite, valid_depth[labels]): raise ValueError("Some nodes in an entrance-connected component are unreachable") usable = float(np.sum(fractions[valid_depth])) depth = float(np.dot(fractions[valid_depth], depths[valid_depth]) / usable) if usable else None max_depth = float(depths[valid_depth].max()) if usable else None gate_distance, unreachable = None, None if gates is not None and len(gates): distances = np.asarray(dijkstra(graph(u, v, lengths, len(x)), directed=False, indices=gates, min_only=True))[tail] reachable = np.isfinite(distances) unreachable = math.fsum(tail_mass[~reachable]) / total_tail_mass if reachable.all(): gate_distance = math.fsum(tail_mass * distances) / total_tail_mass return dict(late_tail_component_count=int(count), late_tail_major_component_count=int(major.sum()), late_tail_component_depth=depth, late_tail_component_depth_max=max_depth, late_tail_component_depth_mass_coverage=usable, near_tie=near, major_component_peak_gap=gap, L_tail_raw=gate_distance, L_tail_over_Rg=gate_distance/rg if gate_distance is not None else None, tail_unreachable_mass_fraction=unreachable, components=[dict(tail_mass_fraction=float(f), peak_T_over_t_set=float(p), major=bool(m), depth_over_Rg=float(d) if valid else None) for f, p, m, d, valid in zip(fractions, maxima, major, depths, valid_depth)]) def compute(job): row, tail, part, geometry = job case = row["case"] source = first.BASE / geometry["preparation_root"] / "geometry.npz" geom_hash = geometry["output_sha256"]["geometry.npz"] with np.load(BytesIO(first.checked_bytes(source, geom_hash)), allow_pickle=False) as z: x = z["points_raw"] tet = z["tetra"] mesh_mass = z["node_volume_raw"] edge = z["edges"] faces = z["boundary_triangles"] ids = z["raw_node_ids"] raw = first.BASE / part["output"] pin = part["output_sha256"] label_mass = checked_array(raw / "mass.npy", pin["mass.npy"]) v5_ids = checked_array(raw / "raw_ids.npy", pin["raw_ids.npy"]) if not (len(tet) == row["tet_count"] and len(x) == part["nodes"] == tail["nodes"] and np.array_equal(ids, v5_ids) and np.all(np.diff(ids) > 0) and np.allclose(label_mass, mesh_mass, rtol=1e-10, atol=0)): raise ValueError(f"Retained label, geometry or mass domain differs: {case}") if (tail["component_id"], tail["fold"], tail["t_set"]) != (row["component_id"], row["fold"], part["t_set"]): raise ValueError(f"Fold or time scale differs: {case}") gates = None if row["gate_mapping_raw_ids"] is not None: gate_ids = np.array(row["gate_mapping_raw_ids"]) gate_ix = np.searchsorted(ids, gate_ids) if np.all(gate_ix < len(ids)) and np.array_equal(ids[gate_ix], gate_ids): gates = gate_ix path = raw / "seconds.npy" with (ROOT / "target-access.jsonl").open("a") as log: log.write(json.dumps(dict(case=case, role="grouped_test", path=str(path), sha256=pin["seconds.npy"], purpose="pre-registered descriptive attribute study")) + "\n") truth = checked_array(path, pin["seconds.npy"]).astype(np.float64) / part["t_set"] membership, witness = tail_membership(truth, weights(label_mass)) if witness != tail["truth_tail"]: raise ValueError(f"Recomputed real-tail definition differs from the frozen prior receipt: {case}") rg = tail["Rg"] out = tail_graph_values(x, edge, label_mass, truth, membership, rg, gates) a, b, c = x[faces].transpose(1, 0, 2) area = math.fsum(.5 * np.linalg.norm(np.cross(b - a, c - a), axis=1)) if not math.isfinite(area) or area <= 0: raise ValueError(f"Invalid retained exterior area: {case}") w = weights(label_mass) centered = x - np.sum(w[:, None] * x, axis=0) spectrum = np.linalg.eigvalsh((centered * w[:, None]).T @ centered)[::-1] if not np.isclose(np.sqrt(spectrum.sum()), rg, rtol=1e-10, atol=1e-10): raise ValueError(f"Mass-weighted Rg differs from the frozen scalar: {case}") distance = out["L_tail_raw"] scale = 2 * math.fsum(label_mass) / area shape = [float(np.sqrt(max(0., ev / spectrum[0]))) if spectrum[0] > 0 else None for ev in spectrum[1:]] out.update(surface_area_raw_squared=area, L_tail_to_h_g=distance/scale if distance is not None else None, h_g_raw=scale, sqrt_lambda2_over_lambda1=shape[0], sqrt_lambda3_over_lambda1=shape[1]) return dict(row, **out, geometry_sha256=geom_hash, true_time_sha256=pin["seconds.npy"]) def worker_limits(): resource.setrlimit(resource.RLIMIT_CPU, (900, 900)) def paired_stats(rows): families = sorted(first.grouped(rows)) draws = np.random.default_rng(first.SEED).integers( len(families), size=(first.REPLICATES, len(families))) miss = [r for r in rows if r["shared_miss"]] other = [r for r in rows if not r["shared_miss"]] result = {} for metric in (*NEW_METRICS, *UNRESOLVED): x, a = first.attribute_rows(families, miss, metric) y, b = first.attribute_rows(families, other, metric) _, pooled = first.attribute_rows(families, rows, metric) diff = first.sample_means(x, draws) - first.sample_means(y, draws) obs = diff[np.isfinite(diff)] l, r = a["family_equal_mean"], b["family_equal_mean"] result[metric] = dict(units=NEW_METRICS.get(metric), missing_reason=UNRESOLVED.get(metric), shared_miss=a, other=b, all_parts=pooled, effect=l-r if l is not None and r is not None else None, interval_95=np.quantile(obs, [.025,.975]).tolist() if len(obs) else None, undefined_replicates=len(diff)-len(obs)) return result def jobs(): meta = first.inputs() prior = first.checked_bytes(first.OUT / "by-part.jsonl", FIRST_ROWS_SHA) first.checked_bytes(first.OUT / "result.json", FIRST_RESULT_SHA) roster = first.unique_index(json.loads(first.checked_bytes(first.BASE / FROZEN_ROSTER, FROZEN_ROSTER_SHA))["parts"]) prior_rows = [json.loads(x) for x in prior.decode().splitlines()] tail = first.unique_index(json.loads(meta["s2_tail"])["rows"]) pilot = json.loads(meta["pilot"])["parts"] if len(prior_rows) != 552 or len(first.unique_index(prior_rows)) != 552: raise ValueError("Frozen 552-part roster required") return [(r, tail[r["case"]], pilot[r["case"]], roster[r["case"]]["preparation"]) for r in prior_rows] def run(): if os.environ.get("CUDA_VISIBLE_DEVICES") != "": raise RuntimeError("A masked CPU-only invocation is required") path = ROOT / "execution-contract.json" c = json.loads(path.read_text()) if (c["authorizing_handoff"] != "h-1149aa75" or c["expected_parts"] != 552 or c["worker_cpu_budget_seconds"] != 3600 or c["worker_limit"] != 4): raise ValueError("Execution scope or pinned implementation differs") for name, digest in c["code_sha256"].items(): first.checked_bytes(first.BASE / name, digest) inputs = jobs() with (ROOT / "started.json").open("x") as stream: json.dump(dict(pid=os.getpid(), cpu_mask=sorted(os.sched_getaffinity(0)), contract_sha256=hashlib.sha256(path.read_bytes()).hexdigest()), stream) rows = [] with (ROOT / "by-part.jsonl").open("x") as stream, ProcessPoolExecutor( max_workers=c["worker_limit"], initializer=worker_limits) as pool: batch = c["worker_limit"] for offset in range(0, len(inputs), batch): for row in pool.map(compute, inputs[offset:offset+batch], chunksize=1): rows.append(row) stream.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n") stream.flush() access = [json.loads(x) for x in (ROOT / "target-access.jsonl").read_text().splitlines()] if len(rows) != 552 or {x["case"] for x in access} != {x["case"] for x in rows} or len(access) != 552: raise ValueError("Expected the exact bounded labelled read set") child = resource.getrusage(resource.RUSAGE_CHILDREN) result = dict(status="COMPLETE", prereg_commit=first.PREREG_COMMIT, baseline_commit="44c1d5a10d4c00d5af2b38f0e5e0e11ed11df611", parts=len(rows), test_role_reads=len(access), new_fits=0, gpu_seconds=0, new_raw_label_decodes=0, prediction_arrays_read=0, bootstrap=dict(seed=first.SEED, replicates=first.REPLICATES, unit="original family, simultaneous"), worker_cpu_seconds=child.ru_utime+child.ru_stime, metrics=paired_stats(rows)) (ROOT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n") if __name__ == "__main__": run()