242 lines
13 KiB
Python
242 lines
13 KiB
Python
"""Second descriptive delivery using only the authorised frozen Greif arrays."""
|
|
|
|
import hashlib
|
|
from io import BytesIO
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
from concurrent.futures import ProcessPoolExecutor
|
|
|
|
import numpy as np
|
|
from scipy.sparse import coo_matrix
|
|
from scipy.sparse.csgraph import connected_components, dijkstra
|
|
|
|
import phase3_h2_shared_miss_attributes as first
|
|
from s2_tail_diagnostic_math import ALPHA, tail_membership, weights
|
|
|
|
|
|
ROOT = first.BASE / "artifacts/phase3-h2-shared-miss-true-tail-2026-11-19"
|
|
FROZEN_ROSTER = "artifacts/s2-main-preparation-2026-11-15/final-roster.json"
|
|
FROZEN_ROSTER_SHA = "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e"
|
|
FIRST_ROWS_SHA = "fd18ac43986d5ecbf60adaaa96fab3c55f8f55da19bccfc3032deae9d76a8f17"
|
|
FIRST_RESULT_SHA = "c6596097021a5d6eb59925aaa261c93e1c51959609ee9a90665ce81505237750"
|
|
UNRESOLVED = {k: first.UNAVAILABLE[k] for k in (
|
|
"physical_branch_count_and_length", "physical_flow_length_to_wall_thickness", "semantic_family_type")}
|
|
NEW_METRICS = {
|
|
"late_tail_component_count": "count",
|
|
"late_tail_major_component_count": "count",
|
|
"late_tail_component_depth": "depth over Rg, valid tail mass weighted",
|
|
"late_tail_component_depth_max": "depth over Rg",
|
|
"late_tail_component_depth_mass_coverage": "fraction of fixed tail mass",
|
|
"near_tie": "proportion",
|
|
"major_component_peak_gap": "T / t_set",
|
|
"L_tail_raw": "raw coordinate, Greif mm convention",
|
|
"L_tail_over_Rg": "distance / Rg",
|
|
"tail_unreachable_mass_fraction": "fraction of fixed tail mass",
|
|
"L_tail_to_h_g": "dimensionless",
|
|
"surface_area_raw_squared": "raw coordinate squared",
|
|
"sqrt_lambda2_over_lambda1": "dimensionless",
|
|
"sqrt_lambda3_over_lambda1": "dimensionless",
|
|
}
|
|
|
|
|
|
def checked_array(path, sha):
|
|
return np.load(BytesIO(first.checked_bytes(path, sha)), allow_pickle=False)
|
|
|
|
|
|
def graph(u, v, lengths, n):
|
|
return coo_matrix((np.tile(lengths, 2), (np.r_[u, v], np.r_[v, u])), shape=(n, n)).tocsr()
|
|
|
|
|
|
def tail_graph_values(x, edge, mass, unit_time, membership, rg, gates):
|
|
u, v = edge.T
|
|
if not (np.all(u < v) and np.all(np.diff(u.astype(np.int64) * len(x) + v) > 0)):
|
|
raise ValueError("Original tet edge identities must be sorted and unique")
|
|
lengths = np.linalg.norm(x[u] - x[v], axis=1)
|
|
if not np.isfinite(lengths).all() or np.any(lengths <= 0):
|
|
raise ValueError("Invalid physical edge length")
|
|
selected = membership > 0
|
|
tail = np.flatnonzero(selected)
|
|
ix = np.full(len(x), -1, dtype=np.int64)
|
|
ix[tail] = np.arange(len(tail))
|
|
inner = selected[u] & selected[v]
|
|
local = graph(ix[u[inner]], ix[v[inner]], lengths[inner], len(tail))
|
|
count, labels = connected_components(local, directed=False)
|
|
tail_mass = mass[tail] * membership[tail]
|
|
total_tail_mass = ALPHA * math.fsum(mass)
|
|
fractions = np.bincount(labels, weights=tail_mass, minlength=count) / total_tail_mass
|
|
major = fractions >= .05
|
|
maxima = np.full(count, -np.inf)
|
|
np.maximum.at(maxima, labels, unit_time[tail])
|
|
peak_order = np.sort(maxima[major])
|
|
gap = float(peak_order[-1] - peak_order[-2]) if len(peak_order) >= 2 else None
|
|
near = None if np.ptp(unit_time) == 0 else int(gap is not None and gap <= .01)
|
|
entering_u, entering_v = selected[u] & ~selected[v], selected[v] & ~selected[u]
|
|
entry_nodes = np.unique(np.r_[ix[u[entering_u]], ix[v[entering_v]]])
|
|
depths = np.zeros(count)
|
|
valid_depth = np.zeros(count, dtype=bool)
|
|
if len(entry_nodes):
|
|
dist = np.asarray(dijkstra(local, directed=False, indices=entry_nodes, min_only=True))
|
|
finite = np.isfinite(dist)
|
|
np.maximum.at(depths, labels[finite], dist[finite] / rg)
|
|
valid_depth[labels[finite]] = True
|
|
if not np.array_equal(finite, valid_depth[labels]):
|
|
raise ValueError("Some nodes in an entrance-connected component are unreachable")
|
|
usable = float(np.sum(fractions[valid_depth]))
|
|
depth = float(np.dot(fractions[valid_depth], depths[valid_depth]) / usable) if usable else None
|
|
max_depth = float(depths[valid_depth].max()) if usable else None
|
|
|
|
gate_distance, unreachable = None, None
|
|
if gates is not None and len(gates):
|
|
distances = np.asarray(dijkstra(graph(u, v, lengths, len(x)), directed=False,
|
|
indices=gates, min_only=True))[tail]
|
|
reachable = np.isfinite(distances)
|
|
unreachable = math.fsum(tail_mass[~reachable]) / total_tail_mass
|
|
if reachable.all():
|
|
gate_distance = math.fsum(tail_mass * distances) / total_tail_mass
|
|
return dict(late_tail_component_count=int(count), late_tail_major_component_count=int(major.sum()),
|
|
late_tail_component_depth=depth, late_tail_component_depth_max=max_depth,
|
|
late_tail_component_depth_mass_coverage=usable, near_tie=near,
|
|
major_component_peak_gap=gap, L_tail_raw=gate_distance,
|
|
L_tail_over_Rg=gate_distance/rg if gate_distance is not None else None,
|
|
tail_unreachable_mass_fraction=unreachable,
|
|
components=[dict(tail_mass_fraction=float(f), peak_T_over_t_set=float(p), major=bool(m),
|
|
depth_over_Rg=float(d) if valid else None)
|
|
for f, p, m, d, valid in zip(fractions, maxima, major, depths, valid_depth)])
|
|
|
|
|
|
def compute(job):
|
|
row, tail, part, geometry = job
|
|
case = row["case"]
|
|
source = first.BASE / geometry["preparation_root"] / "geometry.npz"
|
|
geom_hash = geometry["output_sha256"]["geometry.npz"]
|
|
with np.load(BytesIO(first.checked_bytes(source, geom_hash)), allow_pickle=False) as z:
|
|
x = z["points_raw"]
|
|
tet = z["tetra"]
|
|
mesh_mass = z["node_volume_raw"]
|
|
edge = z["edges"]
|
|
faces = z["boundary_triangles"]
|
|
ids = z["raw_node_ids"]
|
|
raw = first.BASE / part["output"]
|
|
pin = part["output_sha256"]
|
|
label_mass = checked_array(raw / "mass.npy", pin["mass.npy"])
|
|
v5_ids = checked_array(raw / "raw_ids.npy", pin["raw_ids.npy"])
|
|
if not (len(tet) == row["tet_count"] and len(x) == part["nodes"] == tail["nodes"]
|
|
and np.array_equal(ids, v5_ids) and np.all(np.diff(ids) > 0)
|
|
and np.allclose(label_mass, mesh_mass, rtol=1e-10, atol=0)):
|
|
raise ValueError(f"Retained label, geometry or mass domain differs: {case}")
|
|
if (tail["component_id"], tail["fold"], tail["t_set"]) != (row["component_id"], row["fold"], part["t_set"]):
|
|
raise ValueError(f"Fold or time scale differs: {case}")
|
|
gates = None
|
|
if row["gate_mapping_raw_ids"] is not None:
|
|
gate_ids = np.array(row["gate_mapping_raw_ids"])
|
|
gate_ix = np.searchsorted(ids, gate_ids)
|
|
if np.all(gate_ix < len(ids)) and np.array_equal(ids[gate_ix], gate_ids):
|
|
gates = gate_ix
|
|
path = raw / "seconds.npy"
|
|
with (ROOT / "target-access.jsonl").open("a") as log:
|
|
log.write(json.dumps(dict(case=case, role="grouped_test", path=str(path), sha256=pin["seconds.npy"],
|
|
purpose="pre-registered descriptive attribute study")) + "\n")
|
|
truth = checked_array(path, pin["seconds.npy"]).astype(np.float64) / part["t_set"]
|
|
membership, witness = tail_membership(truth, weights(label_mass))
|
|
if witness != tail["truth_tail"]:
|
|
raise ValueError(f"Recomputed real-tail definition differs from the frozen prior receipt: {case}")
|
|
rg = tail["Rg"]
|
|
out = tail_graph_values(x, edge, label_mass, truth, membership, rg, gates)
|
|
a, b, c = x[faces].transpose(1, 0, 2)
|
|
area = math.fsum(.5 * np.linalg.norm(np.cross(b - a, c - a), axis=1))
|
|
if not math.isfinite(area) or area <= 0:
|
|
raise ValueError(f"Invalid retained exterior area: {case}")
|
|
w = weights(label_mass)
|
|
centered = x - np.sum(w[:, None] * x, axis=0)
|
|
spectrum = np.linalg.eigvalsh((centered * w[:, None]).T @ centered)[::-1]
|
|
if not np.isclose(np.sqrt(spectrum.sum()), rg, rtol=1e-10, atol=1e-10):
|
|
raise ValueError(f"Mass-weighted Rg differs from the frozen scalar: {case}")
|
|
distance = out["L_tail_raw"]
|
|
scale = 2 * math.fsum(label_mass) / area
|
|
shape = [float(np.sqrt(max(0., ev / spectrum[0]))) if spectrum[0] > 0 else None for ev in spectrum[1:]]
|
|
out.update(surface_area_raw_squared=area, L_tail_to_h_g=distance/scale if distance is not None else None,
|
|
h_g_raw=scale, sqrt_lambda2_over_lambda1=shape[0], sqrt_lambda3_over_lambda1=shape[1])
|
|
return dict(row, **out, geometry_sha256=geom_hash, true_time_sha256=pin["seconds.npy"])
|
|
|
|
|
|
def worker_limits():
|
|
resource.setrlimit(resource.RLIMIT_CPU, (900, 900))
|
|
|
|
|
|
def paired_stats(rows):
|
|
families = sorted(first.grouped(rows))
|
|
draws = np.random.default_rng(first.SEED).integers(
|
|
len(families), size=(first.REPLICATES, len(families)))
|
|
miss = [r for r in rows if r["shared_miss"]]
|
|
other = [r for r in rows if not r["shared_miss"]]
|
|
result = {}
|
|
for metric in (*NEW_METRICS, *UNRESOLVED):
|
|
x, a = first.attribute_rows(families, miss, metric)
|
|
y, b = first.attribute_rows(families, other, metric)
|
|
_, pooled = first.attribute_rows(families, rows, metric)
|
|
diff = first.sample_means(x, draws) - first.sample_means(y, draws)
|
|
obs = diff[np.isfinite(diff)]
|
|
l, r = a["family_equal_mean"], b["family_equal_mean"]
|
|
result[metric] = dict(units=NEW_METRICS.get(metric), missing_reason=UNRESOLVED.get(metric),
|
|
shared_miss=a, other=b, all_parts=pooled,
|
|
effect=l-r if l is not None and r is not None else None,
|
|
interval_95=np.quantile(obs, [.025,.975]).tolist() if len(obs) else None,
|
|
undefined_replicates=len(diff)-len(obs))
|
|
return result
|
|
|
|
|
|
def jobs():
|
|
meta = first.inputs()
|
|
prior = first.checked_bytes(first.OUT / "by-part.jsonl", FIRST_ROWS_SHA)
|
|
first.checked_bytes(first.OUT / "result.json", FIRST_RESULT_SHA)
|
|
roster = first.unique_index(json.loads(first.checked_bytes(first.BASE / FROZEN_ROSTER,
|
|
FROZEN_ROSTER_SHA))["parts"])
|
|
prior_rows = [json.loads(x) for x in prior.decode().splitlines()]
|
|
tail = first.unique_index(json.loads(meta["s2_tail"])["rows"])
|
|
pilot = json.loads(meta["pilot"])["parts"]
|
|
if len(prior_rows) != 552 or len(first.unique_index(prior_rows)) != 552:
|
|
raise ValueError("Frozen 552-part roster required")
|
|
return [(r, tail[r["case"]], pilot[r["case"]], roster[r["case"]]["preparation"]) for r in prior_rows]
|
|
|
|
|
|
def run():
|
|
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
|
|
raise RuntimeError("A masked CPU-only invocation is required")
|
|
path = ROOT / "execution-contract.json"
|
|
c = json.loads(path.read_text())
|
|
if (c["authorizing_handoff"] != "h-1149aa75" or c["expected_parts"] != 552
|
|
or c["worker_cpu_budget_seconds"] != 3600 or c["worker_limit"] != 4):
|
|
raise ValueError("Execution scope or pinned implementation differs")
|
|
for name, digest in c["code_sha256"].items():
|
|
first.checked_bytes(first.BASE / name, digest)
|
|
inputs = jobs()
|
|
with (ROOT / "started.json").open("x") as stream:
|
|
json.dump(dict(pid=os.getpid(), cpu_mask=sorted(os.sched_getaffinity(0)),
|
|
contract_sha256=hashlib.sha256(path.read_bytes()).hexdigest()), stream)
|
|
rows = []
|
|
with (ROOT / "by-part.jsonl").open("x") as stream, ProcessPoolExecutor(
|
|
max_workers=c["worker_limit"], initializer=worker_limits) as pool:
|
|
batch = c["worker_limit"]
|
|
for offset in range(0, len(inputs), batch):
|
|
for row in pool.map(compute, inputs[offset:offset+batch], chunksize=1):
|
|
rows.append(row)
|
|
stream.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n")
|
|
stream.flush()
|
|
access = [json.loads(x) for x in (ROOT / "target-access.jsonl").read_text().splitlines()]
|
|
if len(rows) != 552 or {x["case"] for x in access} != {x["case"] for x in rows} or len(access) != 552:
|
|
raise ValueError("Expected the exact bounded labelled read set")
|
|
child = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
result = dict(status="COMPLETE", prereg_commit=first.PREREG_COMMIT,
|
|
baseline_commit="44c1d5a10d4c00d5af2b38f0e5e0e11ed11df611", parts=len(rows), test_role_reads=len(access),
|
|
new_fits=0, gpu_seconds=0, new_raw_label_decodes=0, prediction_arrays_read=0,
|
|
bootstrap=dict(seed=first.SEED, replicates=first.REPLICATES, unit="original family, simultaneous"),
|
|
worker_cpu_seconds=child.ru_utime+child.ru_stime, metrics=paired_stats(rows))
|
|
(ROOT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run()
|