Publish separate bounded true-tail graph attributes and exact target-access evidence

This commit is contained in:
NewMeltAssist committed 2026-11-19 14:38:39 +08:00
1 parent 44c1d5a10d
commit 40f9e4ce5d
8 files changed
+2429

No files matched your search

+241
View File
@@ -0,0 +1,241 @@
"""Second descriptive delivery using only the authorised frozen Greif arrays."""
import hashlib
from io import BytesIO
import json
import math
import os
from pathlib import Path
import resource
from concurrent.futures import ProcessPoolExecutor
import numpy as np
from scipy.sparse import coo_matrix
from scipy.sparse.csgraph import connected_components, dijkstra
import phase3_h2_shared_miss_attributes as first
from s2_tail_diagnostic_math import ALPHA, tail_membership, weights
ROOT = first.BASE / "artifacts/phase3-h2-shared-miss-true-tail-2026-11-19"
FROZEN_ROSTER = "artifacts/s2-main-preparation-2026-11-15/final-roster.json"
FROZEN_ROSTER_SHA = "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e"
FIRST_ROWS_SHA = "fd18ac43986d5ecbf60adaaa96fab3c55f8f55da19bccfc3032deae9d76a8f17"
FIRST_RESULT_SHA = "c6596097021a5d6eb59925aaa261c93e1c51959609ee9a90665ce81505237750"
UNRESOLVED = {k: first.UNAVAILABLE[k] for k in (
"physical_branch_count_and_length", "physical_flow_length_to_wall_thickness", "semantic_family_type")}
NEW_METRICS = {
"late_tail_component_count": "count",
"late_tail_major_component_count": "count",
"late_tail_component_depth": "depth over Rg, valid tail mass weighted",
"late_tail_component_depth_max": "depth over Rg",
"late_tail_component_depth_mass_coverage": "fraction of fixed tail mass",
"near_tie": "proportion",
"major_component_peak_gap": "T / t_set",
"L_tail_raw": "raw coordinate, Greif mm convention",
"L_tail_over_Rg": "distance / Rg",
"tail_unreachable_mass_fraction": "fraction of fixed tail mass",
"L_tail_to_h_g": "dimensionless",
"surface_area_raw_squared": "raw coordinate squared",
"sqrt_lambda2_over_lambda1": "dimensionless",
"sqrt_lambda3_over_lambda1": "dimensionless",
}
def checked_array(path, sha):
return np.load(BytesIO(first.checked_bytes(path, sha)), allow_pickle=False)
def graph(u, v, lengths, n):
return coo_matrix((np.tile(lengths, 2), (np.r_[u, v], np.r_[v, u])), shape=(n, n)).tocsr()
def tail_graph_values(x, edge, mass, unit_time, membership, rg, gates):
u, v = edge.T
if not (np.all(u < v) and np.all(np.diff(u.astype(np.int64) * len(x) + v) > 0)):
raise ValueError("Original tet edge identities must be sorted and unique")
lengths = np.linalg.norm(x[u] - x[v], axis=1)
if not np.isfinite(lengths).all() or np.any(lengths <= 0):
raise ValueError("Invalid physical edge length")
selected = membership > 0
tail = np.flatnonzero(selected)
ix = np.full(len(x), -1, dtype=np.int64)
ix[tail] = np.arange(len(tail))
inner = selected[u] & selected[v]
local = graph(ix[u[inner]], ix[v[inner]], lengths[inner], len(tail))
count, labels = connected_components(local, directed=False)
tail_mass = mass[tail] * membership[tail]
total_tail_mass = ALPHA * math.fsum(mass)
fractions = np.bincount(labels, weights=tail_mass, minlength=count) / total_tail_mass
major = fractions >= .05
maxima = np.full(count, -np.inf)
np.maximum.at(maxima, labels, unit_time[tail])
peak_order = np.sort(maxima[major])
gap = float(peak_order[-1] - peak_order[-2]) if len(peak_order) >= 2 else None
near = None if np.ptp(unit_time) == 0 else int(gap is not None and gap <= .01)
entering_u, entering_v = selected[u] & ~selected[v], selected[v] & ~selected[u]
entry_nodes = np.unique(np.r_[ix[u[entering_u]], ix[v[entering_v]]])
depths = np.zeros(count)
valid_depth = np.zeros(count, dtype=bool)
if len(entry_nodes):
dist = np.asarray(dijkstra(local, directed=False, indices=entry_nodes, min_only=True))
finite = np.isfinite(dist)
np.maximum.at(depths, labels[finite], dist[finite] / rg)
valid_depth[labels[finite]] = True
if not np.array_equal(finite, valid_depth[labels]):
raise ValueError("Some nodes in an entrance-connected component are unreachable")
usable = float(np.sum(fractions[valid_depth]))
depth = float(np.dot(fractions[valid_depth], depths[valid_depth]) / usable) if usable else None
max_depth = float(depths[valid_depth].max()) if usable else None
gate_distance, unreachable = None, None
if gates is not None and len(gates):
distances = np.asarray(dijkstra(graph(u, v, lengths, len(x)), directed=False,
indices=gates, min_only=True))[tail]
reachable = np.isfinite(distances)
unreachable = math.fsum(tail_mass[~reachable]) / total_tail_mass
if reachable.all():
gate_distance = math.fsum(tail_mass * distances) / total_tail_mass
return dict(late_tail_component_count=int(count), late_tail_major_component_count=int(major.sum()),
late_tail_component_depth=depth, late_tail_component_depth_max=max_depth,
late_tail_component_depth_mass_coverage=usable, near_tie=near,
major_component_peak_gap=gap, L_tail_raw=gate_distance,
L_tail_over_Rg=gate_distance/rg if gate_distance is not None else None,
tail_unreachable_mass_fraction=unreachable,
components=[dict(tail_mass_fraction=float(f), peak_T_over_t_set=float(p), major=bool(m),
depth_over_Rg=float(d) if valid else None)
for f, p, m, d, valid in zip(fractions, maxima, major, depths, valid_depth)])
def compute(job):
row, tail, part, geometry = job
case = row["case"]
source = first.BASE / geometry["preparation_root"] / "geometry.npz"
geom_hash = geometry["output_sha256"]["geometry.npz"]
with np.load(BytesIO(first.checked_bytes(source, geom_hash)), allow_pickle=False) as z:
x = z["points_raw"]
tet = z["tetra"]
mesh_mass = z["node_volume_raw"]
edge = z["edges"]
faces = z["boundary_triangles"]
ids = z["raw_node_ids"]
raw = first.BASE / part["output"]
pin = part["output_sha256"]
label_mass = checked_array(raw / "mass.npy", pin["mass.npy"])
v5_ids = checked_array(raw / "raw_ids.npy", pin["raw_ids.npy"])
if not (len(tet) == row["tet_count"] and len(x) == part["nodes"] == tail["nodes"]
and np.array_equal(ids, v5_ids) and np.all(np.diff(ids) > 0)
and np.allclose(label_mass, mesh_mass, rtol=1e-10, atol=0)):
raise ValueError(f"Retained label, geometry or mass domain differs: {case}")
if (tail["component_id"], tail["fold"], tail["t_set"]) != (row["component_id"], row["fold"], part["t_set"]):
raise ValueError(f"Fold or time scale differs: {case}")
gates = None
if row["gate_mapping_raw_ids"] is not None:
gate_ids = np.array(row["gate_mapping_raw_ids"])
gate_ix = np.searchsorted(ids, gate_ids)
if np.all(gate_ix < len(ids)) and np.array_equal(ids[gate_ix], gate_ids):
gates = gate_ix
path = raw / "seconds.npy"
with (ROOT / "target-access.jsonl").open("a") as log:
log.write(json.dumps(dict(case=case, role="grouped_test", path=str(path), sha256=pin["seconds.npy"],
purpose="pre-registered descriptive attribute study")) + "\n")
truth = checked_array(path, pin["seconds.npy"]).astype(np.float64) / part["t_set"]
membership, witness = tail_membership(truth, weights(label_mass))
if witness != tail["truth_tail"]:
raise ValueError(f"Recomputed real-tail definition differs from the frozen prior receipt: {case}")
rg = tail["Rg"]
out = tail_graph_values(x, edge, label_mass, truth, membership, rg, gates)
a, b, c = x[faces].transpose(1, 0, 2)
area = math.fsum(.5 * np.linalg.norm(np.cross(b - a, c - a), axis=1))
if not math.isfinite(area) or area <= 0:
raise ValueError(f"Invalid retained exterior area: {case}")
w = weights(label_mass)
centered = x - np.sum(w[:, None] * x, axis=0)
spectrum = np.linalg.eigvalsh((centered * w[:, None]).T @ centered)[::-1]
if not np.isclose(np.sqrt(spectrum.sum()), rg, rtol=1e-10, atol=1e-10):
raise ValueError(f"Mass-weighted Rg differs from the frozen scalar: {case}")
distance = out["L_tail_raw"]
scale = 2 * math.fsum(label_mass) / area
shape = [float(np.sqrt(max(0., ev / spectrum[0]))) if spectrum[0] > 0 else None for ev in spectrum[1:]]
out.update(surface_area_raw_squared=area, L_tail_to_h_g=distance/scale if distance is not None else None,
h_g_raw=scale, sqrt_lambda2_over_lambda1=shape[0], sqrt_lambda3_over_lambda1=shape[1])
return dict(row, **out, geometry_sha256=geom_hash, true_time_sha256=pin["seconds.npy"])
def worker_limits():
resource.setrlimit(resource.RLIMIT_CPU, (900, 900))
def paired_stats(rows):
families = sorted(first.grouped(rows))
draws = np.random.default_rng(first.SEED).integers(
len(families), size=(first.REPLICATES, len(families)))
miss = [r for r in rows if r["shared_miss"]]
other = [r for r in rows if not r["shared_miss"]]
result = {}
for metric in (*NEW_METRICS, *UNRESOLVED):
x, a = first.attribute_rows(families, miss, metric)
y, b = first.attribute_rows(families, other, metric)
_, pooled = first.attribute_rows(families, rows, metric)
diff = first.sample_means(x, draws) - first.sample_means(y, draws)
obs = diff[np.isfinite(diff)]
l, r = a["family_equal_mean"], b["family_equal_mean"]
result[metric] = dict(units=NEW_METRICS.get(metric), missing_reason=UNRESOLVED.get(metric),
shared_miss=a, other=b, all_parts=pooled,
effect=l-r if l is not None and r is not None else None,
interval_95=np.quantile(obs, [.025,.975]).tolist() if len(obs) else None,
undefined_replicates=len(diff)-len(obs))
return result
def jobs():
meta = first.inputs()
prior = first.checked_bytes(first.OUT / "by-part.jsonl", FIRST_ROWS_SHA)
first.checked_bytes(first.OUT / "result.json", FIRST_RESULT_SHA)
roster = first.unique_index(json.loads(first.checked_bytes(first.BASE / FROZEN_ROSTER,
FROZEN_ROSTER_SHA))["parts"])
prior_rows = [json.loads(x) for x in prior.decode().splitlines()]
tail = first.unique_index(json.loads(meta["s2_tail"])["rows"])
pilot = json.loads(meta["pilot"])["parts"]
if len(prior_rows) != 552 or len(first.unique_index(prior_rows)) != 552:
raise ValueError("Frozen 552-part roster required")
return [(r, tail[r["case"]], pilot[r["case"]], roster[r["case"]]["preparation"]) for r in prior_rows]
def run():
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
raise RuntimeError("A masked CPU-only invocation is required")
path = ROOT / "execution-contract.json"
c = json.loads(path.read_text())
if (c["authorizing_handoff"] != "h-1149aa75" or c["expected_parts"] != 552
or c["worker_cpu_budget_seconds"] != 3600 or c["worker_limit"] != 4):
raise ValueError("Execution scope or pinned implementation differs")
for name, digest in c["code_sha256"].items():
first.checked_bytes(first.BASE / name, digest)
inputs = jobs()
with (ROOT / "started.json").open("x") as stream:
json.dump(dict(pid=os.getpid(), cpu_mask=sorted(os.sched_getaffinity(0)),
contract_sha256=hashlib.sha256(path.read_bytes()).hexdigest()), stream)
rows = []
with (ROOT / "by-part.jsonl").open("x") as stream, ProcessPoolExecutor(
max_workers=c["worker_limit"], initializer=worker_limits) as pool:
batch = c["worker_limit"]
for offset in range(0, len(inputs), batch):
for row in pool.map(compute, inputs[offset:offset+batch], chunksize=1):
rows.append(row)
stream.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n")
stream.flush()
access = [json.loads(x) for x in (ROOT / "target-access.jsonl").read_text().splitlines()]
if len(rows) != 552 or {x["case"] for x in access} != {x["case"] for x in rows} or len(access) != 552:
raise ValueError("Expected the exact bounded labelled read set")
child = resource.getrusage(resource.RUSAGE_CHILDREN)
result = dict(status="COMPLETE", prereg_commit=first.PREREG_COMMIT,
baseline_commit="44c1d5a10d4c00d5af2b38f0e5e0e11ed11df611", parts=len(rows), test_role_reads=len(access),
new_fits=0, gpu_seconds=0, new_raw_label_decodes=0, prediction_arrays_read=0,
bootstrap=dict(seed=first.SEED, replicates=first.REPLICATES, unit="original family, simultaneous"),
worker_cpu_seconds=child.ru_utime+child.ru_stime, metrics=paired_stats(rows))
(ROOT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
if __name__ == "__main__":
run()