Publish separate bounded true-tail graph attributes and exact target-access evidence
This commit is contained in:
1 parent
44c1d5a10d
commit
40f9e4ce5d
8 files changed
+2429
No files matched your search
@@ -0,0 +1,241 @@
|
||||
"""Second descriptive delivery using only the authorised frozen Greif arrays."""
|
||||
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
|
||||
import numpy as np
|
||||
from scipy.sparse import coo_matrix
|
||||
from scipy.sparse.csgraph import connected_components, dijkstra
|
||||
|
||||
import phase3_h2_shared_miss_attributes as first
|
||||
from s2_tail_diagnostic_math import ALPHA, tail_membership, weights
|
||||
|
||||
|
||||
ROOT = first.BASE / "artifacts/phase3-h2-shared-miss-true-tail-2026-11-19"
|
||||
FROZEN_ROSTER = "artifacts/s2-main-preparation-2026-11-15/final-roster.json"
|
||||
FROZEN_ROSTER_SHA = "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e"
|
||||
FIRST_ROWS_SHA = "fd18ac43986d5ecbf60adaaa96fab3c55f8f55da19bccfc3032deae9d76a8f17"
|
||||
FIRST_RESULT_SHA = "c6596097021a5d6eb59925aaa261c93e1c51959609ee9a90665ce81505237750"
|
||||
UNRESOLVED = {k: first.UNAVAILABLE[k] for k in (
|
||||
"physical_branch_count_and_length", "physical_flow_length_to_wall_thickness", "semantic_family_type")}
|
||||
NEW_METRICS = {
|
||||
"late_tail_component_count": "count",
|
||||
"late_tail_major_component_count": "count",
|
||||
"late_tail_component_depth": "depth over Rg, valid tail mass weighted",
|
||||
"late_tail_component_depth_max": "depth over Rg",
|
||||
"late_tail_component_depth_mass_coverage": "fraction of fixed tail mass",
|
||||
"near_tie": "proportion",
|
||||
"major_component_peak_gap": "T / t_set",
|
||||
"L_tail_raw": "raw coordinate, Greif mm convention",
|
||||
"L_tail_over_Rg": "distance / Rg",
|
||||
"tail_unreachable_mass_fraction": "fraction of fixed tail mass",
|
||||
"L_tail_to_h_g": "dimensionless",
|
||||
"surface_area_raw_squared": "raw coordinate squared",
|
||||
"sqrt_lambda2_over_lambda1": "dimensionless",
|
||||
"sqrt_lambda3_over_lambda1": "dimensionless",
|
||||
}
|
||||
|
||||
|
||||
def checked_array(path, sha):
|
||||
return np.load(BytesIO(first.checked_bytes(path, sha)), allow_pickle=False)
|
||||
|
||||
|
||||
def graph(u, v, lengths, n):
|
||||
return coo_matrix((np.tile(lengths, 2), (np.r_[u, v], np.r_[v, u])), shape=(n, n)).tocsr()
|
||||
|
||||
|
||||
def tail_graph_values(x, edge, mass, unit_time, membership, rg, gates):
|
||||
u, v = edge.T
|
||||
if not (np.all(u < v) and np.all(np.diff(u.astype(np.int64) * len(x) + v) > 0)):
|
||||
raise ValueError("Original tet edge identities must be sorted and unique")
|
||||
lengths = np.linalg.norm(x[u] - x[v], axis=1)
|
||||
if not np.isfinite(lengths).all() or np.any(lengths <= 0):
|
||||
raise ValueError("Invalid physical edge length")
|
||||
selected = membership > 0
|
||||
tail = np.flatnonzero(selected)
|
||||
ix = np.full(len(x), -1, dtype=np.int64)
|
||||
ix[tail] = np.arange(len(tail))
|
||||
inner = selected[u] & selected[v]
|
||||
local = graph(ix[u[inner]], ix[v[inner]], lengths[inner], len(tail))
|
||||
count, labels = connected_components(local, directed=False)
|
||||
tail_mass = mass[tail] * membership[tail]
|
||||
total_tail_mass = ALPHA * math.fsum(mass)
|
||||
fractions = np.bincount(labels, weights=tail_mass, minlength=count) / total_tail_mass
|
||||
major = fractions >= .05
|
||||
maxima = np.full(count, -np.inf)
|
||||
np.maximum.at(maxima, labels, unit_time[tail])
|
||||
peak_order = np.sort(maxima[major])
|
||||
gap = float(peak_order[-1] - peak_order[-2]) if len(peak_order) >= 2 else None
|
||||
near = None if np.ptp(unit_time) == 0 else int(gap is not None and gap <= .01)
|
||||
entering_u, entering_v = selected[u] & ~selected[v], selected[v] & ~selected[u]
|
||||
entry_nodes = np.unique(np.r_[ix[u[entering_u]], ix[v[entering_v]]])
|
||||
depths = np.zeros(count)
|
||||
valid_depth = np.zeros(count, dtype=bool)
|
||||
if len(entry_nodes):
|
||||
dist = np.asarray(dijkstra(local, directed=False, indices=entry_nodes, min_only=True))
|
||||
finite = np.isfinite(dist)
|
||||
np.maximum.at(depths, labels[finite], dist[finite] / rg)
|
||||
valid_depth[labels[finite]] = True
|
||||
if not np.array_equal(finite, valid_depth[labels]):
|
||||
raise ValueError("Some nodes in an entrance-connected component are unreachable")
|
||||
usable = float(np.sum(fractions[valid_depth]))
|
||||
depth = float(np.dot(fractions[valid_depth], depths[valid_depth]) / usable) if usable else None
|
||||
max_depth = float(depths[valid_depth].max()) if usable else None
|
||||
|
||||
gate_distance, unreachable = None, None
|
||||
if gates is not None and len(gates):
|
||||
distances = np.asarray(dijkstra(graph(u, v, lengths, len(x)), directed=False,
|
||||
indices=gates, min_only=True))[tail]
|
||||
reachable = np.isfinite(distances)
|
||||
unreachable = math.fsum(tail_mass[~reachable]) / total_tail_mass
|
||||
if reachable.all():
|
||||
gate_distance = math.fsum(tail_mass * distances) / total_tail_mass
|
||||
return dict(late_tail_component_count=int(count), late_tail_major_component_count=int(major.sum()),
|
||||
late_tail_component_depth=depth, late_tail_component_depth_max=max_depth,
|
||||
late_tail_component_depth_mass_coverage=usable, near_tie=near,
|
||||
major_component_peak_gap=gap, L_tail_raw=gate_distance,
|
||||
L_tail_over_Rg=gate_distance/rg if gate_distance is not None else None,
|
||||
tail_unreachable_mass_fraction=unreachable,
|
||||
components=[dict(tail_mass_fraction=float(f), peak_T_over_t_set=float(p), major=bool(m),
|
||||
depth_over_Rg=float(d) if valid else None)
|
||||
for f, p, m, d, valid in zip(fractions, maxima, major, depths, valid_depth)])
|
||||
|
||||
|
||||
def compute(job):
|
||||
row, tail, part, geometry = job
|
||||
case = row["case"]
|
||||
source = first.BASE / geometry["preparation_root"] / "geometry.npz"
|
||||
geom_hash = geometry["output_sha256"]["geometry.npz"]
|
||||
with np.load(BytesIO(first.checked_bytes(source, geom_hash)), allow_pickle=False) as z:
|
||||
x = z["points_raw"]
|
||||
tet = z["tetra"]
|
||||
mesh_mass = z["node_volume_raw"]
|
||||
edge = z["edges"]
|
||||
faces = z["boundary_triangles"]
|
||||
ids = z["raw_node_ids"]
|
||||
raw = first.BASE / part["output"]
|
||||
pin = part["output_sha256"]
|
||||
label_mass = checked_array(raw / "mass.npy", pin["mass.npy"])
|
||||
v5_ids = checked_array(raw / "raw_ids.npy", pin["raw_ids.npy"])
|
||||
if not (len(tet) == row["tet_count"] and len(x) == part["nodes"] == tail["nodes"]
|
||||
and np.array_equal(ids, v5_ids) and np.all(np.diff(ids) > 0)
|
||||
and np.allclose(label_mass, mesh_mass, rtol=1e-10, atol=0)):
|
||||
raise ValueError(f"Retained label, geometry or mass domain differs: {case}")
|
||||
if (tail["component_id"], tail["fold"], tail["t_set"]) != (row["component_id"], row["fold"], part["t_set"]):
|
||||
raise ValueError(f"Fold or time scale differs: {case}")
|
||||
gates = None
|
||||
if row["gate_mapping_raw_ids"] is not None:
|
||||
gate_ids = np.array(row["gate_mapping_raw_ids"])
|
||||
gate_ix = np.searchsorted(ids, gate_ids)
|
||||
if np.all(gate_ix < len(ids)) and np.array_equal(ids[gate_ix], gate_ids):
|
||||
gates = gate_ix
|
||||
path = raw / "seconds.npy"
|
||||
with (ROOT / "target-access.jsonl").open("a") as log:
|
||||
log.write(json.dumps(dict(case=case, role="grouped_test", path=str(path), sha256=pin["seconds.npy"],
|
||||
purpose="pre-registered descriptive attribute study")) + "\n")
|
||||
truth = checked_array(path, pin["seconds.npy"]).astype(np.float64) / part["t_set"]
|
||||
membership, witness = tail_membership(truth, weights(label_mass))
|
||||
if witness != tail["truth_tail"]:
|
||||
raise ValueError(f"Recomputed real-tail definition differs from the frozen prior receipt: {case}")
|
||||
rg = tail["Rg"]
|
||||
out = tail_graph_values(x, edge, label_mass, truth, membership, rg, gates)
|
||||
a, b, c = x[faces].transpose(1, 0, 2)
|
||||
area = math.fsum(.5 * np.linalg.norm(np.cross(b - a, c - a), axis=1))
|
||||
if not math.isfinite(area) or area <= 0:
|
||||
raise ValueError(f"Invalid retained exterior area: {case}")
|
||||
w = weights(label_mass)
|
||||
centered = x - np.sum(w[:, None] * x, axis=0)
|
||||
spectrum = np.linalg.eigvalsh((centered * w[:, None]).T @ centered)[::-1]
|
||||
if not np.isclose(np.sqrt(spectrum.sum()), rg, rtol=1e-10, atol=1e-10):
|
||||
raise ValueError(f"Mass-weighted Rg differs from the frozen scalar: {case}")
|
||||
distance = out["L_tail_raw"]
|
||||
scale = 2 * math.fsum(label_mass) / area
|
||||
shape = [float(np.sqrt(max(0., ev / spectrum[0]))) if spectrum[0] > 0 else None for ev in spectrum[1:]]
|
||||
out.update(surface_area_raw_squared=area, L_tail_to_h_g=distance/scale if distance is not None else None,
|
||||
h_g_raw=scale, sqrt_lambda2_over_lambda1=shape[0], sqrt_lambda3_over_lambda1=shape[1])
|
||||
return dict(row, **out, geometry_sha256=geom_hash, true_time_sha256=pin["seconds.npy"])
|
||||
|
||||
|
||||
def worker_limits():
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (900, 900))
|
||||
|
||||
|
||||
def paired_stats(rows):
|
||||
families = sorted(first.grouped(rows))
|
||||
draws = np.random.default_rng(first.SEED).integers(
|
||||
len(families), size=(first.REPLICATES, len(families)))
|
||||
miss = [r for r in rows if r["shared_miss"]]
|
||||
other = [r for r in rows if not r["shared_miss"]]
|
||||
result = {}
|
||||
for metric in (*NEW_METRICS, *UNRESOLVED):
|
||||
x, a = first.attribute_rows(families, miss, metric)
|
||||
y, b = first.attribute_rows(families, other, metric)
|
||||
_, pooled = first.attribute_rows(families, rows, metric)
|
||||
diff = first.sample_means(x, draws) - first.sample_means(y, draws)
|
||||
obs = diff[np.isfinite(diff)]
|
||||
l, r = a["family_equal_mean"], b["family_equal_mean"]
|
||||
result[metric] = dict(units=NEW_METRICS.get(metric), missing_reason=UNRESOLVED.get(metric),
|
||||
shared_miss=a, other=b, all_parts=pooled,
|
||||
effect=l-r if l is not None and r is not None else None,
|
||||
interval_95=np.quantile(obs, [.025,.975]).tolist() if len(obs) else None,
|
||||
undefined_replicates=len(diff)-len(obs))
|
||||
return result
|
||||
|
||||
|
||||
def jobs():
|
||||
meta = first.inputs()
|
||||
prior = first.checked_bytes(first.OUT / "by-part.jsonl", FIRST_ROWS_SHA)
|
||||
first.checked_bytes(first.OUT / "result.json", FIRST_RESULT_SHA)
|
||||
roster = first.unique_index(json.loads(first.checked_bytes(first.BASE / FROZEN_ROSTER,
|
||||
FROZEN_ROSTER_SHA))["parts"])
|
||||
prior_rows = [json.loads(x) for x in prior.decode().splitlines()]
|
||||
tail = first.unique_index(json.loads(meta["s2_tail"])["rows"])
|
||||
pilot = json.loads(meta["pilot"])["parts"]
|
||||
if len(prior_rows) != 552 or len(first.unique_index(prior_rows)) != 552:
|
||||
raise ValueError("Frozen 552-part roster required")
|
||||
return [(r, tail[r["case"]], pilot[r["case"]], roster[r["case"]]["preparation"]) for r in prior_rows]
|
||||
|
||||
|
||||
def run():
|
||||
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
|
||||
raise RuntimeError("A masked CPU-only invocation is required")
|
||||
path = ROOT / "execution-contract.json"
|
||||
c = json.loads(path.read_text())
|
||||
if (c["authorizing_handoff"] != "h-1149aa75" or c["expected_parts"] != 552
|
||||
or c["worker_cpu_budget_seconds"] != 3600 or c["worker_limit"] != 4):
|
||||
raise ValueError("Execution scope or pinned implementation differs")
|
||||
for name, digest in c["code_sha256"].items():
|
||||
first.checked_bytes(first.BASE / name, digest)
|
||||
inputs = jobs()
|
||||
with (ROOT / "started.json").open("x") as stream:
|
||||
json.dump(dict(pid=os.getpid(), cpu_mask=sorted(os.sched_getaffinity(0)),
|
||||
contract_sha256=hashlib.sha256(path.read_bytes()).hexdigest()), stream)
|
||||
rows = []
|
||||
with (ROOT / "by-part.jsonl").open("x") as stream, ProcessPoolExecutor(
|
||||
max_workers=c["worker_limit"], initializer=worker_limits) as pool:
|
||||
batch = c["worker_limit"]
|
||||
for offset in range(0, len(inputs), batch):
|
||||
for row in pool.map(compute, inputs[offset:offset+batch], chunksize=1):
|
||||
rows.append(row)
|
||||
stream.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n")
|
||||
stream.flush()
|
||||
access = [json.loads(x) for x in (ROOT / "target-access.jsonl").read_text().splitlines()]
|
||||
if len(rows) != 552 or {x["case"] for x in access} != {x["case"] for x in rows} or len(access) != 552:
|
||||
raise ValueError("Expected the exact bounded labelled read set")
|
||||
child = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
result = dict(status="COMPLETE", prereg_commit=first.PREREG_COMMIT,
|
||||
baseline_commit="44c1d5a10d4c00d5af2b38f0e5e0e11ed11df611", parts=len(rows), test_role_reads=len(access),
|
||||
new_fits=0, gpu_seconds=0, new_raw_label_decodes=0, prediction_arrays_read=0,
|
||||
bootstrap=dict(seed=first.SEED, replicates=first.REPLICATES, unit="original family, simultaneous"),
|
||||
worker_cpu_seconds=child.ru_utime+child.ru_stime, metrics=paired_stats(rows))
|
||||
(ROOT / "result.json").write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
Reference in new issue
Block a user