97 lines
6.2 KiB
Python
97 lines
6.2 KiB
Python
"""Metadata-only E1 preflight. Never opens VTU, labels, or geometry arrays."""
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import resource
|
|
import time
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
|
|
def main():
|
|
start = time.monotonic()
|
|
out = Path('artifacts/s2-e1-2026-11-14/preflight')
|
|
out.mkdir(parents=True, exist_ok=False)
|
|
census_path = Path('artifacts/intake-2026-11-13/cell_type_census.json')
|
|
groups_path = Path('artifacts/intake-2026-11-13/source_groups.json')
|
|
census = json.loads(census_path.read_text())
|
|
groups = json.loads(groups_path.read_text())
|
|
mixed = {r['case']: r for r in census['mixed_cases']}
|
|
rows = []
|
|
for r in census['rows']:
|
|
n = int(r['points'])
|
|
if r['case'] in mixed:
|
|
n = next(int(x['nodes']) for x in mixed[r['case']]['details'] if x['type'] == 10)
|
|
rows.append(dict(case=r['case'], source_key=r['case'].split('_')[1],
|
|
numeric_prefix=int(r['case'].split('_')[0]),
|
|
tet_nodes=n, tet_cells=int(r['types']['10'])))
|
|
assert len(rows) == 624 and len({r['case'] for r in rows}) == 624
|
|
assert set(r['case'] for r in rows) == {c for cases in groups.values() for c in cases}
|
|
rows.sort(key=lambda r: (r['tet_nodes'], r['numeric_prefix'], r['case']))
|
|
largest = rows[-1]
|
|
assert largest['numeric_prefix'] == 666
|
|
rng = np.random.Generator(np.random.PCG64(2026111411))
|
|
selected = []
|
|
for t, indices in enumerate(np.array_split(np.arange(len(rows)), 3), 1):
|
|
candidates = [int(i) for i in indices if rows[int(i)]['numeric_prefix'] != 666]
|
|
chosen = sorted(map(int, rng.choice(candidates, 3, replace=False)))
|
|
selected.extend(dict(rows[i], tertile=t, selection='random') for i in chosen)
|
|
selected.append(dict(largest, tertile=3, selection='fixed_largest'))
|
|
assert len({r['case'] for r in selected}) == 10
|
|
receipts = []
|
|
for p in sorted(Path('artifacts/s1-panel-precheck-2026-11-14').glob('*/result.json')):
|
|
r = json.loads(p.read_text())
|
|
if r['status'] != 'COUNT_COMPLETE':
|
|
raise ValueError('Unexpected old receipt status')
|
|
receipts.append((p, r))
|
|
assert len(receipts) == 24
|
|
# Ratios of historical WALL seconds, not measured CPU cost or guaranteed bound.
|
|
distance_rates = [r['timings']['write_geometry_and_distance_seconds'] / r['nodes'] for _, r in receipts]
|
|
load_rates = [r['timings']['load_geometry_and_topology_seconds'] / r['tets'] for _, r in receipts]
|
|
cache_sizes = [((p.parent/'geometry.npz').stat().st_size, r['tets']) for p,r in receipts]
|
|
all_nodes = sum(r['tet_nodes'] for r in rows)
|
|
all_tets = sum(r['tet_cells'] for r in rows)
|
|
scenarios = {}
|
|
for q, name in [(0.5, 'panel_median_rate'), (0.95, 'panel_p95_rate'), (1, 'panel_max_rate')]:
|
|
seconds = float(np.quantile(distance_rates,q))*all_nodes + float(np.quantile(load_rates,q))*all_tets
|
|
scenarios[name] = dict(full_corpus_geometry_distance_wall_hours=seconds/3600,
|
|
two_cpu_thread_scenario_core_hours=2*seconds/3600,
|
|
qualification='linear historical wall-time scenario; not measured CPU or bound; no G1 registration/O2 cost included')
|
|
source_counts = Counter(r['source_key'] for r in rows)
|
|
all_pairs = len(rows)*(len(rows)-1)//2
|
|
cross = all_pairs - sum(n*(n-1)//2 for n in source_counts.values())
|
|
cache_scenario = max(size/n for size,n in cache_sizes)*all_tets
|
|
for filename,data in [('geometry_counts.csv',rows),('ten_part_plan.csv',selected)]:
|
|
with (out/filename).open('w') as f:
|
|
w=csv.DictWriter(f,fieldnames=list(data[0]),lineterminator='\n');w.writeheader();w.writerows(data)
|
|
inputs = [census_path, groups_path, Path(__file__)] + [p for p,_ in receipts]
|
|
result = dict(scope='existing label-free metadata/counts/receipts only; new ten-part IDs selected, no new geometry or label arrays read',
|
|
cases=len(rows),source_keys=len(source_counts),tet_nodes=all_nodes,tet_cells=all_tets,
|
|
all_pairs=all_pairs,cross_source_pairs=cross,
|
|
two_route_union_pair_upper_bound_by_k={str(k):min(cross,2*len(rows)*k) for k in [1,2,4,8]},
|
|
candidate_note='upper bounds only; k not selected, no candidate generation; >2000 pairs stops, never truncate or secretly tighten screening',
|
|
distance_scenarios=scenarios,
|
|
geometry_cache_max_panel_bytes_per_tet_scenario=cache_scenario,
|
|
distance_array_bytes=8*all_nodes,
|
|
all_raw_vtu_extraction='not planned; original archive remains immutable; no duplication of label payload',
|
|
g1_registration_cpu_estimate=None,
|
|
g1_registration_reason='no frozen per-pair sampling/start/iteration rules or measured cost; count upper bound alone cannot certify completion within 16 CPU-h',
|
|
ten_part_limits=dict(total_wall_seconds=10800,disk_bytes=5*1024**3,per_part_wall_seconds=1800,tree_rss_bytes=1342177280,sequential=True,no_replacement=True),
|
|
e1_limits=dict(cpu_seconds=57600,disk_bytes=25*1024**3,max_parallel_processes=8,per_job_wall_seconds=1800,tree_rss_bytes=1342177280,max_registration_pairs=2000),
|
|
count_checkpoint_limits=dict(wall_seconds=60,rss_bytes=1342177280,output_bytes=16*1024**2),
|
|
ten_part_cases=[r['case'] for r in selected],
|
|
input_sha256={str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in inputs},
|
|
status='COUNT_CHECKPOINT_ONLY_NOT_FULL_E1_FIT_CERTIFICATION',
|
|
wall_seconds=time.monotonic()-start,
|
|
cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
peak_rss_bytes=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024,
|
|
new_geometry_computations=0,labels_read=False,model_fit=False,gpu_used=False)
|
|
(out/'estimate.json').write_text(json.dumps(result,indent=2,allow_nan=False)+'\n')
|
|
print(json.dumps({k:result[k] for k in ['status','cases','source_keys','tet_nodes','tet_cells','cross_source_pairs','distance_scenarios','geometry_cache_max_panel_bytes_per_tet_scenario','ten_part_cases','wall_seconds','cpu_seconds','peak_rss_bytes']},indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|