Files
MeltMind/tools/estimate_s2_e1.py

97 lines
6.2 KiB
Python

"""Metadata-only E1 preflight. Never opens VTU, labels, or geometry arrays."""
import csv
import hashlib
import json
import resource
import time
from collections import Counter
from pathlib import Path
import numpy as np
def main():
start = time.monotonic()
out = Path('artifacts/s2-e1-2026-11-14/preflight')
out.mkdir(parents=True, exist_ok=False)
census_path = Path('artifacts/intake-2026-11-13/cell_type_census.json')
groups_path = Path('artifacts/intake-2026-11-13/source_groups.json')
census = json.loads(census_path.read_text())
groups = json.loads(groups_path.read_text())
mixed = {r['case']: r for r in census['mixed_cases']}
rows = []
for r in census['rows']:
n = int(r['points'])
if r['case'] in mixed:
n = next(int(x['nodes']) for x in mixed[r['case']]['details'] if x['type'] == 10)
rows.append(dict(case=r['case'], source_key=r['case'].split('_')[1],
numeric_prefix=int(r['case'].split('_')[0]),
tet_nodes=n, tet_cells=int(r['types']['10'])))
assert len(rows) == 624 and len({r['case'] for r in rows}) == 624
assert set(r['case'] for r in rows) == {c for cases in groups.values() for c in cases}
rows.sort(key=lambda r: (r['tet_nodes'], r['numeric_prefix'], r['case']))
largest = rows[-1]
assert largest['numeric_prefix'] == 666
rng = np.random.Generator(np.random.PCG64(2026111411))
selected = []
for t, indices in enumerate(np.array_split(np.arange(len(rows)), 3), 1):
candidates = [int(i) for i in indices if rows[int(i)]['numeric_prefix'] != 666]
chosen = sorted(map(int, rng.choice(candidates, 3, replace=False)))
selected.extend(dict(rows[i], tertile=t, selection='random') for i in chosen)
selected.append(dict(largest, tertile=3, selection='fixed_largest'))
assert len({r['case'] for r in selected}) == 10
receipts = []
for p in sorted(Path('artifacts/s1-panel-precheck-2026-11-14').glob('*/result.json')):
r = json.loads(p.read_text())
if r['status'] != 'COUNT_COMPLETE':
raise ValueError('Unexpected old receipt status')
receipts.append((p, r))
assert len(receipts) == 24
# Ratios of historical WALL seconds, not measured CPU cost or guaranteed bound.
distance_rates = [r['timings']['write_geometry_and_distance_seconds'] / r['nodes'] for _, r in receipts]
load_rates = [r['timings']['load_geometry_and_topology_seconds'] / r['tets'] for _, r in receipts]
cache_sizes = [((p.parent/'geometry.npz').stat().st_size, r['tets']) for p,r in receipts]
all_nodes = sum(r['tet_nodes'] for r in rows)
all_tets = sum(r['tet_cells'] for r in rows)
scenarios = {}
for q, name in [(0.5, 'panel_median_rate'), (0.95, 'panel_p95_rate'), (1, 'panel_max_rate')]:
seconds = float(np.quantile(distance_rates,q))*all_nodes + float(np.quantile(load_rates,q))*all_tets
scenarios[name] = dict(full_corpus_geometry_distance_wall_hours=seconds/3600,
two_cpu_thread_scenario_core_hours=2*seconds/3600,
qualification='linear historical wall-time scenario; not measured CPU or bound; no G1 registration/O2 cost included')
source_counts = Counter(r['source_key'] for r in rows)
all_pairs = len(rows)*(len(rows)-1)//2
cross = all_pairs - sum(n*(n-1)//2 for n in source_counts.values())
cache_scenario = max(size/n for size,n in cache_sizes)*all_tets
for filename,data in [('geometry_counts.csv',rows),('ten_part_plan.csv',selected)]:
with (out/filename).open('w') as f:
w=csv.DictWriter(f,fieldnames=list(data[0]),lineterminator='\n');w.writeheader();w.writerows(data)
inputs = [census_path, groups_path, Path(__file__)] + [p for p,_ in receipts]
result = dict(scope='existing label-free metadata/counts/receipts only; new ten-part IDs selected, no new geometry or label arrays read',
cases=len(rows),source_keys=len(source_counts),tet_nodes=all_nodes,tet_cells=all_tets,
all_pairs=all_pairs,cross_source_pairs=cross,
two_route_union_pair_upper_bound_by_k={str(k):min(cross,2*len(rows)*k) for k in [1,2,4,8]},
candidate_note='upper bounds only; k not selected, no candidate generation; >2000 pairs stops, never truncate or secretly tighten screening',
distance_scenarios=scenarios,
geometry_cache_max_panel_bytes_per_tet_scenario=cache_scenario,
distance_array_bytes=8*all_nodes,
all_raw_vtu_extraction='not planned; original archive remains immutable; no duplication of label payload',
g1_registration_cpu_estimate=None,
g1_registration_reason='no frozen per-pair sampling/start/iteration rules or measured cost; count upper bound alone cannot certify completion within 16 CPU-h',
ten_part_limits=dict(total_wall_seconds=10800,disk_bytes=5*1024**3,per_part_wall_seconds=1800,tree_rss_bytes=1342177280,sequential=True,no_replacement=True),
e1_limits=dict(cpu_seconds=57600,disk_bytes=25*1024**3,max_parallel_processes=8,per_job_wall_seconds=1800,tree_rss_bytes=1342177280,max_registration_pairs=2000),
count_checkpoint_limits=dict(wall_seconds=60,rss_bytes=1342177280,output_bytes=16*1024**2),
ten_part_cases=[r['case'] for r in selected],
input_sha256={str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in inputs},
status='COUNT_CHECKPOINT_ONLY_NOT_FULL_E1_FIT_CERTIFICATION',
wall_seconds=time.monotonic()-start,
cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
peak_rss_bytes=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024,
new_geometry_computations=0,labels_read=False,model_fit=False,gpu_used=False)
(out/'estimate.json').write_text(json.dumps(result,indent=2,allow_nan=False)+'\n')
print(json.dumps({k:result[k] for k in ['status','cases','source_keys','tet_nodes','tet_cells','cross_source_pairs','distance_scenarios','geometry_cache_max_panel_bytes_per_tet_scenario','ten_part_cases','wall_seconds','cpu_seconds','peak_rss_bytes']},indent=2))
if __name__ == '__main__':
main()