Files
MeltMind/tools/build_e6_cost_review.py
T

99 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Allowlisted count/timing evidence; no source geometry or scientific rerun."""
import ast
import csv
import hashlib
import json
from pathlib import Path
import shutil
import zipfile
import numpy as np
def read(p):return json.loads(Path(p).read_text())
def main():
out=Path('artifacts/e6-full-node-cost-2026-11-14');c=read(out/'execution_contract.json');assert read(out/'verification.json')['status']=='PASS'
rows=[];feature_cost=[]
for spec in c['cases']:
r=read(out/(spec['case']+'.json'));t=read(out/(spec['case']+'.calibration.json'));g=r['groups']['all_nodes'];fixed=sum(t['fixed_setup'].values());e=t['estimates']['all_nodes']
rows.append(dict(case=spec['case'][:8],nodes=r['nodes'],positive_nodes=r['groups']['positive_distance_nodes']['nodes'],sources=r['four_point_sources'],R=r['radius'],
all_neighbor_visits=g['neighbor_sum'],positive_neighbor_visits=r['groups']['positive_distance_nodes']['neighbor_sum'],p99_neighbors=g['quantiles']['p99'],max_neighbors=g['quantiles']['max'],
count_only_wall_seconds=r['timings']['wall_seconds'],fixed_setup_seconds=fixed,calibration_loop_seconds=t['measured_query_loop_seconds'],calibration_visits=t['neighbor_visits'],
amortized_microseconds_per_neighbor=t['amortized_seconds_per_neighbor']*1e6,
estimated_full_seconds=e['total_seconds_with_fixed_setup'],fast_scenario_seconds=fixed+e['fast_scenario_seconds'],slow_scenario_seconds=fixed+e['slow_scenario_seconds'],
measured_calibration_peak_MiB=t['memory']['measured_calibration_peak_bytes']/2**20,full_E6_memory_allowance_MiB=t['memory']['full_E6_peak_allowance_bytes']/2**20,
block_output_memory_allowance_MiB=t['memory']['block4096_E6_peak_allowance_bytes']/2**20,
old_budget_pass=g['p99_times_nodes']<=c['kernel_evaluation_budget'] and g['main_plus_pose_pair_budget_pass']))
with zipfile.ZipFile(spec['sample']) as z:
with z.open('edges.npy') as f:
version=np.lib.format.read_magic(f);shape,order,dtype=np.lib.format._read_array_header(f,version)
feature_cost.append(dict(case=spec['case'],nodes=r['nodes'],undirected_edges=shape[0],
bytes_per_additional_float64_node_scalar=8*r['nodes'],bytes_per_additional_float64_undirected_edge_scalar=8*shape[0],
note='Capacity arithmetic only; no O2-rep formula implemented or timed. Directed duplicated edge storage would double the edge term.'))
with (out/'summary.csv').open('w') as f:
w=csv.DictWriter(f,fieldnames=list(rows[0]),lineterminator='\n');w.writeheader();w.writerows(rows)
(out/'feature_storage_units.json').write_text(json.dumps(feature_cost,indent=2)+'\n')
lines=['# E6 全节点成本:实际计数与有界计时校准','',
'结论:在当前四点估计器、半径和缓存下,850 与 666 全节点计算量可行;778 单次约 18.91 亿邻居访问,超过原 5 亿预算。Claude 随后接受校准并给出条件新预算:O2-rep 规范冻结后,每场最多 25 亿次访问或 30 分钟,先达到者为限,三件均可纳入。规范冻结前仍保留已有抽样,不启动全量。18.7 万节点的最大件并非计算瓶颈。本次只完成成本评估,没有执行全节点核重算或实现 O2-rep。','',
'## 实际工作量','',
'| 实件 | 全节点 / 内部节点 | 四点源点 | R | 全节点访问量 | 仅内部访问量 | p99 / 最大邻居数 |','|---|---:|---:|---:|---:|---:|---:|']
for r in rows:lines.append(f"| {r['case']} | {r['nodes']:,} / {r['positive_nodes']:,} | {r['sources']:,} | {r['R']:.6f} | {r['all_neighbor_visits']:,} | {r['positive_neighbor_visits']:,} | {r['p99_neighbors']:.2f} / {r['max_neighbors']:.0f} |")
lines+=['','逐节点计数使用原 E6 四点源坐标及 cKDTree 精确半径查询,workers=1,每批4096个查询,仅返回长度,不构造全邻域列表。5122个旧去重锚点的计数与原预检逐点完全一致;内部/边界节点数量和邻居总量均加和等于全节点。边界判定沿用缓存 d=0;统计其成本不改变既有边界 unscored 语义。','',
'850/666 的 p99×N 和原场+旋转的实际邻居总量均低于原 5e8 门槛。778 单场为门槛的3.78倍,两场为7.56倍;只算内部节点也超过门槛。缩小R、删源点或改核都属于新估计器,本次没有采用。','',
'## 分阶段实测与全量推算','',
'依据 [Claude 的校准批准](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794627909915539),每件64点,内部/边界各分16个邻居数分位层,每层2点,包含最大邻域及旧锚点重叠。合计4,609,064次邻居访问,低于1e7硬上限。原 integrate 算式保持不变,以转发包装记录建树和查询循环时间;56个重叠点的全部输出与旧E6一致。瞬时结果仅用于校准与核对,没有保存新的科学场或给出新科学结论。','',
'| 实件 | 加载+源场+源点+建树实测秒 | 64点积分循环实测秒 | 摊销微秒/邻居 | 全节点单场估计秒 | 两样点快/慢情景秒 |','|---|---:|---:|---:|---:|---:|']
for r in rows:lines.append(f"| {r['case']} | {r['fixed_setup_seconds']:.3f} | {r['calibration_loop_seconds']:.4f} | {r['amortized_microseconds_per_neighbor']:.3f} | {r['estimated_full_seconds']:.2f} | {r['fast_scenario_seconds']:.2f}–{r['slow_scenario_seconds']:.2f} |")
lines+=['','全节点时间是每层节点数×该层两样点平均循环时间,再加一次固定准备。快/慢情景分别使用每层两样点的最小/最大时间;它们不是置信区间,也不是保证上下界。样点是确定性选取,缓存、查询顺序、机器负载和分层内差异都会影响实际时间。另保存非负“每查询固定项+每邻居项”拟合,作为诊断而非替代实测。摊销每邻居耗时包含每查询固定开销,不能跨三件当作统一常数。','',
'仅按旧2000抽样任务总时长外推会重复放大源场准备、建树等固定成本;在666上给出约8分钟的误导性规划。本报告使用分阶段校准后的约52秒估计;旧粗模型保留在计数JSON作来源审计,不作为当前建议。未来O2-rep的新节点/边收缩、序列化和训练成本未计入。','',
'## 内存与执行建议','',
'| 实件 | 校准实测进程峰值 MiB | 全量E6输出规划 MiB | 4096点分块输出规划 MiB | 下一步建议 |','|---|---:|---:|---:|---|']
advice={'00000850':'规范冻结后可全节点;预留30秒/512 MiB','00000778':'规范冻结后按25亿次/30分钟新上限;此前保留抽样','00000666':'规范冻结后可全节点;预留2分钟/1.25 GiB'}
for r in rows:lines.append(f"| {r['case']} | {r['measured_calibration_peak_MiB']:.1f} | {r['full_E6_memory_allowance_MiB']:.1f} | {r['block_output_memory_allowance_MiB']:.1f} | {advice[r['case']]} |")
lines+=['','[Claude 11:48:55 的条件预算决定](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794628135270029):5e8是未知成本前的保护线,并非研究约束;校准后允许规范冻结时采用每场2.5e9实际邻居访问或1800秒,先到者停止。778当前全节点实际计数约占新访问上限75.65%。新预算不回写旧合同,也不把待完善的规范视为已经冻结。全库前须在分层panel先做计数预检,不能由三件外推624。','',
'规划内存=校准峰值+全节点或分块E6输出相对64点的增量+128 MiB余量;是预留,不是严格RSS上界。四点源张量与树保持驻留,逐个查询生成邻居列表;分块输出只节省输出空间,不省掉源张量。778若一次性保存全邻居int64索引就约14.09 GiB,还不含权重与张量,因此不应物化全邻接列表。','',
'以上时间预留只针对单场现有估计器;若以后需要全节点旋转验证,应另计第二场,不把前次64点校准称为已完成全量验证。所有后续运行仍等待对应任务/规范确定;本次没有启动全量。','',
'为尚未冻结的表示提供纯容量换算:','',
'| 实件 | 原始无向边数 | 每新增float64节点标量 MiB | 每新增float64无向边标量 MiB |','|---|---:|---:|---:|']
for r in feature_cost:lines.append(f"| {r['case'][:8]} | {r['undirected_edges']:,} | {r['bytes_per_additional_float64_node_scalar']/2**20:.3f} | {r['bytes_per_additional_float64_undirected_edge_scalar']/2**20:.3f} |")
lines+=['','这些只是从缓存数组头读出的维数和存储算术,不是O2-rep实现;若双向重复存边,边标量项翻倍。没有改变 hard-gap=0.1、类型语义或有效性规则,也没有新距离查询、细化、O1准备、LP、S1或训练。','',
'## 审计与复核','',
'ZIP 包含逐节点邻居计数/内部标记/质量、全部分层样点与计时、合同、源代码、原锚点预检和汇总。24项本地输入哈希核对通过;计数/抽样身份/分层规模/耗时公式/预算算术复核通过。不包含完整源网格、距离或旧场;完整计数/校准重跑需接回合同指向的缓存与既有E6证据。原始执行代码和数据哈希保持可追溯。','',
'解压后可只复核保存的成本数据,无须网格或新积分:','',
'```sh','python tools/verify_e6_cost.py artifacts/e6-full-node-cost-2026-11-14','```','',
'保存结果复核只需NumPy;实际计数/校准还需SciPy。本次Python3.10,OMP/OPENBLAS线程数2、cKDTree计数workers=1。每件独立进程记录峰值。started标记防止意外重复覆盖;完整重跑应使用新的输出目录。','']
(out/'README.md').write_text('\n'.join(lines))
dest=Path('artifacts/e6-cost-review-delivery-2026-11-14');dest.mkdir(exist_ok=False)
include={p for p in out.iterdir() if p.is_file()};include.add(Path('docs/e6-cost-source-2026-11-14.txt'));include.add(Path('docs/e6-cost-budget-decision-2026-11-14.txt'))
for s in c['cases']:include.add(Path(c['prior_directory'])/(s['case']+'_tetra_four_point_anchors.npz'))
pending=[Path('tools')/name for name in ['assess_e6_full_node_cost.py','calibrate_e6_cost.py','verify_e6_cost.py','build_e6_cost_review.py']]
while pending:
path=pending.pop()
if path in include:continue
include.add(path)
for node in ast.walk(ast.parse(path.read_text())):
if isinstance(node,ast.ImportFrom):
names=[node.module.split('.')[-1]] if node.module else [x.name for x in node.names]
for name in names:
candidate=Path('tools')/(name+'.py')
if candidate.exists() and candidate not in include:pending.append(candidate)
manifest=[]
for p in sorted(include):
data=p.read_bytes();assert len(data)<5_000_000
manifest.append(dict(path=str(p),bytes=len(data),sha256=hashlib.sha256(data).hexdigest()))
archive=dest/'MeltMind-E6-cost-20261114.zip'
with zipfile.ZipFile(archive,'x',zipfile.ZIP_DEFLATED,compresslevel=6) as z:
for r in manifest:z.write(r['path'],r['path'])
z.writestr('PACKAGE_MANIFEST.json',json.dumps(manifest,indent=2)+'\n')
(dest/'PACKAGE_MANIFEST.json').write_text(json.dumps(manifest,indent=2)+'\n')
for name in ['README.md','summary.csv','execution_contract.json','calibration_contract.json','calibration_plan.json','conditional_budget.json','feature_storage_units.json','verification.json']:shutil.copy2(out/name,dest/name)
for spec in c['cases']:
for suffix in ['.json','.calibration.json']:shutil.copy2(out/(spec['case']+suffix),dest/(spec['case']+suffix))
hashes={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(dest.iterdir()) if p.is_file()}
(dest/'SHA256SUMS').write_text(''.join(f'{h} {name}\n' for name,h in hashes.items()))
print(json.dumps(dict(destination=str(dest),files=len(manifest),archive_bytes=archive.stat().st_size,archive_sha256=hashes[archive.name]),indent=2))
if __name__=='__main__':main()