99 lines
12 KiB
Python
99 lines
12 KiB
Python
"""Allowlisted count/timing evidence; no source geometry or scientific rerun."""
|
||
import ast
|
||
import csv
|
||
import hashlib
|
||
import json
|
||
from pathlib import Path
|
||
import shutil
|
||
import zipfile
|
||
import numpy as np
|
||
|
||
|
||
def read(p):return json.loads(Path(p).read_text())
|
||
|
||
|
||
def main():
|
||
out=Path('artifacts/e6-full-node-cost-2026-11-14');c=read(out/'execution_contract.json');assert read(out/'verification.json')['status']=='PASS'
|
||
rows=[];feature_cost=[]
|
||
for spec in c['cases']:
|
||
r=read(out/(spec['case']+'.json'));t=read(out/(spec['case']+'.calibration.json'));g=r['groups']['all_nodes'];fixed=sum(t['fixed_setup'].values());e=t['estimates']['all_nodes']
|
||
rows.append(dict(case=spec['case'][:8],nodes=r['nodes'],positive_nodes=r['groups']['positive_distance_nodes']['nodes'],sources=r['four_point_sources'],R=r['radius'],
|
||
all_neighbor_visits=g['neighbor_sum'],positive_neighbor_visits=r['groups']['positive_distance_nodes']['neighbor_sum'],p99_neighbors=g['quantiles']['p99'],max_neighbors=g['quantiles']['max'],
|
||
count_only_wall_seconds=r['timings']['wall_seconds'],fixed_setup_seconds=fixed,calibration_loop_seconds=t['measured_query_loop_seconds'],calibration_visits=t['neighbor_visits'],
|
||
amortized_microseconds_per_neighbor=t['amortized_seconds_per_neighbor']*1e6,
|
||
estimated_full_seconds=e['total_seconds_with_fixed_setup'],fast_scenario_seconds=fixed+e['fast_scenario_seconds'],slow_scenario_seconds=fixed+e['slow_scenario_seconds'],
|
||
measured_calibration_peak_MiB=t['memory']['measured_calibration_peak_bytes']/2**20,full_E6_memory_allowance_MiB=t['memory']['full_E6_peak_allowance_bytes']/2**20,
|
||
block_output_memory_allowance_MiB=t['memory']['block4096_E6_peak_allowance_bytes']/2**20,
|
||
old_budget_pass=g['p99_times_nodes']<=c['kernel_evaluation_budget'] and g['main_plus_pose_pair_budget_pass']))
|
||
with zipfile.ZipFile(spec['sample']) as z:
|
||
with z.open('edges.npy') as f:
|
||
version=np.lib.format.read_magic(f);shape,order,dtype=np.lib.format._read_array_header(f,version)
|
||
feature_cost.append(dict(case=spec['case'],nodes=r['nodes'],undirected_edges=shape[0],
|
||
bytes_per_additional_float64_node_scalar=8*r['nodes'],bytes_per_additional_float64_undirected_edge_scalar=8*shape[0],
|
||
note='Capacity arithmetic only; no O2-rep formula implemented or timed. Directed duplicated edge storage would double the edge term.'))
|
||
with (out/'summary.csv').open('w') as f:
|
||
w=csv.DictWriter(f,fieldnames=list(rows[0]),lineterminator='\n');w.writeheader();w.writerows(rows)
|
||
(out/'feature_storage_units.json').write_text(json.dumps(feature_cost,indent=2)+'\n')
|
||
lines=['# E6 全节点成本:实际计数与有界计时校准','',
|
||
'结论:在当前四点估计器、半径和缓存下,850 与 666 全节点计算量可行;778 单次约 18.91 亿邻居访问,超过原 5 亿预算。Claude 随后接受校准并给出条件新预算:O2-rep 规范冻结后,每场最多 25 亿次访问或 30 分钟,先达到者为限,三件均可纳入。规范冻结前仍保留已有抽样,不启动全量。18.7 万节点的最大件并非计算瓶颈。本次只完成成本评估,没有执行全节点核重算或实现 O2-rep。','',
|
||
'## 实际工作量','',
|
||
'| 实件 | 全节点 / 内部节点 | 四点源点 | R | 全节点访问量 | 仅内部访问量 | p99 / 最大邻居数 |','|---|---:|---:|---:|---:|---:|---:|']
|
||
for r in rows:lines.append(f"| {r['case']} | {r['nodes']:,} / {r['positive_nodes']:,} | {r['sources']:,} | {r['R']:.6f} | {r['all_neighbor_visits']:,} | {r['positive_neighbor_visits']:,} | {r['p99_neighbors']:.2f} / {r['max_neighbors']:.0f} |")
|
||
lines+=['','逐节点计数使用原 E6 四点源坐标及 cKDTree 精确半径查询,workers=1,每批4096个查询,仅返回长度,不构造全邻域列表。5122个旧去重锚点的计数与原预检逐点完全一致;内部/边界节点数量和邻居总量均加和等于全节点。边界判定沿用缓存 d=0;统计其成本不改变既有边界 unscored 语义。','',
|
||
'850/666 的 p99×N 和原场+旋转的实际邻居总量均低于原 5e8 门槛。778 单场为门槛的3.78倍,两场为7.56倍;只算内部节点也超过门槛。缩小R、删源点或改核都属于新估计器,本次没有采用。','',
|
||
'## 分阶段实测与全量推算','',
|
||
'依据 [Claude 的校准批准](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794627909915539),每件64点,内部/边界各分16个邻居数分位层,每层2点,包含最大邻域及旧锚点重叠。合计4,609,064次邻居访问,低于1e7硬上限。原 integrate 算式保持不变,以转发包装记录建树和查询循环时间;56个重叠点的全部输出与旧E6一致。瞬时结果仅用于校准与核对,没有保存新的科学场或给出新科学结论。','',
|
||
'| 实件 | 加载+源场+源点+建树实测秒 | 64点积分循环实测秒 | 摊销微秒/邻居 | 全节点单场估计秒 | 两样点快/慢情景秒 |','|---|---:|---:|---:|---:|---:|']
|
||
for r in rows:lines.append(f"| {r['case']} | {r['fixed_setup_seconds']:.3f} | {r['calibration_loop_seconds']:.4f} | {r['amortized_microseconds_per_neighbor']:.3f} | {r['estimated_full_seconds']:.2f} | {r['fast_scenario_seconds']:.2f}–{r['slow_scenario_seconds']:.2f} |")
|
||
lines+=['','全节点时间是每层节点数×该层两样点平均循环时间,再加一次固定准备。快/慢情景分别使用每层两样点的最小/最大时间;它们不是置信区间,也不是保证上下界。样点是确定性选取,缓存、查询顺序、机器负载和分层内差异都会影响实际时间。另保存非负“每查询固定项+每邻居项”拟合,作为诊断而非替代实测。摊销每邻居耗时包含每查询固定开销,不能跨三件当作统一常数。','',
|
||
'仅按旧2000抽样任务总时长外推会重复放大源场准备、建树等固定成本;在666上给出约8分钟的误导性规划。本报告使用分阶段校准后的约52秒估计;旧粗模型保留在计数JSON作来源审计,不作为当前建议。未来O2-rep的新节点/边收缩、序列化和训练成本未计入。','',
|
||
'## 内存与执行建议','',
|
||
'| 实件 | 校准实测进程峰值 MiB | 全量E6输出规划 MiB | 4096点分块输出规划 MiB | 下一步建议 |','|---|---:|---:|---:|---|']
|
||
advice={'00000850':'规范冻结后可全节点;预留30秒/512 MiB','00000778':'规范冻结后按25亿次/30分钟新上限;此前保留抽样','00000666':'规范冻结后可全节点;预留2分钟/1.25 GiB'}
|
||
for r in rows:lines.append(f"| {r['case']} | {r['measured_calibration_peak_MiB']:.1f} | {r['full_E6_memory_allowance_MiB']:.1f} | {r['block_output_memory_allowance_MiB']:.1f} | {advice[r['case']]} |")
|
||
lines+=['','[Claude 11:48:55 的条件预算决定](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794628135270029):5e8是未知成本前的保护线,并非研究约束;校准后允许规范冻结时采用每场2.5e9实际邻居访问或1800秒,先到者停止。778当前全节点实际计数约占新访问上限75.65%。新预算不回写旧合同,也不把待完善的规范视为已经冻结。全库前须在分层panel先做计数预检,不能由三件外推624。','',
|
||
'规划内存=校准峰值+全节点或分块E6输出相对64点的增量+128 MiB余量;是预留,不是严格RSS上界。四点源张量与树保持驻留,逐个查询生成邻居列表;分块输出只节省输出空间,不省掉源张量。778若一次性保存全邻居int64索引就约14.09 GiB,还不含权重与张量,因此不应物化全邻接列表。','',
|
||
'以上时间预留只针对单场现有估计器;若以后需要全节点旋转验证,应另计第二场,不把前次64点校准称为已完成全量验证。所有后续运行仍等待对应任务/规范确定;本次没有启动全量。','',
|
||
'为尚未冻结的表示提供纯容量换算:','',
|
||
'| 实件 | 原始无向边数 | 每新增float64节点标量 MiB | 每新增float64无向边标量 MiB |','|---|---:|---:|---:|']
|
||
for r in feature_cost:lines.append(f"| {r['case'][:8]} | {r['undirected_edges']:,} | {r['bytes_per_additional_float64_node_scalar']/2**20:.3f} | {r['bytes_per_additional_float64_undirected_edge_scalar']/2**20:.3f} |")
|
||
lines+=['','这些只是从缓存数组头读出的维数和存储算术,不是O2-rep实现;若双向重复存边,边标量项翻倍。没有改变 hard-gap=0.1、类型语义或有效性规则,也没有新距离查询、细化、O1准备、LP、S1或训练。','',
|
||
'## 审计与复核','',
|
||
'ZIP 包含逐节点邻居计数/内部标记/质量、全部分层样点与计时、合同、源代码、原锚点预检和汇总。24项本地输入哈希核对通过;计数/抽样身份/分层规模/耗时公式/预算算术复核通过。不包含完整源网格、距离或旧场;完整计数/校准重跑需接回合同指向的缓存与既有E6证据。原始执行代码和数据哈希保持可追溯。','',
|
||
'解压后可只复核保存的成本数据,无须网格或新积分:','',
|
||
'```sh','python tools/verify_e6_cost.py artifacts/e6-full-node-cost-2026-11-14','```','',
|
||
'保存结果复核只需NumPy;实际计数/校准还需SciPy。本次Python3.10,OMP/OPENBLAS线程数2、cKDTree计数workers=1。每件独立进程记录峰值。started标记防止意外重复覆盖;完整重跑应使用新的输出目录。','']
|
||
(out/'README.md').write_text('\n'.join(lines))
|
||
dest=Path('artifacts/e6-cost-review-delivery-2026-11-14');dest.mkdir(exist_ok=False)
|
||
include={p for p in out.iterdir() if p.is_file()};include.add(Path('docs/e6-cost-source-2026-11-14.txt'));include.add(Path('docs/e6-cost-budget-decision-2026-11-14.txt'))
|
||
for s in c['cases']:include.add(Path(c['prior_directory'])/(s['case']+'_tetra_four_point_anchors.npz'))
|
||
pending=[Path('tools')/name for name in ['assess_e6_full_node_cost.py','calibrate_e6_cost.py','verify_e6_cost.py','build_e6_cost_review.py']]
|
||
while pending:
|
||
path=pending.pop()
|
||
if path in include:continue
|
||
include.add(path)
|
||
for node in ast.walk(ast.parse(path.read_text())):
|
||
if isinstance(node,ast.ImportFrom):
|
||
names=[node.module.split('.')[-1]] if node.module else [x.name for x in node.names]
|
||
for name in names:
|
||
candidate=Path('tools')/(name+'.py')
|
||
if candidate.exists() and candidate not in include:pending.append(candidate)
|
||
manifest=[]
|
||
for p in sorted(include):
|
||
data=p.read_bytes();assert len(data)<5_000_000
|
||
manifest.append(dict(path=str(p),bytes=len(data),sha256=hashlib.sha256(data).hexdigest()))
|
||
archive=dest/'MeltMind-E6-cost-20261114.zip'
|
||
with zipfile.ZipFile(archive,'x',zipfile.ZIP_DEFLATED,compresslevel=6) as z:
|
||
for r in manifest:z.write(r['path'],r['path'])
|
||
z.writestr('PACKAGE_MANIFEST.json',json.dumps(manifest,indent=2)+'\n')
|
||
(dest/'PACKAGE_MANIFEST.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||
for name in ['README.md','summary.csv','execution_contract.json','calibration_contract.json','calibration_plan.json','conditional_budget.json','feature_storage_units.json','verification.json']:shutil.copy2(out/name,dest/name)
|
||
for spec in c['cases']:
|
||
for suffix in ['.json','.calibration.json']:shutil.copy2(out/(spec['case']+suffix),dest/(spec['case']+suffix))
|
||
hashes={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(dest.iterdir()) if p.is_file()}
|
||
(dest/'SHA256SUMS').write_text(''.join(f'{h} {name}\n' for name,h in hashes.items()))
|
||
print(json.dumps(dict(destination=str(dest),files=len(manifest),archive_bytes=archive.stat().st_size,archive_sha256=hashes[archive.name]),indent=2))
|
||
|
||
|
||
if __name__=='__main__':main()
|