Add v4 stored-score threshold fractions and paired medians

This commit is contained in:
Frank14f committed 2026-11-16 23:42:20 +08:00
1 parent 271c47a698
commit 7bf1f922e3
5 files changed
+531

No files matched your search

@@ -0,0 +1,84 @@
"""Display existing scalar scores only; never read labels or prediction arrays."""
import hashlib
import json
import math
from pathlib import Path
import statistics
import time
START_CPU = time.process_time()
BASE = Path('/home/frank14f/optane/MeltMind')
ROOT = BASE / 'artifacts/gre24-v4-stored-score-display-2026-11-16'
SOURCE = BASE / 'artifacts/gre24-v4-reproduction-2026-11-16/protocol/scores.json'
EXPECTED = '7308e1b74d96e1f003766b72fc751c2cb0eedb9956d906d2446924015fcd04a9'
raw = SOURCE.read_bytes()
assert hashlib.sha256(raw).hexdigest() == EXPECTED
rows = json.loads(raw)['rows']
assert len(rows) == 1575
assert len({(r['split'], r['case']) for r in rows}) == len(rows)
metrics = {'rrmse_rms': 'RMS(y)', 'rrmse_maximum': 'max(y)', 'rrmse_mean': 'mean(y)'}
models = ['naive', 'GBRT', 'FNN']
splits = []
for split, expected_count in enumerate([304, 311, 308, 309, 305]):
requested = [r for r in rows if r['split'] == split]
scored = [r for r in requested if r['status'] == 'SCORED']
assert len(requested) == 315 and len(scored) == expected_count
item = {'split': split, 'requested': 315, 'scored': len(scored),
'missing': 315 - len(scored), 'fractions': {}, 'paired_medians': {}}
for metric in metrics:
item['fractions'][metric] = {}
for model in models:
values = [r['scores'][model][metric] for r in scored]
assert all(isinstance(v, (int, float)) and math.isfinite(v) for v in values)
assert all(not r['scores'][model]['undefined_metrics'] for r in scored)
count = sum(v < 0.1 for v in values)
item['fractions'][metric][model] = {'below': count, 'total': len(values),
'fraction': count / len(values)}
differences = [r['scores']['naive'][metric] - r['scores']['GBRT'][metric]
for r in scored]
item['paired_medians'][metric] = {'pairs': len(differences),
'median_naive_minus_GBRT': statistics.median(differences)}
splits.append(item)
means = {metric: {model: statistics.mean(s['fractions'][metric][model]['fraction']
for s in splits) for model in models} for metric in metrics}
result = {'source': str(SOURCE.relative_to(BASE)), 'source_sha256': EXPECTED,
'authorization': 'Claude1794839041.521639', 'threshold': 0.1,
'comparison': 'strictly less than', 'aggregation': 'equal mean of five split fractions',
'paired_definition': 'median of same-case naive minus GBRT scalar RRMSE within each split',
'splits': splits, 'mean_split_fractions': means,
'no_new_fit_prediction_scoring_decode_or_bootstrap': True}
(ROOT / 'supplement.json').write_text(json.dumps(result, indent=2) + '\n')
lines = ['# Gre24 v4:已有评分的阈值及配对差展示补充', '',
'按Claude1794839041.521639要求,仅汇总固定 scores.json 内已有逐件标量。原评分SHA256为 `' + EXPECTED + '`,读取前后相同;未读取标签、预测数组或模型,未重新拟合、预测、评分或bootstrap。原评分和终态账不改。', '',
'## RRMSE严格小于0.1的比例', '',
'五次独立随机315测试/314训练划分,不是五折交叉验证。有效测试依次304/311/308/309/305,缺失11/4/7/6/10;1575次测试访问含38个准备失败行。比例分母为各split有效评分件数,五split比例等权平均,不按节点加权或先池化;14个准备失败及原排除保持不变。', '',
'| 分母 | 模型 | split0 | split1 | split2 | split3 | split4 | 五split比例平均 |',
'| --- | --- | --- | --- | --- | --- | --- | ---: |']
for metric, label in metrics.items():
for model in models:
cells = [s['fractions'][metric][model] for s in splits]
lines.append('| ' + ' | '.join([label, model] + [f"{c['below']}/{c['total']} ({100*c['fraction']:.4f}%)" for c in cells] + [f'{100*means[metric][model]:.6f}%']) + ' |')
lines += ['', '## 同件配对的naive与GBRT差', '',
'先在每件上计算 Δ = RRMSE_naive − RRMSE_GBRT,再在各split取Δ中位数;不是两模型中位数相减。正值表示GBRT误差较低。配对数量为304/311/308/309/305,数值为比例而非百分比。', '',
'| 分母 | split0 median(Δ) | split1 median(Δ) | split2 median(Δ) | split3 median(Δ) | split4 median(Δ) |',
'| --- | ---: | ---: | ---: | ---: | ---: |']
for metric, label in metrics.items():
lines.append('| ' + ' | '.join([label] + [f"{s['paired_medians'][metric]['median_naive_minus_GBRT']:.12f}" for s in splits]) + ' |')
lines += ['', '## 解释边界与费用', '',
'RMS(y)仍为内部预登记主指标;max(y)是保留tet域最后到达时间,作为论文total fill time的代理,作者评分代码尚未取得。三分母完整保留,不据数值接近改主指标。失败覆盖偏差方向未知,v4/v5摘要相近不能建立版本因果配对;t_set替代缺失Q、R1离散补充、fill-only/早停FNN和随机几何近重复限制均保留。', '',
'这是既有分数的描述性展示,不新增统计检验、训练或版本因果结论。原始结果见[固定结果报告](gre24-v4-reproduction-results-2026-11-16.md),独立审计见[MeltAssist审计](meltassist-v4-terminal-audit-2026-11-16.md)。完整计数与配对中位数见[补充JSON](../artifacts/gre24-v4-stored-score-display-2026-11-16/supplement.json)。']
assert hashlib.sha256(SOURCE.read_bytes()).hexdigest() == EXPECTED
cpu = time.process_time() - START_CPU
receipt = {'status': 'COMPLETE', 'created_unix': time.time(), 'display_cpu_seconds': cpu,
'original_worker_cpu_seconds': 140042.73955399985,
'original_plus_display_cpu_seconds': 140042.73955399985 + cpu,
'cpu_cap_seconds': 216000, 'gpu_added_seconds': 0,
'source_before_sha256': EXPECTED, 'source_after_sha256': EXPECTED,
'builder_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
'supplement_sha256': hashlib.sha256((ROOT / 'supplement.json').read_bytes()).hexdigest(),
'accounting_note': 'Display aggregation CPU only; prior independent audit costs remain separately reported.'}
assert receipt['original_plus_display_cpu_seconds'] < receipt['cpu_cap_seconds']
(ROOT / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n')
lines += ['', f'此次展示汇总CPU {cpu:.9f}秒单列;原科学账140042.739554秒不改,加本次展示为{receipt["original_plus_display_cpu_seconds"]:.9f}/216000秒。先前独立审计费用仍另列,非这里的全活动总账。GPU新增0。费用与源码哈希见[回执](../artifacts/gre24-v4-stored-score-display-2026-11-16/receipt.json)。', '']
(BASE / 'docs/gre24-v4-stored-score-display-2026-11-16.md').write_text('\n'.join(lines))
print(json.dumps({'means': means, 'paired': [s['paired_medians'] for s in splits], 'cpu_seconds': cpu}))
@@ -0,0 +1,14 @@
{
"status": "COMPLETE",
"created_unix": 1794843714.0824852,
"display_cpu_seconds": 0.29950983000000003,
"original_worker_cpu_seconds": 140042.73955399985,
"original_plus_display_cpu_seconds": 140043.03906382984,
"cpu_cap_seconds": 216000,
"gpu_added_seconds": 0,
"source_before_sha256": "7308e1b74d96e1f003766b72fc751c2cb0eedb9956d906d2446924015fcd04a9",
"source_after_sha256": "7308e1b74d96e1f003766b72fc751c2cb0eedb9956d906d2446924015fcd04a9",
"builder_sha256": "520fa158334619b3fc155c23c471a71be1699e45288d4e02f41c6077fda4d71a",
"supplement_sha256": "cc75c27be630887bda9430ad5dfab836a8cb1f4c50f94799f91608039326833f",
"accounting_note": "Display aggregation CPU only; prior independent audit costs remain separately reported."
}
@@ -0,0 +1,394 @@
{
"source": "artifacts/gre24-v4-reproduction-2026-11-16/protocol/scores.json",
"source_sha256": "7308e1b74d96e1f003766b72fc751c2cb0eedb9956d906d2446924015fcd04a9",
"authorization": "Claude1794839041.521639",
"threshold": 0.1,
"comparison": "strictly less than",
"aggregation": "equal mean of five split fractions",
"paired_definition": "median of same-case naive minus GBRT scalar RRMSE within each split",
"splits": [
{
"split": 0,
"requested": 315,
"scored": 304,
"missing": 11,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 178,
"total": 304,
"fraction": 0.5855263157894737
},
"GBRT": {
"below": 231,
"total": 304,
"fraction": 0.7598684210526315
},
"FNN": {
"below": 222,
"total": 304,
"fraction": 0.7302631578947368
}
},
"rrmse_maximum": {
"naive": {
"below": 259,
"total": 304,
"fraction": 0.8519736842105263
},
"GBRT": {
"below": 285,
"total": 304,
"fraction": 0.9375
},
"FNN": {
"below": 284,
"total": 304,
"fraction": 0.9342105263157895
}
},
"rrmse_mean": {
"naive": {
"below": 155,
"total": 304,
"fraction": 0.5098684210526315
},
"GBRT": {
"below": 204,
"total": 304,
"fraction": 0.6710526315789473
},
"FNN": {
"below": 189,
"total": 304,
"fraction": 0.6217105263157895
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 304,
"median_naive_minus_GBRT": 0.019882618404954457
},
"rrmse_maximum": {
"pairs": 304,
"median_naive_minus_GBRT": 0.01188850828576504
},
"rrmse_mean": {
"pairs": 304,
"median_naive_minus_GBRT": 0.023021678969099367
}
}
},
{
"split": 1,
"requested": 315,
"scored": 311,
"missing": 4,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 185,
"total": 311,
"fraction": 0.594855305466238
},
"GBRT": {
"below": 239,
"total": 311,
"fraction": 0.7684887459807074
},
"FNN": {
"below": 226,
"total": 311,
"fraction": 0.7266881028938906
}
},
"rrmse_maximum": {
"naive": {
"below": 275,
"total": 311,
"fraction": 0.8842443729903537
},
"GBRT": {
"below": 295,
"total": 311,
"fraction": 0.9485530546623794
},
"FNN": {
"below": 292,
"total": 311,
"fraction": 0.9389067524115756
}
},
"rrmse_mean": {
"naive": {
"below": 161,
"total": 311,
"fraction": 0.5176848874598071
},
"GBRT": {
"below": 216,
"total": 311,
"fraction": 0.6945337620578779
},
"FNN": {
"below": 195,
"total": 311,
"fraction": 0.6270096463022508
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 311,
"median_naive_minus_GBRT": 0.018171540025296536
},
"rrmse_maximum": {
"pairs": 311,
"median_naive_minus_GBRT": 0.010736216578304807
},
"rrmse_mean": {
"pairs": 311,
"median_naive_minus_GBRT": 0.021097379047862533
}
}
},
{
"split": 2,
"requested": 315,
"scored": 308,
"missing": 7,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 191,
"total": 308,
"fraction": 0.6201298701298701
},
"GBRT": {
"below": 246,
"total": 308,
"fraction": 0.7987012987012987
},
"FNN": {
"below": 226,
"total": 308,
"fraction": 0.7337662337662337
}
},
"rrmse_maximum": {
"naive": {
"below": 268,
"total": 308,
"fraction": 0.8701298701298701
},
"GBRT": {
"below": 292,
"total": 308,
"fraction": 0.948051948051948
},
"FNN": {
"below": 287,
"total": 308,
"fraction": 0.9318181818181818
}
},
"rrmse_mean": {
"naive": {
"below": 164,
"total": 308,
"fraction": 0.5324675324675324
},
"GBRT": {
"below": 217,
"total": 308,
"fraction": 0.7045454545454546
},
"FNN": {
"below": 189,
"total": 308,
"fraction": 0.6136363636363636
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 308,
"median_naive_minus_GBRT": 0.018917791542276075
},
"rrmse_maximum": {
"pairs": 308,
"median_naive_minus_GBRT": 0.011273753024802755
},
"rrmse_mean": {
"pairs": 308,
"median_naive_minus_GBRT": 0.021652869330455717
}
}
},
{
"split": 3,
"requested": 315,
"scored": 309,
"missing": 6,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 185,
"total": 309,
"fraction": 0.598705501618123
},
"GBRT": {
"below": 246,
"total": 309,
"fraction": 0.7961165048543689
},
"FNN": {
"below": 220,
"total": 309,
"fraction": 0.7119741100323624
}
},
"rrmse_maximum": {
"naive": {
"below": 272,
"total": 309,
"fraction": 0.8802588996763754
},
"GBRT": {
"below": 294,
"total": 309,
"fraction": 0.9514563106796117
},
"FNN": {
"below": 291,
"total": 309,
"fraction": 0.941747572815534
}
},
"rrmse_mean": {
"naive": {
"below": 155,
"total": 309,
"fraction": 0.5016181229773463
},
"GBRT": {
"below": 215,
"total": 309,
"fraction": 0.6957928802588996
},
"FNN": {
"below": 193,
"total": 309,
"fraction": 0.6245954692556634
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 309,
"median_naive_minus_GBRT": 0.019676381358891645
},
"rrmse_maximum": {
"pairs": 309,
"median_naive_minus_GBRT": 0.011581302653170575
},
"rrmse_mean": {
"pairs": 309,
"median_naive_minus_GBRT": 0.022773028566505635
}
}
},
{
"split": 4,
"requested": 315,
"scored": 305,
"missing": 10,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 186,
"total": 305,
"fraction": 0.6098360655737705
},
"GBRT": {
"below": 243,
"total": 305,
"fraction": 0.7967213114754098
},
"FNN": {
"below": 227,
"total": 305,
"fraction": 0.7442622950819672
}
},
"rrmse_maximum": {
"naive": {
"below": 265,
"total": 305,
"fraction": 0.8688524590163934
},
"GBRT": {
"below": 292,
"total": 305,
"fraction": 0.9573770491803278
},
"FNN": {
"below": 289,
"total": 305,
"fraction": 0.9475409836065574
}
},
"rrmse_mean": {
"naive": {
"below": 159,
"total": 305,
"fraction": 0.521311475409836
},
"GBRT": {
"below": 215,
"total": 305,
"fraction": 0.7049180327868853
},
"FNN": {
"below": 204,
"total": 305,
"fraction": 0.6688524590163935
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 305,
"median_naive_minus_GBRT": 0.017812751846786504
},
"rrmse_maximum": {
"pairs": 305,
"median_naive_minus_GBRT": 0.01106959997313419
},
"rrmse_mean": {
"pairs": 305,
"median_naive_minus_GBRT": 0.02061714424889538
}
}
}
],
"mean_split_fractions": {
"rrmse_rms": {
"naive": 0.601810611715495,
"GBRT": 0.7839792564128832,
"FNN": 0.7293907799338382
},
"rrmse_maximum": {
"naive": 0.8710918572047038,
"GBRT": 0.9485876725148534,
"FNN": 0.9388448033935276
},
"rrmse_mean": {
"naive": 0.5165900878734306,
"GBRT": 0.6941685522456129,
"FNN": 0.6311608929052922
}
},
"no_new_fit_prediction_scoring_decode_or_bootstrap": true
}
@@ -10,6 +10,8 @@
## 已有评分文件的数值
既有分数的三分母严格 `<0.1` 比例、逐split同件配对 naive−GBRT 中位数,已另列于[展示补充](gre24-v4-stored-score-display-2026-11-16.md)。仅汇总原标量,未重新评分;原14个准备失败及所有解释边界保留。
下表直接摘录各split先取中位数、再对五个split取平均的原摘要,未重新聚合预测或评分。原域为全部保留tet节点(含壁面),不是S2查询;这些值不能直接用于R4/S2组件分组归因。
|分母|naive|GBRT|FNN|
@@ -0,0 +1,37 @@
# Gre24 v4:已有评分的阈值及配对差展示补充
按Claude1794839041.521639要求,仅汇总固定 scores.json 内已有逐件标量。原评分SHA256为 `7308e1b74d96e1f003766b72fc751c2cb0eedb9956d906d2446924015fcd04a9`,读取前后相同;未读取标签、预测数组或模型,未重新拟合、预测、评分或bootstrap。原评分和终态账不改。
## RRMSE严格小于0.1的比例
五次独立随机315测试/314训练划分,不是五折交叉验证。有效测试依次304/311/308/309/305,缺失11/4/7/6/10;1575次测试访问含38个准备失败行。比例分母为各split有效评分件数,五split比例等权平均,不按节点加权或先池化;14个准备失败及原排除保持不变。
| 分母 | 模型 | split0 | split1 | split2 | split3 | split4 | 五split比例平均 |
| --- | --- | --- | --- | --- | --- | --- | ---: |
| RMS(y) | naive | 178/304 (58.5526%) | 185/311 (59.4855%) | 191/308 (62.0130%) | 185/309 (59.8706%) | 186/305 (60.9836%) | 60.181061% |
| RMS(y) | GBRT | 231/304 (75.9868%) | 239/311 (76.8489%) | 246/308 (79.8701%) | 246/309 (79.6117%) | 243/305 (79.6721%) | 78.397926% |
| RMS(y) | FNN | 222/304 (73.0263%) | 226/311 (72.6688%) | 226/308 (73.3766%) | 220/309 (71.1974%) | 227/305 (74.4262%) | 72.939078% |
| max(y) | naive | 259/304 (85.1974%) | 275/311 (88.4244%) | 268/308 (87.0130%) | 272/309 (88.0259%) | 265/305 (86.8852%) | 87.109186% |
| max(y) | GBRT | 285/304 (93.7500%) | 295/311 (94.8553%) | 292/308 (94.8052%) | 294/309 (95.1456%) | 292/305 (95.7377%) | 94.858767% |
| max(y) | FNN | 284/304 (93.4211%) | 292/311 (93.8907%) | 287/308 (93.1818%) | 291/309 (94.1748%) | 289/305 (94.7541%) | 93.884480% |
| mean(y) | naive | 155/304 (50.9868%) | 161/311 (51.7685%) | 164/308 (53.2468%) | 155/309 (50.1618%) | 159/305 (52.1311%) | 51.659009% |
| mean(y) | GBRT | 204/304 (67.1053%) | 216/311 (69.4534%) | 217/308 (70.4545%) | 215/309 (69.5793%) | 215/305 (70.4918%) | 69.416855% |
| mean(y) | FNN | 189/304 (62.1711%) | 195/311 (62.7010%) | 189/308 (61.3636%) | 193/309 (62.4595%) | 204/305 (66.8852%) | 63.116089% |
## 同件配对的naive与GBRT差
先在每件上计算 Δ = RRMSE_naive − RRMSE_GBRT,再在各split取Δ中位数;不是两模型中位数相减。正值表示GBRT误差较低。配对数量为304/311/308/309/305,数值为比例而非百分比。
| 分母 | split0 median(Δ) | split1 median(Δ) | split2 median(Δ) | split3 median(Δ) | split4 median(Δ) |
| --- | ---: | ---: | ---: | ---: | ---: |
| RMS(y) | 0.019882618405 | 0.018171540025 | 0.018917791542 | 0.019676381359 | 0.017812751847 |
| max(y) | 0.011888508286 | 0.010736216578 | 0.011273753025 | 0.011581302653 | 0.011069599973 |
| mean(y) | 0.023021678969 | 0.021097379048 | 0.021652869330 | 0.022773028567 | 0.020617144249 |
## 解释边界与费用
RMS(y)仍为内部预登记主指标;max(y)是保留tet域最后到达时间,作为论文total fill time的代理,作者评分代码尚未取得。三分母完整保留,不据数值接近改主指标。失败覆盖偏差方向未知,v4/v5摘要相近不能建立版本因果配对;t_set替代缺失Q、R1离散补充、fill-only/早停FNN和随机几何近重复限制均保留。
这是既有分数的描述性展示,不新增统计检验、训练或版本因果结论。原始结果见[固定结果报告](gre24-v4-reproduction-results-2026-11-16.md),独立审计见[MeltAssist审计](meltassist-v4-terminal-audit-2026-11-16.md)。完整计数与配对中位数见[补充JSON](../artifacts/gre24-v4-stored-score-display-2026-11-16/supplement.json)。
此次展示汇总CPU 0.299509830秒单列;原科学账140042.739554秒不改,加本次展示为140043.039063830/216000秒。先前独立审计费用仍另列,非这里的全活动总账。GPU新增0。费用与源码哈希见[回执](../artifacts/gre24-v4-stored-score-display-2026-11-16/receipt.json)。