Files
MeltMind/artifacts/gre24-v4-stored-score-display-2026-11-16/supplement.json
T

395 lines
9.5 KiB
JSON

{
"source": "artifacts/gre24-v4-reproduction-2026-11-16/protocol/scores.json",
"source_sha256": "7308e1b74d96e1f003766b72fc751c2cb0eedb9956d906d2446924015fcd04a9",
"authorization": "Claude1794839041.521639",
"threshold": 0.1,
"comparison": "strictly less than",
"aggregation": "equal mean of five split fractions",
"paired_definition": "median of same-case naive minus GBRT scalar RRMSE within each split",
"splits": [
{
"split": 0,
"requested": 315,
"scored": 304,
"missing": 11,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 178,
"total": 304,
"fraction": 0.5855263157894737
},
"GBRT": {
"below": 231,
"total": 304,
"fraction": 0.7598684210526315
},
"FNN": {
"below": 222,
"total": 304,
"fraction": 0.7302631578947368
}
},
"rrmse_maximum": {
"naive": {
"below": 259,
"total": 304,
"fraction": 0.8519736842105263
},
"GBRT": {
"below": 285,
"total": 304,
"fraction": 0.9375
},
"FNN": {
"below": 284,
"total": 304,
"fraction": 0.9342105263157895
}
},
"rrmse_mean": {
"naive": {
"below": 155,
"total": 304,
"fraction": 0.5098684210526315
},
"GBRT": {
"below": 204,
"total": 304,
"fraction": 0.6710526315789473
},
"FNN": {
"below": 189,
"total": 304,
"fraction": 0.6217105263157895
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 304,
"median_naive_minus_GBRT": 0.019882618404954457
},
"rrmse_maximum": {
"pairs": 304,
"median_naive_minus_GBRT": 0.01188850828576504
},
"rrmse_mean": {
"pairs": 304,
"median_naive_minus_GBRT": 0.023021678969099367
}
}
},
{
"split": 1,
"requested": 315,
"scored": 311,
"missing": 4,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 185,
"total": 311,
"fraction": 0.594855305466238
},
"GBRT": {
"below": 239,
"total": 311,
"fraction": 0.7684887459807074
},
"FNN": {
"below": 226,
"total": 311,
"fraction": 0.7266881028938906
}
},
"rrmse_maximum": {
"naive": {
"below": 275,
"total": 311,
"fraction": 0.8842443729903537
},
"GBRT": {
"below": 295,
"total": 311,
"fraction": 0.9485530546623794
},
"FNN": {
"below": 292,
"total": 311,
"fraction": 0.9389067524115756
}
},
"rrmse_mean": {
"naive": {
"below": 161,
"total": 311,
"fraction": 0.5176848874598071
},
"GBRT": {
"below": 216,
"total": 311,
"fraction": 0.6945337620578779
},
"FNN": {
"below": 195,
"total": 311,
"fraction": 0.6270096463022508
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 311,
"median_naive_minus_GBRT": 0.018171540025296536
},
"rrmse_maximum": {
"pairs": 311,
"median_naive_minus_GBRT": 0.010736216578304807
},
"rrmse_mean": {
"pairs": 311,
"median_naive_minus_GBRT": 0.021097379047862533
}
}
},
{
"split": 2,
"requested": 315,
"scored": 308,
"missing": 7,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 191,
"total": 308,
"fraction": 0.6201298701298701
},
"GBRT": {
"below": 246,
"total": 308,
"fraction": 0.7987012987012987
},
"FNN": {
"below": 226,
"total": 308,
"fraction": 0.7337662337662337
}
},
"rrmse_maximum": {
"naive": {
"below": 268,
"total": 308,
"fraction": 0.8701298701298701
},
"GBRT": {
"below": 292,
"total": 308,
"fraction": 0.948051948051948
},
"FNN": {
"below": 287,
"total": 308,
"fraction": 0.9318181818181818
}
},
"rrmse_mean": {
"naive": {
"below": 164,
"total": 308,
"fraction": 0.5324675324675324
},
"GBRT": {
"below": 217,
"total": 308,
"fraction": 0.7045454545454546
},
"FNN": {
"below": 189,
"total": 308,
"fraction": 0.6136363636363636
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 308,
"median_naive_minus_GBRT": 0.018917791542276075
},
"rrmse_maximum": {
"pairs": 308,
"median_naive_minus_GBRT": 0.011273753024802755
},
"rrmse_mean": {
"pairs": 308,
"median_naive_minus_GBRT": 0.021652869330455717
}
}
},
{
"split": 3,
"requested": 315,
"scored": 309,
"missing": 6,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 185,
"total": 309,
"fraction": 0.598705501618123
},
"GBRT": {
"below": 246,
"total": 309,
"fraction": 0.7961165048543689
},
"FNN": {
"below": 220,
"total": 309,
"fraction": 0.7119741100323624
}
},
"rrmse_maximum": {
"naive": {
"below": 272,
"total": 309,
"fraction": 0.8802588996763754
},
"GBRT": {
"below": 294,
"total": 309,
"fraction": 0.9514563106796117
},
"FNN": {
"below": 291,
"total": 309,
"fraction": 0.941747572815534
}
},
"rrmse_mean": {
"naive": {
"below": 155,
"total": 309,
"fraction": 0.5016181229773463
},
"GBRT": {
"below": 215,
"total": 309,
"fraction": 0.6957928802588996
},
"FNN": {
"below": 193,
"total": 309,
"fraction": 0.6245954692556634
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 309,
"median_naive_minus_GBRT": 0.019676381358891645
},
"rrmse_maximum": {
"pairs": 309,
"median_naive_minus_GBRT": 0.011581302653170575
},
"rrmse_mean": {
"pairs": 309,
"median_naive_minus_GBRT": 0.022773028566505635
}
}
},
{
"split": 4,
"requested": 315,
"scored": 305,
"missing": 10,
"fractions": {
"rrmse_rms": {
"naive": {
"below": 186,
"total": 305,
"fraction": 0.6098360655737705
},
"GBRT": {
"below": 243,
"total": 305,
"fraction": 0.7967213114754098
},
"FNN": {
"below": 227,
"total": 305,
"fraction": 0.7442622950819672
}
},
"rrmse_maximum": {
"naive": {
"below": 265,
"total": 305,
"fraction": 0.8688524590163934
},
"GBRT": {
"below": 292,
"total": 305,
"fraction": 0.9573770491803278
},
"FNN": {
"below": 289,
"total": 305,
"fraction": 0.9475409836065574
}
},
"rrmse_mean": {
"naive": {
"below": 159,
"total": 305,
"fraction": 0.521311475409836
},
"GBRT": {
"below": 215,
"total": 305,
"fraction": 0.7049180327868853
},
"FNN": {
"below": 204,
"total": 305,
"fraction": 0.6688524590163935
}
}
},
"paired_medians": {
"rrmse_rms": {
"pairs": 305,
"median_naive_minus_GBRT": 0.017812751846786504
},
"rrmse_maximum": {
"pairs": 305,
"median_naive_minus_GBRT": 0.01106959997313419
},
"rrmse_mean": {
"pairs": 305,
"median_naive_minus_GBRT": 0.02061714424889538
}
}
}
],
"mean_split_fractions": {
"rrmse_rms": {
"naive": 0.601810611715495,
"GBRT": 0.7839792564128832,
"FNN": 0.7293907799338382
},
"rrmse_maximum": {
"naive": 0.8710918572047038,
"GBRT": 0.9485876725148534,
"FNN": 0.9388448033935276
},
"rrmse_mean": {
"naive": 0.5165900878734306,
"GBRT": 0.6941685522456129,
"FNN": 0.6311608929052922
}
},
"no_new_fit_prediction_scoring_decode_or_bootstrap": true
}