395 lines
9.5 KiB
JSON
395 lines
9.5 KiB
JSON
{
|
|
"source": "artifacts/gre24-v4-reproduction-2026-11-16/protocol/scores.json",
|
|
"source_sha256": "7308e1b74d96e1f003766b72fc751c2cb0eedb9956d906d2446924015fcd04a9",
|
|
"authorization": "Claude1794839041.521639",
|
|
"threshold": 0.1,
|
|
"comparison": "strictly less than",
|
|
"aggregation": "equal mean of five split fractions",
|
|
"paired_definition": "median of same-case naive minus GBRT scalar RRMSE within each split",
|
|
"splits": [
|
|
{
|
|
"split": 0,
|
|
"requested": 315,
|
|
"scored": 304,
|
|
"missing": 11,
|
|
"fractions": {
|
|
"rrmse_rms": {
|
|
"naive": {
|
|
"below": 178,
|
|
"total": 304,
|
|
"fraction": 0.5855263157894737
|
|
},
|
|
"GBRT": {
|
|
"below": 231,
|
|
"total": 304,
|
|
"fraction": 0.7598684210526315
|
|
},
|
|
"FNN": {
|
|
"below": 222,
|
|
"total": 304,
|
|
"fraction": 0.7302631578947368
|
|
}
|
|
},
|
|
"rrmse_maximum": {
|
|
"naive": {
|
|
"below": 259,
|
|
"total": 304,
|
|
"fraction": 0.8519736842105263
|
|
},
|
|
"GBRT": {
|
|
"below": 285,
|
|
"total": 304,
|
|
"fraction": 0.9375
|
|
},
|
|
"FNN": {
|
|
"below": 284,
|
|
"total": 304,
|
|
"fraction": 0.9342105263157895
|
|
}
|
|
},
|
|
"rrmse_mean": {
|
|
"naive": {
|
|
"below": 155,
|
|
"total": 304,
|
|
"fraction": 0.5098684210526315
|
|
},
|
|
"GBRT": {
|
|
"below": 204,
|
|
"total": 304,
|
|
"fraction": 0.6710526315789473
|
|
},
|
|
"FNN": {
|
|
"below": 189,
|
|
"total": 304,
|
|
"fraction": 0.6217105263157895
|
|
}
|
|
}
|
|
},
|
|
"paired_medians": {
|
|
"rrmse_rms": {
|
|
"pairs": 304,
|
|
"median_naive_minus_GBRT": 0.019882618404954457
|
|
},
|
|
"rrmse_maximum": {
|
|
"pairs": 304,
|
|
"median_naive_minus_GBRT": 0.01188850828576504
|
|
},
|
|
"rrmse_mean": {
|
|
"pairs": 304,
|
|
"median_naive_minus_GBRT": 0.023021678969099367
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"split": 1,
|
|
"requested": 315,
|
|
"scored": 311,
|
|
"missing": 4,
|
|
"fractions": {
|
|
"rrmse_rms": {
|
|
"naive": {
|
|
"below": 185,
|
|
"total": 311,
|
|
"fraction": 0.594855305466238
|
|
},
|
|
"GBRT": {
|
|
"below": 239,
|
|
"total": 311,
|
|
"fraction": 0.7684887459807074
|
|
},
|
|
"FNN": {
|
|
"below": 226,
|
|
"total": 311,
|
|
"fraction": 0.7266881028938906
|
|
}
|
|
},
|
|
"rrmse_maximum": {
|
|
"naive": {
|
|
"below": 275,
|
|
"total": 311,
|
|
"fraction": 0.8842443729903537
|
|
},
|
|
"GBRT": {
|
|
"below": 295,
|
|
"total": 311,
|
|
"fraction": 0.9485530546623794
|
|
},
|
|
"FNN": {
|
|
"below": 292,
|
|
"total": 311,
|
|
"fraction": 0.9389067524115756
|
|
}
|
|
},
|
|
"rrmse_mean": {
|
|
"naive": {
|
|
"below": 161,
|
|
"total": 311,
|
|
"fraction": 0.5176848874598071
|
|
},
|
|
"GBRT": {
|
|
"below": 216,
|
|
"total": 311,
|
|
"fraction": 0.6945337620578779
|
|
},
|
|
"FNN": {
|
|
"below": 195,
|
|
"total": 311,
|
|
"fraction": 0.6270096463022508
|
|
}
|
|
}
|
|
},
|
|
"paired_medians": {
|
|
"rrmse_rms": {
|
|
"pairs": 311,
|
|
"median_naive_minus_GBRT": 0.018171540025296536
|
|
},
|
|
"rrmse_maximum": {
|
|
"pairs": 311,
|
|
"median_naive_minus_GBRT": 0.010736216578304807
|
|
},
|
|
"rrmse_mean": {
|
|
"pairs": 311,
|
|
"median_naive_minus_GBRT": 0.021097379047862533
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"split": 2,
|
|
"requested": 315,
|
|
"scored": 308,
|
|
"missing": 7,
|
|
"fractions": {
|
|
"rrmse_rms": {
|
|
"naive": {
|
|
"below": 191,
|
|
"total": 308,
|
|
"fraction": 0.6201298701298701
|
|
},
|
|
"GBRT": {
|
|
"below": 246,
|
|
"total": 308,
|
|
"fraction": 0.7987012987012987
|
|
},
|
|
"FNN": {
|
|
"below": 226,
|
|
"total": 308,
|
|
"fraction": 0.7337662337662337
|
|
}
|
|
},
|
|
"rrmse_maximum": {
|
|
"naive": {
|
|
"below": 268,
|
|
"total": 308,
|
|
"fraction": 0.8701298701298701
|
|
},
|
|
"GBRT": {
|
|
"below": 292,
|
|
"total": 308,
|
|
"fraction": 0.948051948051948
|
|
},
|
|
"FNN": {
|
|
"below": 287,
|
|
"total": 308,
|
|
"fraction": 0.9318181818181818
|
|
}
|
|
},
|
|
"rrmse_mean": {
|
|
"naive": {
|
|
"below": 164,
|
|
"total": 308,
|
|
"fraction": 0.5324675324675324
|
|
},
|
|
"GBRT": {
|
|
"below": 217,
|
|
"total": 308,
|
|
"fraction": 0.7045454545454546
|
|
},
|
|
"FNN": {
|
|
"below": 189,
|
|
"total": 308,
|
|
"fraction": 0.6136363636363636
|
|
}
|
|
}
|
|
},
|
|
"paired_medians": {
|
|
"rrmse_rms": {
|
|
"pairs": 308,
|
|
"median_naive_minus_GBRT": 0.018917791542276075
|
|
},
|
|
"rrmse_maximum": {
|
|
"pairs": 308,
|
|
"median_naive_minus_GBRT": 0.011273753024802755
|
|
},
|
|
"rrmse_mean": {
|
|
"pairs": 308,
|
|
"median_naive_minus_GBRT": 0.021652869330455717
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"split": 3,
|
|
"requested": 315,
|
|
"scored": 309,
|
|
"missing": 6,
|
|
"fractions": {
|
|
"rrmse_rms": {
|
|
"naive": {
|
|
"below": 185,
|
|
"total": 309,
|
|
"fraction": 0.598705501618123
|
|
},
|
|
"GBRT": {
|
|
"below": 246,
|
|
"total": 309,
|
|
"fraction": 0.7961165048543689
|
|
},
|
|
"FNN": {
|
|
"below": 220,
|
|
"total": 309,
|
|
"fraction": 0.7119741100323624
|
|
}
|
|
},
|
|
"rrmse_maximum": {
|
|
"naive": {
|
|
"below": 272,
|
|
"total": 309,
|
|
"fraction": 0.8802588996763754
|
|
},
|
|
"GBRT": {
|
|
"below": 294,
|
|
"total": 309,
|
|
"fraction": 0.9514563106796117
|
|
},
|
|
"FNN": {
|
|
"below": 291,
|
|
"total": 309,
|
|
"fraction": 0.941747572815534
|
|
}
|
|
},
|
|
"rrmse_mean": {
|
|
"naive": {
|
|
"below": 155,
|
|
"total": 309,
|
|
"fraction": 0.5016181229773463
|
|
},
|
|
"GBRT": {
|
|
"below": 215,
|
|
"total": 309,
|
|
"fraction": 0.6957928802588996
|
|
},
|
|
"FNN": {
|
|
"below": 193,
|
|
"total": 309,
|
|
"fraction": 0.6245954692556634
|
|
}
|
|
}
|
|
},
|
|
"paired_medians": {
|
|
"rrmse_rms": {
|
|
"pairs": 309,
|
|
"median_naive_minus_GBRT": 0.019676381358891645
|
|
},
|
|
"rrmse_maximum": {
|
|
"pairs": 309,
|
|
"median_naive_minus_GBRT": 0.011581302653170575
|
|
},
|
|
"rrmse_mean": {
|
|
"pairs": 309,
|
|
"median_naive_minus_GBRT": 0.022773028566505635
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"split": 4,
|
|
"requested": 315,
|
|
"scored": 305,
|
|
"missing": 10,
|
|
"fractions": {
|
|
"rrmse_rms": {
|
|
"naive": {
|
|
"below": 186,
|
|
"total": 305,
|
|
"fraction": 0.6098360655737705
|
|
},
|
|
"GBRT": {
|
|
"below": 243,
|
|
"total": 305,
|
|
"fraction": 0.7967213114754098
|
|
},
|
|
"FNN": {
|
|
"below": 227,
|
|
"total": 305,
|
|
"fraction": 0.7442622950819672
|
|
}
|
|
},
|
|
"rrmse_maximum": {
|
|
"naive": {
|
|
"below": 265,
|
|
"total": 305,
|
|
"fraction": 0.8688524590163934
|
|
},
|
|
"GBRT": {
|
|
"below": 292,
|
|
"total": 305,
|
|
"fraction": 0.9573770491803278
|
|
},
|
|
"FNN": {
|
|
"below": 289,
|
|
"total": 305,
|
|
"fraction": 0.9475409836065574
|
|
}
|
|
},
|
|
"rrmse_mean": {
|
|
"naive": {
|
|
"below": 159,
|
|
"total": 305,
|
|
"fraction": 0.521311475409836
|
|
},
|
|
"GBRT": {
|
|
"below": 215,
|
|
"total": 305,
|
|
"fraction": 0.7049180327868853
|
|
},
|
|
"FNN": {
|
|
"below": 204,
|
|
"total": 305,
|
|
"fraction": 0.6688524590163935
|
|
}
|
|
}
|
|
},
|
|
"paired_medians": {
|
|
"rrmse_rms": {
|
|
"pairs": 305,
|
|
"median_naive_minus_GBRT": 0.017812751846786504
|
|
},
|
|
"rrmse_maximum": {
|
|
"pairs": 305,
|
|
"median_naive_minus_GBRT": 0.01106959997313419
|
|
},
|
|
"rrmse_mean": {
|
|
"pairs": 305,
|
|
"median_naive_minus_GBRT": 0.02061714424889538
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"mean_split_fractions": {
|
|
"rrmse_rms": {
|
|
"naive": 0.601810611715495,
|
|
"GBRT": 0.7839792564128832,
|
|
"FNN": 0.7293907799338382
|
|
},
|
|
"rrmse_maximum": {
|
|
"naive": 0.8710918572047038,
|
|
"GBRT": 0.9485876725148534,
|
|
"FNN": 0.9388448033935276
|
|
},
|
|
"rrmse_mean": {
|
|
"naive": 0.5165900878734306,
|
|
"GBRT": 0.6941685522456129,
|
|
"FNN": 0.6311608929052922
|
|
}
|
|
},
|
|
"no_new_fit_prediction_scoring_decode_or_bootstrap": true
|
|
}
|