Deliver read-only S2 D1-D4 posthoc diagnostics with oracle boundaries
This commit is contained in:
1 parent
eeeca7c9ea
commit
daed8e873a
14 files changed
+295682
No files matched your search
Binary file not shown.
|
After Width: | Height: | Size: 330 KiB |
@@ -0,0 +1,267 @@
|
||||
{
|
||||
"posthoc": true,
|
||||
"common_components": 167,
|
||||
"common_component_ids": [
|
||||
"component-05eadfa30fdd31df",
|
||||
"component-086ccd76fe9f3fed",
|
||||
"component-0980efc0c7820c75",
|
||||
"component-0a1532712d965221",
|
||||
"component-0a4d19a04b5f0583",
|
||||
"component-0b770e5a683eff09",
|
||||
"component-0c12de0e6de53102",
|
||||
"component-0f6ade6f4665f8b6",
|
||||
"component-0f90cf15fba09a8c",
|
||||
"component-135279f8956d4b36",
|
||||
"component-1497185cb9e5f12f",
|
||||
"component-15aa9a13f4e78694",
|
||||
"component-1630a2a8c6869533",
|
||||
"component-16c0e2fdcbf6371c",
|
||||
"component-16c97672413b1faf",
|
||||
"component-17f281e0efac16a8",
|
||||
"component-19b9b82803f533ca",
|
||||
"component-1afa72491cd77121",
|
||||
"component-1c8e049a12f4d3b1",
|
||||
"component-1eac717346c348e0",
|
||||
"component-1fc0448868cdbd8c",
|
||||
"component-204f543da8ec03e0",
|
||||
"component-25b7ca29603ee46c",
|
||||
"component-2863a75334be074b",
|
||||
"component-2897f9dea32cbfdc",
|
||||
"component-2a31c7815db66b27",
|
||||
"component-2b74b0b7252a513c",
|
||||
"component-2d7817f9639113ea",
|
||||
"component-2e24d6fc316ccd3c",
|
||||
"component-2f99f9a928885bce",
|
||||
"component-35ddaa9dca10c48d",
|
||||
"component-360907e9c694b46c",
|
||||
"component-38514e817d76f7f6",
|
||||
"component-3893a7e1e3b81b86",
|
||||
"component-38ce922f9a8db383",
|
||||
"component-39a7f286baa3cdd5",
|
||||
"component-3a3e340f4dfac391",
|
||||
"component-3c90ebd315532003",
|
||||
"component-3ecf7dbfb533a389",
|
||||
"component-3fcbb5eb18b76e52",
|
||||
"component-429a62ac5b76da81",
|
||||
"component-42f9874b86a10bc0",
|
||||
"component-44ac515ef5215d8a",
|
||||
"component-483a0868ebf4d16d",
|
||||
"component-4890d38c10ec20ba",
|
||||
"component-492baeb8035e5812",
|
||||
"component-49c00b9099dc102e",
|
||||
"component-4bd089778094e746",
|
||||
"component-4c56726e7341825f",
|
||||
"component-4ccbb2d1f81c4078",
|
||||
"component-4e7ae0d24a0e67a3",
|
||||
"component-52ca1ec531fd7005",
|
||||
"component-5313ff7e31a2c998",
|
||||
"component-5ddb0b31bf451678",
|
||||
"component-5f62fd53c0f758d0",
|
||||
"component-61692a6861c9b0b4",
|
||||
"component-618d0a3e69dabc04",
|
||||
"component-61c31493a9b8dfb2",
|
||||
"component-6237152dd2592618",
|
||||
"component-637b8b1b5a28ed6a",
|
||||
"component-6415c1ed614e4bce",
|
||||
"component-689e30d86231d93f",
|
||||
"component-692bbde33c57c76d",
|
||||
"component-6a3fcbe86c9dbe1e",
|
||||
"component-6f635a0665b7d2b3",
|
||||
"component-7062cbdcba767c93",
|
||||
"component-74255929b933c1dc",
|
||||
"component-744a32f26fe4059d",
|
||||
"component-7673ea48942e4d79",
|
||||
"component-777679fa1348062a",
|
||||
"component-787aa2c43ecb5a68",
|
||||
"component-79ab2ef076298903",
|
||||
"component-7cff4e4f1aedfa80",
|
||||
"component-7dd3fc4ec69d7a86",
|
||||
"component-7e051a85ce8f933a",
|
||||
"component-7f438c263898947f",
|
||||
"component-7f859ca5666eab2d",
|
||||
"component-804c15ca69195cb1",
|
||||
"component-81d2d33d574599ba",
|
||||
"component-82c77d19c0cc8461",
|
||||
"component-87a7de951c80fc8e",
|
||||
"component-893af8e11fc55181",
|
||||
"component-8990db5efe78c960",
|
||||
"component-89a1bd0a5dadbab8",
|
||||
"component-92219362862d58d7",
|
||||
"component-93232782623bd98e",
|
||||
"component-932d18a0a5b8d7b1",
|
||||
"component-9526b37d9a704beb",
|
||||
"component-960f4b8fc57bf3cc",
|
||||
"component-969b453b3145f89f",
|
||||
"component-989bd8df811715ce",
|
||||
"component-9a030f8717a2d204",
|
||||
"component-9adce3787800d737",
|
||||
"component-9f963099ca7d17d5",
|
||||
"component-a105ce790f9c0dea",
|
||||
"component-a1955098d6492a04",
|
||||
"component-a368c7804547327c",
|
||||
"component-a369220b23aedf25",
|
||||
"component-a3e7365df9aad52c",
|
||||
"component-a4b8e46c7066097b",
|
||||
"component-a4e1f023a017f3fc",
|
||||
"component-a6fdb266928e1aea",
|
||||
"component-a71ba84fea54e0d0",
|
||||
"component-a7b2fcfc678aa9f9",
|
||||
"component-ab740d6b82ac47d8",
|
||||
"component-abb9ad562b1dfb89",
|
||||
"component-acfd9875725aebe9",
|
||||
"component-aef6b4e1df5fb3cd",
|
||||
"component-af706aec4a88db3c",
|
||||
"component-b053483edb9194ef",
|
||||
"component-b0e48877b6364a0e",
|
||||
"component-b0e4ebdf3c94f676",
|
||||
"component-b1e9ce6299072d9e",
|
||||
"component-b395fc8e469fb243",
|
||||
"component-b438af0ae12f350d",
|
||||
"component-b5f07c4cf8cb2013",
|
||||
"component-b9ab17a1800b2143",
|
||||
"component-bb4a69936b51ac36",
|
||||
"component-bd3826c31a66601f",
|
||||
"component-bfd1b0c8be38a9c7",
|
||||
"component-c212b08feb6b1ab8",
|
||||
"component-c75744670c04befc",
|
||||
"component-ca40411ea4c0e984",
|
||||
"component-cb0dd657a914d5c2",
|
||||
"component-cb13ab081d697206",
|
||||
"component-cb54068c6c0a220c",
|
||||
"component-cc347b30c9570c95",
|
||||
"component-cd01a78bb1ea62fe",
|
||||
"component-cefe97bbfcd4cb7f",
|
||||
"component-cf3ec51e7999ebf0",
|
||||
"component-cf7fed99350d9820",
|
||||
"component-d091cbb9cbeb0b72",
|
||||
"component-d2b6bc4ad2e77bce",
|
||||
"component-d2f6c31433921563",
|
||||
"component-d3510904e8012b1f",
|
||||
"component-d3f0cdc2e2f46576",
|
||||
"component-d422cd53078f40a4",
|
||||
"component-d8891d4da52603dc",
|
||||
"component-d8affb090e93c3e1",
|
||||
"component-d8c9f8e36f243f56",
|
||||
"component-db4ac6a42c76be96",
|
||||
"component-dc923dbd2a1cf831",
|
||||
"component-dce50307bad7cfae",
|
||||
"component-dd06ce02456ccabd",
|
||||
"component-df3507095f6128e9",
|
||||
"component-e26879eab19ef34e",
|
||||
"component-e2693ab70780cc57",
|
||||
"component-e350d15a3d61b541",
|
||||
"component-e3b24d8eecbb278e",
|
||||
"component-e8163664380d3692",
|
||||
"component-e9043ebdff4596a2",
|
||||
"component-e9f05c35576833ab",
|
||||
"component-ea44eb88962f7481",
|
||||
"component-ea92102609a4e413",
|
||||
"component-ec22cc3a7be4b075",
|
||||
"component-ed748026806c1cc9",
|
||||
"component-edd76e8802034b9f",
|
||||
"component-eebf86a5af3c57fd",
|
||||
"component-ef9b28c3cd58cad9",
|
||||
"component-f1287f4c7790fcf4",
|
||||
"component-f3c73d2f0df35d44",
|
||||
"component-f54351987f8e4c4a",
|
||||
"component-f58733faf6b20250",
|
||||
"component-f8858e7d8d25644b",
|
||||
"component-fb5e77783912064e",
|
||||
"component-fc2e6fb56d233838",
|
||||
"component-fe4ff4c933f57600"
|
||||
],
|
||||
"common_folds": [
|
||||
0,
|
||||
2,
|
||||
3,
|
||||
4
|
||||
],
|
||||
"rows": [
|
||||
{
|
||||
"pool_component_fraction": 0.25,
|
||||
"arm": "N",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.12840330448233728,
|
||||
"common_mean": 0.12844555913331876
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 0.25,
|
||||
"arm": "E",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.1277335810020859,
|
||||
"common_mean": 0.1273166545449507
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 0.25,
|
||||
"arm": "R2",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.07695018012838463,
|
||||
"common_mean": 0.07761692739223952
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 0.25,
|
||||
"arm": "B0",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.10401156173394739,
|
||||
"common_mean": 0.10376208391424709
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 0.5,
|
||||
"arm": "N",
|
||||
"original_components": 167,
|
||||
"original_mean": 0.11357598509546316,
|
||||
"common_mean": 0.11357598509546316
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 0.5,
|
||||
"arm": "E",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.11534639592758691,
|
||||
"common_mean": 0.11519367139164288
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 0.5,
|
||||
"arm": "R2",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.07462859520864756,
|
||||
"common_mean": 0.07537031093952994
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 0.5,
|
||||
"arm": "B0",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.10401156173394739,
|
||||
"common_mean": 0.10376208391424709
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 1.0,
|
||||
"arm": "N",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.10796491979030956,
|
||||
"common_mean": 0.10761600807395827
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 1.0,
|
||||
"arm": "E",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.10936022359138735,
|
||||
"common_mean": 0.10893500119740085
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 1.0,
|
||||
"arm": "R2",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.07299265397037516,
|
||||
"common_mean": 0.07362239717604273
|
||||
},
|
||||
{
|
||||
"pool_component_fraction": 1.0,
|
||||
"arm": "B0",
|
||||
"original_components": 209,
|
||||
"original_mean": 0.10401156173394739,
|
||||
"common_mean": 0.10376208391424709
|
||||
}
|
||||
],
|
||||
"N_25_to100_all5_relative_error_reduction": 0.15917335441191216,
|
||||
"note": "Original50pctN denominator differs; common fourfold curve is a separate descriptive diagnostic, not replacement primary analysis"
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
{
|
||||
"posthoc": true,
|
||||
"epoch_display": "one based",
|
||||
"records": [
|
||||
{
|
||||
"name": "fold0-pool100-N128-seed0",
|
||||
"fold": 0,
|
||||
"arm": "N",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 40,
|
||||
"best_validation": 0.1049166511838978,
|
||||
"final_validation": 0.10496740204918055,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00045715988019908344,
|
||||
"final5_endpoint_relative_change": -0.01964236989783663,
|
||||
"first_validation": 0.1509130177240529,
|
||||
"final_training_loss": 0.004722264837997119,
|
||||
"final5_training_relative_change": -0.028116757582468455
|
||||
},
|
||||
{
|
||||
"name": "fold0-pool100-N128-seed1",
|
||||
"fold": 0,
|
||||
"arm": "N",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 38,
|
||||
"best_validation": 0.10407932056721243,
|
||||
"final_validation": 0.10422675334912009,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": 2.652656009030081e-07,
|
||||
"final5_endpoint_relative_change": 0.001416542509157237,
|
||||
"first_validation": 0.25655519932490145,
|
||||
"final_training_loss": 0.004533333337715333,
|
||||
"final5_training_relative_change": -0.02521550027098529
|
||||
},
|
||||
{
|
||||
"name": "fold0-pool100-N128-seed2",
|
||||
"fold": 0,
|
||||
"arm": "N",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 29,
|
||||
"best_validation": 0.10353846067481268,
|
||||
"final_validation": 0.10424100850388006,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00011286143071868351,
|
||||
"final5_endpoint_relative_change": -0.004999973479387854,
|
||||
"first_validation": 0.2215304551083641,
|
||||
"final_training_loss": 0.00440695704137279,
|
||||
"final5_training_relative_change": -0.024549393773133588
|
||||
},
|
||||
{
|
||||
"name": "fold0-pool100-E128-seed0",
|
||||
"fold": 0,
|
||||
"arm": "E",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 33,
|
||||
"best_validation": 0.1070433271875761,
|
||||
"final_validation": 0.10831315365915223,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00021957014156072637,
|
||||
"final5_endpoint_relative_change": -0.00872088564382223,
|
||||
"first_validation": 0.15758904754523437,
|
||||
"final_training_loss": 0.005109943626345516,
|
||||
"final5_training_relative_change": -0.019116007553683922
|
||||
},
|
||||
{
|
||||
"name": "fold0-pool100-E128-seed1",
|
||||
"fold": 0,
|
||||
"arm": "E",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 42,
|
||||
"best_validation": 0.10361167451411284,
|
||||
"final_validation": 0.10361167451411284,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": true,
|
||||
"final5_slope": -0.00032867892150710966,
|
||||
"final5_endpoint_relative_change": -0.016106811180004454,
|
||||
"first_validation": 0.2572014452112838,
|
||||
"final_training_loss": 0.004812080522374757,
|
||||
"final5_training_relative_change": -0.03288377574581991
|
||||
},
|
||||
{
|
||||
"name": "fold0-pool100-E128-seed2",
|
||||
"fold": 0,
|
||||
"arm": "E",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 36,
|
||||
"best_validation": 0.10429503593490197,
|
||||
"final_validation": 0.10467382291867426,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00015268068730843018,
|
||||
"final5_endpoint_relative_change": -0.009048991542484286,
|
||||
"first_validation": 0.19913255553026982,
|
||||
"final_training_loss": 0.004622639868457198,
|
||||
"final5_training_relative_change": -0.027271917154224057
|
||||
},
|
||||
{
|
||||
"name": "fold1-pool100-N128-seed0",
|
||||
"fold": 1,
|
||||
"arm": "N",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 38,
|
||||
"best_validation": 0.11332284754777909,
|
||||
"final_validation": 0.11334357672382486,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": 3.563069714030463e-06,
|
||||
"final5_endpoint_relative_change": 0.00018292141871056877,
|
||||
"first_validation": 0.17901542767802248,
|
||||
"final_training_loss": 0.004641385016132308,
|
||||
"final5_training_relative_change": -0.023692074813476793
|
||||
},
|
||||
{
|
||||
"name": "fold1-pool100-N128-seed1",
|
||||
"fold": 1,
|
||||
"arm": "N",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 40,
|
||||
"best_validation": 0.11295727443793561,
|
||||
"final_validation": 0.11299102665003653,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00011469535694980031,
|
||||
"final5_endpoint_relative_change": -0.003397411547718776,
|
||||
"first_validation": 0.1706622413469755,
|
||||
"final_training_loss": 0.004562276440898171,
|
||||
"final5_training_relative_change": -0.02423232695272337
|
||||
},
|
||||
{
|
||||
"name": "fold1-pool100-N128-seed2",
|
||||
"fold": 1,
|
||||
"arm": "N",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 42,
|
||||
"best_validation": 0.10668976972272697,
|
||||
"final_validation": 0.10668976972272697,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": true,
|
||||
"final5_slope": -8.395794694934279e-05,
|
||||
"final5_endpoint_relative_change": -0.0022431254857503413,
|
||||
"first_validation": 0.1623369935782651,
|
||||
"final_training_loss": 0.004335049312806737,
|
||||
"final5_training_relative_change": -0.02568937692210027
|
||||
},
|
||||
{
|
||||
"name": "fold1-pool100-E128-seed0",
|
||||
"fold": 1,
|
||||
"arm": "E",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 31,
|
||||
"best_validation": 0.11603146362303948,
|
||||
"final_validation": 0.11661034638859284,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00017082187404602548,
|
||||
"final5_endpoint_relative_change": -0.0059569122215826775,
|
||||
"first_validation": 0.17240257463175757,
|
||||
"final_training_loss": 0.004784265762223775,
|
||||
"final5_training_relative_change": -0.024440433839255538
|
||||
},
|
||||
{
|
||||
"name": "fold1-pool100-E128-seed1",
|
||||
"fold": 1,
|
||||
"arm": "E",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 42,
|
||||
"best_validation": 0.11347207660887379,
|
||||
"final_validation": 0.11347207660887379,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": true,
|
||||
"final5_slope": -7.480537445793161e-05,
|
||||
"final5_endpoint_relative_change": -0.0011551600092004621,
|
||||
"first_validation": 0.19980214825927156,
|
||||
"final_training_loss": 0.004570121382958764,
|
||||
"final5_training_relative_change": -0.028113654773548213
|
||||
},
|
||||
{
|
||||
"name": "fold1-pool100-E128-seed2",
|
||||
"fold": 1,
|
||||
"arm": "E",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 42,
|
||||
"best_validation": 0.11097601521081273,
|
||||
"final_validation": 0.11097601521081273,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": true,
|
||||
"final5_slope": -0.00019276669070807803,
|
||||
"final5_endpoint_relative_change": -0.008097702499321291,
|
||||
"first_validation": 0.1851476984374394,
|
||||
"final_training_loss": 0.004496674420950896,
|
||||
"final5_training_relative_change": -0.020690728106584677
|
||||
},
|
||||
{
|
||||
"name": "fold2-pool100-N128-seed0",
|
||||
"fold": 2,
|
||||
"arm": "N",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 41,
|
||||
"best_validation": 0.0990737363781838,
|
||||
"final_validation": 0.09910251571664458,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.0004912672574350816,
|
||||
"final5_endpoint_relative_change": -0.023421765849176768,
|
||||
"first_validation": 0.1703861682805603,
|
||||
"final_training_loss": 0.004328993675090591,
|
||||
"final5_training_relative_change": -0.02241104098857072
|
||||
},
|
||||
{
|
||||
"name": "fold2-pool100-N128-seed1",
|
||||
"fold": 2,
|
||||
"arm": "N",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 40,
|
||||
"best_validation": 0.1013898970394781,
|
||||
"final_validation": 0.101406870241552,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00013742349488998242,
|
||||
"final5_endpoint_relative_change": -0.004139092208572448,
|
||||
"first_validation": 0.18096855780627105,
|
||||
"final_training_loss": 0.004440459902897277,
|
||||
"final5_training_relative_change": -0.029780212595006073
|
||||
},
|
||||
{
|
||||
"name": "fold2-pool100-N128-seed2",
|
||||
"fold": 2,
|
||||
"arm": "N",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 39,
|
||||
"best_validation": 0.10150249938970865,
|
||||
"final_validation": 0.10167555353451982,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.0002748882375111795,
|
||||
"final5_endpoint_relative_change": -0.014741400076694444,
|
||||
"first_validation": 0.21845442187832073,
|
||||
"final_training_loss": 0.004567476129514152,
|
||||
"final5_training_relative_change": -0.02842443063824862
|
||||
},
|
||||
{
|
||||
"name": "fold2-pool100-E128-seed0",
|
||||
"fold": 2,
|
||||
"arm": "E",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 40,
|
||||
"best_validation": 0.1025792798198467,
|
||||
"final_validation": 0.10260812001051592,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -9.140332223147472e-05,
|
||||
"final5_endpoint_relative_change": -0.002439531777327031,
|
||||
"first_validation": 0.23917465169500923,
|
||||
"final_training_loss": 0.004708957616970077,
|
||||
"final5_training_relative_change": -0.023705003657984003
|
||||
},
|
||||
{
|
||||
"name": "fold2-pool100-E128-seed1",
|
||||
"fold": 2,
|
||||
"arm": "E",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 36,
|
||||
"best_validation": 0.10072888419126776,
|
||||
"final_validation": 0.10088989915247112,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": 1.4020182336839486e-05,
|
||||
"final5_endpoint_relative_change": -0.00043235724289258926,
|
||||
"first_validation": 0.19531240187762527,
|
||||
"final_training_loss": 0.004290306225805209,
|
||||
"final5_training_relative_change": -0.023754814283563186
|
||||
},
|
||||
{
|
||||
"name": "fold2-pool100-E128-seed2",
|
||||
"fold": 2,
|
||||
"arm": "E",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 37,
|
||||
"best_validation": 0.10129659064526066,
|
||||
"final_validation": 0.10137647743970307,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00011395334443899845,
|
||||
"final5_endpoint_relative_change": -0.006305233220497475,
|
||||
"first_validation": 0.2285574164161199,
|
||||
"final_training_loss": 0.00429846166037429,
|
||||
"final5_training_relative_change": -0.025397233010670983
|
||||
},
|
||||
{
|
||||
"name": "fold3-pool100-N128-seed0",
|
||||
"fold": 3,
|
||||
"arm": "N",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 30,
|
||||
"best_validation": 0.10354914570078336,
|
||||
"final_validation": 0.10448359681887336,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -2.2736601841234486e-05,
|
||||
"final5_endpoint_relative_change": 0.0005997905719203889,
|
||||
"first_validation": 0.17233980340724167,
|
||||
"final_training_loss": 0.0054052455347419264,
|
||||
"final5_training_relative_change": -0.026059798629568753
|
||||
},
|
||||
{
|
||||
"name": "fold3-pool100-N128-seed1",
|
||||
"fold": 3,
|
||||
"arm": "N",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 35,
|
||||
"best_validation": 0.10315625303830879,
|
||||
"final_validation": 0.10356722665256274,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.0002832698424901282,
|
||||
"final5_endpoint_relative_change": -0.013288043542528216,
|
||||
"first_validation": 0.16577017807119565,
|
||||
"final_training_loss": 0.005527520968955697,
|
||||
"final5_training_relative_change": -0.02038223911258774
|
||||
},
|
||||
{
|
||||
"name": "fold3-pool100-N128-seed2",
|
||||
"fold": 3,
|
||||
"arm": "N",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 38,
|
||||
"best_validation": 0.10053904125896092,
|
||||
"final_validation": 0.10082072226159003,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": 3.307942614645943e-05,
|
||||
"final5_endpoint_relative_change": 0.00280170766601584,
|
||||
"first_validation": 0.18238017429029482,
|
||||
"final_training_loss": 0.00502338712951905,
|
||||
"final5_training_relative_change": -0.027487957707369337
|
||||
},
|
||||
{
|
||||
"name": "fold3-pool100-E128-seed0",
|
||||
"fold": 3,
|
||||
"arm": "E",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 35,
|
||||
"best_validation": 0.10305268729362649,
|
||||
"final_validation": 0.10458062819142573,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.0001425443220065949,
|
||||
"final5_endpoint_relative_change": -0.005305274635308166,
|
||||
"first_validation": 0.1677862998757622,
|
||||
"final_training_loss": 0.005541507372204527,
|
||||
"final5_training_relative_change": -0.024706048516308643
|
||||
},
|
||||
{
|
||||
"name": "fold3-pool100-E128-seed1",
|
||||
"fold": 3,
|
||||
"arm": "E",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 39,
|
||||
"best_validation": 0.10286712329826402,
|
||||
"final_validation": 0.10305291613118085,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00018059030902578698,
|
||||
"final5_endpoint_relative_change": -0.009024037994690426,
|
||||
"first_validation": 0.15057920007700676,
|
||||
"final_training_loss": 0.005525510219212083,
|
||||
"final5_training_relative_change": -0.018864748111938945
|
||||
},
|
||||
{
|
||||
"name": "fold3-pool100-E128-seed2",
|
||||
"fold": 3,
|
||||
"arm": "E",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 35,
|
||||
"best_validation": 0.10352921547817222,
|
||||
"final_validation": 0.10461697446237825,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": 1.412882895297441e-05,
|
||||
"final5_endpoint_relative_change": 0.0033633883322563563,
|
||||
"first_validation": 0.1988066504406308,
|
||||
"final_training_loss": 0.005441256395817325,
|
||||
"final5_training_relative_change": -0.024215822447767765
|
||||
},
|
||||
{
|
||||
"name": "fold4-pool100-N128-seed0",
|
||||
"fold": 4,
|
||||
"arm": "N",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 40,
|
||||
"best_validation": 0.10945123252166526,
|
||||
"final_validation": 0.10945546572681569,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -4.641842288861209e-05,
|
||||
"final5_endpoint_relative_change": -0.0025212388116016005,
|
||||
"first_validation": 0.22377487421780184,
|
||||
"final_training_loss": 0.0043287787334413096,
|
||||
"final5_training_relative_change": -0.03345992390394059
|
||||
},
|
||||
{
|
||||
"name": "fold4-pool100-N128-seed1",
|
||||
"fold": 4,
|
||||
"arm": "N",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 41,
|
||||
"best_validation": 0.10832676057457552,
|
||||
"final_validation": 0.1083284709349328,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.0001580016982651761,
|
||||
"final5_endpoint_relative_change": -0.004228580306688823,
|
||||
"first_validation": 0.1967061241004283,
|
||||
"final_training_loss": 0.004402133440188834,
|
||||
"final5_training_relative_change": -0.02554026592340186
|
||||
},
|
||||
{
|
||||
"name": "fold4-pool100-N128-seed2",
|
||||
"fold": 4,
|
||||
"arm": "N",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 42,
|
||||
"best_validation": 0.10861524747905922,
|
||||
"final_validation": 0.10861524747905922,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": true,
|
||||
"final5_slope": -0.0001220730424474196,
|
||||
"final5_endpoint_relative_change": -0.003946187682261848,
|
||||
"first_validation": 0.18324126694163664,
|
||||
"final_training_loss": 0.004630423621809915,
|
||||
"final5_training_relative_change": -0.02641205763942578
|
||||
},
|
||||
{
|
||||
"name": "fold4-pool100-E128-seed0",
|
||||
"fold": 4,
|
||||
"arm": "E",
|
||||
"seed": 0,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 37,
|
||||
"best_validation": 0.11122744442244252,
|
||||
"final_validation": 0.1112988294426288,
|
||||
"best_in_last5": false,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00011039964604927436,
|
||||
"final5_endpoint_relative_change": -0.0051837083797660055,
|
||||
"first_validation": 0.2626880266940417,
|
||||
"final_training_loss": 0.004536271162139258,
|
||||
"final5_training_relative_change": -0.027650994429610165
|
||||
},
|
||||
{
|
||||
"name": "fold4-pool100-E128-seed1",
|
||||
"fold": 4,
|
||||
"arm": "E",
|
||||
"seed": 1,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 40,
|
||||
"best_validation": 0.1121942992368457,
|
||||
"final_validation": 0.11229075367560004,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": false,
|
||||
"final5_slope": -0.00013946465324565416,
|
||||
"final5_endpoint_relative_change": -0.004198181278038726,
|
||||
"first_validation": 0.18620277595786905,
|
||||
"final_training_loss": 0.004472227382988006,
|
||||
"final5_training_relative_change": -0.027440248292693536
|
||||
},
|
||||
{
|
||||
"name": "fold4-pool100-E128-seed2",
|
||||
"fold": 4,
|
||||
"arm": "E",
|
||||
"seed": 2,
|
||||
"epochs": 42,
|
||||
"best_epoch_1based": 42,
|
||||
"best_validation": 0.11085908193397476,
|
||||
"final_validation": 0.11085908193397476,
|
||||
"best_in_last5": true,
|
||||
"best_at_final": true,
|
||||
"final5_slope": -0.00011996747630761954,
|
||||
"final5_endpoint_relative_change": -0.0034599664047808565,
|
||||
"first_validation": 0.21697312705569205,
|
||||
"final_training_loss": 0.004640844690789634,
|
||||
"final5_training_relative_change": -0.027664646616675337
|
||||
}
|
||||
],
|
||||
"summary": [
|
||||
{
|
||||
"arm": "N",
|
||||
"runs": 15,
|
||||
"all_reached42": true,
|
||||
"median_best_epoch": 40.0,
|
||||
"best_in_last5": 12,
|
||||
"best_at_final": 2,
|
||||
"negative_final5_validation_slope": 12,
|
||||
"median_final5_validation_change": -0.003946187682261848,
|
||||
"median_final5_training_change": -0.02568937692210027
|
||||
},
|
||||
{
|
||||
"arm": "E",
|
||||
"runs": 15,
|
||||
"all_reached42": true,
|
||||
"median_best_epoch": 37.0,
|
||||
"best_in_last5": 7,
|
||||
"best_at_final": 4,
|
||||
"negative_final5_validation_slope": 13,
|
||||
"median_final5_validation_change": -0.005305274635308166,
|
||||
"median_final5_training_change": -0.024706048516308643
|
||||
}
|
||||
],
|
||||
"limitation": "Late best epoch/negative local slope does not prove a longer schedule will beat R2; cosine endpoint and fitting choices are confounded"
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
[
|
||||
{
|
||||
"subset": "all",
|
||||
"pool": 25,
|
||||
"reference": "N",
|
||||
"candidate": "E",
|
||||
"components": 209,
|
||||
"parts": 617,
|
||||
"reference_means": {
|
||||
"e2": 0.021296706742065963,
|
||||
"H": 0.01718452447104169,
|
||||
"S": 0.004112182271024275
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.022766836263711173,
|
||||
"H": 0.017004913378629964,
|
||||
"S": 0.005761922885081207
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.0014701295216452102,
|
||||
"H": -0.00017961109241172724,
|
||||
"S": 0.0016497406140569323
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "all",
|
||||
"pool": 25,
|
||||
"reference": "R2",
|
||||
"candidate": "N",
|
||||
"components": 209,
|
||||
"parts": 617,
|
||||
"reference_means": {
|
||||
"e2": 0.007997060536357327,
|
||||
"H": 0.0074619557647703305,
|
||||
"S": 0.0005351047715869962
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.021296706742065963,
|
||||
"H": 0.01718452447104169,
|
||||
"S": 0.004112182271024275
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.013299646205708636,
|
||||
"H": 0.009722568706271362,
|
||||
"S": 0.0035770774994372785
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "all",
|
||||
"pool": 25,
|
||||
"reference": "B0",
|
||||
"candidate": "N",
|
||||
"components": 209,
|
||||
"parts": 617,
|
||||
"reference_means": {
|
||||
"e2": 0.013878064168416875,
|
||||
"H": 0.011130724449211081,
|
||||
"S": 0.002747339719205788
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.021296706742065963,
|
||||
"H": 0.01718452447104169,
|
||||
"S": 0.004112182271024275
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.007418642573649088,
|
||||
"H": 0.00605380002183061,
|
||||
"S": 0.001364842551818487
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "all",
|
||||
"pool": 50,
|
||||
"reference": "N",
|
||||
"candidate": "E",
|
||||
"components": 167,
|
||||
"parts": 492,
|
||||
"reference_means": {
|
||||
"e2": 0.017242691936768525,
|
||||
"H": 0.01473546011423722,
|
||||
"S": 0.0025072318225313043
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.017632686639609582,
|
||||
"H": 0.015045222569285062,
|
||||
"S": 0.0025874640703245164
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.00038999470284105733,
|
||||
"H": 0.00030976245504784304,
|
||||
"S": 8.023224779321212e-05
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "all",
|
||||
"pool": 50,
|
||||
"reference": "R2",
|
||||
"candidate": "N",
|
||||
"components": 167,
|
||||
"parts": 492,
|
||||
"reference_means": {
|
||||
"e2": 0.007878508226816749,
|
||||
"H": 0.007346307356473599,
|
||||
"S": 0.0005322008703431516
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.017242691936768525,
|
||||
"H": 0.01473546011423722,
|
||||
"S": 0.0025072318225313043
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.009364183709951776,
|
||||
"H": 0.0073891527577636205,
|
||||
"S": 0.0019750309521881526
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "all",
|
||||
"pool": 50,
|
||||
"reference": "B0",
|
||||
"candidate": "N",
|
||||
"components": 167,
|
||||
"parts": 492,
|
||||
"reference_means": {
|
||||
"e2": 0.013818986991014415,
|
||||
"H": 0.011055146212895193,
|
||||
"S": 0.002763840778119213
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.017242691936768525,
|
||||
"H": 0.01473546011423722,
|
||||
"S": 0.0025072318225313043
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.00342370494575411,
|
||||
"H": 0.0036803139013420264,
|
||||
"S": -0.0002566089555879089
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "all",
|
||||
"pool": 100,
|
||||
"reference": "N",
|
||||
"candidate": "E",
|
||||
"components": 209,
|
||||
"parts": 617,
|
||||
"reference_means": {
|
||||
"e2": 0.015649927069400535,
|
||||
"H": 0.013707020691482046,
|
||||
"S": 0.0019429063779184926
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.016042152847056606,
|
||||
"H": 0.013883694505335933,
|
||||
"S": 0.0021584583417206715
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.000392225777656071,
|
||||
"H": 0.00017667381385388752,
|
||||
"S": 0.00021555196380217893
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "all",
|
||||
"pool": 100,
|
||||
"reference": "R2",
|
||||
"candidate": "N",
|
||||
"components": 209,
|
||||
"parts": 617,
|
||||
"reference_means": {
|
||||
"e2": 0.00732010403589583,
|
||||
"H": 0.006882571833199063,
|
||||
"S": 0.0004375322026967659
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.015649927069400535,
|
||||
"H": 0.013707020691482046,
|
||||
"S": 0.0019429063779184926
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.008329823033504706,
|
||||
"H": 0.006824448858282982,
|
||||
"S": 0.0015053741752217267
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "all",
|
||||
"pool": 100,
|
||||
"reference": "B0",
|
||||
"candidate": "N",
|
||||
"components": 209,
|
||||
"parts": 617,
|
||||
"reference_means": {
|
||||
"e2": 0.013878064168416875,
|
||||
"H": 0.011130724449211081,
|
||||
"S": 0.002747339719205788
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.015649927069400535,
|
||||
"H": 0.013707020691482046,
|
||||
"S": 0.0019429063779184926
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.00177186290098366,
|
||||
"H": 0.002576296242270964,
|
||||
"S": -0.0008044333412872953
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "not_known_exposed",
|
||||
"pool": 25,
|
||||
"reference": "N",
|
||||
"candidate": "E",
|
||||
"components": 188,
|
||||
"parts": 517,
|
||||
"reference_means": {
|
||||
"e2": 0.020930344161973438,
|
||||
"H": 0.01682104794622164,
|
||||
"S": 0.004109296215751793
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.022907061736947502,
|
||||
"H": 0.01664713065514149,
|
||||
"S": 0.006259931081806016
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.0019767175749740643,
|
||||
"H": -0.00017391729108015216,
|
||||
"S": 0.0021506348660542225
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "not_known_exposed",
|
||||
"pool": 25,
|
||||
"reference": "R2",
|
||||
"candidate": "N",
|
||||
"components": 188,
|
||||
"parts": 517,
|
||||
"reference_means": {
|
||||
"e2": 0.007987309454011765,
|
||||
"H": 0.00745126959384986,
|
||||
"S": 0.0005360398601619048
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.020930344161973438,
|
||||
"H": 0.01682104794622164,
|
||||
"S": 0.004109296215751793
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.012943034707961673,
|
||||
"H": 0.009369778352371782,
|
||||
"S": 0.0035732563555898886
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "not_known_exposed",
|
||||
"pool": 25,
|
||||
"reference": "B0",
|
||||
"candidate": "N",
|
||||
"components": 188,
|
||||
"parts": 517,
|
||||
"reference_means": {
|
||||
"e2": 0.013717015329354045,
|
||||
"H": 0.01103305552181108,
|
||||
"S": 0.0026839598075429623
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.020930344161973438,
|
||||
"H": 0.01682104794622164,
|
||||
"S": 0.004109296215751793
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.007213328832619393,
|
||||
"H": 0.005787992424410561,
|
||||
"S": 0.001425336408208831
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "not_known_exposed",
|
||||
"pool": 50,
|
||||
"reference": "N",
|
||||
"candidate": "E",
|
||||
"components": 152,
|
||||
"parts": 412,
|
||||
"reference_means": {
|
||||
"e2": 0.016729285628226304,
|
||||
"H": 0.014184998096870705,
|
||||
"S": 0.002544287531355594
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.017122607765309984,
|
||||
"H": 0.01450637045261287,
|
||||
"S": 0.002616237312697116
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.00039332213708367925,
|
||||
"H": 0.0003213723557421652,
|
||||
"S": 7.194978134152184e-05
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "not_known_exposed",
|
||||
"pool": 50,
|
||||
"reference": "R2",
|
||||
"candidate": "N",
|
||||
"components": 152,
|
||||
"parts": 412,
|
||||
"reference_means": {
|
||||
"e2": 0.007847568616163089,
|
||||
"H": 0.007339039582219416,
|
||||
"S": 0.000508529033943674
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.016729285628226304,
|
||||
"H": 0.014184998096870705,
|
||||
"S": 0.002544287531355594
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.008881717012063216,
|
||||
"H": 0.006845958514651289,
|
||||
"S": 0.00203575849741192
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "not_known_exposed",
|
||||
"pool": 50,
|
||||
"reference": "B0",
|
||||
"candidate": "N",
|
||||
"components": 152,
|
||||
"parts": 412,
|
||||
"reference_means": {
|
||||
"e2": 0.013669896180759743,
|
||||
"H": 0.010969913833770342,
|
||||
"S": 0.002699982346989398
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.016729285628226304,
|
||||
"H": 0.014184998096870705,
|
||||
"S": 0.002544287531355594
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.0030593894474665612,
|
||||
"H": 0.0032150842631003627,
|
||||
"S": -0.0001556948156338036
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "not_known_exposed",
|
||||
"pool": 100,
|
||||
"reference": "N",
|
||||
"candidate": "E",
|
||||
"components": 188,
|
||||
"parts": 517,
|
||||
"reference_means": {
|
||||
"e2": 0.014991860954385799,
|
||||
"H": 0.013064645542550542,
|
||||
"S": 0.0019272154118352561
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.015417843204918444,
|
||||
"H": 0.013255187975017797,
|
||||
"S": 0.0021626552299006496
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.0004259822505326451,
|
||||
"H": 0.00019054243246725486,
|
||||
"S": 0.00023543981806539347
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "not_known_exposed",
|
||||
"pool": 100,
|
||||
"reference": "R2",
|
||||
"candidate": "N",
|
||||
"components": 188,
|
||||
"parts": 517,
|
||||
"reference_means": {
|
||||
"e2": 0.007358875600952186,
|
||||
"H": 0.006938696651537931,
|
||||
"S": 0.00042017894941425636
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.014991860954385799,
|
||||
"H": 0.013064645542550542,
|
||||
"S": 0.0019272154118352561
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.007632985353433612,
|
||||
"H": 0.006125948891012612,
|
||||
"S": 0.0015070364624209997
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
},
|
||||
{
|
||||
"subset": "not_known_exposed",
|
||||
"pool": 100,
|
||||
"reference": "B0",
|
||||
"candidate": "N",
|
||||
"components": 188,
|
||||
"parts": 517,
|
||||
"reference_means": {
|
||||
"e2": 0.013717015329354045,
|
||||
"H": 0.01103305552181108,
|
||||
"S": 0.0026839598075429623
|
||||
},
|
||||
"candidate_means": {
|
||||
"e2": 0.014991860954385799,
|
||||
"H": 0.013064645542550542,
|
||||
"S": 0.0019272154118352561
|
||||
},
|
||||
"candidate_minus_reference": {
|
||||
"e2": 0.001274845625031754,
|
||||
"H": 0.002031590020739462,
|
||||
"S": -0.0007567443957077061
|
||||
},
|
||||
"warning": "squared-error diagnostic difference, not primary Delta"
|
||||
}
|
||||
]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"status": "D3_COMPLETE",
|
||||
"rows": 13449,
|
||||
"valid": 13449,
|
||||
"undefined": [],
|
||||
"nonpositive_a_rows": 2,
|
||||
"sign_reversal_rows": 2,
|
||||
"zero_cross_rows": 0,
|
||||
"maximum_identity_absolute_error": 1.4210854715202004e-14,
|
||||
"maximum_frozen_squared_error_difference": 7.105427357601002e-15,
|
||||
"cpu_seconds": 17.133535096,
|
||||
"new_labels_decoded": false,
|
||||
"predictions_modified": false,
|
||||
"calibration": false,
|
||||
"synthetic_checks": 10,
|
||||
"output_sha256": {
|
||||
"D3-part-decomposition.json": "701bf2e52d62388212845aa531908a1f15b7e1572a1cfc04b73ebcf410bd9984",
|
||||
"D3-started.json": "6ac37a4bd961d6a0af559253753a988f5944a657b44c2c3002d062eb6b6fea3d",
|
||||
"D3-comparisons.json": "c2223bf53136efc0fb2a778254707828cdb1a4c86fc74f5070cd84b7ab55eb65",
|
||||
"D3-tables.json": "87786a4afb3ef16c4ebb2414e288d7dbc71dc10d196549a190699b8baf70a327"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"rulings": [
|
||||
"1794731518.379529",
|
||||
"1794731544.355059",
|
||||
"1794731547.266819"
|
||||
],
|
||||
"description": "oracle diagnostic, not a calibration",
|
||||
"synthetic_checks": 10,
|
||||
"code_sha256": "b9aef5e91fd63609e1882e3fedad97e72db05e79232aec7b76c2b5ba53bde351",
|
||||
"zero_pred": "e2=1 retained as original error; a/H/S undefined and excluded from valid decomposition",
|
||||
"tolerance": {
|
||||
"rtol": 1e-10,
|
||||
"atol": 1e-12
|
||||
},
|
||||
"new_labels_decoded": false
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"status": "D1_D2_D3_D4_COMPLETE_FOR_STAGE_REVIEW",
|
||||
"D124_receipt_sha256": "a200e3ffc3ccde1e338edc50bb4259d5107f2b39bb0666b7750ba05868f8326f",
|
||||
"D3_receipt_sha256": "9d4a664baa14106f551169644c93d242266579c82e6f4fc8dd35342dbac3d723",
|
||||
"report": "docs/s2-posthoc-diagnostics-2026-11-15.md",
|
||||
"new_labels_decoded": false,
|
||||
"models_loaded": false,
|
||||
"original_statistics_unchanged": true,
|
||||
"S3_started": false
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"status": "D1_D2_D4_COMPLETE",
|
||||
"cpu_seconds": 2.571860387,
|
||||
"new_labels_decoded": false,
|
||||
"models_loaded": false,
|
||||
"new_training": false,
|
||||
"original_statistics_modified": false,
|
||||
"D3": "WAITING_MATHEMATICAL_BOUNDARIES",
|
||||
"ruling": "1794731320.994789",
|
||||
"code_sha256": "7c205b18756e6f71730ab8872f40558853e6e9aeae9ca4a2d56f3c4b714109d4",
|
||||
"output_sha256": {
|
||||
"D4-geometry-errors.json": "d2f8245bae795579386404d65a422ae20d5aba9261351e4c84b5fa3c026cc133",
|
||||
"D2-epoch-history.json": "fd3ac67ad5a7e34350d239cf525a25d28a263364928550445a628432e62bad6c",
|
||||
"D1-learning-curves.json": "8a789b3e50568ad21b919997bb6fc46aa902f308926a7f2b5add9c7780bb639d",
|
||||
"D1-D2-curves.png": "307e8fea4e50275e2036d467ed0fafa70fec2779fb04fba69fd46947f536eb12"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
# S2 事后诊断 D1–D4(2026-11-15)
|
||||
|
||||
按Claude1794731320.994789执行,仅使用既有预测、已解码标签缓存、训练历史与几何元数据。未重训、未重解码、未修改S2预测或冻结统计。D3数学由Mold1794731518.379529及1794731544.355059、Claude1794731547.266819确定。**D3全表是oracle diagnostic, not a calibration。**
|
||||
|
||||
主要读数:更多数据确实降低N误差,100%池晚期仍有小幅验证改善;但N与R2的均方误差差距约82%留在oracle缩放后的残差项,不能仅归因于全局幅值。D4显示几何大小/厚度代理上的描述性差异,不能当因果证据。S3-A/B尚未执行。
|
||||
|
||||
## D1:学习曲线
|
||||
|
||||
原五折共同有效25%→100%主域:N误差相对下降15.9173%。50%池有既定N缺失,以下另列共同四折167组件的诊断曲线,避免不同分母制造趋势;不替换S2主表。
|
||||
|
||||
| 组件池 | N | E | R2 | B0 |
|
||||
|---|---:|---:|---:|---:|
|
||||
|25%|12.8446%|12.7317%|7.7617%|10.3762%|
|
||||
|50%|11.3576%|11.5194%|7.5370%|10.3762%|
|
||||
|100%|10.7616%|10.8935%|7.3622%|10.3762%|
|
||||
|
||||
N/E随数据增加有所改善,但未追上R2;B0不拟合、曲线恒定。学习比例是组件比例,实际每折件/节点数沿原结果报告。
|
||||
|
||||
## D2:100%池训练历史
|
||||
|
||||
| 臂 | 运行数 | 全部达到42epoch | 最佳epoch中位数 | 最后5epoch内最佳 | 最终epoch最佳 | 最后5epoch验证斜率<0 | 验证末5epoch相对变化中位数 |
|
||||
|---|---:|---|---:|---:|---:|---:|---:|
|
||||
|N|15|True|40|12|2|12|-0.3946%|
|
||||
|E|15|True|37|7|4|13|-0.5305%|
|
||||
|
||||
epoch按1起算。末5验证斜率为5个既有值的描述性直线斜率,非新的模型拟合或显著性测试;完整30条历史摘要在D2-epoch-history.json。晚期最佳与局部下降不证明延长训练可跨过R2,cosine末端、容量和输入选择均混杂。
|
||||
|
||||

|
||||
|
||||
## D3:oracle尺度与残差的均方分解
|
||||
|
||||
每件/臂/已有seed,按原query multiplicity:P=Σcp²,T=Σct²,C=Σcpt,a*=C/P;H=Σc(a*p−t)²/T,S=(1−a*)²P/T,原e²=H+S。计算采用FP64加权单位向量投影,H从残差直接求和,避免近零P或相近数相减。无第二质量权重。
|
||||
|
||||
P=0:oracle undefined,H/S不填入有效表,即使原e²=1;T=0、空集或非有限同样undefined。a≤0保持原值不clip;C=0/负号另标。10个合成边界检查先通过,然后读取冻结缓存。主结果均方分解如下,**mean(e²)不是mean(e)²**,可移除份额只取mean(S)/mean(e²)。
|
||||
|
||||
| 100%池臂 | 组件/件 | mean(e²) | mean(H) | mean(S) | oracle可移除均方份额 |
|
||||
|---|---|---:|---:|---:|---:|
|
||||
|N|209/617|0.0156499271|0.0137070207|0.0019429064|12.4148%|
|
||||
|E|209/617|0.0160421528|0.0138836945|0.0021584583|13.4549%|
|
||||
|R2|209/617|0.0073201040|0.0068825718|0.0004375322|5.9771%|
|
||||
|B0|209/617|0.0138780642|0.0111307244|0.0027473397|19.7963%|
|
||||
|
||||
同209组件/617件的N−R2均方差=0.0083298230,其中残差H差=0.0068244489(81.9279%)、尺度S差=0.0015053742。这说明此均方差主要不属于单一乘数可移除部分;不能直接改写为主L2均值的校准后性能。H还包含相对幅度、局部时序间距和空间误差,不是纯排序误差;正乘数不改变τ或argmax。a*利用测试真值且不等于Tmax比,不可部署。
|
||||
|
||||
全部13,449行D3有效,额外undefined=0;逐行H+S与e²最大绝对差1.421e-14,重构e²与冻结原误差平方最大差7.105e-15(rtol1e-10、atol1e-12)。a≤0共2行、均为负,100%池没有该情形:
|
||||
|
||||
| 件 | 折/池/臂/seed | a* | 原e² | H | S | 解释 |
|
||||
|---|---|---:|---:|---:|---:|---|
|
||||
|00001455_a45f6c99e2154f3b94955175_step_039|1/25%/E/2|-0.15885450|43.57139058|0.18473772|43.38665286|全场反号oracle,不能称幅值校准|
|
||||
|00001455_a45f6c99e2154f3b94955175_step_039|1/50%/E/0|-0.26504824|10.85844953|0.54737383|10.31107571|全场反号oracle,不能称幅值校准|
|
||||
|
||||
这些低池有限负预测仍按原规则计分,不追加NA、不裁剪;两行的组件等权份额在D3-tables.json各臂/池mean.reversal中。原50% E/N配对排fold1,成功E原始结果仍保留。
|
||||
|
||||
## D4:N输给B0的几何分布
|
||||
|
||||
100%池N先对固定3seed等权。N单件误差高于B0的件数312/617;按组件内件等权、组件等权后的败率47.1487%。原主L2 N/B0区间跨零,均值差及本表分层都是描述性,不能声称已确证N劣于B0。
|
||||
|
||||
各几何变量用617件几何经验三分位(linear分位,边界分右箱)分低/中/高;每箱再按件/非空组件等权。组件可出现在多个箱,分箱表不能相加为总体,不作独立样本检验。Rg/t_part沿准备数据原单位,nodes为计数。
|
||||
|
||||
| 变量/箱 | 原值范围 | 件/组件 | N输B0件数 | 组件等权败率 | N−B0误差百分点 |
|
||||
|---|---|---|---:|---:|---:|
|
||||
|Rg/低|[0.34624, 13.26]|206/76|111|49.3901%|-0.3797|
|
||||
|Rg/中|[13.26, 36.344]|205/99|106|50.0172%|+0.6951|
|
||||
|Rg/高|[36.484, 462.2]|206/97|95|46.9136%|+1.2288|
|
||||
|t_part/低|[0.15651, 2.286]|206/89|97|41.1362%|-0.6533|
|
||||
|t_part/中|[2.3141, 4.1988]|205/95|106|48.1123%|-0.0649|
|
||||
|t_part/高|[4.2026, 16]|206/104|109|54.5644%|+1.8587|
|
||||
|nodes/低|[3606, 20087]|206/97|100|48.9863%|+1.0086|
|
||||
|nodes/中|[20105, 32378]|205/104|99|47.9523%|+0.3233|
|
||||
|nodes/高|[32405, 1.8734e+05]|206/90|113|51.4276%|+0.3868|
|
||||
|
||||
较大Rg和较厚t_part箱的平均N−B0差较大,但节点数分箱没有单调趋势;败率与平均差并不相同。分箱、混合组件、尺寸/厚度/网格分辨率相关性不允许因果归因。
|
||||
|
||||
## 执行、限制与待阶段决定
|
||||
|
||||
D1/D2/D4进程CPU2.573114秒;D3 CPU17.133535秒。只读取固定文件;模型未加载,未生成新标签,未改变预测、区间、资格或S2主表。所有D1–D4为事后诊断,不形成新的确认性结论。
|
||||
|
||||
现有证据支持把R2保留为强门槛、把B0信息保留于候选输入设计;不能直接放行更长训练或以oracle宣称校准可修复。S3-A(R2增加相同edge信息)、S3-B(神经修复)仍需Claude综合团队讨论、明确比较与执行合同后才启动。O1继续独立保留。
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Post-hoc D1/D2/D4 from immutable existing outputs; no model or label loading."""
|
||||
from collections import defaultdict
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import numpy as np
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
ROOT=Path('artifacts/s2-posthoc-diagnostics-2026-11-15')
|
||||
EVAL=Path('artifacts/s2-custody-evaluation-2026-11-15')
|
||||
|
||||
|
||||
def read(path):return json.loads(Path(path).read_text())
|
||||
|
||||
|
||||
def grouped(values,ids):
|
||||
groups=defaultdict(list)
|
||||
for value,key in zip(values,ids):groups[key].append(float(value))
|
||||
return float(np.mean([np.mean(v) for v in groups.values()]))
|
||||
|
||||
|
||||
def main():
|
||||
start=time.process_time();assert not ROOT.exists(),'Do not overwrite diagnostic evidence'
|
||||
result=read(EVAL/'custody/result.json')
|
||||
for name in ['component-scores.json','part-scores.json','tables.json']:
|
||||
assert sha(EVAL/'custody'/name)==result['output_sha256'][name]
|
||||
scores=read(EVAL/'custody/component-scores.json');curves={}
|
||||
arms=['N','E','R2','B0'];pools=[25,50,100]
|
||||
for pool in pools:
|
||||
for arm in arms:
|
||||
curves[pool,arm]={r['component_id']:r['value'] for r in scores if r['subset']=='all' and r['metric']=='relative_l2' and r['arm']==arm and r['pool']==pool}
|
||||
common=set.intersection(*(set(v) for v in curves.values()))
|
||||
all_comparison=set(curves[25,'N'])&set(curves[100,'N'])
|
||||
d1=dict(posthoc=True,common_components=len(common),common_component_ids=sorted(common),common_folds=[0,2,3,4],
|
||||
rows=[dict(pool_component_fraction=pool/100,arm=arm,original_components=len(curves[pool,arm]),
|
||||
original_mean=float(np.mean(list(curves[pool,arm].values()))),
|
||||
common_mean=float(np.mean([curves[pool,arm][k] for k in sorted(common)]))) for pool in pools for arm in arms],
|
||||
N_25_to100_all5_relative_error_reduction=float(1-np.mean([curves[100,'N'][k] for k in all_comparison])/np.mean([curves[25,'N'][k] for k in all_comparison])),
|
||||
note='Original50pctN denominator differs; common fourfold curve is a separate descriptive diagnostic, not replacement primary analysis')
|
||||
records=[];histories={}
|
||||
for fold in range(5):
|
||||
for arm in ['N','E']:
|
||||
for seed in [0,1,2]:
|
||||
name=f'fold{fold}-pool100-{arm}128-seed{seed}';base=Path('artifacts/s2-main-matrix-2026-11-15/cells')/name/name
|
||||
rr=read(base/'result.json');h=read(base/'history.json');histories[name]=h
|
||||
y=np.array([v['validation_component_equal_relative_l2'] for v in h]);train=np.array([v['training_global_weighted_mse'] for v in h])
|
||||
n=min(5,len(y));slope=float(np.polyfit(np.arange(n),y[-n:],1)[0])
|
||||
records.append(dict(name=name,fold=fold,arm=arm,seed=seed,epochs=len(h),best_epoch_1based=rr['best_epoch']+1,
|
||||
best_validation=rr['best_validation_score'],final_validation=float(y[-1]),
|
||||
best_in_last5=rr['best_epoch']>=len(h)-5,best_at_final=rr['best_epoch']==len(h)-1,
|
||||
final5_slope=slope,final5_endpoint_relative_change=float(y[-1]/y[-n]-1),
|
||||
first_validation=float(y[0]),final_training_loss=float(train[-1]),final5_training_relative_change=float(train[-1]/train[-n]-1)))
|
||||
summary=[]
|
||||
for arm in ['N','E']:
|
||||
rr=[r for r in records if r['arm']==arm]
|
||||
summary.append(dict(arm=arm,runs=len(rr),all_reached42=all(r['epochs']==42 for r in rr),
|
||||
median_best_epoch=float(np.median([r['best_epoch_1based'] for r in rr])),
|
||||
best_in_last5=sum(r['best_in_last5'] for r in rr),best_at_final=sum(r['best_at_final'] for r in rr),
|
||||
negative_final5_validation_slope=sum(r['final5_slope']<0 for r in rr),
|
||||
median_final5_validation_change=float(np.median([r['final5_endpoint_relative_change'] for r in rr])),
|
||||
median_final5_training_change=float(np.median([r['final5_training_relative_change'] for r in rr]))))
|
||||
d2=dict(posthoc=True,epoch_display='one based',records=records,summary=summary,
|
||||
limitation='Late best epoch/negative local slope does not prove a longer schedule will beat R2; cosine endpoint and fitting choices are confounded')
|
||||
roster=read('artifacts/s2-main-preparation-2026-11-15/final-roster.json')['parts']
|
||||
partrows=read(EVAL/'custody/part-scores.json');by=defaultdict(list)
|
||||
for r in partrows:
|
||||
if r['arm']=='B0' or r['pool']==100:by[r['case'],r['arm']].append(r['relative_l2']['value'])
|
||||
geom=[]
|
||||
for r in roster:
|
||||
if not r['included']:continue
|
||||
p=Path(r['preparation']['preparation_root'])/'result.json';assert sha(p)==r['preparation']['result_sha256'];s=read(p)['scales']
|
||||
means={arm:float(np.mean(by[r['case'],arm])) for arm in arms}
|
||||
assert len(by[r['case'],'N'])==len(by[r['case'],'E'])==3
|
||||
geom.append(dict(case=r['case'],component_id=r['component_id'],nodes=r['nodes'],Rg=s['Rg'],t_part=s['t_part'],
|
||||
errors=means,N_minus_B0=means['N']-means['B0'],N_loses_to_B0=means['N']>means['B0']))
|
||||
bins=[]
|
||||
for feature in ['Rg','t_part','nodes']:
|
||||
x=np.array([r[feature] for r in geom]);cuts=np.quantile(x,[1/3,2/3],method='linear');assignment=np.searchsorted(cuts,x,side='right')
|
||||
for b in range(3):
|
||||
rr=[r for i,r in enumerate(geom) if assignment[i]==b];ids=[r['component_id'] for r in rr]
|
||||
bins.append(dict(feature=feature,bin=b,tertile_cutpoints=cuts.tolist(),min=min(r[feature] for r in rr),max=max(r[feature] for r in rr),
|
||||
parts=len(rr),components=len(set(ids)),N_loses_parts=sum(r['N_loses_to_B0'] for r in rr),
|
||||
component_equal_loss_fraction=grouped([r['N_loses_to_B0'] for r in rr],ids),
|
||||
component_equal_difference=grouped([r['N_minus_B0'] for r in rr],ids),
|
||||
component_equal_N=grouped([r['errors']['N'] for r in rr],ids),
|
||||
component_equal_B0=grouped([r['errors']['B0'] for r in rr],ids)))
|
||||
d4=dict(posthoc=True,parts=617,geometry_units='original prepared geometry units; nodes count',
|
||||
bins=bins,part_records=geom,N_loses_parts=sum(r['N_loses_to_B0'] for r in geom),
|
||||
component_equal_loss_fraction=grouped([r['N_loses_to_B0'] for r in geom],[r['component_id'] for r in geom]),
|
||||
limitation='Geometry-only empirical tertiles on617parts, posthoc descriptive; within-bin parts/component equal; no causal claim or multiplicity test')
|
||||
ROOT.mkdir();save(ROOT/'D1-learning-curves.json',d1);save(ROOT/'D2-epoch-history.json',d2);save(ROOT/'D4-geometry-errors.json',d4)
|
||||
# Standalone scientific figure from existing tabulations, no new fitted model.
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
fig,axes=plt.subplots(1,3,figsize=(13,3.8))
|
||||
for arm in arms:
|
||||
axes[0].plot([.25,.5,1],[100*next(r['common_mean'] for r in d1['rows'] if r['arm']==arm and r['pool_component_fraction']==p) for p in [.25,.5,1]],'-o',label=arm)
|
||||
axes[0].set(xlabel='Learning component fraction',ylabel='Component-equal relative L2 (%)',title='D1: common four-fold diagnostic');axes[0].legend()
|
||||
for ax,arm in zip(axes[1:],['N','E']):
|
||||
hs=[h for name,h in histories.items() if f'-{arm}128-' in name]
|
||||
for h in hs:ax.plot([r['epoch']+1 for r in h],[r['validation_component_equal_relative_l2'] for r in h],alpha=.3,lw=.8)
|
||||
ax.set(xlabel='Epoch',ylabel='Validation relative L2',title=f'D2: {arm}, 100% pool, 15 runs')
|
||||
fig.tight_layout();fig.savefig(ROOT/'D1-D2-curves.png',dpi=200);plt.close(fig)
|
||||
save(ROOT/'receipt.json',dict(status='D1_D2_D4_COMPLETE',cpu_seconds=time.process_time()-start,
|
||||
new_labels_decoded=False,models_loaded=False,new_training=False,original_statistics_modified=False,
|
||||
D3='WAITING_MATHEMATICAL_BOUNDARIES',ruling='1794731320.994789',code_sha256=sha(__file__),
|
||||
output_sha256={p.name:sha(p) for p in ROOT.iterdir() if p.is_file()}))
|
||||
print(json.dumps(dict(D1_reduction=d1['N_25_to100_all5_relative_error_reduction'],D2=summary,D4_loses=d4['N_loses_parts'],CPU=time.process_time()-start)))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,119 @@
|
||||
"""D3 oracle diagnostic, not a calibration; only frozen prediction/label caches."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import numpy as np
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
from s2_custody_common import read_json,checked,test_rows,expected_models,prediction_path
|
||||
from s2_custody_statistics import component_values
|
||||
|
||||
ROOT=Path('artifacts/s2-posthoc-diagnostics-2026-11-15')
|
||||
EVAL=Path('artifacts/s2-custody-evaluation-2026-11-15')
|
||||
|
||||
|
||||
def decompose(prediction,truth,counts):
|
||||
p=np.asarray(prediction,dtype=np.float64);t=np.asarray(truth,dtype=np.float64);c=np.asarray(counts,dtype=np.float64)
|
||||
if p.ndim!=1 or p.shape!=t.shape or p.shape!=c.shape or not len(p):return dict(valid=False,reason='empty or invalid shape')
|
||||
if not all(np.isfinite(v).all() for v in [p,t,c]) or np.any(c<=0) or np.any(c!=np.floor(c)):
|
||||
return dict(valid=False,reason='nonfinite eligible item or invalid counts')
|
||||
ps=float(np.max(np.abs(p)));ts=float(np.max(np.abs(t)))
|
||||
if ts==0:return dict(valid=False,reason='zero_truth',e2=None,H=None,S=None,a=None)
|
||||
if ps==0:return dict(valid=False,reason='zero_pred',e2=1.,H=None,S=None,a=None)
|
||||
# Unit-vector projection avoids squaring tiny P or subtracting close energies.
|
||||
wp=np.sqrt(c)*(p/ps);wt=np.sqrt(c)*(t/ts)
|
||||
pn=np.linalg.norm(wp);tn=np.linalg.norm(wt);u=wp/pn;v=wt/tn
|
||||
cosine=float(u@v)
|
||||
beta_ld=np.longdouble(ps)/np.longdouble(ts)*np.longdouble(pn)/np.longdouble(tn)
|
||||
beta=float(beta_ld)
|
||||
a_ld=np.longdouble(cosine)/beta_ld
|
||||
a=float(a_ld) if abs(a_ld)<=np.finfo(float).max else None
|
||||
e2=float(np.sum((beta*u-v)**2));H=float(np.sum((cosine*u-v)**2));S=float((beta-cosine)**2)
|
||||
if not np.isfinite([e2,H,S]).all():return dict(valid=False,reason='unrepresentable normalized energy',a=a)
|
||||
if not np.isclose(e2,H+S,rtol=1e-10,atol=1e-12):raise ArithmeticError('Orthogonal decomposition failed')
|
||||
return dict(valid=True,reason=None,a=a,a_scientific=np.format_float_scientific(a_ld,precision=17),
|
||||
e2=e2,H=H,S=S,identity_absolute_error=abs(e2-H-S),
|
||||
sign_reversal=cosine<0,nonpositive_a=cosine<=0,zero_cross=cosine==0,
|
||||
perfect_prediction=e2==0,negative_prediction_items=int(np.sum(p<0)))
|
||||
|
||||
|
||||
def synthetic_checks():
|
||||
for p,t,want in [([2,4],[1,2],(.5,0.,1.,1.)),([1,2],[1,2],(1.,0.,0.,0.)),
|
||||
([-1,-2],[1,2],(-1.,0.,4.,4.)),([1,0],[0,1],(0.,1.,1.,2.)),
|
||||
([1e-200,2e-200],[1,2],(1e200,0.,1.,1.))]:
|
||||
d=decompose(p,t,[1,1]);assert d['valid']
|
||||
np.testing.assert_allclose([d['a'],d['H'],d['S'],d['e2']],want,rtol=1e-12,atol=1e-12)
|
||||
z=decompose([0,0],[1,2],[1,1]);assert not z['valid'] and z['reason']=='zero_pred' and z['e2']==1 and z['H'] is None
|
||||
for p,t,c in [([1,2],[0,0],[1,1]),([],[],[]),([1,np.nan],[1,2],[1,1])]:assert not decompose(p,t,c)['valid']
|
||||
p=np.array([.1,.4,.9]);t=np.array([.2,.3,1.]);c=np.array([3,1,7]);d=decompose(p,t,c)
|
||||
np.testing.assert_allclose(d['a'],np.sum(c*p*t)/np.sum(c*p*p),rtol=1e-12)
|
||||
np.testing.assert_allclose(d['e2'],np.sum(c*(p-t)**2)/np.sum(c*t*t),rtol=1e-12)
|
||||
return 10
|
||||
|
||||
|
||||
def main():
|
||||
start=time.process_time();assert not (ROOT/'D3-started.json').exists(),'No duplicate diagnostic pass'
|
||||
tests=synthetic_checks();c=read_json(EVAL/'contract.json');original=read_json(EVAL/'custody/result.json')
|
||||
for name in ['label-index.json','part-scores.json']:checked(EVAL/'custody'/name,original['output_sha256'][name])
|
||||
predindex=read_json(checked(EVAL/'prediction-index.json',read_json(EVAL/'predictions-complete.json')['prediction_index_sha256']))
|
||||
labels={r['case']:r for r in read_json(EVAL/'custody/label-index.json')}
|
||||
original_scores={(r['case'],r['arm'],r['pool'],r['seed']):r for r in read_json(EVAL/'custody/part-scores.json')}
|
||||
save(ROOT/'D3-started.json',dict(rulings=['1794731518.379529','1794731544.355059','1794731547.266819'],
|
||||
description='oracle diagnostic, not a calibration',synthetic_checks=tests,code_sha256=sha(__file__),
|
||||
zero_pred='e2=1 retained as original error; a/H/S undefined and excluded from valid decomposition',
|
||||
tolerance=dict(rtol=1e-10,atol=1e-12),new_labels_decoded=False))
|
||||
rows=[];metric_rows=[];maximum_identity_error=maximum_frozen_error=0.
|
||||
for fold in range(5):
|
||||
for row in test_rows(c,fold):
|
||||
label=labels[row['case']];checked(label['path'],label['sha256'])
|
||||
with np.load(label['path'],allow_pickle=False) as z:truth=z['seconds'];ids=z['raw_ids'];counts=z['counts']
|
||||
for model in expected_models(c,fold):
|
||||
p=prediction_path(EVAL,model,row['case']);checked(p,predindex[str(p)])
|
||||
with np.load(p,allow_pickle=False) as z:
|
||||
assert np.array_equal(z['raw_ids'],ids) and np.array_equal(z['counts'],counts)
|
||||
d=decompose(z['seconds'],truth,counts)
|
||||
old=original_scores[row['case'],model['arm'],model['pool'],model['seed']]['relative_l2']['value']
|
||||
if d['valid']:
|
||||
error=abs(d['e2']-old**2);maximum_frozen_error=max(maximum_frozen_error,error)
|
||||
assert np.isclose(d['e2'],old**2,rtol=1e-10,atol=1e-12)
|
||||
maximum_identity_error=max(maximum_identity_error,d['identity_absolute_error'])
|
||||
record=dict(case=row['case'],fold=fold,component_id=row['component_id'],arm=model['arm'],pool=model['pool'],seed=model['seed'],**d)
|
||||
rows.append(record)
|
||||
metric_rows.append(dict(case=row['case'],arm=model['arm'],pool=model['pool'],seed=model['seed'],
|
||||
**{key:dict(value=d[key] if d['valid'] else None,reason=d['reason']) for key in ['e2','H','S']},
|
||||
nonpositive=dict(value=float(d['nonpositive_a']) if d['valid'] else None,reason=d['reason']),
|
||||
reversal=dict(value=float(d['sign_reversal']) if d['valid'] else None,reason=d['reason'])))
|
||||
roster=read_json(c['roster'])['parts'];members={p for f in read_json(checked(c['exposure_membership'],c['exposure_membership_sha256'])) for p in f['parts']}
|
||||
tables=[];comparisons=[]
|
||||
for subset in ['all','not_known_exposed']:
|
||||
for pool in [25,50,100]:
|
||||
byarm={}
|
||||
for arm in ['N','E','R2','B0']:
|
||||
values={};details={};excluded={}
|
||||
for metric in ['e2','H','S','nonpositive','reversal']:
|
||||
values[metric],excluded,details=component_values(metric_rows,roster,arm,pool,metric,subset!='all',members)
|
||||
keys=set.intersection(*(set(v) for v in values.values()));means={k:float(np.mean([v[x] for x in sorted(keys)])) if keys else None for k,v in values.items()}
|
||||
if keys:assert np.isclose(means['e2'],means['H']+means['S'],rtol=1e-10,atol=1e-12)
|
||||
tables.append(dict(subset=subset,pool=pool,arm=arm,components=len(keys),parts=sum(details[k]['parts'] for k in keys),
|
||||
excluded_components=excluded,mean=means,oracle_removable_squared_error_share=means['S']/means['e2'] if keys and means['e2']>0 else None,
|
||||
warning='oracle diagnostic, not a calibration; mean squared error is not square of primary mean L2'))
|
||||
byarm[arm]=(values,details)
|
||||
for ref,cand in [('N','E'),('R2','N'),('B0','N')]:
|
||||
aa,details=byarm[ref];bb,_=byarm[cand];keys=set(aa['e2'])&set(bb['e2'])
|
||||
am={k:float(np.mean([aa[k][x] for x in sorted(keys)])) for k in ['e2','H','S']}
|
||||
bm={k:float(np.mean([bb[k][x] for x in sorted(keys)])) for k in ['e2','H','S']}
|
||||
comparisons.append(dict(subset=subset,pool=pool,reference=ref,candidate=cand,components=len(keys),
|
||||
parts=sum(details[x]['parts'] for x in keys),reference_means=am,candidate_means=bm,
|
||||
candidate_minus_reference={k:bm[k]-am[k] for k in am},warning='squared-error diagnostic difference, not primary Delta'))
|
||||
save(ROOT/'D3-part-decomposition.json',rows);save(ROOT/'D3-tables.json',tables);save(ROOT/'D3-comparisons.json',comparisons)
|
||||
summary=dict(status='D3_COMPLETE',rows=len(rows),valid=sum(r['valid'] for r in rows),undefined=[r for r in rows if not r['valid']],
|
||||
nonpositive_a_rows=sum(r.get('nonpositive_a',False) for r in rows),sign_reversal_rows=sum(r.get('sign_reversal',False) for r in rows),
|
||||
zero_cross_rows=sum(r.get('zero_cross',False) for r in rows),maximum_identity_absolute_error=maximum_identity_error,
|
||||
maximum_frozen_squared_error_difference=maximum_frozen_error,cpu_seconds=time.process_time()-start,
|
||||
new_labels_decoded=False,predictions_modified=False,calibration=False,synthetic_checks=tests,
|
||||
output_sha256={p.name:sha(p) for p in ROOT.glob('D3-*.json')})
|
||||
save(ROOT/'D3-receipt.json',summary)
|
||||
print(json.dumps(summary));print(json.dumps([x for x in tables if x['subset']=='all' and x['pool']==100]))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
Reference in new issue
Block a user