diff --git a/artifacts/phase3-item1b-fold0-screen-2026-11-19/comparison-contract.json b/artifacts/phase3-item1b-fold0-screen-2026-11-19/comparison-contract.json new file mode 100644 index 000000000..dffef86ad --- /dev/null +++ b/artifacts/phase3-item1b-fold0-screen-2026-11-19/comparison-contract.json @@ -0,0 +1,130 @@ +{ + "ruling": "3a48b123da54b0bcff0e7fa672fc74e8920235a0", + "fold": 0, + "validation_cases": [ + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000190_185ecbc783f34408aef52119_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001040_74c42207fa764831b03f7658_step_000", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001107_78566795c3144719804f8d34_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000" + ], + "validation_components": 35, + "score": "Relative L2 on the frozen validation query with original PPS multiplicity, then equal means by component. Negative predictions remain visible.", + "paired_references": [ + "long_seed0", + "long_seed1" + ], + "delta_short_at_most": -0.001, + "effect_strictly_greater_than_abs_seed_delta": true, + "duration": "100 complete epochs, final epoch only", + "warmup": "Original 5-epoch warmup and full cosine decay over 95 epochs", + "train_fields": "Original seed0, order, 19 inputs, optimizer, MSE and numerical recipe", + "test_allowed": false, + "automatic_continuation": false +} diff --git a/artifacts/phase3-item1b-fold0-screen-2026-11-19/contract.json b/artifacts/phase3-item1b-fold0-screen-2026-11-19/contract.json new file mode 100644 index 000000000..50109c5ca --- /dev/null +++ b/artifacts/phase3-item1b-fold0-screen-2026-11-19/contract.json @@ -0,0 +1,729 @@ +{ + "version": "phase3-item1b-fold0-screen-v1", + "ruling": "3a48b123da54b0bcff0e7fa672fc74e8920235a0", + "fold": 0, + "screen_fold": 0, + "seed": 0, + "epochs": 100, + "max_fits": 1, + "retry": false, + "test_targets_allowed": false, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "worker_cpus": [ + 22 + ], + "total_cpu_seconds": 30000, + "max_processes": 1, + "rss_bytes": 4294967296, + "disk_bytes": 8589934592, + "device_vram_bytes": 4294967296, + "recorded_vram_bytes": 2147483648, + "job": { + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000049_aa208c45e8734584a147f595_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000198_826494e894454504b0971fd1_step_001", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000658_a40da330321b44f286a27361_step_000", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000696_32309198fce9481896bd0a99_step_000", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001300_b9951da825954791b59b979d_step_000", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001348_318b81a8e412438183201be7_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005", + "00001586_763da6a559544098acbf4154_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000" + ], + "validation": [ + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000190_185ecbc783f34408aef52119_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001040_74c42207fa764831b03f7658_step_000", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001107_78566795c3144719804f8d34_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000" + ], + "test": [ + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000796_dec80701adc84239b485449d_step_002", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001034_085828cd621d404a90ccd883_step_000", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "c": 1.026602777749673, + "clock_sha256": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96", + "learning_nodes": 8742060, + "id": 0, + "target_steps": 32000, + "cpu_seconds": 30000, + "validation_seconds_bound": 12.383006267948076, + "rss_bytes": 4294967296, + "recorded_vram_bytes": 2147483648, + "process_vram_bytes": 4294967296, + "disk_bytes": 8589934592 + }, + "budget": { + "item_gpu_limit_seconds": 90000, + "phase3_gpu_limit_seconds": 720000, + "prior_item_gpu_seconds": 19262.057604262838, + "prior_phase3_gpu_seconds": 408630.4104626046, + "train_epoch_seconds_basis": 99.90287962602451, + "validation_seconds_basis": 16.66734165791422, + "train_gpu_seconds": 12817, + "screen_gpu_seconds": 183, + "training_setup_tail_seconds": 120, + "idle_wait_seconds": 60, + "finalization_gpu_seconds": 120, + "projected_new_gpu_seconds": 13180 + }, + "references": { + "long_seed0": { + "checkpoint": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/last-epoch200.pt", + "checkpoint_sha256": "35cc9e41a4197e186570089ac305c4dee100f3d37611e0ca17e01db441b93756", + "standardization": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/standardization.json", + "standardization_sha256": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421", + "seed": 0, + "epochs": 200, + "training_contract_sha256": "9ea2ad987ce165660c1c1f648a3edf8069c40ef270e233f869fd881e42d7c382" + }, + "long_seed1": { + "checkpoint": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/last-epoch200.pt", + "checkpoint_sha256": "b223cb3eb36efff52e5905f7c6a39a40a0e32e38f64fe9947d2597f68e7c3923", + "standardization": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/standardization.json", + "standardization_sha256": "98eabfb586fb7689bd31990b49301ccaceec34f84e16d220f3e3e7ee34f8e681", + "seed": 1, + "epochs": 200, + "training_contract_sha256": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981" + } + }, + "pilot_contract": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json", + "query_roster_contract": "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/contract.json", + "comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-screen-2026-11-19/comparison-contract.json", + "comparison_contract_sha256": "bd71a5a6a0f3a288267f568cbb85d1cea5ab6fa0485e7fd212cd55e36f664954", + "source_sha256": { + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/contract.json": "d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/complete.json": "430247e44f301f984f396c662e6fa5419d09649476f6b2a81076444731f55dde", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/post-terminal-cost.json": "7241fe629e3227b81655886c6c208336deef443d67c8d6efd7530000de8477bf", + "/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json": "ca576e271715c862ee21e75b2dec7b03e45317e5f8bb073adae016a8001a0a59", + "/home/frank14f/MeltMind-runs/phase3-seed1-postfit-2026-11-18/post-terminal-cost.json": "927546a8991fc2d39127067c4907ae74276d6ec6d5d0a82e0f57db7951813bbe", + "/home/frank14f/MeltMind-runs/phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json": "2de53b3aca1ae21bc53480ad7425094f30273f798cd292a895526094c993b41f", + "/home/frank14f/MeltMind-runs/phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json": "ea99e46a6ffe687004f174df58fdd0f1c13d0231bac7e28e6cbdd988b7fe5c84", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/worker/history/001.json": "579391f43569643a4ea8ac1af6e2043ebeb9bf149b2c17b3ca3b3ef0aa2c43e5", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/assignment.json": "050e8b6748a3e2a287eb905f75179daa9941bd5690660f2c2716298fbf71bd1d", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/assignment.json": "a2699b9e9174eb7ba8a4b54a4764f58f28fb794be0008083f36765c738035b26", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/worker/worker-identity.json": "98094ad553095111798c43d8b33ec14cbe95955b0df5110e233e176856a0a881", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/worker/worker-identity.json": "2953a472cdff619d52235420837880bfc551ce65b5699db77124b3f1a0e147ca", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.started.json": "c4985f4c4b5d24c81a29feb1b275d41213fc1f2a9583d01c48130007564ac401", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.started.json": "b4c43b504190fe334d049fad6889f6d998e64140ef760958a1308612cf538bf0", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.receipt.json": "36b9c44bcf2248be07314300ee8c99d214232869e24399314e0907c9c2469b5b", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.receipt.json": "640f14ff26fcba3c6213b111c81463a1261efd6992ff7fb3c827d7da06b6195f", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/gpu-sample.json": "9e92c761638268f48c1436243a18de13e41d67b15a335ff18cd9bc5a037790bc", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/epoch-host-observations/026.json": "5f3ccb39ea8669bb5fed7260b3b786ace6366ff59a0a4a0f135681e6311dd830", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/history/026.json": "0ce765d2bc3cc68640971afaa9f2a78a3f59ae0602c2c60d0f690533891b43f4", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/epoch-host-observations/020.json": "91f70761c95aa8f211b050c39685ae2a1981891f8151f17aa75805eaaa1c72e6", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/history/020.json": "776c76e9882a3470fe1264d1c61b4307e0f6a9f0fcfc320d04a6d6b05becfd01", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/contract.json": "0bd03f42c3b2db5bef26708b2ef873edfe674f724b59bc4b4de8641980d8d330", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/contract.json": "9ea2ad987ce165660c1c1f648a3edf8069c40ef270e233f869fd881e42d7c382", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/result.json": "f2bbe0d6ff36bcdbcfb36cc9cb17459c65e4f7b5d47fa7252a2c73fbbfb8387e", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/standardization.json": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/last-epoch200.pt": "35cc9e41a4197e186570089ac305c4dee100f3d37611e0ca17e01db441b93756", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/contract.json": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/result.json": "79b38fd28b590a4ff99b378d76f32cbf6f959d056cfa8e3c92b4a9d5e9b611ac", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/standardization.json": "98eabfb586fb7689bd31990b49301ccaceec34f84e16d220f3e3e7ee34f8e681", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/last-epoch200.pt": "b223cb3eb36efff52e5905f7c6a39a40a0e32e38f64fe9947d2597f68e7c3923", + "/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-screen-2026-11-19/timing-and-budget.json": "33a2908ae8785b405869d7ef957445bd93566c5d73c1a26dd86489fe077a4232" + }, + "code_sha256": { + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_fold0.py": "1446919f2274bc6207527ad80cbd634e47761f9d6391543d420d4f8cd6498527", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "06cb97594c3a87f42f712dcde6cec8c2d90e0b50a89a6be7954bdd2f5c18d5de", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "b781e56c519401bd38af6a522deb24506cba6993d2af008f85ea4e1715f1d9c6", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b_fold0.py": "f48ebe3e3aeca03e5d4cbdd532061cfb27362ec84e85dae70e77e22b4e40b8e0", + "/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609", + "/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61", + "/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75", + "/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4", + "/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0", + "/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "08b7d8ab2ac773fab8b749f9f61af259254422128e9634f8b715c81efce2651f", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "31dfd71369c63512c1d6fe176fd22e8db646520c92da533bb5530ca3843c388d", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "c7bc9f3b7cf4137f52e1efad19dcb7045282d4e9a9c1aae50f066c9e488f7c01", + "/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "b4d78f4384651711fe83ff00eb6719c6c61664699d801ecb34978b9546439da9", + "/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496", + "/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a", + "/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447", + "/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "3115aa9049974eac86657aa4cee7b5d716f6ee93268555cc86242029ebefce4c", + "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062", + "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073" + } +} diff --git a/artifacts/phase3-item1b-fold0-screen-2026-11-19/cpu-guard-preflight.json b/artifacts/phase3-item1b-fold0-screen-2026-11-19/cpu-guard-preflight.json new file mode 100644 index 000000000..3233cb941 --- /dev/null +++ b/artifacts/phase3-item1b-fold0-screen-2026-11-19/cpu-guard-preflight.json @@ -0,0 +1,26 @@ +{ + "status": "PASS", + "actual_contract_sha256": "942323ddfceecf47515eb48628c45157073cbb909867a606ef4d3764a13be7d4", + "cases": [ + { + "case": "idle-dispatch", + "exit_code": 0, + "trigger": "SCREEN_PASS", + "mock_dispatches": [ + "train", + "screen" + ], + "gate_wait_seconds": 20.0 + }, + { + "case": "busy-timeout", + "exit_code": 1, + "trigger": "IDLE_TIMEOUT", + "mock_dispatches": [], + "gate_wait_seconds": 60.0 + } + ], + "real_gpu_or_worker_started": false, + "scientific_arrays_read": false, + "review_status": "PENDING" +} diff --git a/artifacts/phase3-item1b-fold0-screen-2026-11-19/timing-and-budget.json b/artifacts/phase3-item1b-fold0-screen-2026-11-19/timing-and-budget.json new file mode 100644 index 000000000..b094bafd4 --- /dev/null +++ b/artifacts/phase3-item1b-fold0-screen-2026-11-19/timing-and-budget.json @@ -0,0 +1,53 @@ +{ + "prior_phase3_costs": { + "item1": 72421.31599938683, + "seed1": 114756.77075582312, + "item1b": 19262.057604262838, + "c": 105994.92276056076, + "b": 96195.34334257105 + }, + "budget": { + "item_gpu_limit_seconds": 90000, + "phase3_gpu_limit_seconds": 720000, + "prior_item_gpu_seconds": 19262.057604262838, + "prior_phase3_gpu_seconds": 408630.4104626046, + "train_epoch_seconds_basis": 99.90287962602451, + "validation_seconds_basis": 16.66734165791422, + "train_gpu_seconds": 12817, + "screen_gpu_seconds": 183, + "training_setup_tail_seconds": 120, + "idle_wait_seconds": 60, + "finalization_gpu_seconds": 120, + "projected_new_gpu_seconds": 13180 + }, + "remaining_item_gpu_seconds": 70737.94239573716, + "same_card_fold0_epoch_seconds": 96.1752110850066, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "same_card_train_peers_instantiated": { + "0": 2025071, + "2": 2025091 + }, + "sampled_cuda_pids": [ + 1765688, + 2025071, + 2025091 + ], + "sample_unix": 1795021405.2712727, + "worker_intervals": [ + { + "fold": 0, + "started_unix": 1795021298.0488324, + "ended_unix": 1795021406.151041 + }, + { + "fold": 2, + "started_unix": 1795021298.5668569, + "ended_unix": 1795021405.9669158 + } + ], + "same_fold_solo_epoch_records": 199, + "longest_solo_train_seconds": 99.90287962602451, + "longest_solo_validation_seconds": 16.66734165791422, + "limitation": "The two workers started together. Shared occupancy is sampled near epoch completion. The third CUDA context is observed only at that sample.", + "target_admission": "Selected V100 starts idle, with stop on foreign occupancy. The co-located trace is a conservative budget input." +} diff --git a/docs/phase3-item1b-fold0-screen-2026-11-19.md b/docs/phase3-item1b-fold0-screen-2026-11-19.md new file mode 100644 index 000000000..9f65a2d3f --- /dev/null +++ b/docs/phase3-item1b-fold0-screen-2026-11-19.md @@ -0,0 +1,10 @@ +# Phase 3 item 1b 折 0 筛选准备 + +Claude 裁定 h-c53c94a6 批准一个新标记的短日程折 0。旧五折队列因总成本投影停止,所留训练、失败和结算继续有效。[裁定 3a48b12](https://gitea.frank14f.online/Frank14f/MeltMind/src/commit/3a48b123da54b0bcff0e7fa672fc74e8920235a0/docs/claude/stage-review-2026-11-19.md) + +- **同卡成本证据。** 原折 0 完成第 1 轮,训练 96.175211 秒。折 2 在同一 V100 晚约 0.52 秒启动,在回收前的同一 GPU 快照中可见两名 worker 和第三个 CUDA 进程。第三个进程只有该时刻的占用证据。另核同卡同折 199 条独占采样匹配的 seed 1 轮回执。预算取较大的轮时 99.902880 秒和验证 16.667342 秒,保守覆盖已有直接并发计时。新入口从空卡起步,发现外部 CUDA 进程时只停止自己的 worker。 +- **结算。** 100 轮与 10 次诊断的实测投影加 25%,再计 120 秒训练准备与收尾,训练帽 12,817 GPU 秒。三臂验证和准备帽 183 GPU 秒。排队最多 60 秒,另留 120 秒入口和终态开销。总帽 13,180 GPU 秒(3.6611 小时),低于原 25 小时扣除已结算 19,262.057604 秒后剩余的 70,737.942396 秒。等待单列并计入总账一次。首轮再次核投影,命中任一边界留停止回执。 +- **对照。** 同 320 件 learning、19 输入、seed 0、固定逐轮样件顺序、5 轮 warmup 和完成的 95 轮余弦退火。只取第 100 轮。用同域 111 件 validation 比较第 200 轮 seed 0 与 seed 1。先冻结三臂预测,再按固定查询点及原 PPS multiplicity 算场相对 L2,件均值汇成组件等权。沿原筛选门,短日程差值须 ≤ −0.001 且绝对值严格大于同域种子差。终态只报告筛选。测试访问、其他折和自动续跑均未获本入口授权。 +- **上线证据。** 新入口用原训练函数本体和原学习日程。针对真实生成合同运行了真实调度守卫的 CPU 模拟,空卡路径模拟一次训练和一次筛选,忙卡 60 秒超时且无派发。模拟无真实 GPU worker,无科学数组访问。精确独立 PASS 待收。 + +[合同](../artifacts/phase3-item1b-fold0-screen-2026-11-19/contract.json) · [比较合同](../artifacts/phase3-item1b-fold0-screen-2026-11-19/comparison-contract.json) · [计时与预算](../artifacts/phase3-item1b-fold0-screen-2026-11-19/timing-and-budget.json) · [实际守卫 CPU 回执](../artifacts/phase3-item1b-fold0-screen-2026-11-19/cpu-guard-preflight.json) diff --git a/home-runs/phase3-item-1b-2026-11-18/jobs/0/gpu-sample.json b/home-runs/phase3-item-1b-2026-11-18/jobs/0/gpu-sample.json new file mode 100644 index 000000000..a58144171 --- /dev/null +++ b/home-runs/phase3-item-1b-2026-11-18/jobs/0/gpu-sample.json @@ -0,0 +1,20 @@ +{ + "unix": 1795021405.2712727, + "rows": [ + { + "pid": 1765688, + "bytes": 1918894080 + }, + { + "pid": 2025071, + "bytes": 1790967808 + }, + { + "pid": 2025091, + "bytes": 1918894080 + } + ], + "own_pids": [ + 2025071 + ] +} diff --git a/home-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.receipt.json b/home-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.receipt.json new file mode 100644 index 000000000..88e651d1e --- /dev/null +++ b/home-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.receipt.json @@ -0,0 +1,39 @@ +{ + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_item1b_worker.py", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18", + "0", + "050e8b6748a3e2a287eb905f75179daa9941bd5690660f2c2716298fbf71bd1d" + ], + "returncode": -9, + "stop_reason": null, + "cpu_seconds": 104.23424299999999, + "wall_seconds": 108.10220872296486, + "peak_tree_rss_bytes": 1431515136, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 8192, + "max_poll_gap_seconds": 0.8025802979245782, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null, + "monitor_policy": { + "rule": "three_consecutive_failed_samples_per_operation", + "failure_limit": 3, + "failed": false, + "errors": 0, + "streaks": { + "initial_disk": 0, + "tree_usage": 0, + "live_disk": 0, + "final_disk": 0 + }, + "max_streaks": {} + } +} diff --git a/home-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.started.json b/home-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.started.json new file mode 100644 index 000000000..2fcfdda84 --- /dev/null +++ b/home-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.started.json @@ -0,0 +1,18 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_item1b_worker.py", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18", + "0", + "050e8b6748a3e2a287eb905f75179daa9941bd5690660f2c2716298fbf71bd1d" + ], + "limits": { + "total_cpu_seconds": 29940, + "job_cpu_seconds": 29940, + "job_wall_seconds": 26341.294980686158, + "tree_rss_bytes": 4294967296, + "disk_bytes": 8589934592, + "max_processes": 1 + }, + "started_unix": 1795021298.0488324 +} diff --git a/home-runs/phase3-item-1b-2026-11-18/jobs/0/worker/history/001.json b/home-runs/phase3-item-1b-2026-11-18/jobs/0/worker/history/001.json new file mode 100644 index 000000000..fefc6436f --- /dev/null +++ b/home-runs/phase3-item-1b-2026-11-18/jobs/0/worker/history/001.json @@ -0,0 +1,15 @@ +{ + "fold": 0, + "stage": "EPOCH_COMPLETE", + "epoch": 1, + "optimizer_steps": 320, + "wall_seconds": 101.60979303403292, + "cpu_seconds": 101.23031225700001, + "train_mean_weighted_loss": 0.004886068560188051, + "train_epoch_seconds": 96.1752110850066, + "epoch_parts": 320, + "epoch_nodes": 8742060, + "lr": 0.0001, + "peak_allocated_bytes": 1334860288, + "peak_reserved_bytes": 1388314624 +} diff --git a/home-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.receipt.json b/home-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.receipt.json new file mode 100644 index 000000000..371cc9108 --- /dev/null +++ b/home-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.receipt.json @@ -0,0 +1,39 @@ +{ + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_item1b_worker.py", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18", + "2", + "a2699b9e9174eb7ba8a4b54a4764f58f28fb794be0008083f36765c738035b26" + ], + "returncode": -9, + "stop_reason": null, + "cpu_seconds": 105.068737, + "wall_seconds": 107.40005897299852, + "peak_tree_rss_bytes": 1471344640, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.9828807739540935, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null, + "monitor_policy": { + "rule": "three_consecutive_failed_samples_per_operation", + "failure_limit": 3, + "failed": false, + "errors": 0, + "streaks": { + "initial_disk": 0, + "tree_usage": 0, + "live_disk": 0, + "final_disk": 0 + }, + "max_streaks": {} + } +} diff --git a/home-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.started.json b/home-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.started.json new file mode 100644 index 000000000..21ab4a478 --- /dev/null +++ b/home-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.started.json @@ -0,0 +1,18 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_item1b_worker.py", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18", + "2", + "a2699b9e9174eb7ba8a4b54a4764f58f28fb794be0008083f36765c738035b26" + ], + "limits": { + "total_cpu_seconds": 29940, + "job_cpu_seconds": 29940, + "job_wall_seconds": 26340.25473072508, + "tree_rss_bytes": 4294967296, + "disk_bytes": 8589934592, + "max_processes": 1 + }, + "started_unix": 1795021298.5668569 +} diff --git a/tools/phase3_item1b_fold0.py b/tools/phase3_item1b_fold0.py new file mode 100644 index 000000000..a9fbf351a --- /dev/null +++ b/tools/phase3_item1b_fold0.py @@ -0,0 +1,390 @@ +"""One reviewed fold-0 attempt and its validation-only screen.""" + +import argparse +from collections import defaultdict +import ctypes +import fcntl +import json +import math +import os +from pathlib import Path +import shutil +import signal +import sys +import threading +import time +from types import FunctionType + +from phase2_gate_outcome_review import validate_review +from phase2_h1_worker_v2 import checked, save, sha +from phase3_item1b_common import fold_view, read +from phase3_shared_capacity import process_identity +from s2_gpu_idle_gate import snapshot, wait_idle +from s2_main_memory_guard import PeakRecords, active_command_pids +from s2_recovery_gpu_supervisor import stop_group, supervise + + +ROOT = Path(__file__).resolve().parents[1] / "artifacts/phase3-item1b-fold0-screen-2026-11-19" +RULING = "3a48b123da54b0bcff0e7fa672fc74e8920235a0" +ALLOCATOR = "expandable_segments:True" +GIB = 2**30 +ARMS = ("short_seed0", "long_seed0", "long_seed1") + + +def contract(path): + c = read(path) + if (c["ruling"] != RULING or c["fold"] != 0 or c["screen_fold"] != 0 + or c["epochs"] != 100 or c["seed"] != 0 or c["max_fits"] != 1 + or c["retry"] is not False or c["test_targets_allowed"] is not False): + raise PermissionError("Only the new fold-0 validation scope is authorized") + for p, h in (c["code_sha256"] | c["source_sha256"]).items(): + checked(p, h) + comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"])) + if comparison["validation_cases"] != c["job"]["validation"]: + raise ValueError("Fold-0 validation identity differs from comparison") + budget = c["budget"] + reserves = sum(budget[k] for k in ("train_gpu_seconds", "screen_gpu_seconds", + "idle_wait_seconds", "finalization_gpu_seconds")) + if (budget["projected_new_gpu_seconds"] != reserves + or budget["prior_item_gpu_seconds"] + reserves > budget["item_gpu_limit_seconds"] + or budget["prior_phase3_gpu_seconds"] + reserves > budget["phase3_gpu_limit_seconds"]): + raise PermissionError("Phase 3 or 1b remaining budget exceeded") + return c + + +def folded_inputs(c): + pilot = read(c["pilot_contract"]) + view = fold_view(pilot, 0) + original = view["folds"][0] + job = c["job"] + for k in ("learning", "validation", "test", "c", "clock_sha256", "learning_nodes"): + if job[k] != original[k]: + raise ValueError(f"Original fold-0 identity changed: {k}") + if job["id"] != 0 or job["target_steps"] != 100 * len(original["learning"]): + raise ValueError("Full fold-0 learning schedule required") + if set(original["validation"]) & set(original["test"]): + raise ValueError("Validation and test overlap") + return pilot, view + + +def train(root, c): + import numpy as np + import torch + from phase3_item1b_worker import run as frozen_worker + + pilot, _ = folded_inputs(c) + if torch.__version__ != pilot["versions"]["torch"] or np.__version__ != pilot["versions"]["numpy"]: + raise ValueError("Original array and Torch versions required") + job = dict(c["job"], gpu_uuid=c["gpu_uuid"], worker_cpus=c["worker_cpus"]) + digest = sha(root / "contract.json") + + def first_epoch_release(queue, fold): + if queue != root or fold != 0: + raise PermissionError("Wrong worker route") + row = read(root / "jobs/0/worker/history/001.json") + budget = c["budget"] + epoch_seconds = max(row["train_epoch_seconds"], budget["train_epoch_seconds_basis"]) + projection = (row["wall_seconds"] + 99 * epoch_seconds + + 10 * budget["validation_seconds_basis"] + budget["training_setup_tail_seconds"]) + passed = math.isfinite(projection) and projection <= budget["train_gpu_seconds"] + save(root / "first-epoch-projection.json", dict( + status="PASS" if passed else "STOP_BUDGET", contract_sha256=digest, + measured_epoch_seconds=row["train_epoch_seconds"], projected_train_gpu_seconds=projection)) + if not passed: + raise RuntimeError("First-epoch projection exceeds fold-0 training cap") + return True + + # Reuse the immutable training body. The private namespace changes only + # its root/assignment adapters and the one-fold budget gate. + namespace = dict(frozen_worker.__globals__, config=lambda _: (c, pilot), + assigned_job=lambda *_: job, first_epoch_release=first_epoch_release) + runner = FunctionType(frozen_worker.__code__, namespace, frozen_worker.__name__) + runner(root, 0, digest) + + +def screen(root, c): + import numpy as np + import torch + from phase2_h2_data import arguments, cached, to_device + from phase2_h2_model import H2Geo + from phase2_h2_pilot_worker import peaks + from phase2_residual_math import decode_seconds + from phase3_item1_postfit_data import load_evaluation_part + from s2_custody_common import prediction_inputs + from s2_models.objectives import relative_l2 + + pilot, view = folded_inputs(c) + comparison = read(c["comparison_contract"]) + fold = view["folds"][0] + cases = comparison["validation_cases"] + if (cases != fold["validation"] or torch.__version__ != pilot["versions"]["torch"] + or np.__version__ != pilot["versions"]["numpy"] + or os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"] + or os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR): + raise PermissionError("Screen scope or environment differs") + train_path = root / "jobs/0/worker/result.json" + adopted = read(root / "training-adopted.json") + short = read(checked(train_path, adopted["result_sha256"])) + models = dict(c["references"]) + models["short_seed0"] = dict( + checkpoint=short["checkpoint"], checkpoint_sha256=short["checkpoint_sha256"], + standardization=str(train_path.with_name("standardization.json")), + standardization_sha256=short["standardization_sha256"], seed=0, epochs=100, + training_contract_sha256=sha(root / "contract.json")) + roster = {r["case"]: r for r in read(c["query_roster_contract"])["rows"] if r["case"] in cases} + if set(roster) != set(cases) or any( + roster[case]["component_id"] != pilot["parts"][case]["clock_group"] + or roster[case]["v5_cache"]["output_sha256"] != pilot["parts"][case]["output_sha256"] + for case in cases + ): + raise ValueError("Frozen validation query-to-cache identity differs") + + os.sched_setaffinity(0, c["worker_cpus"]) + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + torch.cuda.set_device(0) + torch.cuda.reset_peak_memory_stats() + out = root / "screen" + out.mkdir(exist_ok=False) + predictions = {} + with torch.no_grad(): + for arm in ARMS: + ref = models[arm] + stats = read(checked(ref["standardization"], ref["standardization_sha256"])) + blob = torch.load(checked(ref["checkpoint"], ref["checkpoint_sha256"]), + map_location="cpu", weights_only=True) + required = dict(completed_epochs=ref["epochs"], fold=0, seed=ref["seed"], + role="last_epoch_only", contract_sha256=ref["training_contract_sha256"], + optimizer_steps=ref["epochs"] * len(fold["learning"])) + if any(blob.get(k) != v for k, v in required.items()): + raise ValueError(f"Final checkpoint identity: {arm}") + if stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]: + raise ValueError("Screen learning-only normalization mismatch") + model = H2Geo().cuda() + model.load_state_dict(blob["model"], strict=True) + del blob + model.eval() + folder = out / arm + folder.mkdir() + for case in cases: + data = to_device(load_evaluation_part(view, case, stats, "validation"), "cuda") + values = model(*arguments(data), checkpoint_blocks=False).cpu().numpy() + seconds = decode_seconds(values, data["F"], fold["c"], data["t_set"]) + if seconds.shape != (data["nodes"],) or not np.isfinite(seconds).all(): + raise ValueError("Full validation prediction is missing or nonfinite") + path = folder / (case + ".npy") + np.save(path, seconds, allow_pickle=False) + predictions.setdefault(case, {})[arm] = {"file": str(path), "sha256": sha(path)} + save(out / "progress.json", dict(stage="VALIDATION_PREDICT", arm=arm, case=case, **peaks())) + del data + del model + manifest = out / "predictions-frozen.json" + save(manifest, predictions) + manifest_hash = sha(manifest) + predictions = read(checked(manifest, manifest_hash)) + rows = [] + grouped = defaultdict(list) + for case in cases: + q = prediction_inputs(roster[case]) + ids = cached(view, case, "raw_ids") + ix = np.searchsorted(ids, q["raw_ids"]) + if np.any(ix >= len(ids)) or not np.array_equal(ids[ix], q["raw_ids"]): + raise ValueError("Validation query does not map to original raw IDs") + with (out / "validation-target-access.jsonl").open("a") as log: + log.write(json.dumps(dict(case=case, role="validation", predictions_sha256=manifest_hash, + target_sha256=pilot["parts"][case]["output_sha256"]["seconds.npy"])) + "\n") + truth = cached(view, case, "seconds")[ix] + scores = {} + for arm in ARMS: + row = predictions[case][arm] + pred = np.load(checked(row["file"], row["sha256"]), allow_pickle=False)[ix] + value = relative_l2(pred, truth, q["counts"], np.ones(len(ix), dtype=bool))["value"] + if value is None or not math.isfinite(value): + raise ValueError("Undefined validation part metric") + scores[arm] = value + component = roster[case]["component_id"] + grouped[component].append([scores[arm] for arm in ARMS]) + rows.append(dict(case=case, component_id=component, relative_l2=scores)) + means = dict(zip(ARMS, np.mean([np.mean(a, axis=0) for a in grouped.values()], axis=0).tolist())) + delta = means["short_seed0"] - means["long_seed0"] + seed_delta = means["long_seed1"] - means["long_seed0"] + passed = delta <= comparison["delta_short_at_most"] and abs(delta) > abs(seed_delta) + save(out / "result.json", dict( + status="COMPLETE", decision="SCREEN_PASS" if passed else "SCREEN_FAIL", + parts=len(rows), components=len(grouped), component_equal_relative_l2=means, + delta_short=delta, delta_seed=seed_delta, rows=rows, predictions_sha256=manifest_hash, + test_targets_read=False, automatic_continuation=False, **peaks())) + + +def dispatch_guard(root, contract_path): + """The same entry consumes the generated contract in production and preflight.""" + began_entry = time.monotonic() + c = contract(contract_path) + folded_inputs(c) + budget = c["budget"] + gpu = c["gpu_uuid"] + charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0., + screen_gpu_seconds=0., finalization_gpu_seconds=0.) + receipts = [] + status = "STOPPED" + trigger = "STOP_BEFORE_DISPATCH" + try: + if shutil.disk_usage(root).free < c["disk_bytes"]: + raise RuntimeError("Insufficient free disk for the fixed attempt") + began_wait = time.monotonic() + try: + idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot) + except BaseException: + charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait + raise + charges["queue_wait_gpu_seconds"] = idle["wall_seconds"] + save(root / "idle-gate.json", idle) + trigger = idle["status"] + if idle["status"] != "IDLE_CONFIRMED" or not snapshot(gpu)["idle"]: + if idle["status"] == "IDLE_CONFIRMED": + trigger = "SELECTED_GPU_BUSY_AFTER_IDLE_GATE" + return 1 + (root / "jobs/0").mkdir(parents=True, exist_ok=False) + for mode, cap_key, charge_key, output in ( + ("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"), + ("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"), + ): + began = time.monotonic() + if not snapshot(gpu)["idle"]: + trigger = "SELECTED_GPU_BUSY_BEFORE_" + mode.upper() + charges[charge_key] = time.monotonic() - began + return 1 + command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)] + tracker = PeakRecords() + done = threading.Event() + reasons = [] + + def watch(): + while not done.is_set(): + try: + tracker.scan(output) + state = snapshot(gpu) + pids = set(active_command_pids(command)) + if max(tracker.maxima.values()) > c["recorded_vram_bytes"]: + reasons.append("RECORDED_VRAM_LIMIT") + if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]: + reasons.append("DEVICE_VRAM_LIMIT") + if any(p["pid"] not in pids for p in state["occupants"]): + reasons.append("FOREIGN_GPU_OCCUPANT") + except Exception as exc: + reasons.append(f"MONITOR_FAILURE: {exc}") + if reasons: + for pid in active_command_pids(command): + stop_group(pid) + # Also catch a child spawned just after the failing + # sample while the supervisor was taking its lock. + done.wait(.1) + continue + done.wait(.5) + + monitor = threading.Thread(target=watch, daemon=True) + monitor.start() + try: + if reasons: + raise RuntimeError(reasons[0]) + cap = budget[cap_key] - (time.monotonic() - began) - 10 + receipt = supervise(command, root, mode, dict( + total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key], + job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"], + disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]), + poll=.2, gpu_uuid=gpu) + finally: + done.set() + monitor.join(timeout=8) + charges[charge_key] = time.monotonic() - began + if monitor.is_alive(): + reasons.append("MONITOR_DID_NOT_EXIT") + else: + tracker.scan(output) + if max(tracker.maxima.values()) > c["recorded_vram_bytes"]: + reasons.append("RECORDED_VRAM_LIMIT") + if set(tracker.witnesses) != set(tracker.maxima): + reasons.append("MISSING_RECORDED_VRAM_PEAKS") + receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima)) + if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0 + or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]): + worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT" + trigger = reasons[0] if reasons else worker_reason + return 1 + result_path = output / "result.json" + result = read(result_path) + if result["status"] != "COMPLETE" or result["test_targets_read"] is not False: + raise RuntimeError("Unexpected worker result") + if mode == "train": + if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"] + or result["optimizer_steps"] != c["job"]["target_steps"]): + raise RuntimeError("Fold-0 final checkpoint identity differs") + checked(result["checkpoint"], result["checkpoint_sha256"]) + save(root / "training-adopted.json", dict(result_sha256=sha(result_path))) + elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]: + raise RuntimeError("Validation screen scope differs") + overhead = time.monotonic() - began_entry - sum(charges.values()) + if overhead > budget["finalization_gpu_seconds"]: + raise RuntimeError("Setup and finalization cost exceeded the fixed allowance") + status, trigger = "COMPLETE", result["decision"] + return 0 + except Exception as exc: + trigger = f"{type(exc).__name__}: {exc}" + return 1 + except KeyboardInterrupt: + trigger = "COORDINATOR_INTERRUPTED" + return 1 + finally: + charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values())) + new_gpu = sum(charges.values()) + save(root / "complete.json", dict( + status=status, trigger=trigger, receipts=receipts, **charges, + cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu, + cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu, + no_retry=True, test_targets_read=False, automatic_continuation=False)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("run", "train", "screen")) + parser.add_argument("root", type=Path) + args = parser.parse_args() + root = args.root.resolve() + if root != ROOT: + raise PermissionError("Only the new fixed fold-0 root is accepted") + expected_python = str(ROOT.parents[1] / "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python") + if sys.executable != expected_python: + raise PermissionError("Use the fixed original H2 environment") + validate_review(root) + contract_path = root / "contract.json" + if args.mode != "run": + coordinator = read(root / "started.json") + if process_identity(os.getppid()) != coordinator["coordinator"]: + raise PermissionError("Worker requires the live guard as its parent") + if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "Cannot bind worker lifetime to guard") + if os.getppid() != coordinator["coordinator"]["pid"]: + raise RuntimeError("Guard disappeared before worker setup") + return globals()[args.mode](root, contract(contract_path)) + if os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR: + raise PermissionError("Explicit original CUDA allocator setup required") + with (root / "entry.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if (root / "started.json").exists(): + raise FileExistsError("Fold-0 attempt already started") + save(root / "started.json", dict(coordinator=process_identity(os.getpid()), + contract_sha256=sha(contract_path), unix=time.time())) + def request_stop(signum, _frame): + raise KeyboardInterrupt(signal.Signals(signum).name) + + old_handler = signal.signal(signal.SIGTERM, request_stop) + try: + return dispatch_guard(root, contract_path) + finally: + signal.signal(signal.SIGTERM, old_handler) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/prepare_phase3_item1b_fold0.py b/tools/prepare_phase3_item1b_fold0.py new file mode 100644 index 000000000..b9e06c20d --- /dev/null +++ b/tools/prepare_phase3_item1b_fold0.py @@ -0,0 +1,234 @@ +"""Freeze the new fold-0 contract and exercise its actual guard on CPU.""" + +from functools import partial +import math +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from phase2_h1_worker_v2 import checked, save, sha +from phase3_item1b_common import ROOT as OLD_ROOT, read +import phase3_item1b_fold0 as entry +from s2_gpu_idle_gate import wait_idle +from track_a_h2_prepare import local_code_closure + + +BASE = entry.ROOT.parents[1] +RUNS = OLD_ROOT.parent +SEED1 = RUNS / "phase3-seed1-v2-2026-11-18" + + +def prepare(): + root = entry.ROOT + if root.exists(): + raise FileExistsError("New fold-0 preparation already exists") + pins = {} + + def pin(path): + p = Path(path) + pins[str(p)] = sha(p) + return read(p) + + old = pin(OLD_ROOT / "contract.json") + stop = pin(OLD_ROOT / "complete.json") + settled = pin(OLD_ROOT / "post-terminal-cost.json") + if (stop["failure"] != "RuntimeError: TOTAL_PROJECTION_OR_ACTUAL_CAP" + or settled["training_complete_sha256"] != pins[str(OLD_ROOT / "complete.json")] + or settled["postfit_started"] is not False or not settled["freeze_tail_charged_once"]): + raise ValueError("Historical budget-stop record differs") + totals = { + "item1": pin(RUNS / "phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json")[ + "cumulative_item_gpu_seconds"], + "seed1": pin(RUNS / "phase3-seed1-postfit-2026-11-18/post-terminal-cost.json")[ + "cumulative_seed1_gpu_seconds"], + "item1b": settled["cumulative_gpu_seconds"], + "c": pin(RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[ + "cumulative_item_gpu_seconds"], + "b": pin(RUNS / "phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[ + "cumulative_item_gpu_seconds"], + } + history_path = OLD_ROOT / "jobs/0/worker/history/001.json" + epoch = pin(history_path) + assignments = {i: pin(OLD_ROOT / f"jobs/{i}/assignment.json") for i in (0, 2)} + identities = {i: pin(OLD_ROOT / f"jobs/{i}/worker/worker-identity.json") for i in (0, 2)} + start = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.started.json")["started_unix"] for i in (0, 2)} + elapsed = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.receipt.json")["wall_seconds"] for i in (0, 2)} + gpu = assignments[0]["gpu_uuid"] + sample = pin(OLD_ROOT / "jobs/0/gpu-sample.json") + observed = {row["pid"] for row in sample["rows"]} + if (assignments[2]["gpu_uuid"] != gpu or len(observed) != 3 + or not {p["pid"] for p in identities.values()} <= observed + or not max(start.values()) < sample["unix"] < min(start[i] + elapsed[i] for i in start) + or epoch["optimizer_steps"] != len(old["jobs"][0]["learning"]) + or epoch["stage"] != "EPOCH_COMPLETE"): + raise ValueError("Missing same-card fold-0 timing witness") + solo = [] + for obs_path in sorted((SEED1 / "jobs/0/epoch-host-observations").glob("*.json")): + obs = read(obs_path) + on_card = [p for p in obs["gpu_processes"] if p["on_assigned_gpu"]] + if (obs["status"] != "OBSERVED" or obs["errors"] or obs["assigned_gpu_uuid"] != gpu + or len(on_card) != 1 or not on_card[0]["own_v2_worker"]): + continue + row_path = Path(obs["receipt_path"]) + row = read(checked(row_path, obs["receipt_sha256"])) + solo.append((obs_path, row_path, row)) + if not solo: + raise ValueError("Missing fold-0 diagnostic duration and comparison envelope") + longest_train = max(solo, key=lambda x: x[2]["train_epoch_seconds"]) + longest_val = max(solo, key=lambda x: x[2].get("validation_seconds", 0)) + for observation, receipt, _ in (longest_train, longest_val): + pin(observation) + pin(receipt) + train_basis = max(epoch["train_epoch_seconds"], longest_train[2]["train_epoch_seconds"]) + val_basis = max(old["jobs"][0]["validation_seconds_bound"], longest_val[2]["validation_seconds"]) + # The explicit compute margin applies once. Setup, bounded waiting and the + # terminal allowance do not inflate an individual training-epoch time. + train_setup = 120 + train_cap = math.ceil(1.25 * (100 * train_basis + 10 * val_basis)) + train_setup + screen_cap = math.ceil(1.25 * 3 * val_basis) + 120 + budget = dict( + item_gpu_limit_seconds=90000, phase3_gpu_limit_seconds=720000, + prior_item_gpu_seconds=totals["item1b"], prior_phase3_gpu_seconds=sum(totals.values()), + train_epoch_seconds_basis=train_basis, validation_seconds_basis=val_basis, + train_gpu_seconds=train_cap, screen_gpu_seconds=screen_cap, + training_setup_tail_seconds=train_setup, idle_wait_seconds=60, finalization_gpu_seconds=120, + projected_new_gpu_seconds=train_cap + screen_cap + 60 + 120, + ) + if budget["projected_new_gpu_seconds"] > 90000 - totals["item1b"]: + raise ValueError(f"Fold-0 cap exceeds remaining authorization: {budget}") + + pilot = pin(old["pilot_contract"]) + roster = pin(old["query_roster_contract"]) + fold = pilot["folds"][0] + rows = {r["case"]: r for r in roster["rows"] if r["case"] in fold["validation"]} + if set(rows) != set(fold["validation"]): + raise ValueError("Fold-0 validation roster incomplete") + references = {} + baseline = old["baseline_models"][0] + workers = [Path(baseline["result"]).parent, SEED1 / "jobs/0/worker"] + roots = [workers[0].parent, SEED1] + for seed, worker in enumerate(workers): + run_contract = roots[seed] / "contract.json" + run_meta = pin(run_contract) + result = pin(worker / "result.json") + stats_path = worker / "standardization.json" + stats = pin(stats_path) + checked(stats_path, result["standardization_sha256"]) + checkpoint = checked(result["checkpoint"], result["checkpoint_sha256"]) + pins[str(checkpoint)] = result["checkpoint_sha256"] + if (run_meta["seed"] != seed or result["completed_epochs"] != 200 + or result["optimizer_steps"] != 200 * len(fold["learning"]) + or stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]): + raise ValueError("Reference final checkpoint scope differs") + references[f"long_seed{seed}"] = dict( + checkpoint=str(checkpoint), checkpoint_sha256=result["checkpoint_sha256"], + standardization=str(stats_path), standardization_sha256=result["standardization_sha256"], + seed=seed, epochs=200, training_contract_sha256=pins[str(run_contract)]) + + root.mkdir(exist_ok=False) + comparison = root / "comparison-contract.json" + save(comparison, dict( + ruling=entry.RULING, fold=0, validation_cases=fold["validation"], + validation_components=len({row["component_id"] for row in rows.values()}), + score="Relative L2 on the frozen validation query with original PPS multiplicity, then " + "equal means by component. Negative predictions remain visible.", + paired_references=list(references), delta_short_at_most=-.001, + effect_strictly_greater_than_abs_seed_delta=True, duration="100 complete epochs, final epoch only", + warmup="Original 5-epoch warmup and full cosine decay over 95 epochs", + train_fields="Original seed0, order, 19 inputs, optimizer, MSE and numerical recipe", + test_allowed=False, automatic_continuation=False)) + codes = local_code_closure([Path(__file__), Path(entry.__file__)] + + list((BASE / "tools/s2_models").glob("*.py"))) + for path, digest in old["code_sha256"].items(): + checked(path, digest) + codes.setdefault(path, digest) + source = root / "timing-and-budget.json" + save(source, dict( + prior_phase3_costs=totals, budget=budget, remaining_item_gpu_seconds=90000 - totals["item1b"], + same_card_fold0_epoch_seconds=epoch["train_epoch_seconds"], gpu_uuid=gpu, + same_card_train_peers_instantiated={str(i): p["pid"] for i, p in identities.items()}, + sampled_cuda_pids=sorted(observed), sample_unix=sample["unix"], worker_intervals=[ + dict(fold=i, started_unix=start[i], ended_unix=start[i] + elapsed[i]) for i in start], + same_fold_solo_epoch_records=len(solo), longest_solo_train_seconds=longest_train[2]["train_epoch_seconds"], + longest_solo_validation_seconds=longest_val[2]["validation_seconds"], + limitation="The two workers started together. Shared occupancy is sampled near epoch " + "completion. The third CUDA context is observed only at that sample.", + target_admission="Selected V100 starts idle, with stop on foreign occupancy. " + "The co-located trace is a conservative budget input.")) + pins[str(source)] = sha(source) + contract = root / "contract.json" + save(contract, dict( + version="phase3-item1b-fold0-screen-v1", ruling=entry.RULING, fold=0, screen_fold=0, seed=0, + epochs=100, max_fits=1, retry=False, test_targets_allowed=False, gpu_uuid=gpu, + worker_cpus=assignments[0]["worker_cpus"], total_cpu_seconds=30000, max_processes=1, + rss_bytes=4 * entry.GIB, disk_bytes=8 * entry.GIB, device_vram_bytes=4 * entry.GIB, + recorded_vram_bytes=2 * entry.GIB, job=old["jobs"][0], budget=budget, references=references, + pilot_contract=old["pilot_contract"], query_roster_contract=old["query_roster_contract"], + comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison), + source_sha256=pins, code_sha256=codes)) + return contract + + +def preflight(contract): + c = entry.contract(contract) + results = [] + for busy in (False, True): + simulated_time = 0. + + def sleep(seconds): + nonlocal simulated_time + simulated_time += seconds + + def snapshot(uuid): + if uuid != c["gpu_uuid"]: + raise ValueError("Incorrect GPU identity") + return dict(uuid=uuid, idle=not busy, memory_MiB=1500 if busy else 5, occupants=[]) + + def simulate(command, guard_root, mode, limits, *, poll, gpu_uuid): + if (guard_root != root or command[2:] != [mode, str(root)] or gpu_uuid != c["gpu_uuid"] + or not 0 < limits["job_wall_seconds"] < c["budget"][f"{mode}_gpu_seconds"] + or limits["tree_rss_bytes"] != c["rss_bytes"] or limits["disk_bytes"] != c["disk_bytes"] + or limits["max_processes"] != c["max_processes"] + or limits["total_cpu_seconds"] != c["total_cpu_seconds"]): + raise ValueError("Actual guard did not propagate frozen fields") + record = dict(status="COMPLETE", test_targets_read=False, peak_allocated_bytes=1024, + peak_reserved_bytes=2048) + if mode == "train": + out = root / "jobs/0/worker" + out.mkdir() + checkpoint = out / "simulated-checkpoint" + checkpoint.write_bytes(b"CPU mock only") + record.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"], + checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint)) + else: + out = root / "screen" + out.mkdir() + record.update(parts=len(c["job"]["validation"]), automatic_continuation=False, + decision="SCREEN_PASS") + save(out / "result.json", record) + return dict(status="PASS", returncode=0, monitor_errors=0) + + with TemporaryDirectory(prefix="meltmind-fold0-guard-") as folder: + root = Path(folder) + with (patch.object(entry, "snapshot", side_effect=snapshot), + patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: simulated_time, sleep=sleep)), + patch.object(entry, "supervise", side_effect=simulate) as dispatch): + exit_code = entry.dispatch_guard(root, contract) + terminal = read(root / "complete.json") + gate = read(root / "idle-gate.json") + expected = (1, "IDLE_TIMEOUT", []) if busy else (0, "SCREEN_PASS", ["train", "screen"]) + actual = (exit_code, terminal["trigger"], [call.args[2] for call in dispatch.call_args_list]) + if actual != expected: + raise RuntimeError(f"Actual guard CPU simulation differed: {actual}") + results.append(dict(case="busy-timeout" if busy else "idle-dispatch", + exit_code=exit_code, trigger=terminal["trigger"], mock_dispatches=actual[2], + gate_wait_seconds=gate["wall_seconds"])) + return dict(status="PASS", actual_contract_sha256=sha(contract), cases=results, + real_gpu_or_worker_started=False, scientific_arrays_read=False, review_status="PENDING") + + +if __name__ == "__main__": + path = prepare() + receipt = preflight(path) + save(path.parent / "cpu-guard-preflight.json", receipt) + print(receipt)