diff --git a/artifacts/phase3-item1b-fold0-deadline-2026-11-19/actual-guard-cpu-replay.json b/artifacts/phase3-item1b-fold0-deadline-2026-11-19/actual-guard-cpu-replay.json new file mode 100644 index 000000000..e4e1bcdd1 --- /dev/null +++ b/artifacts/phase3-item1b-fold0-deadline-2026-11-19/actual-guard-cpu-replay.json @@ -0,0 +1,171 @@ +{ + "status": "PASS", + "contract_sha256": "25cb8ce8ba1b43695fb5c229459033d6c5f0c79a9e67e156b9e78353b9a89971", + "old_binding_sha256": "ed6bfe113f9c0bf925684a62cf0c1bb09bfc7b1277e06fad628fc0e9e5bcd859", + "late_recheck": { + "gate": "IDLE_CONFIRMED", + "gate_seconds": 59.0, + "snapshot_start_seconds": [ + 0.0, + 11.5, + 23.0, + 34.5, + 46.0, + 57.5, + 59.0 + ], + "mock_dispatches": [], + "returned": 1, + "terminal": { + "status": "STOPPED", + "trigger": "TRAIN_DISPATCH_DEADLINE", + "underlying_trigger": "TRAIN_DISPATCH_DEADLINE", + "queue_wait_gpu_seconds": 59.0, + "training_gpu_seconds": 0.0, + "screen_gpu_seconds": 0.0, + "finalization_gpu_seconds": 1.5, + "new_gpu_seconds": 60.5, + "new_gpu_cap_seconds": 13180, + "cap_overage_gpu_seconds": 0.0, + "cumulative_item_gpu_seconds": 19322.557604262838, + "cumulative_phase3_gpu_seconds": 408690.9104626046, + "no_retry": true, + "test_targets_read": false, + "automatic_continuation": false + } + }, + "final_settlement": { + "gate": "IDLE_CONFIRMED", + "gate_seconds": 59.0, + "snapshot_start_seconds": [ + 0.0, + 11.5, + 23.0, + 34.5, + 46.0, + 57.5, + 59.0, + 59.0, + 12984.9 + ], + "mock_dispatches": [ + { + "mode": "train", + "seconds": 59.0, + "dispatch_deadline": 60.0 + }, + { + "mode": "screen", + "seconds": 12984.9, + "dispatch_deadline": null + } + ], + "returned": 1, + "terminal": { + "status": "STOPPED", + "trigger": "TOTAL_GPU_CAP_AT_SETTLEMENT", + "underlying_trigger": "SCREEN_PASS", + "queue_wait_gpu_seconds": 59.0, + "training_gpu_seconds": 12806.9, + "screen_gpu_seconds": 172.89999999999964, + "finalization_gpu_seconds": 142.0, + "new_gpu_seconds": 13180.8, + "new_gpu_cap_seconds": 13180, + "cap_overage_gpu_seconds": 0.7999999999992724, + "cumulative_item_gpu_seconds": 32442.857604262837, + "cumulative_phase3_gpu_seconds": 421811.2104626046, + "no_retry": true, + "test_targets_read": false, + "automatic_continuation": false + } + }, + "timely_control": { + "gate": "IDLE_CONFIRMED", + "gate_seconds": 59.0, + "snapshot_start_seconds": [ + 0.0, + 11.5, + 23.0, + 34.5, + 46.0, + 57.5, + 59.0, + 59.0, + 12984.9 + ], + "mock_dispatches": [ + { + "mode": "train", + "seconds": 59.0, + "dispatch_deadline": 60.0 + }, + { + "mode": "screen", + "seconds": 12984.9, + "dispatch_deadline": null + } + ], + "returned": 0, + "terminal": { + "status": "COMPLETE", + "trigger": "SCREEN_PASS", + "underlying_trigger": "SCREEN_PASS", + "queue_wait_gpu_seconds": 59.0, + "training_gpu_seconds": 12806.9, + "screen_gpu_seconds": 172.89999999999964, + "finalization_gpu_seconds": 119.0, + "new_gpu_seconds": 13157.8, + "new_gpu_cap_seconds": 13180, + "cap_overage_gpu_seconds": 0.0, + "cumulative_item_gpu_seconds": 32419.857604262837, + "cumulative_phase3_gpu_seconds": 421788.2104626046, + "no_retry": true, + "test_targets_read": false, + "automatic_continuation": false + } + }, + "busy_control": { + "gate": "IDLE_TIMEOUT", + "gate_seconds": 62.0, + "snapshot_start_seconds": [ + 0.0, + 12.0, + 24.0, + 36.0, + 48.0, + 60.0 + ], + "mock_dispatches": [], + "returned": 1, + "terminal": { + "status": "STOPPED", + "trigger": "IDLE_TIMEOUT", + "underlying_trigger": "IDLE_TIMEOUT", + "queue_wait_gpu_seconds": 62.0, + "training_gpu_seconds": 0.0, + "screen_gpu_seconds": 0.0, + "finalization_gpu_seconds": 0.0, + "new_gpu_seconds": 62.0, + "new_gpu_cap_seconds": 13180, + "cap_overage_gpu_seconds": 0.0, + "cumulative_item_gpu_seconds": 19324.057604262838, + "cumulative_phase3_gpu_seconds": 408692.4104626046, + "no_retry": true, + "test_targets_read": false, + "automatic_continuation": false + } + }, + "in_supervisor_deadline": { + "marker": { + "trigger": "TRAIN_DISPATCH_DEADLINE", + "worker_started": false, + "dispatch_deadline_monotonic": 60.0, + "rejected_monotonic": 60.5 + }, + "real_popen_calls": 0 + }, + "real_gpu_queries": 0, + "real_workers": 0, + "targets_read": false, + "independent_review": "PENDING" +} diff --git a/artifacts/phase3-item1b-fold0-deadline-2026-11-19/contract.json b/artifacts/phase3-item1b-fold0-deadline-2026-11-19/contract.json new file mode 100644 index 000000000..60de85024 --- /dev/null +++ b/artifacts/phase3-item1b-fold0-deadline-2026-11-19/contract.json @@ -0,0 +1,736 @@ +{ + "version": "phase3-item1b-fold0-deadline-settlement-v2", + "ruling": "3a48b123da54b0bcff0e7fa672fc74e8920235a0", + "fold": 0, + "screen_fold": 0, + "seed": 0, + "epochs": 100, + "max_fits": 1, + "retry": false, + "test_targets_allowed": false, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "worker_cpus": [ + 22 + ], + "total_cpu_seconds": 30000, + "max_processes": 1, + "rss_bytes": 4294967296, + "disk_bytes": 8589934592, + "device_vram_bytes": 4294967296, + "recorded_vram_bytes": 2147483648, + "job": { + "learning": [ + "00000003_1ffb81a71e5b402e966b9341_step_002", + "00000004_1ffb81a71e5b402e966b9341_step_003", + "00000049_aa208c45e8734584a147f595_step_000", + "00000057_666139e3bff64d4e8a6ce183_step_006", + "00000058_666139e3bff64d4e8a6ce183_step_007", + "00000060_767e4372b5f94a88a7a17d90_step_001", + "00000061_767e4372b5f94a88a7a17d90_step_002", + "00000065_767e4372b5f94a88a7a17d90_step_006", + "00000066_767e4372b5f94a88a7a17d90_step_007", + "00000067_767e4372b5f94a88a7a17d90_step_008", + "00000068_767e4372b5f94a88a7a17d90_step_009", + "00000070_767e4372b5f94a88a7a17d90_step_011", + "00000072_767e4372b5f94a88a7a17d90_step_013", + "00000077_767e4372b5f94a88a7a17d90_step_018", + "00000078_767e4372b5f94a88a7a17d90_step_019", + "00000079_767e4372b5f94a88a7a17d90_step_020", + "00000087_b4f9335a60a14cecaa599159_step_002", + "00000088_169bec34317b46e98eb1a403_step_000", + "00000093_169bec34317b46e98eb1a403_step_005", + "00000098_72cf0f9ca5324822a4d41a80_step_000", + "00000103_52f7124261ae4c558f800775_step_000", + "00000104_52f7124261ae4c558f800775_step_001", + "00000105_52f7124261ae4c558f800775_step_002", + "00000106_52f7124261ae4c558f800775_step_003", + "00000125_e18c7feacfb54f3880659b43_step_000", + "00000128_e18c7feacfb54f3880659b43_step_003", + "00000129_e18c7feacfb54f3880659b43_step_004", + "00000130_e18c7feacfb54f3880659b43_step_005", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000142_2fc54fcd110d4f49969163c4_step_005", + "00000144_2fc54fcd110d4f49969163c4_step_007", + "00000145_2fc54fcd110d4f49969163c4_step_008", + "00000152_a951550b8a1043a68423c541_step_002", + "00000153_a951550b8a1043a68423c541_step_003", + "00000154_a951550b8a1043a68423c541_step_004", + "00000157_8208fd8b44b64dce91322f84_step_000", + "00000161_a583c6c3cdc143bab3f6c856_step_000", + "00000162_4dcda9fab6dd46cf98fb194e_step_000", + "00000163_ff9b2ed255484db78bb0e6da_step_000", + "00000164_91adbf180e114b698160c9a4_step_000", + "00000165_4070a99d6ebe45f5bd5d71eb_step_000", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00000170_b21aa794b6b64d87a57c245e_step_000", + "00000171_b21aa794b6b64d87a57c245e_step_001", + "00000173_b21aa794b6b64d87a57c245e_step_003", + "00000174_b21aa794b6b64d87a57c245e_step_004", + "00000175_b21aa794b6b64d87a57c245e_step_005", + "00000176_b21aa794b6b64d87a57c245e_step_006", + "00000181_b21aa794b6b64d87a57c245e_step_011", + "00000198_826494e894454504b0971fd1_step_001", + "00000223_97b354a907114b7183faa9c4_step_001", + "00000224_97b354a907114b7183faa9c4_step_002", + "00000229_682d529d5e8c4387bc59b3a1_step_000", + "00000250_641f03cbe6a145508c6ae055_step_000", + "00000272_648a7c23a21f4e88bf25ffeb_step_000", + "00000273_648a7c23a21f4e88bf25ffeb_step_001", + "00000274_648a7c23a21f4e88bf25ffeb_step_002", + "00000275_648a7c23a21f4e88bf25ffeb_step_003", + "00000276_648a7c23a21f4e88bf25ffeb_step_004", + "00000280_5a6ef3285b89452ea6b390e7_step_001", + "00000282_5a6ef3285b89452ea6b390e7_step_003", + "00000285_d215b7bc60ae468288e5f1ea_step_000", + "00000287_58f8ff46b7564e989f9695b7_step_000", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00000291_58f8ff46b7564e989f9695b7_step_004", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00000295_52a153d2417747b5926ff9df_step_000", + "00000296_52a153d2417747b5926ff9df_step_001", + "00000301_52a153d2417747b5926ff9df_step_006", + "00000302_39b08d62a677414f86fdae52_step_000", + "00000329_12a935cd9b684f879c0505fe_step_000", + "00000330_12a935cd9b684f879c0505fe_step_001", + "00000331_12a935cd9b684f879c0505fe_step_002", + "00000336_1a821c3883f7459a9615fca4_step_000", + "00000337_153efbd85ca64c4fb6b7ed5f_step_000", + "00000346_4185972a944744d8a7a0f2b4_step_002", + "00000348_4185972a944744d8a7a0f2b4_step_004", + "00000352_4185972a944744d8a7a0f2b4_step_008", + "00000355_68d1a5c2319a4a419ff625c6_step_001", + "00000356_68d1a5c2319a4a419ff625c6_step_002", + "00000357_68d1a5c2319a4a419ff625c6_step_003", + "00000364_38d899e2706a4d09960705dd_step_001", + "00000367_23c957e8e7b2428282a13ae2_step_000", + "00000368_23c957e8e7b2428282a13ae2_step_001", + "00000369_23c957e8e7b2428282a13ae2_step_002", + "00000374_23c957e8e7b2428282a13ae2_step_007", + "00000375_23c957e8e7b2428282a13ae2_step_008", + "00000376_23c957e8e7b2428282a13ae2_step_009", + "00000377_23c957e8e7b2428282a13ae2_step_010", + "00000378_23c957e8e7b2428282a13ae2_step_011", + "00000379_239bbf0e92974a32bd214366_step_000", + "00000382_239bbf0e92974a32bd214366_step_003", + "00000383_239bbf0e92974a32bd214366_step_004", + "00000386_239bbf0e92974a32bd214366_step_007", + "00000387_239bbf0e92974a32bd214366_step_008", + "00000388_239bbf0e92974a32bd214366_step_009", + "00000418_ce5b31679ee845a69ba89d6f_step_000", + "00000422_8e8a96d437584c77b3cfd84f_step_000", + "00000423_8e8a96d437584c77b3cfd84f_step_001", + "00000424_8e8a96d437584c77b3cfd84f_step_002", + "00000459_516b1784884d4facbc4c0d74_step_007", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000466_11951e51b89249dba38ded3e_step_001", + "00000469_11951e51b89249dba38ded3e_step_004", + "00000470_11951e51b89249dba38ded3e_step_005", + "00000474_a5639f2b84c445aea5da18ad_step_000", + "00000477_0fc8cc17c6af48dabe36c8f2_step_002", + "00000478_0fc8cc17c6af48dabe36c8f2_step_003", + "00000479_0fc8cc17c6af48dabe36c8f2_step_004", + "00000485_a7d0c851b5f24a97a7cac950_step_002", + "00000488_a7d0c851b5f24a97a7cac950_step_005", + "00000489_a7d0c851b5f24a97a7cac950_step_006", + "00000492_a7d0c851b5f24a97a7cac950_step_009", + "00000513_f78bf7cf156f4231988af062_step_001", + "00000519_5b8fa6b193d14279a77a0e50_step_000", + "00000523_1f8d7f343e634c11a80d23ac_step_000", + "00000526_f9d6559308dc4c85aab9fa38_step_000", + "00000530_bc0d6937206b43bc96521413_step_002", + "00000531_bc0d6937206b43bc96521413_step_003", + "00000543_8cf6554d440d40c4823dad6c_step_000", + "00000559_a1621e156ade4317ab6a5bc5_step_000", + "00000565_5b5119548f7b421e8f597438_step_000", + "00000609_04e799936ae446b9a3d8d2b8_step_000", + "00000610_04e799936ae446b9a3d8d2b8_step_001", + "00000611_04e799936ae446b9a3d8d2b8_step_002", + "00000613_189d1310aa8c4a2a92a21020_step_001", + "00000615_c710d414bbbc4340b682c58a_step_001", + "00000616_c710d414bbbc4340b682c58a_step_002", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000618_c710d414bbbc4340b682c58a_step_004", + "00000619_c710d414bbbc4340b682c58a_step_005", + "00000620_c710d414bbbc4340b682c58a_step_006", + "00000621_c710d414bbbc4340b682c58a_step_007", + "00000622_c710d414bbbc4340b682c58a_step_008", + "00000624_1eacb269d52a47f5b265c1ab_step_000", + "00000626_f686126d229f47aeb125b5ba_step_000", + "00000627_53bfa7e2e04c4d9dbf4a3db2_step_000", + "00000632_4f322b34e87c43289916c298_step_001", + "00000658_a40da330321b44f286a27361_step_000", + "00000683_8863c3497d6d4e05b0f41ddc_step_000", + "00000685_24615f5ff44d4cd5809e77a5_step_001", + "00000686_24615f5ff44d4cd5809e77a5_step_002", + "00000688_b9f3563b403143dbb86ab2b8_step_001", + "00000691_b9f3563b403143dbb86ab2b8_step_004", + "00000693_b9f3563b403143dbb86ab2b8_step_006", + "00000694_45f6100bfa3948e69405bb21_step_000", + "00000696_32309198fce9481896bd0a99_step_000", + "00000698_6f0fc559e8f3422c908b6060_step_001", + "00000699_cae88f0aaaf34de597863810_step_000", + "00000708_c08c1c78b86f4ee3bcb7d9de_step_004", + "00000709_c08c1c78b86f4ee3bcb7d9de_step_005", + "00000710_c08c1c78b86f4ee3bcb7d9de_step_006", + "00000721_9a461e8ae7dc4d33a55883eb_step_000", + "00000723_8ed79b6bcb024d1a9f2aa5e7_step_000", + "00000737_989e0f807cde438face0cb00_step_002", + "00000738_989e0f807cde438face0cb00_step_003", + "00000751_6f0d90054d40400593050930_step_000", + "00000752_6f0d90054d40400593050930_step_001", + "00000764_9cab8335065e4d439bbff471_step_000", + "00000765_9cab8335065e4d439bbff471_step_001", + "00000766_9cab8335065e4d439bbff471_step_002", + "00000767_9cab8335065e4d439bbff471_step_003", + "00000775_a25fb37392a44447b9684f16_step_000", + "00000776_a25fb37392a44447b9684f16_step_001", + "00000777_a25fb37392a44447b9684f16_step_002", + "00000778_a25fb37392a44447b9684f16_step_003", + "00000779_a25fb37392a44447b9684f16_step_004", + "00000780_a25fb37392a44447b9684f16_step_005", + "00000781_a25fb37392a44447b9684f16_step_006", + "00000782_a25fb37392a44447b9684f16_step_007", + "00000783_a25fb37392a44447b9684f16_step_008", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000785_a25fb37392a44447b9684f16_step_010", + "00000786_a25fb37392a44447b9684f16_step_011", + "00000787_a25fb37392a44447b9684f16_step_012", + "00000788_a25fb37392a44447b9684f16_step_013", + "00000789_a25fb37392a44447b9684f16_step_014", + "00000790_a25fb37392a44447b9684f16_step_015", + "00000791_a25fb37392a44447b9684f16_step_016", + "00000792_a25fb37392a44447b9684f16_step_017", + "00000793_a25fb37392a44447b9684f16_step_018", + "00000797_e7533c87b9a347cdafd0c390_step_000", + "00000799_ad6107d9fc994ccca38cc4e4_step_001", + "00000800_ad6107d9fc994ccca38cc4e4_step_002", + "00000801_ad6107d9fc994ccca38cc4e4_step_003", + "00000802_ad6107d9fc994ccca38cc4e4_step_004", + "00000818_6bbad2756d7845bd8d731d60_step_000", + "00000819_6bbad2756d7845bd8d731d60_step_001", + "00000821_6bbad2756d7845bd8d731d60_step_003", + "00000822_c98978894f544f89aa10633f_step_000", + "00000823_c98978894f544f89aa10633f_step_001", + "00000827_b0d209f7a8ad4b02910c317f_step_003", + "00000829_889a065d51a24bb8ad704c69_step_000", + "00000830_889a065d51a24bb8ad704c69_step_001", + "00000832_889a065d51a24bb8ad704c69_step_003", + "00000834_889a065d51a24bb8ad704c69_step_005", + "00000837_b5409201f9a5495fa169af85_step_000", + "00000889_e2d3a4dc5f7c438bb240e9d1_step_000", + "00000890_e2d3a4dc5f7c438bb240e9d1_step_001", + "00000900_4df6ab9924674556bb4a509d_step_000", + "00000904_4df6ab9924674556bb4a509d_step_004", + "00000905_4df6ab9924674556bb4a509d_step_005", + "00000906_4df6ab9924674556bb4a509d_step_006", + "00000942_f297a24e1a4446cf9d74efd9_step_001", + "00000948_f297a24e1a4446cf9d74efd9_step_007", + "00000949_f297a24e1a4446cf9d74efd9_step_008", + "00000953_ccc14f0fcff54b5985cc9113_step_000", + "00000958_04906edf324a4bce8fc3f417_step_000", + "00000961_a98d481268ea4524a7769731_step_000", + "00000962_a98d481268ea4524a7769731_step_001", + "00000963_a98d481268ea4524a7769731_step_002", + "00000973_209426c5054945fab8a4af1b_step_000", + "00000976_b85ec2ff588148c4abd8c8d7_step_000", + "00001024_21adcacb9707488ea6655fd2_step_000", + "00001025_21adcacb9707488ea6655fd2_step_001", + "00001029_cde47df6e28543d1b434f206_step_001", + "00001030_cde47df6e28543d1b434f206_step_002", + "00001031_cde47df6e28543d1b434f206_step_003", + "00001033_18151464e6434759b7c2ece7_step_000", + "00001035_be3ed2c9696e4e6a91b34f79_step_000", + "00001036_be3ed2c9696e4e6a91b34f79_step_001", + "00001041_6baf25f5fe2947e8927797d8_step_000", + "00001048_2b2638f82b3946b1867f2d11_step_004", + "00001049_2b2638f82b3946b1867f2d11_step_005", + "00001050_2b2638f82b3946b1867f2d11_step_006", + "00001061_284effca5a19488fa8a228e2_step_002", + "00001062_c04956ddff6c4324b5cad67c_step_000", + "00001106_ba88e7579ced4a0aa2de629c_step_000", + "00001108_412675e2ea594e70bbd8cf6c_step_000", + "00001114_5f5cf6feacbc41fb88b2d873_step_000", + "00001121_da4037e40839451dbbc2bd1f_step_000", + "00001130_c2db313c2ce749d784c610d2_step_002", + "00001139_0b0ed9b0ccbb455d82c60d69_step_000", + "00001140_0b0ed9b0ccbb455d82c60d69_step_001", + "00001141_0b0ed9b0ccbb455d82c60d69_step_002", + "00001146_274866da478e4a009bed8750_step_001", + "00001159_f112d6f3e0c644ae9de2a724_step_000", + "00001165_4b65bb9a02f6494aa897a53e_step_000", + "00001169_0889c458f3f74dc288d2d531_step_000", + "00001170_0889c458f3f74dc288d2d531_step_001", + "00001182_bc0b5846e3d84904ad02e547_step_001", + "00001185_507b60b57791425bbb73273e_step_000", + "00001186_507b60b57791425bbb73273e_step_001", + "00001187_507b60b57791425bbb73273e_step_002", + "00001190_507b60b57791425bbb73273e_step_005", + "00001191_507b60b57791425bbb73273e_step_006", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00001200_e44a0f9afbe84b918f789f0f_step_000", + "00001206_e44a0f9afbe84b918f789f0f_step_006", + "00001210_4be610c137ba42ae9933e69b_step_000", + "00001211_4be610c137ba42ae9933e69b_step_001", + "00001212_4be610c137ba42ae9933e69b_step_002", + "00001213_4be610c137ba42ae9933e69b_step_003", + "00001214_9edb90c9f417497fb70bddf6_step_000", + "00001228_2fd836ae38b44c41bfbd9584_step_002", + "00001232_73b40f9a4cd3419ab3b42b73_step_001", + "00001233_73b40f9a4cd3419ab3b42b73_step_002", + "00001235_17ea801126da4654b34f6c52_step_000", + "00001254_c2da273d137e475abb1ab1f4_step_000", + "00001255_3df9015915ff4fa7b57f0a71_step_000", + "00001257_a4715d363f88400c8f0238d1_step_000", + "00001260_a4715d363f88400c8f0238d1_step_003", + "00001266_b53ece83d8964b44bbf1f8ed_step_002", + "00001273_b53ece83d8964b44bbf1f8ed_step_009", + "00001274_b53ece83d8964b44bbf1f8ed_step_010", + "00001276_b53ece83d8964b44bbf1f8ed_step_012", + "00001285_b53ece83d8964b44bbf1f8ed_step_021", + "00001294_b53ece83d8964b44bbf1f8ed_step_030", + "00001298_b53ece83d8964b44bbf1f8ed_step_034", + "00001300_b9951da825954791b59b979d_step_000", + "00001301_a54ee779467b42ea8709e1ca_step_000", + "00001311_e95b8713cc0b4436909c78e9_step_000", + "00001319_6a5a7dbad03b4a57b33f2c0d_step_002", + "00001320_6a5a7dbad03b4a57b33f2c0d_step_003", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00001324_5bf4a74af99f405392406e40_step_000", + "00001344_d8d97699a52842539e08d117_step_002", + "00001345_d8d97699a52842539e08d117_step_003", + "00001346_d8d97699a52842539e08d117_step_004", + "00001348_318b81a8e412438183201be7_step_000", + "00001349_27327ddba7f64417ae1e69a3_step_000", + "00001356_aa0de7c5e9e84d73816bdbde_step_000", + "00001380_c49484307f434da78fa06b1f_step_000", + "00001381_c49484307f434da78fa06b1f_step_001", + "00001389_21b525da345e4e62aebd906b_step_000", + "00001391_406eb61c554a429ebd8a4d4f_step_000", + "00001392_406eb61c554a429ebd8a4d4f_step_001", + "00001393_406eb61c554a429ebd8a4d4f_step_002", + "00001394_406eb61c554a429ebd8a4d4f_step_003", + "00001414_b38f661707674b8eb983d992_step_000", + "00001490_6c8ef6d2f5564d139a5077f8_step_001", + "00001491_6c8ef6d2f5564d139a5077f8_step_002", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001496_eb4e2b01aaca4131bfc64adb_step_000", + "00001497_eb4e2b01aaca4131bfc64adb_step_001", + "00001500_eb4e2b01aaca4131bfc64adb_step_004", + "00001501_6a0f221e85e04411afe23f51_step_000", + "00001503_6a0f221e85e04411afe23f51_step_002", + "00001505_6d02705716e04d9081f4980b_step_000", + "00001527_9214869844f14e83a9dce3ec_step_000", + "00001534_e1d62958d7ca4ad18fefd54d_step_001", + "00001535_e1d62958d7ca4ad18fefd54d_step_002", + "00001538_e1d62958d7ca4ad18fefd54d_step_005", + "00001543_e1d62958d7ca4ad18fefd54d_step_010", + "00001550_a94d978cf62948ce80b417a9_step_000", + "00001554_30de00089a3f4a339eddb68d_step_000", + "00001555_30de00089a3f4a339eddb68d_step_001", + "00001556_30de00089a3f4a339eddb68d_step_002", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00001563_2e9dc7e9a1a54dd7ab901478_step_000", + "00001567_9280741c840d4aa1af32658e_step_002", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00001578_a3b89691665043a08931897f_step_001", + "00001579_a3b89691665043a08931897f_step_002", + "00001582_a3b89691665043a08931897f_step_005", + "00001586_763da6a559544098acbf4154_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00001594_2128dbbd4bf34879913b3a99_step_000", + "00001596_2128dbbd4bf34879913b3a99_step_002", + "00001597_13bd23e0e4c740969b0c71ab_step_000" + ], + "validation": [ + "00000111_ce24e0cbec8e45c89735d148_step_003", + "00000115_ce24e0cbec8e45c89735d148_step_007", + "00000116_ce24e0cbec8e45c89735d148_step_008", + "00000117_ce24e0cbec8e45c89735d148_step_009", + "00000119_ce24e0cbec8e45c89735d148_step_011", + "00000120_ce24e0cbec8e45c89735d148_step_012", + "00000131_945244874ed84e599987daa9_step_000", + "00000132_438507a38e404bfdbd51a16b_step_000", + "00000133_08b739e0511f4305b161cd05_step_000", + "00000190_185ecbc783f34408aef52119_step_000", + "00000196_c9b71ecdd0654716891b0db8_step_000", + "00000206_27417003e762409185cb2b69_step_000", + "00000238_4829bc0645d6414d9b5cec31_step_001", + "00000241_4829bc0645d6414d9b5cec31_step_004", + "00000243_4829bc0645d6414d9b5cec31_step_006", + "00000248_e87554b09cdc48c3add39db9_step_000", + "00000257_845fb06d7f2f4802a806a600_step_002", + "00000258_845fb06d7f2f4802a806a600_step_003", + "00000259_845fb06d7f2f4802a806a600_step_004", + "00000261_845fb06d7f2f4802a806a600_step_006", + "00000262_845fb06d7f2f4802a806a600_step_007", + "00000264_845fb06d7f2f4802a806a600_step_009", + "00000265_845fb06d7f2f4802a806a600_step_010", + "00000266_845fb06d7f2f4802a806a600_step_011", + "00000267_845fb06d7f2f4802a806a600_step_012", + "00000269_845fb06d7f2f4802a806a600_step_014", + "00000391_fd490b564a44413884fab719_step_000", + "00000405_f3759bbb4682423caf207e8e_step_002", + "00000406_f3759bbb4682423caf207e8e_step_003", + "00000444_4ed4c78d6d754aac90876fc2_step_003", + "00000445_4ed4c78d6d754aac90876fc2_step_004", + "00000501_5499914343bd4f549b38046d_step_002", + "00000502_5499914343bd4f549b38046d_step_003", + "00000574_5a4cef68211d4706b1ec8586_step_003", + "00000578_5a4cef68211d4706b1ec8586_step_007", + "00000579_5a4cef68211d4706b1ec8586_step_008", + "00000580_5a4cef68211d4706b1ec8586_step_009", + "00000582_5a4cef68211d4706b1ec8586_step_011", + "00000583_5a4cef68211d4706b1ec8586_step_012", + "00000597_7fe894812c4244e18241aa08_step_003", + "00000601_7fe894812c4244e18241aa08_step_007", + "00000602_7fe894812c4244e18241aa08_step_008", + "00000603_7fe894812c4244e18241aa08_step_009", + "00000605_7fe894812c4244e18241aa08_step_011", + "00000606_7fe894812c4244e18241aa08_step_012", + "00000654_d87d11d9c3ca4ee5a88797ae_step_002", + "00000758_8ab50540e9804365a8717ec0_step_005", + "00000759_8ab50540e9804365a8717ec0_step_006", + "00000760_8ab50540e9804365a8717ec0_step_007", + "00000763_8ab50540e9804365a8717ec0_step_010", + "00000816_36e6258346004cabb4c091b2_step_004", + "00000817_36e6258346004cabb4c091b2_step_005", + "00000892_be26dc83f78947738b2bfabd_step_001", + "00000894_be26dc83f78947738b2bfabd_step_003", + "00000897_be26dc83f78947738b2bfabd_step_006", + "00000919_cdcebf32644443b9812097ba_step_003", + "00000923_cdcebf32644443b9812097ba_step_007", + "00000927_cdcebf32644443b9812097ba_step_011", + "00000928_cdcebf32644443b9812097ba_step_012", + "00000955_a741c7966d2c40a8a511263e_step_001", + "00000984_0dd1386597104265ab67e6cf_step_007", + "00000986_0dd1386597104265ab67e6cf_step_009", + "00000987_0dd1386597104265ab67e6cf_step_010", + "00000988_0dd1386597104265ab67e6cf_step_011", + "00000996_6d71695ba3fd475c927a00a7_step_000", + "00001019_ca4cf9693d6d4a98b515ba99_step_000", + "00001020_ca4cf9693d6d4a98b515ba99_step_001", + "00001021_ca4cf9693d6d4a98b515ba99_step_002", + "00001026_0f133d8e7fd448708a554e28_step_000", + "00001027_0f133d8e7fd448708a554e28_step_001", + "00001040_74c42207fa764831b03f7658_step_000", + "00001052_2fd6155fa34f418083d0bd0f_step_001", + "00001053_2fd6155fa34f418083d0bd0f_step_002", + "00001097_256ee7ec3a4b4972a8b87c8d_step_001", + "00001099_256ee7ec3a4b4972a8b87c8d_step_003", + "00001102_256ee7ec3a4b4972a8b87c8d_step_006", + "00001107_78566795c3144719804f8d34_step_000", + "00001167_107f61d0402f4c9a9d9d1342_step_000", + "00001168_107f61d0402f4c9a9d9d1342_step_001", + "00001217_e517c61f1c49433bb81cf980_step_000", + "00001238_96503b6a26c64074ad598927_step_000", + "00001239_96503b6a26c64074ad598927_step_001", + "00001240_96503b6a26c64074ad598927_step_002", + "00001241_96503b6a26c64074ad598927_step_003", + "00001242_96503b6a26c64074ad598927_step_004", + "00001243_96503b6a26c64074ad598927_step_005", + "00001244_96503b6a26c64074ad598927_step_006", + "00001246_96503b6a26c64074ad598927_step_008", + "00001247_96503b6a26c64074ad598927_step_009", + "00001249_96503b6a26c64074ad598927_step_011", + "00001252_96503b6a26c64074ad598927_step_014", + "00001253_96503b6a26c64074ad598927_step_015", + "00001314_a9ed5e56a6a8414f91b39bdb_step_000", + "00001333_beab79941e884455a62af40c_step_001", + "00001334_beab79941e884455a62af40c_step_002", + "00001335_beab79941e884455a62af40c_step_003", + "00001336_beab79941e884455a62af40c_step_004", + "00001338_beab79941e884455a62af40c_step_006", + "00001339_beab79941e884455a62af40c_step_007", + "00001340_beab79941e884455a62af40c_step_008", + "00001352_4e1d56bc2cef49ea84626b11_step_000", + "00001353_4e1d56bc2cef49ea84626b11_step_001", + "00001354_4e1d56bc2cef49ea84626b11_step_002", + "00001379_5b1d6e2599ea47d498a04256_step_000", + "00001387_60cf3ad636dd42e0bcec6946_step_000", + "00001511_5cbe58077e4b4885a5c213b3_step_004", + "00001512_5cbe58077e4b4885a5c213b3_step_005", + "00001525_e0d35a03b46745259cf8878f_step_000", + "00001532_9f0a4a641e8f4cc588eb4315_step_000", + "00001552_b0a9c4d836294846891615fe_step_000", + "00001553_adfa1c33ae4d4d0aa441ddc3_step_000" + ], + "test": [ + "00000006_d4fe04f0f5f84b52bd4f10e4_step_001", + "00000008_9b3d6a97e8de4aa193b81000_step_000", + "00000009_9b3d6a97e8de4aa193b81000_step_001", + "00000011_e909f412cda24521865fac0f_step_000", + "00000102_5ed74bccca6f4e89829bcb5e_step_002", + "00000183_6c4172e8a17140f89d3b1c0a_step_000", + "00000220_5324c79b172f41e8a49afc6f_step_007", + "00000221_5324c79b172f41e8a49afc6f_step_008", + "00000249_07a2da801a60492fb1e5eeb3_step_000", + "00000408_36bcb4030e36417b8dc74b87_step_000", + "00000410_30afd12eede44fa8a1def049_step_000", + "00000411_305e45c9479f43979b0d7b53_step_000", + "00000413_94f421b1b33d47c592c9fe1d_step_000", + "00000421_bd0972033f464f8f961a150a_step_002", + "00000508_d1c0a44331fd41faa91d4c2e_step_000", + "00000509_d1c0a44331fd41faa91d4c2e_step_001", + "00000510_d1c0a44331fd41faa91d4c2e_step_002", + "00000522_a56dd4152b7e4ccf91212d7b_step_000", + "00000527_db75f3a38d9746a1bd66168d_step_000", + "00000542_4b1756f6e72b45c29a8e4eb4_step_000", + "00000549_41a24c0b74a34217832f5af0_step_000", + "00000551_41a24c0b74a34217832f5af0_step_002", + "00000562_1724088724bf4e53a36e363e_step_000", + "00000566_2f2b015ef4904c1caef56154_step_000", + "00000637_5ae45a997c76423aba2fcc11_step_002", + "00000638_5ae45a997c76423aba2fcc11_step_003", + "00000643_bb55eaa4e9e24c2ea9d18ee0_step_000", + "00000659_6fae5591b3034f949b7e8eae_step_000", + "00000695_3f46763af0744ab2a90e6b48_step_000", + "00000746_6cb3ddbb49974d7db0e97135_step_001", + "00000796_dec80701adc84239b485449d_step_002", + "00000806_05cadeb104be4c8aa75ccc71_step_000", + "00000807_05cadeb104be4c8aa75ccc71_step_001", + "00000808_05cadeb104be4c8aa75ccc71_step_002", + "00000809_05cadeb104be4c8aa75ccc71_step_003", + "00000840_2c1efcc32d094e9fb9e1a88e_step_000", + "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "00000843_2c1efcc32d094e9fb9e1a88e_step_003", + "00000844_2c1efcc32d094e9fb9e1a88e_step_004", + "00000849_2c1efcc32d094e9fb9e1a88e_step_009", + "00000850_2c1efcc32d094e9fb9e1a88e_step_010", + "00000851_2c1efcc32d094e9fb9e1a88e_step_011", + "00000855_2c1efcc32d094e9fb9e1a88e_step_015", + "00000856_2c1efcc32d094e9fb9e1a88e_step_016", + "00000859_58ac6a347bbe4cf5ae07a565_step_000", + "00000863_029c2575dd474650ba1b8d21_step_003", + "00000864_029c2575dd474650ba1b8d21_step_004", + "00000866_029c2575dd474650ba1b8d21_step_006", + "00000867_029c2575dd474650ba1b8d21_step_007", + "00000868_029c2575dd474650ba1b8d21_step_008", + "00000870_029c2575dd474650ba1b8d21_step_010", + "00000871_029c2575dd474650ba1b8d21_step_011", + "00000872_029c2575dd474650ba1b8d21_step_012", + "00000873_029c2575dd474650ba1b8d21_step_013", + "00000874_029c2575dd474650ba1b8d21_step_014", + "00000877_029c2575dd474650ba1b8d21_step_017", + "00000879_029c2575dd474650ba1b8d21_step_019", + "00000880_029c2575dd474650ba1b8d21_step_020", + "00000882_029c2575dd474650ba1b8d21_step_022", + "00000883_029c2575dd474650ba1b8d21_step_023", + "00000884_029c2575dd474650ba1b8d21_step_024", + "00000885_029c2575dd474650ba1b8d21_step_025", + "00000886_029c2575dd474650ba1b8d21_step_026", + "00000887_029c2575dd474650ba1b8d21_step_027", + "00000938_4f351b53bfd84bb4a2f7809c_step_003", + "00000970_edacb215ec6949d8af6c8c68_step_000", + "00000993_ac4ba032d4b5431ca04f542c_step_000", + "00000994_ac4ba032d4b5431ca04f542c_step_001", + "00000998_6962bc8aad2c4e27b046d5b4_step_000", + "00001000_6962bc8aad2c4e27b046d5b4_step_002", + "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "00001003_6962bc8aad2c4e27b046d5b4_step_005", + "00001004_6962bc8aad2c4e27b046d5b4_step_006", + "00001005_6962bc8aad2c4e27b046d5b4_step_007", + "00001006_6962bc8aad2c4e27b046d5b4_step_008", + "00001009_6962bc8aad2c4e27b046d5b4_step_011", + "00001010_6962bc8aad2c4e27b046d5b4_step_012", + "00001011_6962bc8aad2c4e27b046d5b4_step_013", + "00001013_6962bc8aad2c4e27b046d5b4_step_015", + "00001014_6962bc8aad2c4e27b046d5b4_step_016", + "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "00001034_085828cd621d404a90ccd883_step_000", + "00001063_c6fda10dc58448d1951c5cc3_step_000", + "00001064_c6fda10dc58448d1951c5cc3_step_001", + "00001065_c6fda10dc58448d1951c5cc3_step_002", + "00001067_8a2edcf36bcd435092f4c8ae_step_001", + "00001069_8a2edcf36bcd435092f4c8ae_step_003", + "00001071_8a2edcf36bcd435092f4c8ae_step_005", + "00001112_3bd4a008a3a0483d86ccab3f_step_000", + "00001177_9f1ce06d5e334fe88e1581aa_step_001", + "00001178_515d7e1aeb364a2d88083cf3_step_000", + "00001262_157e5348bf6545e282472856_step_000", + "00001263_157e5348bf6545e282472856_step_001", + "00001305_3f3a389e9d5a4370b952bdce_step_001", + "00001310_3f3a389e9d5a4370b952bdce_step_006", + "00001358_2e314c6e080b40fcb6a6db96_step_001", + "00001360_2e314c6e080b40fcb6a6db96_step_003", + "00001368_2e314c6e080b40fcb6a6db96_step_011", + "00001369_2e314c6e080b40fcb6a6db96_step_012", + "00001371_2e314c6e080b40fcb6a6db96_step_014", + "00001373_2e314c6e080b40fcb6a6db96_step_016", + "00001375_2e314c6e080b40fcb6a6db96_step_018", + "00001377_2e314c6e080b40fcb6a6db96_step_020", + "00001398_49414db2dd594dc6a3907cf3_step_001", + "00001399_49414db2dd594dc6a3907cf3_step_002", + "00001400_49414db2dd594dc6a3907cf3_step_003", + "00001401_49414db2dd594dc6a3907cf3_step_004", + "00001402_49414db2dd594dc6a3907cf3_step_005", + "00001403_49414db2dd594dc6a3907cf3_step_006", + "00001404_49414db2dd594dc6a3907cf3_step_007", + "00001405_49414db2dd594dc6a3907cf3_step_008", + "00001406_49414db2dd594dc6a3907cf3_step_009", + "00001407_49414db2dd594dc6a3907cf3_step_010", + "00001408_49414db2dd594dc6a3907cf3_step_011", + "00001409_49414db2dd594dc6a3907cf3_step_012", + "00001410_49414db2dd594dc6a3907cf3_step_013", + "00001411_49414db2dd594dc6a3907cf3_step_014", + "00001412_49414db2dd594dc6a3907cf3_step_015", + "00001413_4642a707b7894949ba816427_step_000", + "00001522_7079a898746d41ba8ae13d69_step_003", + "00001598_c118664e9f6f4943855293fa_step_000" + ], + "c": 1.026602777749673, + "clock_sha256": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96", + "learning_nodes": 8742060, + "id": 0, + "target_steps": 32000, + "cpu_seconds": 30000, + "validation_seconds_bound": 12.383006267948076, + "rss_bytes": 4294967296, + "recorded_vram_bytes": 2147483648, + "process_vram_bytes": 4294967296, + "disk_bytes": 8589934592 + }, + "budget": { + "item_gpu_limit_seconds": 90000, + "phase3_gpu_limit_seconds": 720000, + "prior_item_gpu_seconds": 19262.057604262838, + "prior_phase3_gpu_seconds": 408630.4104626046, + "train_epoch_seconds_basis": 99.90287962602451, + "validation_seconds_basis": 16.66734165791422, + "train_gpu_seconds": 12817, + "screen_gpu_seconds": 183, + "training_setup_tail_seconds": 120, + "idle_wait_seconds": 60, + "finalization_gpu_seconds": 120, + "projected_new_gpu_seconds": 13180 + }, + "references": { + "long_seed0": { + "checkpoint": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/last-epoch200.pt", + "checkpoint_sha256": "35cc9e41a4197e186570089ac305c4dee100f3d37611e0ca17e01db441b93756", + "standardization": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/standardization.json", + "standardization_sha256": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421", + "seed": 0, + "epochs": 200, + "training_contract_sha256": "9ea2ad987ce165660c1c1f648a3edf8069c40ef270e233f869fd881e42d7c382" + }, + "long_seed1": { + "checkpoint": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/last-epoch200.pt", + "checkpoint_sha256": "b223cb3eb36efff52e5905f7c6a39a40a0e32e38f64fe9947d2597f68e7c3923", + "standardization": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/standardization.json", + "standardization_sha256": "98eabfb586fb7689bd31990b49301ccaceec34f84e16d220f3e3e7ee34f8e681", + "seed": 1, + "epochs": 200, + "training_contract_sha256": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981" + } + }, + "pilot_contract": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json", + "query_roster_contract": "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/contract.json", + "comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-screen-2026-11-19/comparison-contract.json", + "comparison_contract_sha256": "bd71a5a6a0f3a288267f568cbb85d1cea5ab6fa0485e7fd212cd55e36f664954", + "source_sha256": { + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/contract.json": "d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/complete.json": "430247e44f301f984f396c662e6fa5419d09649476f6b2a81076444731f55dde", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/post-terminal-cost.json": "7241fe629e3227b81655886c6c208336deef443d67c8d6efd7530000de8477bf", + "/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json": "ca576e271715c862ee21e75b2dec7b03e45317e5f8bb073adae016a8001a0a59", + "/home/frank14f/MeltMind-runs/phase3-seed1-postfit-2026-11-18/post-terminal-cost.json": "927546a8991fc2d39127067c4907ae74276d6ec6d5d0a82e0f57db7951813bbe", + "/home/frank14f/MeltMind-runs/phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json": "2de53b3aca1ae21bc53480ad7425094f30273f798cd292a895526094c993b41f", + "/home/frank14f/MeltMind-runs/phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json": "ea99e46a6ffe687004f174df58fdd0f1c13d0231bac7e28e6cbdd988b7fe5c84", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/worker/history/001.json": "579391f43569643a4ea8ac1af6e2043ebeb9bf149b2c17b3ca3b3ef0aa2c43e5", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/assignment.json": "050e8b6748a3e2a287eb905f75179daa9941bd5690660f2c2716298fbf71bd1d", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/assignment.json": "a2699b9e9174eb7ba8a4b54a4764f58f28fb794be0008083f36765c738035b26", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/worker/worker-identity.json": "98094ad553095111798c43d8b33ec14cbe95955b0df5110e233e176856a0a881", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/worker/worker-identity.json": "2953a472cdff619d52235420837880bfc551ce65b5699db77124b3f1a0e147ca", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.started.json": "c4985f4c4b5d24c81a29feb1b275d41213fc1f2a9583d01c48130007564ac401", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.started.json": "b4c43b504190fe334d049fad6889f6d998e64140ef760958a1308612cf538bf0", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.receipt.json": "36b9c44bcf2248be07314300ee8c99d214232869e24399314e0907c9c2469b5b", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.receipt.json": "640f14ff26fcba3c6213b111c81463a1261efd6992ff7fb3c827d7da06b6195f", + "/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/gpu-sample.json": "9e92c761638268f48c1436243a18de13e41d67b15a335ff18cd9bc5a037790bc", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/epoch-host-observations/026.json": "5f3ccb39ea8669bb5fed7260b3b786ace6366ff59a0a4a0f135681e6311dd830", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/history/026.json": "0ce765d2bc3cc68640971afaa9f2a78a3f59ae0602c2c60d0f690533891b43f4", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/epoch-host-observations/020.json": "91f70761c95aa8f211b050c39685ae2a1981891f8151f17aa75805eaaa1c72e6", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/history/020.json": "776c76e9882a3470fe1264d1c61b4307e0f6a9f0fcfc320d04a6d6b05becfd01", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b", + "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/contract.json": "0bd03f42c3b2db5bef26708b2ef873edfe674f724b59bc4b4de8641980d8d330", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/contract.json": "9ea2ad987ce165660c1c1f648a3edf8069c40ef270e233f869fd881e42d7c382", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/result.json": "f2bbe0d6ff36bcdbcfb36cc9cb17459c65e4f7b5d47fa7252a2c73fbbfb8387e", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/standardization.json": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421", + "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/last-epoch200.pt": "35cc9e41a4197e186570089ac305c4dee100f3d37611e0ca17e01db441b93756", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/contract.json": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/result.json": "79b38fd28b590a4ff99b378d76f32cbf6f959d056cfa8e3c92b4a9d5e9b611ac", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/standardization.json": "98eabfb586fb7689bd31990b49301ccaceec34f84e16d220f3e3e7ee34f8e681", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/last-epoch200.pt": "b223cb3eb36efff52e5905f7c6a39a40a0e32e38f64fe9947d2597f68e7c3923", + "/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-screen-2026-11-19/timing-and-budget.json": "33a2908ae8785b405869d7ef957445bd93566c5d73c1a26dd86489fe077a4232", + "/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-screen-2026-11-19/contract.json": "942323ddfceecf47515eb48628c45157073cbb909867a606ef4d3764a13be7d4", + "/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-screen-2026-11-19/expected-review-binding.json": "ed6bfe113f9c0bf925684a62cf0c1bb09bfc7b1277e06fad628fc0e9e5bcd859", + "/home/frank14f/optane/MeltMind/artifacts/meltassist-item1b-fold0-screen-review-2026-11-19/review.json": "0fadd70d365e1015f835fb953691797811a032766d3a9c1b75f3b8e4678faf61", + "/home/frank14f/optane/MeltMind/artifacts/meltassist-item1b-fold0-final-deadline-review-2026-11-19/review.json": "d45ac966f612236656f9b1a0d59ed2f60493c2da0b3c31279e189d7dfa217e35" + }, + "code_sha256": { + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_fold0.py": "1446919f2274bc6207527ad80cbd634e47761f9d6391543d420d4f8cd6498527", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "06cb97594c3a87f42f712dcde6cec8c2d90e0b50a89a6be7954bdd2f5c18d5de", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "b781e56c519401bd38af6a522deb24506cba6993d2af008f85ea4e1715f1d9c6", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b_fold0.py": "f48ebe3e3aeca03e5d4cbdd532061cfb27362ec84e85dae70e77e22b4e40b8e0", + "/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609", + "/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61", + "/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75", + "/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4", + "/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0", + "/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "08b7d8ab2ac773fab8b749f9f61af259254422128e9634f8b715c81efce2651f", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "31dfd71369c63512c1d6fe176fd22e8db646520c92da533bb5530ca3843c388d", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "c7bc9f3b7cf4137f52e1efad19dcb7045282d4e9a9c1aae50f066c9e488f7c01", + "/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "b4d78f4384651711fe83ff00eb6719c6c61664699d801ecb34978b9546439da9", + "/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496", + "/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb", + "/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a", + "/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447", + "/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "3115aa9049974eac86657aa4cee7b5d716f6ee93268555cc86242029ebefce4c", + "/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062", + "/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6", + "/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5", + "/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_fold0_deadline.py": "5a4cac9846433f3661b87f3ffbe2c04015a3cfd7c885ad06a2b8f1e2a0a4d388", + "/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b_fold0_deadline.py": "be7fb0c5feb39d31386742f4691cc3bf2c675e11f72534853363fbc605841ab1" + }, + "engineering_ruling": "h-5064d81d" +} diff --git a/docs/phase3-item1b-fold0-deadline-2026-11-19.md b/docs/phase3-item1b-fold0-deadline-2026-11-19.md new file mode 100644 index 000000000..c92ae36e9 --- /dev/null +++ b/docs/phase3-item1b-fold0-deadline-2026-11-19.md @@ -0,0 +1,16 @@ +# 1b 折 0|派发截止与最终结算 + +Claude `h-5064d81d` 撤回此前豁免。新入口绑定相同的折 0、训练日程、筛选口径和 **13,180 GPU 秒**总帽。原绑定 `ed6bfe11` 与两份独立 `BLOCKED` 均由新合同引用,旧工件继续可读。[原版阻塞及审阅](./phase3-item1b-fold0-waiver-blocker-2026-11-19.md) + +- 从初始空闲等待开始,只使用一个 60 秒派发截止。确认后的两次训练前复查均在采样前后查同一截止,训练 supervisor 在内部锁和磁盘检查之后、创建进程之前再次核对。筛选仍走原单独预算。共享守卫和运行入口文件的原字节都保留。 +- 最终一次单调时钟结算决定终态与返回码。超过 13,180 GPU 秒时记录 `TOTAL_GPU_CAP_AT_SETTLEMENT`,原筛选结果、分段实账、超额和累计账都保留。 + +| 真实守卫的 CPU 回放 | 观察 | +| --- | --- | +| 第三次空闲确认 59 秒,复查 1.5 秒 | 60.5 秒停止,训练和筛选零派发 | +| supervisor 内部准备从 59.5 秒跨到 60.5 秒 | 创建进程调用为零 | +| 原终态调度延迟反例 | 13,180.8 GPU 秒实账,超额约 0.8 秒,STOPPED,退出 1 | +| 及时完成对照 | 13,157.8 GPU 秒,COMPLETE,退出 0 | +| 原忙卡止损对照 | 62 秒据实记入等待,IDLE_TIMEOUT,零派发 | + +新入口待独立窄审。回放无真实 GPU 查询、worker 启动或目标访问。[新合同](../artifacts/phase3-item1b-fold0-deadline-2026-11-19/contract.json) · [原反例 CPU 回放](../artifacts/phase3-item1b-fold0-deadline-2026-11-19/actual-guard-cpu-replay.json) diff --git a/tools/phase3_item1b_fold0_deadline.py b/tools/phase3_item1b_fold0_deadline.py new file mode 100644 index 000000000..d92e3f590 --- /dev/null +++ b/tools/phase3_item1b_fold0_deadline.py @@ -0,0 +1,215 @@ +"""Fold-0 1b recovery scoped to dispatch deadline and terminal settlement.""" + +import ctypes +import shutil +import subprocess +import sys +import threading +import time +from pathlib import Path +from types import FunctionType, SimpleNamespace + +import phase3_item1b_fold0 as legacy +from phase3_item1b_fold0 import contract, folded_inputs, read, checked, save, sha +from s2_gpu_idle_gate import snapshot, wait_idle +from s2_main_memory_guard import PeakRecords, active_command_pids +from s2_recovery_gpu_supervisor import stop_group, supervise as shared_supervise + + +ROOT = legacy.ROOT.with_name("phase3-item1b-fold0-deadline-2026-11-19") + + +class GuardStop(RuntimeError): + """A named terminal stop that retains the actual accounting.""" + + +def require_training_window(deadline): + if time.monotonic() >= deadline: + raise GuardStop("TRAIN_DISPATCH_DEADLINE") + + +def supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline=None): + if dispatch_deadline is None: + return shared_supervise(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid) + + require_training_window(dispatch_deadline) + libc = ctypes.CDLL(None, use_errno=True) + previous_subreaper = ctypes.c_int() + if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "Cannot read subreaper state") + + def spawn(*args, **kwargs): + require_training_window(dispatch_deadline) + return subprocess.Popen(*args, **kwargs) + + # Keep the published shared supervisor intact. This entry checks again + # after its internal locking, disk check and receipt, right before spawn. + namespace = dict(shared_supervise.__globals__, + subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT)) + supervisor = FunctionType(shared_supervise.__code__, namespace, shared_supervise.__name__) + try: + return supervisor(command, root, mode, limits, poll=poll, gpu_uuid=gpu_uuid) + except GuardStop: + if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "Cannot restore subreaper state") + save(root / "train-spawn-blocked.json", dict( + trigger="TRAIN_DISPATCH_DEADLINE", worker_started=False, + dispatch_deadline_monotonic=dispatch_deadline, rejected_monotonic=time.monotonic())) + raise + + +def recheck_idle(gpu, busy_trigger, deadline=None): + if deadline is not None: + require_training_window(deadline) + idle = snapshot(gpu)["idle"] + if deadline is not None: + require_training_window(deadline) + if not idle: + raise GuardStop(busy_trigger) + + +def dispatch_guard(root, contract_path): + """The same entry consumes the generated contract in production and preflight.""" + began_entry = time.monotonic() + c = contract(contract_path) + folded_inputs(c) + budget = c["budget"] + gpu = c["gpu_uuid"] + charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0., + screen_gpu_seconds=0., finalization_gpu_seconds=0.) + receipts = [] + status = "STOPPED" + trigger = "STOP_BEFORE_DISPATCH" + try: + if shutil.disk_usage(root).free < c["disk_bytes"]: + raise RuntimeError("Insufficient free disk for the fixed attempt") + began_wait = time.monotonic() + dispatch_deadline = began_wait + budget["idle_wait_seconds"] + try: + idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot) + except BaseException: + charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait + raise + charges["queue_wait_gpu_seconds"] = idle["wall_seconds"] + save(root / "idle-gate.json", idle) + if idle["status"] != "IDLE_CONFIRMED": + raise GuardStop(idle["status"]) + recheck_idle(gpu, "SELECTED_GPU_BUSY_AFTER_IDLE_GATE", dispatch_deadline) + (root / "jobs/0").mkdir(parents=True, exist_ok=False) + for mode, cap_key, charge_key, output in ( + ("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"), + ("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"), + ): + began = time.monotonic() + deadline = dispatch_deadline if mode == "train" else None + try: + recheck_idle(gpu, "SELECTED_GPU_BUSY_BEFORE_" + mode.upper(), deadline) + except GuardStop: + charges[charge_key] = time.monotonic() - began + raise + command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)] + tracker = PeakRecords() + done = threading.Event() + reasons = [] + + def watch(): + while not done.is_set(): + try: + tracker.scan(output) + state = snapshot(gpu) + pids = set(active_command_pids(command)) + if max(tracker.maxima.values()) > c["recorded_vram_bytes"]: + reasons.append("RECORDED_VRAM_LIMIT") + if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]: + reasons.append("DEVICE_VRAM_LIMIT") + if any(p["pid"] not in pids for p in state["occupants"]): + reasons.append("FOREIGN_GPU_OCCUPANT") + except Exception as exc: + reasons.append(f"MONITOR_FAILURE: {exc}") + if reasons: + for pid in active_command_pids(command): + stop_group(pid) + # Also catch a child spawned just after the failing + # sample while the supervisor was taking its lock. + done.wait(.1) + continue + done.wait(.5) + + monitor = threading.Thread(target=watch, daemon=True) + monitor.start() + try: + if reasons: + raise RuntimeError(reasons[0]) + cap = budget[cap_key] - (time.monotonic() - began) - 10 + if deadline is not None: + require_training_window(deadline) + receipt = supervise(command, root, mode, dict( + total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key], + job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"], + disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]), + poll=.2, gpu_uuid=gpu, dispatch_deadline=deadline) + finally: + done.set() + monitor.join(timeout=8) + charges[charge_key] = time.monotonic() - began + if monitor.is_alive(): + reasons.append("MONITOR_DID_NOT_EXIT") + else: + tracker.scan(output) + if max(tracker.maxima.values()) > c["recorded_vram_bytes"]: + reasons.append("RECORDED_VRAM_LIMIT") + if set(tracker.witnesses) != set(tracker.maxima): + reasons.append("MISSING_RECORDED_VRAM_PEAKS") + receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima)) + if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0 + or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]): + worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT" + raise GuardStop(reasons[0] if reasons else worker_reason) + result_path = output / "result.json" + result = read(result_path) + if result["status"] != "COMPLETE" or result["test_targets_read"] is not False: + raise RuntimeError("Unexpected worker result") + if mode == "train": + if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"] + or result["optimizer_steps"] != c["job"]["target_steps"]): + raise RuntimeError("Fold-0 final checkpoint identity differs") + checked(result["checkpoint"], result["checkpoint_sha256"]) + save(root / "training-adopted.json", dict(result_sha256=sha(result_path))) + elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]: + raise RuntimeError("Validation screen scope differs") + overhead = time.monotonic() - began_entry - sum(charges.values()) + if overhead > budget["finalization_gpu_seconds"]: + raise RuntimeError("Setup and finalization cost exceeded the fixed allowance") + status, trigger = "COMPLETE", result["decision"] + except GuardStop as exc: + trigger = str(exc) + except Exception as exc: + trigger = f"{type(exc).__name__}: {exc}" + except KeyboardInterrupt: + trigger = "COORDINATOR_INTERRUPTED" + finally: + charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values())) + new_gpu = sum(charges.values()) + limit = budget["projected_new_gpu_seconds"] + overage = max(0., new_gpu - limit) + underlying_trigger = trigger + if overage > 0: + status, trigger = "STOPPED", "TOTAL_GPU_CAP_AT_SETTLEMENT" + save(root / "complete.json", dict( + status=status, trigger=trigger, underlying_trigger=underlying_trigger, receipts=receipts, **charges, + new_gpu_seconds=new_gpu, new_gpu_cap_seconds=limit, cap_overage_gpu_seconds=overage, + cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu, + cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu, + no_retry=True, test_targets_read=False, automatic_continuation=False)) + return 0 if status == "COMPLETE" else 1 + + +def main(): + # The original entry and worker functions enforce the same exact review, + # process lifetime, environment and one-attempt rules at the new root. + namespace = dict(legacy.main.__globals__, ROOT=ROOT, dispatch_guard=dispatch_guard) + return FunctionType(legacy.main.__code__, namespace, legacy.main.__name__)() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/prepare_phase3_item1b_fold0_deadline.py b/tools/prepare_phase3_item1b_fold0_deadline.py new file mode 100644 index 000000000..82ec80eac --- /dev/null +++ b/tools/prepare_phase3_item1b_fold0_deadline.py @@ -0,0 +1,183 @@ +"""Freeze the narrow 1b repair and replay the independent review's boundary cases.""" + +from copy import deepcopy +from functools import partial +import inspect +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import phase3_item1b_fold0 as legacy +import phase3_item1b_fold0_deadline as entry +from phase2_h1_worker_v2 import checked, save, sha +from s2_gpu_idle_gate import wait_idle + + +OLD_BINDING = "ed6bfe113f9c0bf925684a62cf0c1bb09bfc7b1277e06fad628fc0e9e5bcd859" +OLD_CONTRACT = "942323ddfceecf47515eb48628c45157073cbb909867a606ef4d3764a13be7d4" +ORIGINAL_BLOCKS = { + "meltassist-item1b-fold0-screen-review-2026-11-19": "0fadd70d365e1015f835fb953691797811a032766d3a9c1b75f3b8e4678faf61", + "meltassist-item1b-fold0-final-deadline-review-2026-11-19": "d45ac966f612236656f9b1a0d59ed2f60493c2da0b3c31279e189d7dfa217e35", +} + + +class InertMonitor: + def __init__(self, **kwargs): + pass + + def start(self): + pass + + def join(self, **kwargs): + pass + + def is_alive(self): + return False + + +def prepare(): + old_path = checked(legacy.ROOT / "contract.json", OLD_CONTRACT) + binding = checked(legacy.ROOT / "expected-review-binding.json", OLD_BINDING) + old = legacy.contract(old_path) + if (legacy.ROOT / "started.json").exists(): + raise PermissionError("The original fold-0 entry already has a start record") + sources = {str(old_path): OLD_CONTRACT, str(binding): OLD_BINDING} + for name, digest in ORIGINAL_BLOCKS.items(): + path = checked(entry.ROOT.parent / name / "review.json", digest) + if entry.read(path)["verdict"] != "BLOCKED": + raise ValueError("Original blocking review must be preserved") + sources[str(path)] = digest + new = deepcopy(old) + new["version"] = "phase3-item1b-fold0-deadline-settlement-v2" + new["engineering_ruling"] = "h-5064d81d" + new["source_sha256"].update(sources) + for path in (Path(entry.__file__), Path(__file__).resolve()): + new["code_sha256"][str(path)] = sha(path) + if new["budget"] != old["budget"] or new["budget"]["projected_new_gpu_seconds"] != 13180: + raise ValueError("Original budget must remain 13,180 GPU seconds") + entry.ROOT.mkdir(exist_ok=False) + path = entry.ROOT / "contract.json" + save(path, new) + return path + + +def simulate(path, *, latency, recheck_latency, settle_delay=None): + c = entry.contract(path) + now = 0. + samples = [] + dispatches = [] + source, first = inspect.getsourcelines(entry.dispatch_guard) + settle_line = next(first + i for i, line in enumerate(source) + if 'charges["finalization_gpu_seconds"] = max(' in line) + + def sleep(seconds): + nonlocal now + now += seconds + + def clock(): + if settle_delay is not None and inspect.currentframe().f_back.f_lineno == settle_line: + sleep(settle_delay) + return now + + def sample(uuid): + index = len(samples) + samples.append(now) + sleep(latency if index < 6 else recheck_latency) + return dict(uuid=uuid, idle=index >= 3, memory_MiB=5 if index >= 3 else 1500, occupants=[]) + + def record(path, data): + save(path, data) + if settle_delay is not None and path.name == "training-adopted.json": + sleep(119.) + + def mock_supervise(command, root, mode, limits, *, poll, gpu_uuid, dispatch_deadline): + dispatches.append(dict(mode=mode, seconds=now, dispatch_deadline=dispatch_deadline)) + if settle_delay is None: + raise AssertionError("Late-admission case must stop before the supervisor") + if dispatch_deadline != (60. if mode == "train" else None): + raise AssertionError("The original deadline must follow training only") + sleep(limits["job_wall_seconds"] - .1) + output = root / ("jobs/0/worker" if mode == "train" else "screen") + output.mkdir() + result = dict(status="COMPLETE", test_targets_read=False, + peak_allocated_bytes=32, peak_reserved_bytes=64) + if mode == "train": + checkpoint = output / "mock.pt" + checkpoint.write_bytes(b"synthetic") + result.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"], + checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint)) + else: + result.update(parts=len(c["job"]["validation"]), automatic_continuation=False, decision="SCREEN_PASS") + save(output / "result.json", result) + return dict(status="PASS", returncode=0, monitor_errors=0) + + with TemporaryDirectory(prefix="meltmind-1b-deadline-replay-") as scratch: + root = Path(scratch) + with (patch.object(entry, "snapshot", side_effect=sample), + patch.object(entry, "wait_idle", partial(wait_idle, clock=clock, sleep=sleep)), + patch.object(entry, "time", SimpleNamespace(monotonic=clock)), + patch.object(entry, "threading", SimpleNamespace(Thread=InertMonitor, Event=entry.threading.Event)), + patch.object(entry, "supervise", side_effect=mock_supervise), + patch.object(entry, "save", side_effect=record)): + returned = entry.dispatch_guard(root, path) + terminal = entry.read(root / "complete.json") + gate = entry.read(root / "idle-gate.json") + terminal.pop("receipts") + return dict(gate=gate["status"], gate_seconds=gate["wall_seconds"], + snapshot_start_seconds=samples, mock_dispatches=dispatches, + returned=returned, terminal=terminal) + + +def supervisor_boundary(path): + c = entry.contract(path) + now = 59.5 + + def disk_check(_): + nonlocal now + now = 60.5 + return 0 + + blocked_spawn = Mock(side_effect=AssertionError("Popen invoked after the training deadline")) + with TemporaryDirectory(prefix="meltmind-1b-supervisor-deadline-") as scratch: + root = Path(scratch) + with (patch.object(entry, "time", SimpleNamespace(monotonic=lambda: now)), + patch.dict(entry.shared_supervise.__globals__, {"disk_usage": disk_check}), + patch.object(entry, "subprocess", SimpleNamespace(Popen=blocked_spawn, STDOUT=entry.subprocess.STDOUT))): + try: + entry.supervise(["synthetic-no-spawn"], root, "train", dict( + total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=c["budget"]["train_gpu_seconds"], + job_wall_seconds=10., tree_rss_bytes=c["rss_bytes"], disk_bytes=c["disk_bytes"], + max_processes=c["max_processes"]), poll=.2, gpu_uuid=c["gpu_uuid"], dispatch_deadline=60.) + except entry.GuardStop as exc: + assert str(exc) == "TRAIN_DISPATCH_DEADLINE" + else: + raise AssertionError("The actual supervisor did not stop before spawn") + marker = entry.read(root / "train-spawn-blocked.json") + assert blocked_spawn.call_count == 0 and not marker["worker_started"] + return dict(marker=marker, real_popen_calls=blocked_spawn.call_count) + + +def preflight(path): + late = simulate(path, latency=1.5, recheck_latency=1.5) + cap = simulate(path, latency=1.5, recheck_latency=0., settle_delay=23.) + baseline = simulate(path, latency=1.5, recheck_latency=0., settle_delay=0.) + busy = simulate(path, latency=2., recheck_latency=2.) + assert late["gate_seconds"] == 59. and late["terminal"]["trigger"] == "TRAIN_DISPATCH_DEADLINE" + assert late["mock_dispatches"] == [] and late["returned"] == 1 + assert round(cap["terminal"]["new_gpu_seconds"], 1) == 13180.8 + assert cap["terminal"]["status"] == "STOPPED" and cap["returned"] == 1 + assert cap["terminal"]["trigger"] == "TOTAL_GPU_CAP_AT_SETTLEMENT" + assert baseline["returned"] == 0 and baseline["terminal"]["status"] == "COMPLETE" + assert busy["terminal"]["trigger"] == "IDLE_TIMEOUT" and busy["mock_dispatches"] == [] + return dict(status="PASS", contract_sha256=sha(path), old_binding_sha256=OLD_BINDING, + late_recheck=late, final_settlement=cap, timely_control=baseline, busy_control=busy, + in_supervisor_deadline=supervisor_boundary(path), real_gpu_queries=0, real_workers=0, + targets_read=False, independent_review="PENDING") + + +if __name__ == "__main__": + path = prepare() + receipt = preflight(path) + save(path.parent / "actual-guard-cpu-replay.json", receipt) + print(receipt)