Freeze Phase 3 item 1b fold-zero review entry

This commit is contained in:
NewMeltAssist committed 2026-11-19 12:15:05 +08:00
1 parent 52c74d25aa
commit a92d0cf774
13 files changed
+1721

No files matched your search

@@ -0,0 +1,130 @@
{
"ruling": "3a48b123da54b0bcff0e7fa672fc74e8920235a0",
"fold": 0,
"validation_cases": [
"00000111_ce24e0cbec8e45c89735d148_step_003",
"00000115_ce24e0cbec8e45c89735d148_step_007",
"00000116_ce24e0cbec8e45c89735d148_step_008",
"00000117_ce24e0cbec8e45c89735d148_step_009",
"00000119_ce24e0cbec8e45c89735d148_step_011",
"00000120_ce24e0cbec8e45c89735d148_step_012",
"00000131_945244874ed84e599987daa9_step_000",
"00000132_438507a38e404bfdbd51a16b_step_000",
"00000133_08b739e0511f4305b161cd05_step_000",
"00000190_185ecbc783f34408aef52119_step_000",
"00000196_c9b71ecdd0654716891b0db8_step_000",
"00000206_27417003e762409185cb2b69_step_000",
"00000238_4829bc0645d6414d9b5cec31_step_001",
"00000241_4829bc0645d6414d9b5cec31_step_004",
"00000243_4829bc0645d6414d9b5cec31_step_006",
"00000248_e87554b09cdc48c3add39db9_step_000",
"00000257_845fb06d7f2f4802a806a600_step_002",
"00000258_845fb06d7f2f4802a806a600_step_003",
"00000259_845fb06d7f2f4802a806a600_step_004",
"00000261_845fb06d7f2f4802a806a600_step_006",
"00000262_845fb06d7f2f4802a806a600_step_007",
"00000264_845fb06d7f2f4802a806a600_step_009",
"00000265_845fb06d7f2f4802a806a600_step_010",
"00000266_845fb06d7f2f4802a806a600_step_011",
"00000267_845fb06d7f2f4802a806a600_step_012",
"00000269_845fb06d7f2f4802a806a600_step_014",
"00000391_fd490b564a44413884fab719_step_000",
"00000405_f3759bbb4682423caf207e8e_step_002",
"00000406_f3759bbb4682423caf207e8e_step_003",
"00000444_4ed4c78d6d754aac90876fc2_step_003",
"00000445_4ed4c78d6d754aac90876fc2_step_004",
"00000501_5499914343bd4f549b38046d_step_002",
"00000502_5499914343bd4f549b38046d_step_003",
"00000574_5a4cef68211d4706b1ec8586_step_003",
"00000578_5a4cef68211d4706b1ec8586_step_007",
"00000579_5a4cef68211d4706b1ec8586_step_008",
"00000580_5a4cef68211d4706b1ec8586_step_009",
"00000582_5a4cef68211d4706b1ec8586_step_011",
"00000583_5a4cef68211d4706b1ec8586_step_012",
"00000597_7fe894812c4244e18241aa08_step_003",
"00000601_7fe894812c4244e18241aa08_step_007",
"00000602_7fe894812c4244e18241aa08_step_008",
"00000603_7fe894812c4244e18241aa08_step_009",
"00000605_7fe894812c4244e18241aa08_step_011",
"00000606_7fe894812c4244e18241aa08_step_012",
"00000654_d87d11d9c3ca4ee5a88797ae_step_002",
"00000758_8ab50540e9804365a8717ec0_step_005",
"00000759_8ab50540e9804365a8717ec0_step_006",
"00000760_8ab50540e9804365a8717ec0_step_007",
"00000763_8ab50540e9804365a8717ec0_step_010",
"00000816_36e6258346004cabb4c091b2_step_004",
"00000817_36e6258346004cabb4c091b2_step_005",
"00000892_be26dc83f78947738b2bfabd_step_001",
"00000894_be26dc83f78947738b2bfabd_step_003",
"00000897_be26dc83f78947738b2bfabd_step_006",
"00000919_cdcebf32644443b9812097ba_step_003",
"00000923_cdcebf32644443b9812097ba_step_007",
"00000927_cdcebf32644443b9812097ba_step_011",
"00000928_cdcebf32644443b9812097ba_step_012",
"00000955_a741c7966d2c40a8a511263e_step_001",
"00000984_0dd1386597104265ab67e6cf_step_007",
"00000986_0dd1386597104265ab67e6cf_step_009",
"00000987_0dd1386597104265ab67e6cf_step_010",
"00000988_0dd1386597104265ab67e6cf_step_011",
"00000996_6d71695ba3fd475c927a00a7_step_000",
"00001019_ca4cf9693d6d4a98b515ba99_step_000",
"00001020_ca4cf9693d6d4a98b515ba99_step_001",
"00001021_ca4cf9693d6d4a98b515ba99_step_002",
"00001026_0f133d8e7fd448708a554e28_step_000",
"00001027_0f133d8e7fd448708a554e28_step_001",
"00001040_74c42207fa764831b03f7658_step_000",
"00001052_2fd6155fa34f418083d0bd0f_step_001",
"00001053_2fd6155fa34f418083d0bd0f_step_002",
"00001097_256ee7ec3a4b4972a8b87c8d_step_001",
"00001099_256ee7ec3a4b4972a8b87c8d_step_003",
"00001102_256ee7ec3a4b4972a8b87c8d_step_006",
"00001107_78566795c3144719804f8d34_step_000",
"00001167_107f61d0402f4c9a9d9d1342_step_000",
"00001168_107f61d0402f4c9a9d9d1342_step_001",
"00001217_e517c61f1c49433bb81cf980_step_000",
"00001238_96503b6a26c64074ad598927_step_000",
"00001239_96503b6a26c64074ad598927_step_001",
"00001240_96503b6a26c64074ad598927_step_002",
"00001241_96503b6a26c64074ad598927_step_003",
"00001242_96503b6a26c64074ad598927_step_004",
"00001243_96503b6a26c64074ad598927_step_005",
"00001244_96503b6a26c64074ad598927_step_006",
"00001246_96503b6a26c64074ad598927_step_008",
"00001247_96503b6a26c64074ad598927_step_009",
"00001249_96503b6a26c64074ad598927_step_011",
"00001252_96503b6a26c64074ad598927_step_014",
"00001253_96503b6a26c64074ad598927_step_015",
"00001314_a9ed5e56a6a8414f91b39bdb_step_000",
"00001333_beab79941e884455a62af40c_step_001",
"00001334_beab79941e884455a62af40c_step_002",
"00001335_beab79941e884455a62af40c_step_003",
"00001336_beab79941e884455a62af40c_step_004",
"00001338_beab79941e884455a62af40c_step_006",
"00001339_beab79941e884455a62af40c_step_007",
"00001340_beab79941e884455a62af40c_step_008",
"00001352_4e1d56bc2cef49ea84626b11_step_000",
"00001353_4e1d56bc2cef49ea84626b11_step_001",
"00001354_4e1d56bc2cef49ea84626b11_step_002",
"00001379_5b1d6e2599ea47d498a04256_step_000",
"00001387_60cf3ad636dd42e0bcec6946_step_000",
"00001511_5cbe58077e4b4885a5c213b3_step_004",
"00001512_5cbe58077e4b4885a5c213b3_step_005",
"00001525_e0d35a03b46745259cf8878f_step_000",
"00001532_9f0a4a641e8f4cc588eb4315_step_000",
"00001552_b0a9c4d836294846891615fe_step_000",
"00001553_adfa1c33ae4d4d0aa441ddc3_step_000"
],
"validation_components": 35,
"score": "Relative L2 on the frozen validation query with original PPS multiplicity, then equal means by component. Negative predictions remain visible.",
"paired_references": [
"long_seed0",
"long_seed1"
],
"delta_short_at_most": -0.001,
"effect_strictly_greater_than_abs_seed_delta": true,
"duration": "100 complete epochs, final epoch only",
"warmup": "Original 5-epoch warmup and full cosine decay over 95 epochs",
"train_fields": "Original seed0, order, 19 inputs, optimizer, MSE and numerical recipe",
"test_allowed": false,
"automatic_continuation": false
}
@@ -0,0 +1,729 @@
{
"version": "phase3-item1b-fold0-screen-v1",
"ruling": "3a48b123da54b0bcff0e7fa672fc74e8920235a0",
"fold": 0,
"screen_fold": 0,
"seed": 0,
"epochs": 100,
"max_fits": 1,
"retry": false,
"test_targets_allowed": false,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"worker_cpus": [
22
],
"total_cpu_seconds": 30000,
"max_processes": 1,
"rss_bytes": 4294967296,
"disk_bytes": 8589934592,
"device_vram_bytes": 4294967296,
"recorded_vram_bytes": 2147483648,
"job": {
"learning": [
"00000003_1ffb81a71e5b402e966b9341_step_002",
"00000004_1ffb81a71e5b402e966b9341_step_003",
"00000049_aa208c45e8734584a147f595_step_000",
"00000057_666139e3bff64d4e8a6ce183_step_006",
"00000058_666139e3bff64d4e8a6ce183_step_007",
"00000060_767e4372b5f94a88a7a17d90_step_001",
"00000061_767e4372b5f94a88a7a17d90_step_002",
"00000065_767e4372b5f94a88a7a17d90_step_006",
"00000066_767e4372b5f94a88a7a17d90_step_007",
"00000067_767e4372b5f94a88a7a17d90_step_008",
"00000068_767e4372b5f94a88a7a17d90_step_009",
"00000070_767e4372b5f94a88a7a17d90_step_011",
"00000072_767e4372b5f94a88a7a17d90_step_013",
"00000077_767e4372b5f94a88a7a17d90_step_018",
"00000078_767e4372b5f94a88a7a17d90_step_019",
"00000079_767e4372b5f94a88a7a17d90_step_020",
"00000087_b4f9335a60a14cecaa599159_step_002",
"00000088_169bec34317b46e98eb1a403_step_000",
"00000093_169bec34317b46e98eb1a403_step_005",
"00000098_72cf0f9ca5324822a4d41a80_step_000",
"00000103_52f7124261ae4c558f800775_step_000",
"00000104_52f7124261ae4c558f800775_step_001",
"00000105_52f7124261ae4c558f800775_step_002",
"00000106_52f7124261ae4c558f800775_step_003",
"00000125_e18c7feacfb54f3880659b43_step_000",
"00000128_e18c7feacfb54f3880659b43_step_003",
"00000129_e18c7feacfb54f3880659b43_step_004",
"00000130_e18c7feacfb54f3880659b43_step_005",
"00000140_2fc54fcd110d4f49969163c4_step_003",
"00000142_2fc54fcd110d4f49969163c4_step_005",
"00000144_2fc54fcd110d4f49969163c4_step_007",
"00000145_2fc54fcd110d4f49969163c4_step_008",
"00000152_a951550b8a1043a68423c541_step_002",
"00000153_a951550b8a1043a68423c541_step_003",
"00000154_a951550b8a1043a68423c541_step_004",
"00000157_8208fd8b44b64dce91322f84_step_000",
"00000161_a583c6c3cdc143bab3f6c856_step_000",
"00000162_4dcda9fab6dd46cf98fb194e_step_000",
"00000163_ff9b2ed255484db78bb0e6da_step_000",
"00000164_91adbf180e114b698160c9a4_step_000",
"00000165_4070a99d6ebe45f5bd5d71eb_step_000",
"00000166_2246cab58dbe4a2fb00fed50_step_000",
"00000170_b21aa794b6b64d87a57c245e_step_000",
"00000171_b21aa794b6b64d87a57c245e_step_001",
"00000173_b21aa794b6b64d87a57c245e_step_003",
"00000174_b21aa794b6b64d87a57c245e_step_004",
"00000175_b21aa794b6b64d87a57c245e_step_005",
"00000176_b21aa794b6b64d87a57c245e_step_006",
"00000181_b21aa794b6b64d87a57c245e_step_011",
"00000198_826494e894454504b0971fd1_step_001",
"00000223_97b354a907114b7183faa9c4_step_001",
"00000224_97b354a907114b7183faa9c4_step_002",
"00000229_682d529d5e8c4387bc59b3a1_step_000",
"00000250_641f03cbe6a145508c6ae055_step_000",
"00000272_648a7c23a21f4e88bf25ffeb_step_000",
"00000273_648a7c23a21f4e88bf25ffeb_step_001",
"00000274_648a7c23a21f4e88bf25ffeb_step_002",
"00000275_648a7c23a21f4e88bf25ffeb_step_003",
"00000276_648a7c23a21f4e88bf25ffeb_step_004",
"00000280_5a6ef3285b89452ea6b390e7_step_001",
"00000282_5a6ef3285b89452ea6b390e7_step_003",
"00000285_d215b7bc60ae468288e5f1ea_step_000",
"00000287_58f8ff46b7564e989f9695b7_step_000",
"00000290_58f8ff46b7564e989f9695b7_step_003",
"00000291_58f8ff46b7564e989f9695b7_step_004",
"00000293_58f8ff46b7564e989f9695b7_step_006",
"00000295_52a153d2417747b5926ff9df_step_000",
"00000296_52a153d2417747b5926ff9df_step_001",
"00000301_52a153d2417747b5926ff9df_step_006",
"00000302_39b08d62a677414f86fdae52_step_000",
"00000329_12a935cd9b684f879c0505fe_step_000",
"00000330_12a935cd9b684f879c0505fe_step_001",
"00000331_12a935cd9b684f879c0505fe_step_002",
"00000336_1a821c3883f7459a9615fca4_step_000",
"00000337_153efbd85ca64c4fb6b7ed5f_step_000",
"00000346_4185972a944744d8a7a0f2b4_step_002",
"00000348_4185972a944744d8a7a0f2b4_step_004",
"00000352_4185972a944744d8a7a0f2b4_step_008",
"00000355_68d1a5c2319a4a419ff625c6_step_001",
"00000356_68d1a5c2319a4a419ff625c6_step_002",
"00000357_68d1a5c2319a4a419ff625c6_step_003",
"00000364_38d899e2706a4d09960705dd_step_001",
"00000367_23c957e8e7b2428282a13ae2_step_000",
"00000368_23c957e8e7b2428282a13ae2_step_001",
"00000369_23c957e8e7b2428282a13ae2_step_002",
"00000374_23c957e8e7b2428282a13ae2_step_007",
"00000375_23c957e8e7b2428282a13ae2_step_008",
"00000376_23c957e8e7b2428282a13ae2_step_009",
"00000377_23c957e8e7b2428282a13ae2_step_010",
"00000378_23c957e8e7b2428282a13ae2_step_011",
"00000379_239bbf0e92974a32bd214366_step_000",
"00000382_239bbf0e92974a32bd214366_step_003",
"00000383_239bbf0e92974a32bd214366_step_004",
"00000386_239bbf0e92974a32bd214366_step_007",
"00000387_239bbf0e92974a32bd214366_step_008",
"00000388_239bbf0e92974a32bd214366_step_009",
"00000418_ce5b31679ee845a69ba89d6f_step_000",
"00000422_8e8a96d437584c77b3cfd84f_step_000",
"00000423_8e8a96d437584c77b3cfd84f_step_001",
"00000424_8e8a96d437584c77b3cfd84f_step_002",
"00000459_516b1784884d4facbc4c0d74_step_007",
"00000462_516b1784884d4facbc4c0d74_step_010",
"00000466_11951e51b89249dba38ded3e_step_001",
"00000469_11951e51b89249dba38ded3e_step_004",
"00000470_11951e51b89249dba38ded3e_step_005",
"00000474_a5639f2b84c445aea5da18ad_step_000",
"00000477_0fc8cc17c6af48dabe36c8f2_step_002",
"00000478_0fc8cc17c6af48dabe36c8f2_step_003",
"00000479_0fc8cc17c6af48dabe36c8f2_step_004",
"00000485_a7d0c851b5f24a97a7cac950_step_002",
"00000488_a7d0c851b5f24a97a7cac950_step_005",
"00000489_a7d0c851b5f24a97a7cac950_step_006",
"00000492_a7d0c851b5f24a97a7cac950_step_009",
"00000513_f78bf7cf156f4231988af062_step_001",
"00000519_5b8fa6b193d14279a77a0e50_step_000",
"00000523_1f8d7f343e634c11a80d23ac_step_000",
"00000526_f9d6559308dc4c85aab9fa38_step_000",
"00000530_bc0d6937206b43bc96521413_step_002",
"00000531_bc0d6937206b43bc96521413_step_003",
"00000543_8cf6554d440d40c4823dad6c_step_000",
"00000559_a1621e156ade4317ab6a5bc5_step_000",
"00000565_5b5119548f7b421e8f597438_step_000",
"00000609_04e799936ae446b9a3d8d2b8_step_000",
"00000610_04e799936ae446b9a3d8d2b8_step_001",
"00000611_04e799936ae446b9a3d8d2b8_step_002",
"00000613_189d1310aa8c4a2a92a21020_step_001",
"00000615_c710d414bbbc4340b682c58a_step_001",
"00000616_c710d414bbbc4340b682c58a_step_002",
"00000617_c710d414bbbc4340b682c58a_step_003",
"00000618_c710d414bbbc4340b682c58a_step_004",
"00000619_c710d414bbbc4340b682c58a_step_005",
"00000620_c710d414bbbc4340b682c58a_step_006",
"00000621_c710d414bbbc4340b682c58a_step_007",
"00000622_c710d414bbbc4340b682c58a_step_008",
"00000624_1eacb269d52a47f5b265c1ab_step_000",
"00000626_f686126d229f47aeb125b5ba_step_000",
"00000627_53bfa7e2e04c4d9dbf4a3db2_step_000",
"00000632_4f322b34e87c43289916c298_step_001",
"00000658_a40da330321b44f286a27361_step_000",
"00000683_8863c3497d6d4e05b0f41ddc_step_000",
"00000685_24615f5ff44d4cd5809e77a5_step_001",
"00000686_24615f5ff44d4cd5809e77a5_step_002",
"00000688_b9f3563b403143dbb86ab2b8_step_001",
"00000691_b9f3563b403143dbb86ab2b8_step_004",
"00000693_b9f3563b403143dbb86ab2b8_step_006",
"00000694_45f6100bfa3948e69405bb21_step_000",
"00000696_32309198fce9481896bd0a99_step_000",
"00000698_6f0fc559e8f3422c908b6060_step_001",
"00000699_cae88f0aaaf34de597863810_step_000",
"00000708_c08c1c78b86f4ee3bcb7d9de_step_004",
"00000709_c08c1c78b86f4ee3bcb7d9de_step_005",
"00000710_c08c1c78b86f4ee3bcb7d9de_step_006",
"00000721_9a461e8ae7dc4d33a55883eb_step_000",
"00000723_8ed79b6bcb024d1a9f2aa5e7_step_000",
"00000737_989e0f807cde438face0cb00_step_002",
"00000738_989e0f807cde438face0cb00_step_003",
"00000751_6f0d90054d40400593050930_step_000",
"00000752_6f0d90054d40400593050930_step_001",
"00000764_9cab8335065e4d439bbff471_step_000",
"00000765_9cab8335065e4d439bbff471_step_001",
"00000766_9cab8335065e4d439bbff471_step_002",
"00000767_9cab8335065e4d439bbff471_step_003",
"00000775_a25fb37392a44447b9684f16_step_000",
"00000776_a25fb37392a44447b9684f16_step_001",
"00000777_a25fb37392a44447b9684f16_step_002",
"00000778_a25fb37392a44447b9684f16_step_003",
"00000779_a25fb37392a44447b9684f16_step_004",
"00000780_a25fb37392a44447b9684f16_step_005",
"00000781_a25fb37392a44447b9684f16_step_006",
"00000782_a25fb37392a44447b9684f16_step_007",
"00000783_a25fb37392a44447b9684f16_step_008",
"00000784_a25fb37392a44447b9684f16_step_009",
"00000785_a25fb37392a44447b9684f16_step_010",
"00000786_a25fb37392a44447b9684f16_step_011",
"00000787_a25fb37392a44447b9684f16_step_012",
"00000788_a25fb37392a44447b9684f16_step_013",
"00000789_a25fb37392a44447b9684f16_step_014",
"00000790_a25fb37392a44447b9684f16_step_015",
"00000791_a25fb37392a44447b9684f16_step_016",
"00000792_a25fb37392a44447b9684f16_step_017",
"00000793_a25fb37392a44447b9684f16_step_018",
"00000797_e7533c87b9a347cdafd0c390_step_000",
"00000799_ad6107d9fc994ccca38cc4e4_step_001",
"00000800_ad6107d9fc994ccca38cc4e4_step_002",
"00000801_ad6107d9fc994ccca38cc4e4_step_003",
"00000802_ad6107d9fc994ccca38cc4e4_step_004",
"00000818_6bbad2756d7845bd8d731d60_step_000",
"00000819_6bbad2756d7845bd8d731d60_step_001",
"00000821_6bbad2756d7845bd8d731d60_step_003",
"00000822_c98978894f544f89aa10633f_step_000",
"00000823_c98978894f544f89aa10633f_step_001",
"00000827_b0d209f7a8ad4b02910c317f_step_003",
"00000829_889a065d51a24bb8ad704c69_step_000",
"00000830_889a065d51a24bb8ad704c69_step_001",
"00000832_889a065d51a24bb8ad704c69_step_003",
"00000834_889a065d51a24bb8ad704c69_step_005",
"00000837_b5409201f9a5495fa169af85_step_000",
"00000889_e2d3a4dc5f7c438bb240e9d1_step_000",
"00000890_e2d3a4dc5f7c438bb240e9d1_step_001",
"00000900_4df6ab9924674556bb4a509d_step_000",
"00000904_4df6ab9924674556bb4a509d_step_004",
"00000905_4df6ab9924674556bb4a509d_step_005",
"00000906_4df6ab9924674556bb4a509d_step_006",
"00000942_f297a24e1a4446cf9d74efd9_step_001",
"00000948_f297a24e1a4446cf9d74efd9_step_007",
"00000949_f297a24e1a4446cf9d74efd9_step_008",
"00000953_ccc14f0fcff54b5985cc9113_step_000",
"00000958_04906edf324a4bce8fc3f417_step_000",
"00000961_a98d481268ea4524a7769731_step_000",
"00000962_a98d481268ea4524a7769731_step_001",
"00000963_a98d481268ea4524a7769731_step_002",
"00000973_209426c5054945fab8a4af1b_step_000",
"00000976_b85ec2ff588148c4abd8c8d7_step_000",
"00001024_21adcacb9707488ea6655fd2_step_000",
"00001025_21adcacb9707488ea6655fd2_step_001",
"00001029_cde47df6e28543d1b434f206_step_001",
"00001030_cde47df6e28543d1b434f206_step_002",
"00001031_cde47df6e28543d1b434f206_step_003",
"00001033_18151464e6434759b7c2ece7_step_000",
"00001035_be3ed2c9696e4e6a91b34f79_step_000",
"00001036_be3ed2c9696e4e6a91b34f79_step_001",
"00001041_6baf25f5fe2947e8927797d8_step_000",
"00001048_2b2638f82b3946b1867f2d11_step_004",
"00001049_2b2638f82b3946b1867f2d11_step_005",
"00001050_2b2638f82b3946b1867f2d11_step_006",
"00001061_284effca5a19488fa8a228e2_step_002",
"00001062_c04956ddff6c4324b5cad67c_step_000",
"00001106_ba88e7579ced4a0aa2de629c_step_000",
"00001108_412675e2ea594e70bbd8cf6c_step_000",
"00001114_5f5cf6feacbc41fb88b2d873_step_000",
"00001121_da4037e40839451dbbc2bd1f_step_000",
"00001130_c2db313c2ce749d784c610d2_step_002",
"00001139_0b0ed9b0ccbb455d82c60d69_step_000",
"00001140_0b0ed9b0ccbb455d82c60d69_step_001",
"00001141_0b0ed9b0ccbb455d82c60d69_step_002",
"00001146_274866da478e4a009bed8750_step_001",
"00001159_f112d6f3e0c644ae9de2a724_step_000",
"00001165_4b65bb9a02f6494aa897a53e_step_000",
"00001169_0889c458f3f74dc288d2d531_step_000",
"00001170_0889c458f3f74dc288d2d531_step_001",
"00001182_bc0b5846e3d84904ad02e547_step_001",
"00001185_507b60b57791425bbb73273e_step_000",
"00001186_507b60b57791425bbb73273e_step_001",
"00001187_507b60b57791425bbb73273e_step_002",
"00001190_507b60b57791425bbb73273e_step_005",
"00001191_507b60b57791425bbb73273e_step_006",
"00001193_46b17dd88ed24f55bca2eb69_step_000",
"00001200_e44a0f9afbe84b918f789f0f_step_000",
"00001206_e44a0f9afbe84b918f789f0f_step_006",
"00001210_4be610c137ba42ae9933e69b_step_000",
"00001211_4be610c137ba42ae9933e69b_step_001",
"00001212_4be610c137ba42ae9933e69b_step_002",
"00001213_4be610c137ba42ae9933e69b_step_003",
"00001214_9edb90c9f417497fb70bddf6_step_000",
"00001228_2fd836ae38b44c41bfbd9584_step_002",
"00001232_73b40f9a4cd3419ab3b42b73_step_001",
"00001233_73b40f9a4cd3419ab3b42b73_step_002",
"00001235_17ea801126da4654b34f6c52_step_000",
"00001254_c2da273d137e475abb1ab1f4_step_000",
"00001255_3df9015915ff4fa7b57f0a71_step_000",
"00001257_a4715d363f88400c8f0238d1_step_000",
"00001260_a4715d363f88400c8f0238d1_step_003",
"00001266_b53ece83d8964b44bbf1f8ed_step_002",
"00001273_b53ece83d8964b44bbf1f8ed_step_009",
"00001274_b53ece83d8964b44bbf1f8ed_step_010",
"00001276_b53ece83d8964b44bbf1f8ed_step_012",
"00001285_b53ece83d8964b44bbf1f8ed_step_021",
"00001294_b53ece83d8964b44bbf1f8ed_step_030",
"00001298_b53ece83d8964b44bbf1f8ed_step_034",
"00001300_b9951da825954791b59b979d_step_000",
"00001301_a54ee779467b42ea8709e1ca_step_000",
"00001311_e95b8713cc0b4436909c78e9_step_000",
"00001319_6a5a7dbad03b4a57b33f2c0d_step_002",
"00001320_6a5a7dbad03b4a57b33f2c0d_step_003",
"00001321_7cef9b79a6b04db894fd74e5_step_000",
"00001324_5bf4a74af99f405392406e40_step_000",
"00001344_d8d97699a52842539e08d117_step_002",
"00001345_d8d97699a52842539e08d117_step_003",
"00001346_d8d97699a52842539e08d117_step_004",
"00001348_318b81a8e412438183201be7_step_000",
"00001349_27327ddba7f64417ae1e69a3_step_000",
"00001356_aa0de7c5e9e84d73816bdbde_step_000",
"00001380_c49484307f434da78fa06b1f_step_000",
"00001381_c49484307f434da78fa06b1f_step_001",
"00001389_21b525da345e4e62aebd906b_step_000",
"00001391_406eb61c554a429ebd8a4d4f_step_000",
"00001392_406eb61c554a429ebd8a4d4f_step_001",
"00001393_406eb61c554a429ebd8a4d4f_step_002",
"00001394_406eb61c554a429ebd8a4d4f_step_003",
"00001414_b38f661707674b8eb983d992_step_000",
"00001490_6c8ef6d2f5564d139a5077f8_step_001",
"00001491_6c8ef6d2f5564d139a5077f8_step_002",
"00001495_6c8ef6d2f5564d139a5077f8_step_006",
"00001496_eb4e2b01aaca4131bfc64adb_step_000",
"00001497_eb4e2b01aaca4131bfc64adb_step_001",
"00001500_eb4e2b01aaca4131bfc64adb_step_004",
"00001501_6a0f221e85e04411afe23f51_step_000",
"00001503_6a0f221e85e04411afe23f51_step_002",
"00001505_6d02705716e04d9081f4980b_step_000",
"00001527_9214869844f14e83a9dce3ec_step_000",
"00001534_e1d62958d7ca4ad18fefd54d_step_001",
"00001535_e1d62958d7ca4ad18fefd54d_step_002",
"00001538_e1d62958d7ca4ad18fefd54d_step_005",
"00001543_e1d62958d7ca4ad18fefd54d_step_010",
"00001550_a94d978cf62948ce80b417a9_step_000",
"00001554_30de00089a3f4a339eddb68d_step_000",
"00001555_30de00089a3f4a339eddb68d_step_001",
"00001556_30de00089a3f4a339eddb68d_step_002",
"00001557_16c191aef5034779aaab6e3f_step_000",
"00001563_2e9dc7e9a1a54dd7ab901478_step_000",
"00001567_9280741c840d4aa1af32658e_step_002",
"00001569_6fc579f3224d4d53a1b60710_step_000",
"00001578_a3b89691665043a08931897f_step_001",
"00001579_a3b89691665043a08931897f_step_002",
"00001582_a3b89691665043a08931897f_step_005",
"00001586_763da6a559544098acbf4154_step_002",
"00001592_35f4abaacc9447e4ac0b964f_step_001",
"00001594_2128dbbd4bf34879913b3a99_step_000",
"00001596_2128dbbd4bf34879913b3a99_step_002",
"00001597_13bd23e0e4c740969b0c71ab_step_000"
],
"validation": [
"00000111_ce24e0cbec8e45c89735d148_step_003",
"00000115_ce24e0cbec8e45c89735d148_step_007",
"00000116_ce24e0cbec8e45c89735d148_step_008",
"00000117_ce24e0cbec8e45c89735d148_step_009",
"00000119_ce24e0cbec8e45c89735d148_step_011",
"00000120_ce24e0cbec8e45c89735d148_step_012",
"00000131_945244874ed84e599987daa9_step_000",
"00000132_438507a38e404bfdbd51a16b_step_000",
"00000133_08b739e0511f4305b161cd05_step_000",
"00000190_185ecbc783f34408aef52119_step_000",
"00000196_c9b71ecdd0654716891b0db8_step_000",
"00000206_27417003e762409185cb2b69_step_000",
"00000238_4829bc0645d6414d9b5cec31_step_001",
"00000241_4829bc0645d6414d9b5cec31_step_004",
"00000243_4829bc0645d6414d9b5cec31_step_006",
"00000248_e87554b09cdc48c3add39db9_step_000",
"00000257_845fb06d7f2f4802a806a600_step_002",
"00000258_845fb06d7f2f4802a806a600_step_003",
"00000259_845fb06d7f2f4802a806a600_step_004",
"00000261_845fb06d7f2f4802a806a600_step_006",
"00000262_845fb06d7f2f4802a806a600_step_007",
"00000264_845fb06d7f2f4802a806a600_step_009",
"00000265_845fb06d7f2f4802a806a600_step_010",
"00000266_845fb06d7f2f4802a806a600_step_011",
"00000267_845fb06d7f2f4802a806a600_step_012",
"00000269_845fb06d7f2f4802a806a600_step_014",
"00000391_fd490b564a44413884fab719_step_000",
"00000405_f3759bbb4682423caf207e8e_step_002",
"00000406_f3759bbb4682423caf207e8e_step_003",
"00000444_4ed4c78d6d754aac90876fc2_step_003",
"00000445_4ed4c78d6d754aac90876fc2_step_004",
"00000501_5499914343bd4f549b38046d_step_002",
"00000502_5499914343bd4f549b38046d_step_003",
"00000574_5a4cef68211d4706b1ec8586_step_003",
"00000578_5a4cef68211d4706b1ec8586_step_007",
"00000579_5a4cef68211d4706b1ec8586_step_008",
"00000580_5a4cef68211d4706b1ec8586_step_009",
"00000582_5a4cef68211d4706b1ec8586_step_011",
"00000583_5a4cef68211d4706b1ec8586_step_012",
"00000597_7fe894812c4244e18241aa08_step_003",
"00000601_7fe894812c4244e18241aa08_step_007",
"00000602_7fe894812c4244e18241aa08_step_008",
"00000603_7fe894812c4244e18241aa08_step_009",
"00000605_7fe894812c4244e18241aa08_step_011",
"00000606_7fe894812c4244e18241aa08_step_012",
"00000654_d87d11d9c3ca4ee5a88797ae_step_002",
"00000758_8ab50540e9804365a8717ec0_step_005",
"00000759_8ab50540e9804365a8717ec0_step_006",
"00000760_8ab50540e9804365a8717ec0_step_007",
"00000763_8ab50540e9804365a8717ec0_step_010",
"00000816_36e6258346004cabb4c091b2_step_004",
"00000817_36e6258346004cabb4c091b2_step_005",
"00000892_be26dc83f78947738b2bfabd_step_001",
"00000894_be26dc83f78947738b2bfabd_step_003",
"00000897_be26dc83f78947738b2bfabd_step_006",
"00000919_cdcebf32644443b9812097ba_step_003",
"00000923_cdcebf32644443b9812097ba_step_007",
"00000927_cdcebf32644443b9812097ba_step_011",
"00000928_cdcebf32644443b9812097ba_step_012",
"00000955_a741c7966d2c40a8a511263e_step_001",
"00000984_0dd1386597104265ab67e6cf_step_007",
"00000986_0dd1386597104265ab67e6cf_step_009",
"00000987_0dd1386597104265ab67e6cf_step_010",
"00000988_0dd1386597104265ab67e6cf_step_011",
"00000996_6d71695ba3fd475c927a00a7_step_000",
"00001019_ca4cf9693d6d4a98b515ba99_step_000",
"00001020_ca4cf9693d6d4a98b515ba99_step_001",
"00001021_ca4cf9693d6d4a98b515ba99_step_002",
"00001026_0f133d8e7fd448708a554e28_step_000",
"00001027_0f133d8e7fd448708a554e28_step_001",
"00001040_74c42207fa764831b03f7658_step_000",
"00001052_2fd6155fa34f418083d0bd0f_step_001",
"00001053_2fd6155fa34f418083d0bd0f_step_002",
"00001097_256ee7ec3a4b4972a8b87c8d_step_001",
"00001099_256ee7ec3a4b4972a8b87c8d_step_003",
"00001102_256ee7ec3a4b4972a8b87c8d_step_006",
"00001107_78566795c3144719804f8d34_step_000",
"00001167_107f61d0402f4c9a9d9d1342_step_000",
"00001168_107f61d0402f4c9a9d9d1342_step_001",
"00001217_e517c61f1c49433bb81cf980_step_000",
"00001238_96503b6a26c64074ad598927_step_000",
"00001239_96503b6a26c64074ad598927_step_001",
"00001240_96503b6a26c64074ad598927_step_002",
"00001241_96503b6a26c64074ad598927_step_003",
"00001242_96503b6a26c64074ad598927_step_004",
"00001243_96503b6a26c64074ad598927_step_005",
"00001244_96503b6a26c64074ad598927_step_006",
"00001246_96503b6a26c64074ad598927_step_008",
"00001247_96503b6a26c64074ad598927_step_009",
"00001249_96503b6a26c64074ad598927_step_011",
"00001252_96503b6a26c64074ad598927_step_014",
"00001253_96503b6a26c64074ad598927_step_015",
"00001314_a9ed5e56a6a8414f91b39bdb_step_000",
"00001333_beab79941e884455a62af40c_step_001",
"00001334_beab79941e884455a62af40c_step_002",
"00001335_beab79941e884455a62af40c_step_003",
"00001336_beab79941e884455a62af40c_step_004",
"00001338_beab79941e884455a62af40c_step_006",
"00001339_beab79941e884455a62af40c_step_007",
"00001340_beab79941e884455a62af40c_step_008",
"00001352_4e1d56bc2cef49ea84626b11_step_000",
"00001353_4e1d56bc2cef49ea84626b11_step_001",
"00001354_4e1d56bc2cef49ea84626b11_step_002",
"00001379_5b1d6e2599ea47d498a04256_step_000",
"00001387_60cf3ad636dd42e0bcec6946_step_000",
"00001511_5cbe58077e4b4885a5c213b3_step_004",
"00001512_5cbe58077e4b4885a5c213b3_step_005",
"00001525_e0d35a03b46745259cf8878f_step_000",
"00001532_9f0a4a641e8f4cc588eb4315_step_000",
"00001552_b0a9c4d836294846891615fe_step_000",
"00001553_adfa1c33ae4d4d0aa441ddc3_step_000"
],
"test": [
"00000006_d4fe04f0f5f84b52bd4f10e4_step_001",
"00000008_9b3d6a97e8de4aa193b81000_step_000",
"00000009_9b3d6a97e8de4aa193b81000_step_001",
"00000011_e909f412cda24521865fac0f_step_000",
"00000102_5ed74bccca6f4e89829bcb5e_step_002",
"00000183_6c4172e8a17140f89d3b1c0a_step_000",
"00000220_5324c79b172f41e8a49afc6f_step_007",
"00000221_5324c79b172f41e8a49afc6f_step_008",
"00000249_07a2da801a60492fb1e5eeb3_step_000",
"00000408_36bcb4030e36417b8dc74b87_step_000",
"00000410_30afd12eede44fa8a1def049_step_000",
"00000411_305e45c9479f43979b0d7b53_step_000",
"00000413_94f421b1b33d47c592c9fe1d_step_000",
"00000421_bd0972033f464f8f961a150a_step_002",
"00000508_d1c0a44331fd41faa91d4c2e_step_000",
"00000509_d1c0a44331fd41faa91d4c2e_step_001",
"00000510_d1c0a44331fd41faa91d4c2e_step_002",
"00000522_a56dd4152b7e4ccf91212d7b_step_000",
"00000527_db75f3a38d9746a1bd66168d_step_000",
"00000542_4b1756f6e72b45c29a8e4eb4_step_000",
"00000549_41a24c0b74a34217832f5af0_step_000",
"00000551_41a24c0b74a34217832f5af0_step_002",
"00000562_1724088724bf4e53a36e363e_step_000",
"00000566_2f2b015ef4904c1caef56154_step_000",
"00000637_5ae45a997c76423aba2fcc11_step_002",
"00000638_5ae45a997c76423aba2fcc11_step_003",
"00000643_bb55eaa4e9e24c2ea9d18ee0_step_000",
"00000659_6fae5591b3034f949b7e8eae_step_000",
"00000695_3f46763af0744ab2a90e6b48_step_000",
"00000746_6cb3ddbb49974d7db0e97135_step_001",
"00000796_dec80701adc84239b485449d_step_002",
"00000806_05cadeb104be4c8aa75ccc71_step_000",
"00000807_05cadeb104be4c8aa75ccc71_step_001",
"00000808_05cadeb104be4c8aa75ccc71_step_002",
"00000809_05cadeb104be4c8aa75ccc71_step_003",
"00000840_2c1efcc32d094e9fb9e1a88e_step_000",
"00000842_2c1efcc32d094e9fb9e1a88e_step_002",
"00000843_2c1efcc32d094e9fb9e1a88e_step_003",
"00000844_2c1efcc32d094e9fb9e1a88e_step_004",
"00000849_2c1efcc32d094e9fb9e1a88e_step_009",
"00000850_2c1efcc32d094e9fb9e1a88e_step_010",
"00000851_2c1efcc32d094e9fb9e1a88e_step_011",
"00000855_2c1efcc32d094e9fb9e1a88e_step_015",
"00000856_2c1efcc32d094e9fb9e1a88e_step_016",
"00000859_58ac6a347bbe4cf5ae07a565_step_000",
"00000863_029c2575dd474650ba1b8d21_step_003",
"00000864_029c2575dd474650ba1b8d21_step_004",
"00000866_029c2575dd474650ba1b8d21_step_006",
"00000867_029c2575dd474650ba1b8d21_step_007",
"00000868_029c2575dd474650ba1b8d21_step_008",
"00000870_029c2575dd474650ba1b8d21_step_010",
"00000871_029c2575dd474650ba1b8d21_step_011",
"00000872_029c2575dd474650ba1b8d21_step_012",
"00000873_029c2575dd474650ba1b8d21_step_013",
"00000874_029c2575dd474650ba1b8d21_step_014",
"00000877_029c2575dd474650ba1b8d21_step_017",
"00000879_029c2575dd474650ba1b8d21_step_019",
"00000880_029c2575dd474650ba1b8d21_step_020",
"00000882_029c2575dd474650ba1b8d21_step_022",
"00000883_029c2575dd474650ba1b8d21_step_023",
"00000884_029c2575dd474650ba1b8d21_step_024",
"00000885_029c2575dd474650ba1b8d21_step_025",
"00000886_029c2575dd474650ba1b8d21_step_026",
"00000887_029c2575dd474650ba1b8d21_step_027",
"00000938_4f351b53bfd84bb4a2f7809c_step_003",
"00000970_edacb215ec6949d8af6c8c68_step_000",
"00000993_ac4ba032d4b5431ca04f542c_step_000",
"00000994_ac4ba032d4b5431ca04f542c_step_001",
"00000998_6962bc8aad2c4e27b046d5b4_step_000",
"00001000_6962bc8aad2c4e27b046d5b4_step_002",
"00001002_6962bc8aad2c4e27b046d5b4_step_004",
"00001003_6962bc8aad2c4e27b046d5b4_step_005",
"00001004_6962bc8aad2c4e27b046d5b4_step_006",
"00001005_6962bc8aad2c4e27b046d5b4_step_007",
"00001006_6962bc8aad2c4e27b046d5b4_step_008",
"00001009_6962bc8aad2c4e27b046d5b4_step_011",
"00001010_6962bc8aad2c4e27b046d5b4_step_012",
"00001011_6962bc8aad2c4e27b046d5b4_step_013",
"00001013_6962bc8aad2c4e27b046d5b4_step_015",
"00001014_6962bc8aad2c4e27b046d5b4_step_016",
"00001015_6962bc8aad2c4e27b046d5b4_step_017",
"00001034_085828cd621d404a90ccd883_step_000",
"00001063_c6fda10dc58448d1951c5cc3_step_000",
"00001064_c6fda10dc58448d1951c5cc3_step_001",
"00001065_c6fda10dc58448d1951c5cc3_step_002",
"00001067_8a2edcf36bcd435092f4c8ae_step_001",
"00001069_8a2edcf36bcd435092f4c8ae_step_003",
"00001071_8a2edcf36bcd435092f4c8ae_step_005",
"00001112_3bd4a008a3a0483d86ccab3f_step_000",
"00001177_9f1ce06d5e334fe88e1581aa_step_001",
"00001178_515d7e1aeb364a2d88083cf3_step_000",
"00001262_157e5348bf6545e282472856_step_000",
"00001263_157e5348bf6545e282472856_step_001",
"00001305_3f3a389e9d5a4370b952bdce_step_001",
"00001310_3f3a389e9d5a4370b952bdce_step_006",
"00001358_2e314c6e080b40fcb6a6db96_step_001",
"00001360_2e314c6e080b40fcb6a6db96_step_003",
"00001368_2e314c6e080b40fcb6a6db96_step_011",
"00001369_2e314c6e080b40fcb6a6db96_step_012",
"00001371_2e314c6e080b40fcb6a6db96_step_014",
"00001373_2e314c6e080b40fcb6a6db96_step_016",
"00001375_2e314c6e080b40fcb6a6db96_step_018",
"00001377_2e314c6e080b40fcb6a6db96_step_020",
"00001398_49414db2dd594dc6a3907cf3_step_001",
"00001399_49414db2dd594dc6a3907cf3_step_002",
"00001400_49414db2dd594dc6a3907cf3_step_003",
"00001401_49414db2dd594dc6a3907cf3_step_004",
"00001402_49414db2dd594dc6a3907cf3_step_005",
"00001403_49414db2dd594dc6a3907cf3_step_006",
"00001404_49414db2dd594dc6a3907cf3_step_007",
"00001405_49414db2dd594dc6a3907cf3_step_008",
"00001406_49414db2dd594dc6a3907cf3_step_009",
"00001407_49414db2dd594dc6a3907cf3_step_010",
"00001408_49414db2dd594dc6a3907cf3_step_011",
"00001409_49414db2dd594dc6a3907cf3_step_012",
"00001410_49414db2dd594dc6a3907cf3_step_013",
"00001411_49414db2dd594dc6a3907cf3_step_014",
"00001412_49414db2dd594dc6a3907cf3_step_015",
"00001413_4642a707b7894949ba816427_step_000",
"00001522_7079a898746d41ba8ae13d69_step_003",
"00001598_c118664e9f6f4943855293fa_step_000"
],
"c": 1.026602777749673,
"clock_sha256": "c7b76bd79e17565b05669ec31a0e9fbf16e0113e6d6a3d913514075e5aedea96",
"learning_nodes": 8742060,
"id": 0,
"target_steps": 32000,
"cpu_seconds": 30000,
"validation_seconds_bound": 12.383006267948076,
"rss_bytes": 4294967296,
"recorded_vram_bytes": 2147483648,
"process_vram_bytes": 4294967296,
"disk_bytes": 8589934592
},
"budget": {
"item_gpu_limit_seconds": 90000,
"phase3_gpu_limit_seconds": 720000,
"prior_item_gpu_seconds": 19262.057604262838,
"prior_phase3_gpu_seconds": 408630.4104626046,
"train_epoch_seconds_basis": 99.90287962602451,
"validation_seconds_basis": 16.66734165791422,
"train_gpu_seconds": 12817,
"screen_gpu_seconds": 183,
"training_setup_tail_seconds": 120,
"idle_wait_seconds": 60,
"finalization_gpu_seconds": 120,
"projected_new_gpu_seconds": 13180
},
"references": {
"long_seed0": {
"checkpoint": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/last-epoch200.pt",
"checkpoint_sha256": "35cc9e41a4197e186570089ac305c4dee100f3d37611e0ca17e01db441b93756",
"standardization": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/standardization.json",
"standardization_sha256": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421",
"seed": 0,
"epochs": 200,
"training_contract_sha256": "9ea2ad987ce165660c1c1f648a3edf8069c40ef270e233f869fd881e42d7c382"
},
"long_seed1": {
"checkpoint": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/last-epoch200.pt",
"checkpoint_sha256": "b223cb3eb36efff52e5905f7c6a39a40a0e32e38f64fe9947d2597f68e7c3923",
"standardization": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/standardization.json",
"standardization_sha256": "98eabfb586fb7689bd31990b49301ccaceec34f84e16d220f3e3e7ee34f8e681",
"seed": 1,
"epochs": 200,
"training_contract_sha256": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981"
}
},
"pilot_contract": "/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json",
"query_roster_contract": "/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/contract.json",
"comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-screen-2026-11-19/comparison-contract.json",
"comparison_contract_sha256": "bd71a5a6a0f3a288267f568cbb85d1cea5ab6fa0485e7fd212cd55e36f664954",
"source_sha256": {
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/contract.json": "d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/complete.json": "430247e44f301f984f396c662e6fa5419d09649476f6b2a81076444731f55dde",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/post-terminal-cost.json": "7241fe629e3227b81655886c6c208336deef443d67c8d6efd7530000de8477bf",
"/home/frank14f/MeltMind-runs/phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json": "ca576e271715c862ee21e75b2dec7b03e45317e5f8bb073adae016a8001a0a59",
"/home/frank14f/MeltMind-runs/phase3-seed1-postfit-2026-11-18/post-terminal-cost.json": "927546a8991fc2d39127067c4907ae74276d6ec6d5d0a82e0f57db7951813bbe",
"/home/frank14f/MeltMind-runs/phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json": "2de53b3aca1ae21bc53480ad7425094f30273f798cd292a895526094c993b41f",
"/home/frank14f/MeltMind-runs/phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json": "ea99e46a6ffe687004f174df58fdd0f1c13d0231bac7e28e6cbdd988b7fe5c84",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/worker/history/001.json": "579391f43569643a4ea8ac1af6e2043ebeb9bf149b2c17b3ca3b3ef0aa2c43e5",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/assignment.json": "050e8b6748a3e2a287eb905f75179daa9941bd5690660f2c2716298fbf71bd1d",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/assignment.json": "a2699b9e9174eb7ba8a4b54a4764f58f28fb794be0008083f36765c738035b26",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/worker/worker-identity.json": "98094ad553095111798c43d8b33ec14cbe95955b0df5110e233e176856a0a881",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/worker/worker-identity.json": "2953a472cdff619d52235420837880bfc551ce65b5699db77124b3f1a0e147ca",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.started.json": "c4985f4c4b5d24c81a29feb1b275d41213fc1f2a9583d01c48130007564ac401",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.started.json": "b4c43b504190fe334d049fad6889f6d998e64140ef760958a1308612cf538bf0",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/guards/train.receipt.json": "36b9c44bcf2248be07314300ee8c99d214232869e24399314e0907c9c2469b5b",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/2/guards/train.receipt.json": "640f14ff26fcba3c6213b111c81463a1261efd6992ff7fb3c827d7da06b6195f",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18/jobs/0/gpu-sample.json": "9e92c761638268f48c1436243a18de13e41d67b15a335ff18cd9bc5a037790bc",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/epoch-host-observations/026.json": "5f3ccb39ea8669bb5fed7260b3b786ace6366ff59a0a4a0f135681e6311dd830",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/history/026.json": "0ce765d2bc3cc68640971afaa9f2a78a3f59ae0602c2c60d0f690533891b43f4",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/epoch-host-observations/020.json": "91f70761c95aa8f211b050c39685ae2a1981891f8151f17aa75805eaaa1c72e6",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/history/020.json": "776c76e9882a3470fe1264d1c61b4307e0f6a9f0fcfc320d04a6d6b05becfd01",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-pilot-2026-11-17/contract.json": "9f616974fb01f8f5cab6233273d541b3e53f5975f3632db45ab4676f911fba7b",
"/home/frank14f/MeltMind-runs/phase2-h2-s2-postfit-2026-11-17/contract.json": "0bd03f42c3b2db5bef26708b2ef873edfe674f724b59bc4b4de8641980d8d330",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/contract.json": "9ea2ad987ce165660c1c1f648a3edf8069c40ef270e233f869fd881e42d7c382",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/result.json": "f2bbe0d6ff36bcdbcfb36cc9cb17459c65e4f7b5d47fa7252a2c73fbbfb8387e",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/standardization.json": "8f8038dcbf9dbf3e9fc3dbbb439ecedf51d04a30ce31aa52a2dacce926b24421",
"/home/frank14f/optane/MeltMind/artifacts/phase2-h2-s1-2026-11-17/worker/last-epoch200.pt": "35cc9e41a4197e186570089ac305c4dee100f3d37611e0ca17e01db441b93756",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/contract.json": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/result.json": "79b38fd28b590a4ff99b378d76f32cbf6f959d056cfa8e3c92b4a9d5e9b611ac",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/standardization.json": "98eabfb586fb7689bd31990b49301ccaceec34f84e16d220f3e3e7ee34f8e681",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18/jobs/0/worker/last-epoch200.pt": "b223cb3eb36efff52e5905f7c6a39a40a0e32e38f64fe9947d2597f68e7c3923",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-screen-2026-11-19/timing-and-budget.json": "33a2908ae8785b405869d7ef957445bd93566c5d73c1a26dd86489fe077a4232"
},
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_fold0.py": "1446919f2274bc6207527ad80cbd634e47761f9d6391543d420d4f8cd6498527",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "06cb97594c3a87f42f712dcde6cec8c2d90e0b50a89a6be7954bdd2f5c18d5de",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "b781e56c519401bd38af6a522deb24506cba6993d2af008f85ea4e1715f1d9c6",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b_fold0.py": "f48ebe3e3aeca03e5d4cbdd532061cfb27362ec84e85dae70e77e22b4e40b8e0",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "08b7d8ab2ac773fab8b749f9f61af259254422128e9634f8b715c81efce2651f",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "31dfd71369c63512c1d6fe176fd22e8db646520c92da533bb5530ca3843c388d",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "c7bc9f3b7cf4137f52e1efad19dcb7045282d4e9a9c1aae50f066c9e488f7c01",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "b4d78f4384651711fe83ff00eb6719c6c61664699d801ecb34978b9546439da9",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "3115aa9049974eac86657aa4cee7b5d716f6ee93268555cc86242029ebefce4c",
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073"
}
}
@@ -0,0 +1,26 @@
{
"status": "PASS",
"actual_contract_sha256": "942323ddfceecf47515eb48628c45157073cbb909867a606ef4d3764a13be7d4",
"cases": [
{
"case": "idle-dispatch",
"exit_code": 0,
"trigger": "SCREEN_PASS",
"mock_dispatches": [
"train",
"screen"
],
"gate_wait_seconds": 20.0
},
{
"case": "busy-timeout",
"exit_code": 1,
"trigger": "IDLE_TIMEOUT",
"mock_dispatches": [],
"gate_wait_seconds": 60.0
}
],
"real_gpu_or_worker_started": false,
"scientific_arrays_read": false,
"review_status": "PENDING"
}
@@ -0,0 +1,53 @@
{
"prior_phase3_costs": {
"item1": 72421.31599938683,
"seed1": 114756.77075582312,
"item1b": 19262.057604262838,
"c": 105994.92276056076,
"b": 96195.34334257105
},
"budget": {
"item_gpu_limit_seconds": 90000,
"phase3_gpu_limit_seconds": 720000,
"prior_item_gpu_seconds": 19262.057604262838,
"prior_phase3_gpu_seconds": 408630.4104626046,
"train_epoch_seconds_basis": 99.90287962602451,
"validation_seconds_basis": 16.66734165791422,
"train_gpu_seconds": 12817,
"screen_gpu_seconds": 183,
"training_setup_tail_seconds": 120,
"idle_wait_seconds": 60,
"finalization_gpu_seconds": 120,
"projected_new_gpu_seconds": 13180
},
"remaining_item_gpu_seconds": 70737.94239573716,
"same_card_fold0_epoch_seconds": 96.1752110850066,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"same_card_train_peers_instantiated": {
"0": 2025071,
"2": 2025091
},
"sampled_cuda_pids": [
1765688,
2025071,
2025091
],
"sample_unix": 1795021405.2712727,
"worker_intervals": [
{
"fold": 0,
"started_unix": 1795021298.0488324,
"ended_unix": 1795021406.151041
},
{
"fold": 2,
"started_unix": 1795021298.5668569,
"ended_unix": 1795021405.9669158
}
],
"same_fold_solo_epoch_records": 199,
"longest_solo_train_seconds": 99.90287962602451,
"longest_solo_validation_seconds": 16.66734165791422,
"limitation": "The two workers started together. Shared occupancy is sampled near epoch completion. The third CUDA context is observed only at that sample.",
"target_admission": "Selected V100 starts idle, with stop on foreign occupancy. The co-located trace is a conservative budget input."
}
@@ -0,0 +1,10 @@
# Phase 3 item 1b 折 0 筛选准备
Claude 裁定 h-c53c94a6 批准一个新标记的短日程折 0。旧五折队列因总成本投影停止,所留训练、失败和结算继续有效。[裁定 3a48b12](https://gitea.frank14f.online/Frank14f/MeltMind/src/commit/3a48b123da54b0bcff0e7fa672fc74e8920235a0/docs/claude/stage-review-2026-11-19.md)
- **同卡成本证据。** 原折 0 完成第 1 轮,训练 96.175211 秒。折 2 在同一 V100 晚约 0.52 秒启动,在回收前的同一 GPU 快照中可见两名 worker 和第三个 CUDA 进程。第三个进程只有该时刻的占用证据。另核同卡同折 199 条独占采样匹配的 seed 1 轮回执。预算取较大的轮时 99.902880 秒和验证 16.667342 秒,保守覆盖已有直接并发计时。新入口从空卡起步,发现外部 CUDA 进程时只停止自己的 worker。
- **结算。** 100 轮与 10 次诊断的实测投影加 25%,再计 120 秒训练准备与收尾,训练帽 12,817 GPU 秒。三臂验证和准备帽 183 GPU 秒。排队最多 60 秒,另留 120 秒入口和终态开销。总帽 13,180 GPU 秒(3.6611 小时),低于原 25 小时扣除已结算 19,262.057604 秒后剩余的 70,737.942396 秒。等待单列并计入总账一次。首轮再次核投影,命中任一边界留停止回执。
- **对照。** 同 320 件 learning、19 输入、seed 0、固定逐轮样件顺序、5 轮 warmup 和完成的 95 轮余弦退火。只取第 100 轮。用同域 111 件 validation 比较第 200 轮 seed 0 与 seed 1。先冻结三臂预测,再按固定查询点及原 PPS multiplicity 算场相对 L2,件均值汇成组件等权。沿原筛选门,短日程差值须 ≤ −0.001 且绝对值严格大于同域种子差。终态只报告筛选。测试访问、其他折和自动续跑均未获本入口授权。
- **上线证据。** 新入口用原训练函数本体和原学习日程。针对真实生成合同运行了真实调度守卫的 CPU 模拟,空卡路径模拟一次训练和一次筛选,忙卡 60 秒超时且无派发。模拟无真实 GPU worker,无科学数组访问。精确独立 PASS 待收。
[合同](../artifacts/phase3-item1b-fold0-screen-2026-11-19/contract.json) · [比较合同](../artifacts/phase3-item1b-fold0-screen-2026-11-19/comparison-contract.json) · [计时与预算](../artifacts/phase3-item1b-fold0-screen-2026-11-19/timing-and-budget.json) · [实际守卫 CPU 回执](../artifacts/phase3-item1b-fold0-screen-2026-11-19/cpu-guard-preflight.json)
@@ -0,0 +1,20 @@
{
"unix": 1795021405.2712727,
"rows": [
{
"pid": 1765688,
"bytes": 1918894080
},
{
"pid": 2025071,
"bytes": 1790967808
},
{
"pid": 2025091,
"bytes": 1918894080
}
],
"own_pids": [
2025071
]
}
@@ -0,0 +1,39 @@
{
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_item1b_worker.py",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18",
"0",
"050e8b6748a3e2a287eb905f75179daa9941bd5690660f2c2716298fbf71bd1d"
],
"returncode": -9,
"stop_reason": null,
"cpu_seconds": 104.23424299999999,
"wall_seconds": 108.10220872296486,
"peak_tree_rss_bytes": 1431515136,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 8192,
"max_poll_gap_seconds": 0.8025802979245782,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null,
"monitor_policy": {
"rule": "three_consecutive_failed_samples_per_operation",
"failure_limit": 3,
"failed": false,
"errors": 0,
"streaks": {
"initial_disk": 0,
"tree_usage": 0,
"live_disk": 0,
"final_disk": 0
},
"max_streaks": {}
}
}
@@ -0,0 +1,18 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_item1b_worker.py",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18",
"0",
"050e8b6748a3e2a287eb905f75179daa9941bd5690660f2c2716298fbf71bd1d"
],
"limits": {
"total_cpu_seconds": 29940,
"job_cpu_seconds": 29940,
"job_wall_seconds": 26341.294980686158,
"tree_rss_bytes": 4294967296,
"disk_bytes": 8589934592,
"max_processes": 1
},
"started_unix": 1795021298.0488324
}
@@ -0,0 +1,15 @@
{
"fold": 0,
"stage": "EPOCH_COMPLETE",
"epoch": 1,
"optimizer_steps": 320,
"wall_seconds": 101.60979303403292,
"cpu_seconds": 101.23031225700001,
"train_mean_weighted_loss": 0.004886068560188051,
"train_epoch_seconds": 96.1752110850066,
"epoch_parts": 320,
"epoch_nodes": 8742060,
"lr": 0.0001,
"peak_allocated_bytes": 1334860288,
"peak_reserved_bytes": 1388314624
}
@@ -0,0 +1,39 @@
{
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_item1b_worker.py",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18",
"2",
"a2699b9e9174eb7ba8a4b54a4764f58f28fb794be0008083f36765c738035b26"
],
"returncode": -9,
"stop_reason": null,
"cpu_seconds": 105.068737,
"wall_seconds": 107.40005897299852,
"peak_tree_rss_bytes": 1471344640,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.9828807739540935,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null,
"monitor_policy": {
"rule": "three_consecutive_failed_samples_per_operation",
"failure_limit": 3,
"failed": false,
"errors": 0,
"streaks": {
"initial_disk": 0,
"tree_usage": 0,
"live_disk": 0,
"final_disk": 0
},
"max_streaks": {}
}
}
@@ -0,0 +1,18 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_item1b_worker.py",
"/home/frank14f/MeltMind-runs/phase3-item-1b-2026-11-18",
"2",
"a2699b9e9174eb7ba8a4b54a4764f58f28fb794be0008083f36765c738035b26"
],
"limits": {
"total_cpu_seconds": 29940,
"job_cpu_seconds": 29940,
"job_wall_seconds": 26340.25473072508,
"tree_rss_bytes": 4294967296,
"disk_bytes": 8589934592,
"max_processes": 1
},
"started_unix": 1795021298.5668569
}
+390
View File
@@ -0,0 +1,390 @@
"""One reviewed fold-0 attempt and its validation-only screen."""
import argparse
from collections import defaultdict
import ctypes
import fcntl
import json
import math
import os
from pathlib import Path
import shutil
import signal
import sys
import threading
import time
from types import FunctionType
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase3_item1b_common import fold_view, read
from phase3_shared_capacity import process_identity
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords, active_command_pids
from s2_recovery_gpu_supervisor import stop_group, supervise
ROOT = Path(__file__).resolve().parents[1] / "artifacts/phase3-item1b-fold0-screen-2026-11-19"
RULING = "3a48b123da54b0bcff0e7fa672fc74e8920235a0"
ALLOCATOR = "expandable_segments:True"
GIB = 2**30
ARMS = ("short_seed0", "long_seed0", "long_seed1")
def contract(path):
c = read(path)
if (c["ruling"] != RULING or c["fold"] != 0 or c["screen_fold"] != 0
or c["epochs"] != 100 or c["seed"] != 0 or c["max_fits"] != 1
or c["retry"] is not False or c["test_targets_allowed"] is not False):
raise PermissionError("Only the new fold-0 validation scope is authorized")
for p, h in (c["code_sha256"] | c["source_sha256"]).items():
checked(p, h)
comparison = read(checked(c["comparison_contract"], c["comparison_contract_sha256"]))
if comparison["validation_cases"] != c["job"]["validation"]:
raise ValueError("Fold-0 validation identity differs from comparison")
budget = c["budget"]
reserves = sum(budget[k] for k in ("train_gpu_seconds", "screen_gpu_seconds",
"idle_wait_seconds", "finalization_gpu_seconds"))
if (budget["projected_new_gpu_seconds"] != reserves
or budget["prior_item_gpu_seconds"] + reserves > budget["item_gpu_limit_seconds"]
or budget["prior_phase3_gpu_seconds"] + reserves > budget["phase3_gpu_limit_seconds"]):
raise PermissionError("Phase 3 or 1b remaining budget exceeded")
return c
def folded_inputs(c):
pilot = read(c["pilot_contract"])
view = fold_view(pilot, 0)
original = view["folds"][0]
job = c["job"]
for k in ("learning", "validation", "test", "c", "clock_sha256", "learning_nodes"):
if job[k] != original[k]:
raise ValueError(f"Original fold-0 identity changed: {k}")
if job["id"] != 0 or job["target_steps"] != 100 * len(original["learning"]):
raise ValueError("Full fold-0 learning schedule required")
if set(original["validation"]) & set(original["test"]):
raise ValueError("Validation and test overlap")
return pilot, view
def train(root, c):
import numpy as np
import torch
from phase3_item1b_worker import run as frozen_worker
pilot, _ = folded_inputs(c)
if torch.__version__ != pilot["versions"]["torch"] or np.__version__ != pilot["versions"]["numpy"]:
raise ValueError("Original array and Torch versions required")
job = dict(c["job"], gpu_uuid=c["gpu_uuid"], worker_cpus=c["worker_cpus"])
digest = sha(root / "contract.json")
def first_epoch_release(queue, fold):
if queue != root or fold != 0:
raise PermissionError("Wrong worker route")
row = read(root / "jobs/0/worker/history/001.json")
budget = c["budget"]
epoch_seconds = max(row["train_epoch_seconds"], budget["train_epoch_seconds_basis"])
projection = (row["wall_seconds"] + 99 * epoch_seconds
+ 10 * budget["validation_seconds_basis"] + budget["training_setup_tail_seconds"])
passed = math.isfinite(projection) and projection <= budget["train_gpu_seconds"]
save(root / "first-epoch-projection.json", dict(
status="PASS" if passed else "STOP_BUDGET", contract_sha256=digest,
measured_epoch_seconds=row["train_epoch_seconds"], projected_train_gpu_seconds=projection))
if not passed:
raise RuntimeError("First-epoch projection exceeds fold-0 training cap")
return True
# Reuse the immutable training body. The private namespace changes only
# its root/assignment adapters and the one-fold budget gate.
namespace = dict(frozen_worker.__globals__, config=lambda _: (c, pilot),
assigned_job=lambda *_: job, first_epoch_release=first_epoch_release)
runner = FunctionType(frozen_worker.__code__, namespace, frozen_worker.__name__)
runner(root, 0, digest)
def screen(root, c):
import numpy as np
import torch
from phase2_h2_data import arguments, cached, to_device
from phase2_h2_model import H2Geo
from phase2_h2_pilot_worker import peaks
from phase2_residual_math import decode_seconds
from phase3_item1_postfit_data import load_evaluation_part
from s2_custody_common import prediction_inputs
from s2_models.objectives import relative_l2
pilot, view = folded_inputs(c)
comparison = read(c["comparison_contract"])
fold = view["folds"][0]
cases = comparison["validation_cases"]
if (cases != fold["validation"] or torch.__version__ != pilot["versions"]["torch"]
or np.__version__ != pilot["versions"]["numpy"]
or os.environ.get("CUDA_VISIBLE_DEVICES") != c["gpu_uuid"]
or os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR):
raise PermissionError("Screen scope or environment differs")
train_path = root / "jobs/0/worker/result.json"
adopted = read(root / "training-adopted.json")
short = read(checked(train_path, adopted["result_sha256"]))
models = dict(c["references"])
models["short_seed0"] = dict(
checkpoint=short["checkpoint"], checkpoint_sha256=short["checkpoint_sha256"],
standardization=str(train_path.with_name("standardization.json")),
standardization_sha256=short["standardization_sha256"], seed=0, epochs=100,
training_contract_sha256=sha(root / "contract.json"))
roster = {r["case"]: r for r in read(c["query_roster_contract"])["rows"] if r["case"] in cases}
if set(roster) != set(cases) or any(
roster[case]["component_id"] != pilot["parts"][case]["clock_group"]
or roster[case]["v5_cache"]["output_sha256"] != pilot["parts"][case]["output_sha256"]
for case in cases
):
raise ValueError("Frozen validation query-to-cache identity differs")
os.sched_setaffinity(0, c["worker_cpus"])
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
torch.cuda.set_device(0)
torch.cuda.reset_peak_memory_stats()
out = root / "screen"
out.mkdir(exist_ok=False)
predictions = {}
with torch.no_grad():
for arm in ARMS:
ref = models[arm]
stats = read(checked(ref["standardization"], ref["standardization_sha256"]))
blob = torch.load(checked(ref["checkpoint"], ref["checkpoint_sha256"]),
map_location="cpu", weights_only=True)
required = dict(completed_epochs=ref["epochs"], fold=0, seed=ref["seed"],
role="last_epoch_only", contract_sha256=ref["training_contract_sha256"],
optimizer_steps=ref["epochs"] * len(fold["learning"]))
if any(blob.get(k) != v for k, v in required.items()):
raise ValueError(f"Final checkpoint identity: {arm}")
if stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]:
raise ValueError("Screen learning-only normalization mismatch")
model = H2Geo().cuda()
model.load_state_dict(blob["model"], strict=True)
del blob
model.eval()
folder = out / arm
folder.mkdir()
for case in cases:
data = to_device(load_evaluation_part(view, case, stats, "validation"), "cuda")
values = model(*arguments(data), checkpoint_blocks=False).cpu().numpy()
seconds = decode_seconds(values, data["F"], fold["c"], data["t_set"])
if seconds.shape != (data["nodes"],) or not np.isfinite(seconds).all():
raise ValueError("Full validation prediction is missing or nonfinite")
path = folder / (case + ".npy")
np.save(path, seconds, allow_pickle=False)
predictions.setdefault(case, {})[arm] = {"file": str(path), "sha256": sha(path)}
save(out / "progress.json", dict(stage="VALIDATION_PREDICT", arm=arm, case=case, **peaks()))
del data
del model
manifest = out / "predictions-frozen.json"
save(manifest, predictions)
manifest_hash = sha(manifest)
predictions = read(checked(manifest, manifest_hash))
rows = []
grouped = defaultdict(list)
for case in cases:
q = prediction_inputs(roster[case])
ids = cached(view, case, "raw_ids")
ix = np.searchsorted(ids, q["raw_ids"])
if np.any(ix >= len(ids)) or not np.array_equal(ids[ix], q["raw_ids"]):
raise ValueError("Validation query does not map to original raw IDs")
with (out / "validation-target-access.jsonl").open("a") as log:
log.write(json.dumps(dict(case=case, role="validation", predictions_sha256=manifest_hash,
target_sha256=pilot["parts"][case]["output_sha256"]["seconds.npy"])) + "\n")
truth = cached(view, case, "seconds")[ix]
scores = {}
for arm in ARMS:
row = predictions[case][arm]
pred = np.load(checked(row["file"], row["sha256"]), allow_pickle=False)[ix]
value = relative_l2(pred, truth, q["counts"], np.ones(len(ix), dtype=bool))["value"]
if value is None or not math.isfinite(value):
raise ValueError("Undefined validation part metric")
scores[arm] = value
component = roster[case]["component_id"]
grouped[component].append([scores[arm] for arm in ARMS])
rows.append(dict(case=case, component_id=component, relative_l2=scores))
means = dict(zip(ARMS, np.mean([np.mean(a, axis=0) for a in grouped.values()], axis=0).tolist()))
delta = means["short_seed0"] - means["long_seed0"]
seed_delta = means["long_seed1"] - means["long_seed0"]
passed = delta <= comparison["delta_short_at_most"] and abs(delta) > abs(seed_delta)
save(out / "result.json", dict(
status="COMPLETE", decision="SCREEN_PASS" if passed else "SCREEN_FAIL",
parts=len(rows), components=len(grouped), component_equal_relative_l2=means,
delta_short=delta, delta_seed=seed_delta, rows=rows, predictions_sha256=manifest_hash,
test_targets_read=False, automatic_continuation=False, **peaks()))
def dispatch_guard(root, contract_path):
"""The same entry consumes the generated contract in production and preflight."""
began_entry = time.monotonic()
c = contract(contract_path)
folded_inputs(c)
budget = c["budget"]
gpu = c["gpu_uuid"]
charges = dict(queue_wait_gpu_seconds=0., training_gpu_seconds=0.,
screen_gpu_seconds=0., finalization_gpu_seconds=0.)
receipts = []
status = "STOPPED"
trigger = "STOP_BEFORE_DISPATCH"
try:
if shutil.disk_usage(root).free < c["disk_bytes"]:
raise RuntimeError("Insufficient free disk for the fixed attempt")
began_wait = time.monotonic()
try:
idle = wait_idle(gpu, budget["idle_wait_seconds"], lambda _: None, sample=snapshot)
except BaseException:
charges["queue_wait_gpu_seconds"] = time.monotonic() - began_wait
raise
charges["queue_wait_gpu_seconds"] = idle["wall_seconds"]
save(root / "idle-gate.json", idle)
trigger = idle["status"]
if idle["status"] != "IDLE_CONFIRMED" or not snapshot(gpu)["idle"]:
if idle["status"] == "IDLE_CONFIRMED":
trigger = "SELECTED_GPU_BUSY_AFTER_IDLE_GATE"
return 1
(root / "jobs/0").mkdir(parents=True, exist_ok=False)
for mode, cap_key, charge_key, output in (
("train", "train_gpu_seconds", "training_gpu_seconds", root / "jobs/0/worker"),
("screen", "screen_gpu_seconds", "screen_gpu_seconds", root / "screen"),
):
began = time.monotonic()
if not snapshot(gpu)["idle"]:
trigger = "SELECTED_GPU_BUSY_BEFORE_" + mode.upper()
charges[charge_key] = time.monotonic() - began
return 1
command = [sys.executable, str(Path(__file__).resolve()), mode, str(root)]
tracker = PeakRecords()
done = threading.Event()
reasons = []
def watch():
while not done.is_set():
try:
tracker.scan(output)
state = snapshot(gpu)
pids = set(active_command_pids(command))
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if state["memory_MiB"] * 2**20 > c["device_vram_bytes"]:
reasons.append("DEVICE_VRAM_LIMIT")
if any(p["pid"] not in pids for p in state["occupants"]):
reasons.append("FOREIGN_GPU_OCCUPANT")
except Exception as exc:
reasons.append(f"MONITOR_FAILURE: {exc}")
if reasons:
for pid in active_command_pids(command):
stop_group(pid)
# Also catch a child spawned just after the failing
# sample while the supervisor was taking its lock.
done.wait(.1)
continue
done.wait(.5)
monitor = threading.Thread(target=watch, daemon=True)
monitor.start()
try:
if reasons:
raise RuntimeError(reasons[0])
cap = budget[cap_key] - (time.monotonic() - began) - 10
receipt = supervise(command, root, mode, dict(
total_cpu_seconds=c["total_cpu_seconds"], job_cpu_seconds=budget[cap_key],
job_wall_seconds=cap, tree_rss_bytes=c["rss_bytes"],
disk_bytes=c["disk_bytes"], max_processes=c["max_processes"]),
poll=.2, gpu_uuid=gpu)
finally:
done.set()
monitor.join(timeout=8)
charges[charge_key] = time.monotonic() - began
if monitor.is_alive():
reasons.append("MONITOR_DID_NOT_EXIT")
else:
tracker.scan(output)
if max(tracker.maxima.values()) > c["recorded_vram_bytes"]:
reasons.append("RECORDED_VRAM_LIMIT")
if set(tracker.witnesses) != set(tracker.maxima):
reasons.append("MISSING_RECORDED_VRAM_PEAKS")
receipts.append(dict(mode=mode, worker=receipt, reasons=reasons, peaks=tracker.maxima))
if (reasons or receipt["status"] != "PASS" or receipt["returncode"] != 0
or receipt["monitor_errors"] != 0 or charges[charge_key] > budget[cap_key]):
worker_reason = receipt.get("stop_reason") or "WORKER_FAILED_OR_TIME_LIMIT"
trigger = reasons[0] if reasons else worker_reason
return 1
result_path = output / "result.json"
result = read(result_path)
if result["status"] != "COMPLETE" or result["test_targets_read"] is not False:
raise RuntimeError("Unexpected worker result")
if mode == "train":
if (result["fold"] != 0 or result["completed_epochs"] != c["epochs"]
or result["optimizer_steps"] != c["job"]["target_steps"]):
raise RuntimeError("Fold-0 final checkpoint identity differs")
checked(result["checkpoint"], result["checkpoint_sha256"])
save(root / "training-adopted.json", dict(result_sha256=sha(result_path)))
elif result["parts"] != len(c["job"]["validation"]) or result["automatic_continuation"]:
raise RuntimeError("Validation screen scope differs")
overhead = time.monotonic() - began_entry - sum(charges.values())
if overhead > budget["finalization_gpu_seconds"]:
raise RuntimeError("Setup and finalization cost exceeded the fixed allowance")
status, trigger = "COMPLETE", result["decision"]
return 0
except Exception as exc:
trigger = f"{type(exc).__name__}: {exc}"
return 1
except KeyboardInterrupt:
trigger = "COORDINATOR_INTERRUPTED"
return 1
finally:
charges["finalization_gpu_seconds"] = max(0., time.monotonic() - began_entry - sum(charges.values()))
new_gpu = sum(charges.values())
save(root / "complete.json", dict(
status=status, trigger=trigger, receipts=receipts, **charges,
cumulative_item_gpu_seconds=budget["prior_item_gpu_seconds"] + new_gpu,
cumulative_phase3_gpu_seconds=budget["prior_phase3_gpu_seconds"] + new_gpu,
no_retry=True, test_targets_read=False, automatic_continuation=False))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("run", "train", "screen"))
parser.add_argument("root", type=Path)
args = parser.parse_args()
root = args.root.resolve()
if root != ROOT:
raise PermissionError("Only the new fixed fold-0 root is accepted")
expected_python = str(ROOT.parents[1] / "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python")
if sys.executable != expected_python:
raise PermissionError("Use the fixed original H2 environment")
validate_review(root)
contract_path = root / "contract.json"
if args.mode != "run":
coordinator = read(root / "started.json")
if process_identity(os.getppid()) != coordinator["coordinator"]:
raise PermissionError("Worker requires the live guard as its parent")
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), "Cannot bind worker lifetime to guard")
if os.getppid() != coordinator["coordinator"]["pid"]:
raise RuntimeError("Guard disappeared before worker setup")
return globals()[args.mode](root, contract(contract_path))
if os.environ.get("PYTORCH_CUDA_ALLOC_CONF") != ALLOCATOR:
raise PermissionError("Explicit original CUDA allocator setup required")
with (root / "entry.lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / "started.json").exists():
raise FileExistsError("Fold-0 attempt already started")
save(root / "started.json", dict(coordinator=process_identity(os.getpid()),
contract_sha256=sha(contract_path), unix=time.time()))
def request_stop(signum, _frame):
raise KeyboardInterrupt(signal.Signals(signum).name)
old_handler = signal.signal(signal.SIGTERM, request_stop)
try:
return dispatch_guard(root, contract_path)
finally:
signal.signal(signal.SIGTERM, old_handler)
if __name__ == "__main__":
raise SystemExit(main())
+234
View File
@@ -0,0 +1,234 @@
"""Freeze the new fold-0 contract and exercise its actual guard on CPU."""
from functools import partial
import math
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
from phase2_h1_worker_v2 import checked, save, sha
from phase3_item1b_common import ROOT as OLD_ROOT, read
import phase3_item1b_fold0 as entry
from s2_gpu_idle_gate import wait_idle
from track_a_h2_prepare import local_code_closure
BASE = entry.ROOT.parents[1]
RUNS = OLD_ROOT.parent
SEED1 = RUNS / "phase3-seed1-v2-2026-11-18"
def prepare():
root = entry.ROOT
if root.exists():
raise FileExistsError("New fold-0 preparation already exists")
pins = {}
def pin(path):
p = Path(path)
pins[str(p)] = sha(p)
return read(p)
old = pin(OLD_ROOT / "contract.json")
stop = pin(OLD_ROOT / "complete.json")
settled = pin(OLD_ROOT / "post-terminal-cost.json")
if (stop["failure"] != "RuntimeError: TOTAL_PROJECTION_OR_ACTUAL_CAP"
or settled["training_complete_sha256"] != pins[str(OLD_ROOT / "complete.json")]
or settled["postfit_started"] is not False or not settled["freeze_tail_charged_once"]):
raise ValueError("Historical budget-stop record differs")
totals = {
"item1": pin(RUNS / "phase3-item1-postfit-recovery-2026-11-18/post-terminal-cost.json")[
"cumulative_item_gpu_seconds"],
"seed1": pin(RUNS / "phase3-seed1-postfit-2026-11-18/post-terminal-cost.json")[
"cumulative_seed1_gpu_seconds"],
"item1b": settled["cumulative_gpu_seconds"],
"c": pin(RUNS / "phase3-item-c-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
"cumulative_item_gpu_seconds"],
"b": pin(RUNS / "phase3-item-b-postfit-idle-recovery-2026-11-19/post-terminal-cost.json")[
"cumulative_item_gpu_seconds"],
}
history_path = OLD_ROOT / "jobs/0/worker/history/001.json"
epoch = pin(history_path)
assignments = {i: pin(OLD_ROOT / f"jobs/{i}/assignment.json") for i in (0, 2)}
identities = {i: pin(OLD_ROOT / f"jobs/{i}/worker/worker-identity.json") for i in (0, 2)}
start = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.started.json")["started_unix"] for i in (0, 2)}
elapsed = {i: pin(OLD_ROOT / f"jobs/{i}/guards/train.receipt.json")["wall_seconds"] for i in (0, 2)}
gpu = assignments[0]["gpu_uuid"]
sample = pin(OLD_ROOT / "jobs/0/gpu-sample.json")
observed = {row["pid"] for row in sample["rows"]}
if (assignments[2]["gpu_uuid"] != gpu or len(observed) != 3
or not {p["pid"] for p in identities.values()} <= observed
or not max(start.values()) < sample["unix"] < min(start[i] + elapsed[i] for i in start)
or epoch["optimizer_steps"] != len(old["jobs"][0]["learning"])
or epoch["stage"] != "EPOCH_COMPLETE"):
raise ValueError("Missing same-card fold-0 timing witness")
solo = []
for obs_path in sorted((SEED1 / "jobs/0/epoch-host-observations").glob("*.json")):
obs = read(obs_path)
on_card = [p for p in obs["gpu_processes"] if p["on_assigned_gpu"]]
if (obs["status"] != "OBSERVED" or obs["errors"] or obs["assigned_gpu_uuid"] != gpu
or len(on_card) != 1 or not on_card[0]["own_v2_worker"]):
continue
row_path = Path(obs["receipt_path"])
row = read(checked(row_path, obs["receipt_sha256"]))
solo.append((obs_path, row_path, row))
if not solo:
raise ValueError("Missing fold-0 diagnostic duration and comparison envelope")
longest_train = max(solo, key=lambda x: x[2]["train_epoch_seconds"])
longest_val = max(solo, key=lambda x: x[2].get("validation_seconds", 0))
for observation, receipt, _ in (longest_train, longest_val):
pin(observation)
pin(receipt)
train_basis = max(epoch["train_epoch_seconds"], longest_train[2]["train_epoch_seconds"])
val_basis = max(old["jobs"][0]["validation_seconds_bound"], longest_val[2]["validation_seconds"])
# The explicit compute margin applies once. Setup, bounded waiting and the
# terminal allowance do not inflate an individual training-epoch time.
train_setup = 120
train_cap = math.ceil(1.25 * (100 * train_basis + 10 * val_basis)) + train_setup
screen_cap = math.ceil(1.25 * 3 * val_basis) + 120
budget = dict(
item_gpu_limit_seconds=90000, phase3_gpu_limit_seconds=720000,
prior_item_gpu_seconds=totals["item1b"], prior_phase3_gpu_seconds=sum(totals.values()),
train_epoch_seconds_basis=train_basis, validation_seconds_basis=val_basis,
train_gpu_seconds=train_cap, screen_gpu_seconds=screen_cap,
training_setup_tail_seconds=train_setup, idle_wait_seconds=60, finalization_gpu_seconds=120,
projected_new_gpu_seconds=train_cap + screen_cap + 60 + 120,
)
if budget["projected_new_gpu_seconds"] > 90000 - totals["item1b"]:
raise ValueError(f"Fold-0 cap exceeds remaining authorization: {budget}")
pilot = pin(old["pilot_contract"])
roster = pin(old["query_roster_contract"])
fold = pilot["folds"][0]
rows = {r["case"]: r for r in roster["rows"] if r["case"] in fold["validation"]}
if set(rows) != set(fold["validation"]):
raise ValueError("Fold-0 validation roster incomplete")
references = {}
baseline = old["baseline_models"][0]
workers = [Path(baseline["result"]).parent, SEED1 / "jobs/0/worker"]
roots = [workers[0].parent, SEED1]
for seed, worker in enumerate(workers):
run_contract = roots[seed] / "contract.json"
run_meta = pin(run_contract)
result = pin(worker / "result.json")
stats_path = worker / "standardization.json"
stats = pin(stats_path)
checked(stats_path, result["standardization_sha256"])
checkpoint = checked(result["checkpoint"], result["checkpoint_sha256"])
pins[str(checkpoint)] = result["checkpoint_sha256"]
if (run_meta["seed"] != seed or result["completed_epochs"] != 200
or result["optimizer_steps"] != 200 * len(fold["learning"])
or stats["parts"] != fold["learning"] or stats["nodes"] != fold["learning_nodes"]):
raise ValueError("Reference final checkpoint scope differs")
references[f"long_seed{seed}"] = dict(
checkpoint=str(checkpoint), checkpoint_sha256=result["checkpoint_sha256"],
standardization=str(stats_path), standardization_sha256=result["standardization_sha256"],
seed=seed, epochs=200, training_contract_sha256=pins[str(run_contract)])
root.mkdir(exist_ok=False)
comparison = root / "comparison-contract.json"
save(comparison, dict(
ruling=entry.RULING, fold=0, validation_cases=fold["validation"],
validation_components=len({row["component_id"] for row in rows.values()}),
score="Relative L2 on the frozen validation query with original PPS multiplicity, then "
"equal means by component. Negative predictions remain visible.",
paired_references=list(references), delta_short_at_most=-.001,
effect_strictly_greater_than_abs_seed_delta=True, duration="100 complete epochs, final epoch only",
warmup="Original 5-epoch warmup and full cosine decay over 95 epochs",
train_fields="Original seed0, order, 19 inputs, optimizer, MSE and numerical recipe",
test_allowed=False, automatic_continuation=False))
codes = local_code_closure([Path(__file__), Path(entry.__file__)] +
list((BASE / "tools/s2_models").glob("*.py")))
for path, digest in old["code_sha256"].items():
checked(path, digest)
codes.setdefault(path, digest)
source = root / "timing-and-budget.json"
save(source, dict(
prior_phase3_costs=totals, budget=budget, remaining_item_gpu_seconds=90000 - totals["item1b"],
same_card_fold0_epoch_seconds=epoch["train_epoch_seconds"], gpu_uuid=gpu,
same_card_train_peers_instantiated={str(i): p["pid"] for i, p in identities.items()},
sampled_cuda_pids=sorted(observed), sample_unix=sample["unix"], worker_intervals=[
dict(fold=i, started_unix=start[i], ended_unix=start[i] + elapsed[i]) for i in start],
same_fold_solo_epoch_records=len(solo), longest_solo_train_seconds=longest_train[2]["train_epoch_seconds"],
longest_solo_validation_seconds=longest_val[2]["validation_seconds"],
limitation="The two workers started together. Shared occupancy is sampled near epoch "
"completion. The third CUDA context is observed only at that sample.",
target_admission="Selected V100 starts idle, with stop on foreign occupancy. "
"The co-located trace is a conservative budget input."))
pins[str(source)] = sha(source)
contract = root / "contract.json"
save(contract, dict(
version="phase3-item1b-fold0-screen-v1", ruling=entry.RULING, fold=0, screen_fold=0, seed=0,
epochs=100, max_fits=1, retry=False, test_targets_allowed=False, gpu_uuid=gpu,
worker_cpus=assignments[0]["worker_cpus"], total_cpu_seconds=30000, max_processes=1,
rss_bytes=4 * entry.GIB, disk_bytes=8 * entry.GIB, device_vram_bytes=4 * entry.GIB,
recorded_vram_bytes=2 * entry.GIB, job=old["jobs"][0], budget=budget, references=references,
pilot_contract=old["pilot_contract"], query_roster_contract=old["query_roster_contract"],
comparison_contract=str(comparison), comparison_contract_sha256=sha(comparison),
source_sha256=pins, code_sha256=codes))
return contract
def preflight(contract):
c = entry.contract(contract)
results = []
for busy in (False, True):
simulated_time = 0.
def sleep(seconds):
nonlocal simulated_time
simulated_time += seconds
def snapshot(uuid):
if uuid != c["gpu_uuid"]:
raise ValueError("Incorrect GPU identity")
return dict(uuid=uuid, idle=not busy, memory_MiB=1500 if busy else 5, occupants=[])
def simulate(command, guard_root, mode, limits, *, poll, gpu_uuid):
if (guard_root != root or command[2:] != [mode, str(root)] or gpu_uuid != c["gpu_uuid"]
or not 0 < limits["job_wall_seconds"] < c["budget"][f"{mode}_gpu_seconds"]
or limits["tree_rss_bytes"] != c["rss_bytes"] or limits["disk_bytes"] != c["disk_bytes"]
or limits["max_processes"] != c["max_processes"]
or limits["total_cpu_seconds"] != c["total_cpu_seconds"]):
raise ValueError("Actual guard did not propagate frozen fields")
record = dict(status="COMPLETE", test_targets_read=False, peak_allocated_bytes=1024,
peak_reserved_bytes=2048)
if mode == "train":
out = root / "jobs/0/worker"
out.mkdir()
checkpoint = out / "simulated-checkpoint"
checkpoint.write_bytes(b"CPU mock only")
record.update(fold=0, completed_epochs=100, optimizer_steps=c["job"]["target_steps"],
checkpoint=str(checkpoint), checkpoint_sha256=sha(checkpoint))
else:
out = root / "screen"
out.mkdir()
record.update(parts=len(c["job"]["validation"]), automatic_continuation=False,
decision="SCREEN_PASS")
save(out / "result.json", record)
return dict(status="PASS", returncode=0, monitor_errors=0)
with TemporaryDirectory(prefix="meltmind-fold0-guard-") as folder:
root = Path(folder)
with (patch.object(entry, "snapshot", side_effect=snapshot),
patch.object(entry, "wait_idle", partial(wait_idle, clock=lambda: simulated_time, sleep=sleep)),
patch.object(entry, "supervise", side_effect=simulate) as dispatch):
exit_code = entry.dispatch_guard(root, contract)
terminal = read(root / "complete.json")
gate = read(root / "idle-gate.json")
expected = (1, "IDLE_TIMEOUT", []) if busy else (0, "SCREEN_PASS", ["train", "screen"])
actual = (exit_code, terminal["trigger"], [call.args[2] for call in dispatch.call_args_list])
if actual != expected:
raise RuntimeError(f"Actual guard CPU simulation differed: {actual}")
results.append(dict(case="busy-timeout" if busy else "idle-dispatch",
exit_code=exit_code, trigger=terminal["trigger"], mock_dispatches=actual[2],
gate_wait_seconds=gate["wall_seconds"]))
return dict(status="PASS", actual_contract_sha256=sha(contract), cases=results,
real_gpu_or_worker_started=False, scientific_arrays_read=False, review_status="PENDING")
if __name__ == "__main__":
path = prepare()
receipt = preflight(path)
save(path.parent / "cpu-guard-preflight.json", receipt)
print(receipt)