Preserve the sole 1b fold0 foreign-GPU stop and bounded evidence
This commit is contained in:
1 parent
2c0dd614f8
commit
3772b765cf
7 files changed
+4330
No files matched your search
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"status": "STOPPED",
|
||||
"trigger": "FOREIGN_GPU_OCCUPANT",
|
||||
"underlying_trigger": "FOREIGN_GPU_OCCUPANT",
|
||||
"receipts": [
|
||||
{
|
||||
"mode": "train",
|
||||
"worker": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_fold0_deadline.py",
|
||||
"train",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-deadline-2026-11-19"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 720.046806,
|
||||
"wall_seconds": 720.661197388079,
|
||||
"peak_tree_rss_bytes": 1465208832,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 2924544,
|
||||
"max_poll_gap_seconds": 0.6629281869390979,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
},
|
||||
"reasons": [
|
||||
"FOREIGN_GPU_OCCUPANT",
|
||||
"FOREIGN_GPU_OCCUPANT"
|
||||
],
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
}
|
||||
}
|
||||
],
|
||||
"queue_wait_gpu_seconds": 20.293689728947356,
|
||||
"training_gpu_seconds": 720.7542707340326,
|
||||
"screen_gpu_seconds": 0.0,
|
||||
"finalization_gpu_seconds": 0.11176366300787777,
|
||||
"new_gpu_seconds": 741.1597241259879,
|
||||
"new_gpu_cap_seconds": 13180,
|
||||
"cap_overage_gpu_seconds": 0.0,
|
||||
"cumulative_item_gpu_seconds": 20003.217328388826,
|
||||
"cumulative_phase3_gpu_seconds": 409371.5701867306,
|
||||
"no_retry": true,
|
||||
"test_targets_read": false,
|
||||
"automatic_continuation": false
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"fold": 0,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 13,
|
||||
"optimizer_steps": 4024,
|
||||
"wall_seconds": 716.2925684670918,
|
||||
"cpu_seconds": 715.89480798,
|
||||
"case": "00001597_13bd23e0e4c740969b0c71ab_step_000",
|
||||
"epoch_parts": 184,
|
||||
"epoch_nodes": 4916146,
|
||||
"lr": 0.0009827784105872876,
|
||||
"loss": 0.0026505172718316317,
|
||||
"gradient_norm_before_clip": 0.03279545158147812,
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
{"case": "00000111_ce24e0cbec8e45c89735d148_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "bf1d84e7796d30bea6158803178475cd1588bec3c00342d57ec3c73a74d78eb2"}
|
||||
{"case": "00000115_ce24e0cbec8e45c89735d148_step_007", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "8a94423dcd2c2258e5b89c25e8c59135c529e00d917052fb3beba18211cda6b1"}
|
||||
{"case": "00000116_ce24e0cbec8e45c89735d148_step_008", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "cbba293577b04dacfb8778d57cf83e288448c56dddf31f2423f19e0ee3ef7918"}
|
||||
{"case": "00000117_ce24e0cbec8e45c89735d148_step_009", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "e990c5253100702f1cb3cb15f5c4755e3016b9fba9868ae0d7e28f582f08efbf"}
|
||||
{"case": "00000119_ce24e0cbec8e45c89735d148_step_011", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "fc329cc3022f2e7a1b6a6af52fe31052c4677727bce0aa61263533389deee828"}
|
||||
{"case": "00000120_ce24e0cbec8e45c89735d148_step_012", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "58137595925282fe87e94b2cd6c51218ea5887471c0c494419eb63da606dce4f"}
|
||||
{"case": "00000131_945244874ed84e599987daa9_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "c2eb307b067760491447266edbf3da7add5f54a1dc6f0f882d7390607db2b7a9"}
|
||||
{"case": "00000132_438507a38e404bfdbd51a16b_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "f330f567158b38d64b3a10c9524a6e106d84b7fefa3d45d8d7b2b30c19a0fe7d"}
|
||||
{"case": "00000133_08b739e0511f4305b161cd05_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "21db69db3e48d09e8e2153196637b8f393470f45091d67d7f2c0387a7bd5bd46"}
|
||||
{"case": "00000190_185ecbc783f34408aef52119_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "94d3c4fb92117069aa115fc94763b19a001c70ef2043db79a5ecd81b9cad6259"}
|
||||
{"case": "00000196_c9b71ecdd0654716891b0db8_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "72fbd3975dd98949608fdee1c27f617478ebf372081e99918a9e3f5be46e8b28"}
|
||||
{"case": "00000206_27417003e762409185cb2b69_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "11e89644e7cece10b89e6416bbb11f3d4db7c1f7cc83adbcc88d0126ce9bef52"}
|
||||
{"case": "00000238_4829bc0645d6414d9b5cec31_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "5e7f4c2dc05c2b0711878fd4d229b8eebe5824fcbe48fa8af65243217771f994"}
|
||||
{"case": "00000241_4829bc0645d6414d9b5cec31_step_004", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "c5f546cda134c928048eee14776b29dd47fbc0bde6614060a89801d8b333ee99"}
|
||||
{"case": "00000243_4829bc0645d6414d9b5cec31_step_006", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "e2ceebd779ee84a81d638c5f7494eaa7891851017f37fd24d3d92d40d3a10a08"}
|
||||
{"case": "00000248_e87554b09cdc48c3add39db9_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "8ca477507c126737eefc6707a07a67033365eafa4435a0cd604b4e4eb68e2135"}
|
||||
{"case": "00000257_845fb06d7f2f4802a806a600_step_002", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "da827ef693daff2f6648cecf349c0ede7424d54a890cdba147ebdefb865ceb92"}
|
||||
{"case": "00000258_845fb06d7f2f4802a806a600_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "840059c6d43e1105e153f0fe172354800207bc6ae687f4dbb7cc1f25ddcce92f"}
|
||||
{"case": "00000259_845fb06d7f2f4802a806a600_step_004", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "625f799edf3fe54fc028057f10897f6ac7c0bf647f2437b1886b4a33389fa471"}
|
||||
{"case": "00000261_845fb06d7f2f4802a806a600_step_006", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "f8250ed416f7437356f6d489b2a522bb195695b218fd6a2b9a18245e555f3fb1"}
|
||||
{"case": "00000262_845fb06d7f2f4802a806a600_step_007", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "bd409e7e550707db4bdf90159f3dc5a382f1605f76d683026ce0037009dd7209"}
|
||||
{"case": "00000264_845fb06d7f2f4802a806a600_step_009", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "4865f642e7a1fbaa4bd419a23fc6179bc414157835259a9b4d9b43bd163fc1b7"}
|
||||
{"case": "00000265_845fb06d7f2f4802a806a600_step_010", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "ce6d01b2ff0cc2cbb4a4fd15dad6a130436cd6efdfff1c6ab8e8f2d727cb6442"}
|
||||
{"case": "00000266_845fb06d7f2f4802a806a600_step_011", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "d02e2c960e567dff49436857a9880a2c47c000149b6db41b6cc989ea4bcdba4a"}
|
||||
{"case": "00000267_845fb06d7f2f4802a806a600_step_012", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "85ae1234e6852c6980aac90ae4f48186235460785e3086a3743bb67c1b5f5cdf"}
|
||||
{"case": "00000269_845fb06d7f2f4802a806a600_step_014", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "1222f7dfa8a7060dded02c8c680073550d0248ec8d489ab32ee618077281627a"}
|
||||
{"case": "00000391_fd490b564a44413884fab719_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "964710b92345041b05b4c4e9c8f8461565424c5a7fe257c3b1daad23b272ffe4"}
|
||||
{"case": "00000405_f3759bbb4682423caf207e8e_step_002", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "ff46298b38e1e2615da49005ffa3a9dc778f06bade5608bf498358fc6ac9713f"}
|
||||
{"case": "00000406_f3759bbb4682423caf207e8e_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "3bafb2869d80cf70eccf0388f9261a1e63bdc1aed5f76a97eab7abc81a68f39c"}
|
||||
{"case": "00000444_4ed4c78d6d754aac90876fc2_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "f45bc24f6a41026795e2fae364523d0052d4dba2b37083ed189c37083972575c"}
|
||||
{"case": "00000445_4ed4c78d6d754aac90876fc2_step_004", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "de03f20a206a42e28f348a927d32461a64e6d80331e801072e15641ab11fd18a"}
|
||||
{"case": "00000501_5499914343bd4f549b38046d_step_002", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "d5c38482fd3fb4658004d47cfb97e7f302ebfbf52fd037627575106faa7d62ed"}
|
||||
{"case": "00000502_5499914343bd4f549b38046d_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "7b046e9ef8fda96203e294b34e6f7d9e8224c93aa22ae2b33160e2f962d59af1"}
|
||||
{"case": "00000574_5a4cef68211d4706b1ec8586_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "d6d49f7214be9bd741cfb29c1db9ed32296355fd54599b6b6c4596e828070950"}
|
||||
{"case": "00000578_5a4cef68211d4706b1ec8586_step_007", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "2f0166ddf5883a0831084985fd27c1a47cc9e487d5e07b3747f95606b5dd534f"}
|
||||
{"case": "00000579_5a4cef68211d4706b1ec8586_step_008", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "d5ad1ad6160204903182bae4829c8a703593b77102d667f26e63ce8f773fd6d7"}
|
||||
{"case": "00000580_5a4cef68211d4706b1ec8586_step_009", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "505d1b2cbe5ea6f2d89804225a2afe649620708f291288fcadd607c4f8eaa380"}
|
||||
{"case": "00000582_5a4cef68211d4706b1ec8586_step_011", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "d2622abdd54e8c757f37127c8cbb69a2b98b7f7726195109b0f8e169eb40c8c5"}
|
||||
{"case": "00000583_5a4cef68211d4706b1ec8586_step_012", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "abf687bf1b72bfdcb8be0a0e6dd727e58cbea355f86674af7e8d78527fc51123"}
|
||||
{"case": "00000597_7fe894812c4244e18241aa08_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "afd15d291607085b0328aa39a56a12dc5249d7aa83378ac6ea202f6efaf8f479"}
|
||||
{"case": "00000601_7fe894812c4244e18241aa08_step_007", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "6ce6c4b4a9433d9b764497b47c436ca43707318897e0e23bda65b3ab44ff5601"}
|
||||
{"case": "00000602_7fe894812c4244e18241aa08_step_008", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "9220f78fc709b73d272fad866130721122b5f2e508f0b4689fb0266d3de96e49"}
|
||||
{"case": "00000603_7fe894812c4244e18241aa08_step_009", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "bccc8d04635ee4880d6518cb770f5498fe5bb9d0dec45e3a06a9ca7ca2044bfc"}
|
||||
{"case": "00000605_7fe894812c4244e18241aa08_step_011", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "ceb7c0e01a0da9cc536420bdbabefd8a479cd04b066867f41da2466bb5e20f08"}
|
||||
{"case": "00000606_7fe894812c4244e18241aa08_step_012", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "b01867d33249904cef3cbe6ebc7beb9e5bb3c2f7a69302c8a586e32e8a013a40"}
|
||||
{"case": "00000654_d87d11d9c3ca4ee5a88797ae_step_002", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "b80a50bf70e720b064ae42332f4badfff8a6b75d7532ca7c532514894b8006fa"}
|
||||
{"case": "00000758_8ab50540e9804365a8717ec0_step_005", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "d1973bdf55f99100032b3f356d65ae2d0e955cae7b11ebfaf3fd94b4ee054aa3"}
|
||||
{"case": "00000759_8ab50540e9804365a8717ec0_step_006", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "bd733710658431dfbb30a1c1ce4796539f8fb044163fd4f64cf5958c42ecebf2"}
|
||||
{"case": "00000760_8ab50540e9804365a8717ec0_step_007", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "92c7f5a9fd9c1b0e2b15b94ea572f53daa86e5c1de52bd7e86f24204aaea0176"}
|
||||
{"case": "00000763_8ab50540e9804365a8717ec0_step_010", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "5f29264829e8d86ab70a2534a1b99c20aa7d5c171b756f5580e50bfee4d9cf1f"}
|
||||
{"case": "00000816_36e6258346004cabb4c091b2_step_004", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "4faba0383fe78bc22060e8db38f5bbdc6169b78b39dc7203cf5fc1c8113c5286"}
|
||||
{"case": "00000817_36e6258346004cabb4c091b2_step_005", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "c9dcd13a4b79db70c3b8f6c4ddf5b5197af07546aa0aa43442c2017935b31f5e"}
|
||||
{"case": "00000892_be26dc83f78947738b2bfabd_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "28079815308e72ed68c8faccb6914c3a857dd4eff7d16f6a5f92b8f63402d983"}
|
||||
{"case": "00000894_be26dc83f78947738b2bfabd_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "ea5269e956de50fe37da47118f5886c30cf61e9fa44e72e8527f43300fb557a3"}
|
||||
{"case": "00000897_be26dc83f78947738b2bfabd_step_006", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "9d66172e8ecbf16d9c42aaa572fe95c5dad74368dfb4bc6d71483ac8eb91977f"}
|
||||
{"case": "00000919_cdcebf32644443b9812097ba_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "bacd8592b20789627e248b456d6ddbc2235c503c8832db5b1c9fde07de5c6212"}
|
||||
{"case": "00000923_cdcebf32644443b9812097ba_step_007", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "211d5abe63eb9319234ab659fb89d51811ac36fc4510d24311eca181bcd85297"}
|
||||
{"case": "00000927_cdcebf32644443b9812097ba_step_011", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "7fe89e9ec49185193b31f2aa1226b6425fdc26ca7e11f07c68c1be012d765267"}
|
||||
{"case": "00000928_cdcebf32644443b9812097ba_step_012", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "591ea63775c4d9a6316f11e73c6d9eafd5455f7be2e7f56e054314f110067cc2"}
|
||||
{"case": "00000955_a741c7966d2c40a8a511263e_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "dff1f65cf40e27567bfe2aea32347fb8e78463e1e847375419a57bd229a024a0"}
|
||||
{"case": "00000984_0dd1386597104265ab67e6cf_step_007", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "548b63c5f15dbdb91d590e527fcbfbecf98786d3c38270d3f98bc4b344bf1c94"}
|
||||
{"case": "00000986_0dd1386597104265ab67e6cf_step_009", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "8615c37d0336e2327287838d141578d78349c519cdb6dbaa6e5ae28c82c26839"}
|
||||
{"case": "00000987_0dd1386597104265ab67e6cf_step_010", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "564896adb0e100fb99782eba139123c22847cbfa566f79db5fb304bdb8330a54"}
|
||||
{"case": "00000988_0dd1386597104265ab67e6cf_step_011", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "3fca1aae7fbca096536a8bf71ec6dc3e8e98264eb36dc0d6781a967bc58e7304"}
|
||||
{"case": "00000996_6d71695ba3fd475c927a00a7_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "f7a28d6dc2db420c1aa7422c7b42b9ede3419038d2b2335a693e67e2b9de64e2"}
|
||||
{"case": "00001019_ca4cf9693d6d4a98b515ba99_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "8184fc8a04471b8053e56ed3c833768a5fa7c3dc6bb3e9cc094a3a20a1b7f8e7"}
|
||||
{"case": "00001020_ca4cf9693d6d4a98b515ba99_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "07e04f066eb62fe208a1681585bbda18a01719009a4b84819bd1babef7264ec1"}
|
||||
{"case": "00001021_ca4cf9693d6d4a98b515ba99_step_002", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "7d82fa1d39b531c1a783b53626aa8c3a3d8fb62fd694c1eee99e74165f3c7712"}
|
||||
{"case": "00001026_0f133d8e7fd448708a554e28_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "48f865b03819b29c7a50908e5662de3b44e15a0e56aab72769c7a2beb9264176"}
|
||||
{"case": "00001027_0f133d8e7fd448708a554e28_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "59ca28fdaeec40bbc7d7307fc6404193970e0158fad97cea69d561f4cfcde63f"}
|
||||
{"case": "00001040_74c42207fa764831b03f7658_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "d3205f81cec9af7e472904a8d07bee1f3f3ee5fad4f3fe36f83e5a6c25903bbc"}
|
||||
{"case": "00001052_2fd6155fa34f418083d0bd0f_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "f01d683fbd8d7387ce6892945c8f074c81196fc5dc08adbd08991fcdae7e1f3d"}
|
||||
{"case": "00001053_2fd6155fa34f418083d0bd0f_step_002", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "cbf69d06b0e237fa8d2ce62254967dc9cbc35c3ba46dcefedc61ef7c5fe4c22a"}
|
||||
{"case": "00001097_256ee7ec3a4b4972a8b87c8d_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "f37baaabb768e92fa8fc91f845f2ec9454a43adfca8be1af089b88ee62512f8e"}
|
||||
{"case": "00001099_256ee7ec3a4b4972a8b87c8d_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "1605ed2f721c1375fc380f5ac17cd15113e6fe7ae3993674823dbad8f7652812"}
|
||||
{"case": "00001102_256ee7ec3a4b4972a8b87c8d_step_006", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "65ddf1df58390e7961046c56c901a9557079838322fbfcf36e65562124eeef94"}
|
||||
{"case": "00001107_78566795c3144719804f8d34_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "c6d878d3efdce5c79b74e434363535641f0bd8004154ed3286769f5887f0a6b8"}
|
||||
{"case": "00001167_107f61d0402f4c9a9d9d1342_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "2e696b9f2981b8e2581e471a08b2dc0c449ea7f58bada878bb3dbb0204a8c1e1"}
|
||||
{"case": "00001168_107f61d0402f4c9a9d9d1342_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "fc87af02eeab2f81e57eabb0acee667310958c1c355b71da87165d5b5ca8b1fd"}
|
||||
{"case": "00001217_e517c61f1c49433bb81cf980_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "72d387617ebf9796a9df64c1753b631c14f1c90d394df2dc569834b9f28e51a5"}
|
||||
{"case": "00001238_96503b6a26c64074ad598927_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "f556f4273f1d4f22db57b1ab56dc521c3e8843fc8f049d6aced398fb18874428"}
|
||||
{"case": "00001239_96503b6a26c64074ad598927_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "a1f7c8f12df88c28679d8a1e585ac5698f1f921faf7f76c72688485fb3b93db5"}
|
||||
{"case": "00001240_96503b6a26c64074ad598927_step_002", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "61fc61280fb2b8857af12b34f7db88a777fc99fe9449b5d5959ab653172f2941"}
|
||||
{"case": "00001241_96503b6a26c64074ad598927_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "178940552ff89af3f30005fec9cbf1744c6add200ad82df83d18e02756c0d576"}
|
||||
{"case": "00001242_96503b6a26c64074ad598927_step_004", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "5a1a13b941565e7456e43f9bad59b866578aa36e3baf9c8e40b345865283ca61"}
|
||||
{"case": "00001243_96503b6a26c64074ad598927_step_005", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "8fb8111188ff13277dc155770fdd6f208d449d96802c1fcea6020bc939fdd6b1"}
|
||||
{"case": "00001244_96503b6a26c64074ad598927_step_006", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "e5d79fcc4b63745ec41aa38077212ad2ff3a9ed0d08afe221249628e19731a56"}
|
||||
{"case": "00001246_96503b6a26c64074ad598927_step_008", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "0db24b45c6a0a2a3da57aa0e673007c2e75c9da5089b66e66271f9440c3648e6"}
|
||||
{"case": "00001247_96503b6a26c64074ad598927_step_009", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "5e9d362295f197380cb37fd3a99cbb0a48dc439d4e642bc1d38eb697d34c47cb"}
|
||||
{"case": "00001249_96503b6a26c64074ad598927_step_011", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "34b0c8f979fba3fa5a8bf86d563ef76f3770986c06e30a091960b9d8131c0169"}
|
||||
{"case": "00001252_96503b6a26c64074ad598927_step_014", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "77ce5e946178874a5f74f3e1f3070d303630d4c5f82f505eb2e39beb7452425e"}
|
||||
{"case": "00001253_96503b6a26c64074ad598927_step_015", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "fe117a5653d11d9164c159f59b553994dcc110de7562aac3ca349c4dd002a675"}
|
||||
{"case": "00001314_a9ed5e56a6a8414f91b39bdb_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "0bac221869e64d737242e75ef8ab0fd3f5bf5d58da51e88777d02875ff6b0444"}
|
||||
{"case": "00001333_beab79941e884455a62af40c_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "2808c1499057f885181f48740db6eba4a924d622997562f71974c66cc547f68c"}
|
||||
{"case": "00001334_beab79941e884455a62af40c_step_002", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "c1540efe2781ee3b793b90217190c92f5a2249b4239f413757fd76ee97771a8d"}
|
||||
{"case": "00001335_beab79941e884455a62af40c_step_003", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "f47fdfddfd997cba31c21db35319ed73eb9f88575ab5b01eb79557b126c21d76"}
|
||||
{"case": "00001336_beab79941e884455a62af40c_step_004", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "c422b37ff51f68d6e4deebdaa22250ff89d68f78f2bc3efca89b5c539ce81af5"}
|
||||
{"case": "00001338_beab79941e884455a62af40c_step_006", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "425e1914f87a6e534c32e740067dd31fcb66b7ec31690f8a9a4383019dbfef21"}
|
||||
{"case": "00001339_beab79941e884455a62af40c_step_007", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "81a6e78e0a327b59acc7fb431617c27a340b57bbc0ce6f0e1e10417b017d1c3a"}
|
||||
{"case": "00001340_beab79941e884455a62af40c_step_008", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "5224c1b1735eab76da350cafddb79b0e5c8b615d216c3bbaf1953c4c5aa56161"}
|
||||
{"case": "00001352_4e1d56bc2cef49ea84626b11_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "00c1b8455d18f84e255e5263419726b5d178d43af69e0b0695458cfc78cfa9dc"}
|
||||
{"case": "00001353_4e1d56bc2cef49ea84626b11_step_001", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "8c2de3b1c8097620b80ae883747d51c4819c7139ad5bf46806dae0a3351ee4e8"}
|
||||
{"case": "00001354_4e1d56bc2cef49ea84626b11_step_002", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "fb51f547c824682119a7de20d8a27b6eb1830554cbe2240a200efb8eb3b7210a"}
|
||||
{"case": "00001379_5b1d6e2599ea47d498a04256_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "b0263173ae145d86a52b2006ef3694c5144dfb11606860b88f9e417048340752"}
|
||||
{"case": "00001387_60cf3ad636dd42e0bcec6946_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "697cafca17c906d17e8fd085e29ebbc0578229356688273a2ed5e9ba2cae2186"}
|
||||
{"case": "00001511_5cbe58077e4b4885a5c213b3_step_004", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "06212a3933f7f72c7e1b72fcaaa2adbbbd97f3f5d21e0419cfb74fc70989b83f"}
|
||||
{"case": "00001512_5cbe58077e4b4885a5c213b3_step_005", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "f0f33fd65a14ecd47004b7882a37f12fe46dba23688f758228f86769d180d5a3"}
|
||||
{"case": "00001525_e0d35a03b46745259cf8878f_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "ba0e7de8d57df58ec731c2652323efed4395ca497e359be3f995b522d3d400c9"}
|
||||
{"case": "00001532_9f0a4a641e8f4cc588eb4315_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "47049626a5605ce89932100a832c254066497b0a4583bf46a59fd8e6f6c8bcf1"}
|
||||
{"case": "00001552_b0a9c4d836294846891615fe_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "d9c1e57ae408fde8a19d6722dbf4d5bfcad52cad37310c5868808c27546c0180"}
|
||||
{"case": "00001553_adfa1c33ae4d4d0aa441ddc3_step_000", "epoch": 10, "role": "validation_diagnostic_only", "target_sha256": "681f14b143d1c5250f761d8d1a8835a7335d2b846dc13eae7d60d5e6b38ed486"}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 720.046806,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "train",
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"cpu_seconds": 720.046806
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"audited_utc": "2026-11-19T05:56:33.444485+00:00",
|
||||
"status": "STOPPED",
|
||||
"trigger": "FOREIGN_GPU_OCCUPANT",
|
||||
"training_attempts": 1,
|
||||
"no_retry": true,
|
||||
"service": "meltmind-1b-fold0-deadline-20261119.service",
|
||||
"process_exit": {
|
||||
"service_exit": 1,
|
||||
"train_returncode": -9,
|
||||
"coordinator_absent": 2506340,
|
||||
"worker_absent": 2506399
|
||||
},
|
||||
"later_observed_gpu_query": "Result=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nGPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 2512405, 820 MiB\n PID STAT COMMAND\n",
|
||||
"observed_current_occupant_attribution": "Later read-only snapshot; the exact triggering sample PID was not persisted.",
|
||||
"completed_epochs": [
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
11,
|
||||
12
|
||||
],
|
||||
"last_training_progress": {
|
||||
"fold": 0,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 13,
|
||||
"optimizer_steps": 4024,
|
||||
"wall_seconds": 716.2925684670918,
|
||||
"cpu_seconds": 715.89480798,
|
||||
"case": "00001597_13bd23e0e4c740969b0c71ab_step_000",
|
||||
"epoch_parts": 184,
|
||||
"epoch_nodes": 4916146,
|
||||
"lr": 0.0009827784105872876,
|
||||
"loss": 0.0026505172718316317,
|
||||
"gradient_norm_before_clip": 0.03279545158147812,
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
},
|
||||
"training_checkpoint_adopted": false,
|
||||
"diagnostics": [
|
||||
{
|
||||
"epoch": 10,
|
||||
"role": "validation_diagnostic_only",
|
||||
"part_records": 111
|
||||
}
|
||||
],
|
||||
"final_validation_screen_started": false,
|
||||
"test_targets_read": false,
|
||||
"gpu_seconds": {
|
||||
"queue_wait_gpu_seconds": 20.293689728947356,
|
||||
"training_gpu_seconds": 720.7542707340326,
|
||||
"screen_gpu_seconds": 0.0,
|
||||
"finalization_gpu_seconds": 0.11176366300787777,
|
||||
"new_gpu_seconds": 741.1597241259879,
|
||||
"new_gpu_cap_seconds": 13180,
|
||||
"cap_overage_gpu_seconds": 0.0,
|
||||
"cumulative_item_gpu_seconds": 20003.217328388826,
|
||||
"cumulative_phase3_gpu_seconds": 409371.5701867306
|
||||
},
|
||||
"pinned_contract_sha256": "25cb8ce8ba1b43695fb5c229459033d6c5f0c79a9e67e156b9e78353b9a89971",
|
||||
"source_evidence_sha256": {
|
||||
"artifacts/phase3-item1b-fold0-deadline-2026-11-19/complete.json": "703cac3af80d53b13f908b18d86c260d52562797dcd0610cd307a5af92e56f2d",
|
||||
"artifacts/phase3-item1b-fold0-deadline-2026-11-19/train.receipt.json": "3e130980d5ca7a93ff990835739e5974b49b098766d17548947e3d171b5363ab",
|
||||
"artifacts/phase3-item1b-fold0-deadline-2026-11-19/ledger.json": "a236be6dfd60d0be13d87ea5f190f7e3b7aa3e4eddad43b843cd6f522fe737c6",
|
||||
"artifacts/phase3-item1b-fold0-deadline-2026-11-19/jobs/0/worker/progress.json": "7d86c6f16d02c4e4e8c97734b47e56adedea290d6150fadd8ac489332b21a58b",
|
||||
"artifacts/phase3-item1b-fold0-deadline-2026-11-19/jobs/0/worker/history.jsonl": "0e3daec1890e2b4f6b1b980f9072f51adac15132d12337a3749db65960b924a3",
|
||||
"artifacts/phase3-item1b-fold0-deadline-2026-11-19/jobs/0/worker/validation-target-access.jsonl": "711cee1461dd72c6f3551a2908718c1a919ddbbea49ee15a91267cc0e20f45d1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_fold0_deadline.py",
|
||||
"train",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-deadline-2026-11-19"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 720.046806,
|
||||
"wall_seconds": 720.661197388079,
|
||||
"peak_tree_rss_bytes": 1465208832,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 2924544,
|
||||
"max_poll_gap_seconds": 0.6629281869390979,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
Reference in new issue
Block a user