220 lines
6.9 KiB
JSON
220 lines
6.9 KiB
JSON
{
|
|
"status": "FOUR_GPU_TRAINING_OBSERVED_NOT_COMPLETE",
|
|
"observed_unix": 1794706688.0787027,
|
|
"session_id": 92182,
|
|
"coordinator_pid": 223006,
|
|
"implementation_commit": "e3fa0af79a9bd4ecf74015e14a706fab0ea231a7",
|
|
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56",
|
|
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
|
|
"live_training_workers": [
|
|
{
|
|
"pid": 223145,
|
|
"command": [
|
|
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
|
"tools/run_s2_main_training.py",
|
|
"artifacts/s2-main-matrix-2026-11-15",
|
|
"6"
|
|
],
|
|
"cpu_affinity": [
|
|
6,
|
|
7
|
|
],
|
|
"status": "running"
|
|
},
|
|
{
|
|
"pid": 223131,
|
|
"command": [
|
|
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
|
"tools/run_s2_main_training.py",
|
|
"artifacts/s2-main-matrix-2026-11-15",
|
|
"4"
|
|
],
|
|
"cpu_affinity": [
|
|
4,
|
|
5
|
|
],
|
|
"status": "running"
|
|
},
|
|
{
|
|
"pid": 223141,
|
|
"command": [
|
|
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
|
"tools/run_s2_main_training.py",
|
|
"artifacts/s2-main-matrix-2026-11-15",
|
|
"2"
|
|
],
|
|
"cpu_affinity": [
|
|
2,
|
|
3
|
|
],
|
|
"status": "running"
|
|
},
|
|
{
|
|
"pid": 223127,
|
|
"command": [
|
|
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
|
"tools/run_s2_main_training.py",
|
|
"artifacts/s2-main-matrix-2026-11-15",
|
|
"0"
|
|
],
|
|
"cpu_affinity": [
|
|
0,
|
|
1
|
|
],
|
|
"status": "running"
|
|
}
|
|
],
|
|
"training_progress": {
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/progress.json": {
|
|
"stage": "training",
|
|
"epoch": 6,
|
|
"parts_done": 33,
|
|
"parts_total": 145,
|
|
"updates": 901,
|
|
"elapsed_seconds": 79.2007255940116,
|
|
"peak_allocated_bytes": 3137353216,
|
|
"peak_reserved_bytes": 3292528640
|
|
},
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/progress.json": {
|
|
"stage": "training",
|
|
"epoch": 6,
|
|
"parts_done": 49,
|
|
"parts_total": 145,
|
|
"updates": 917,
|
|
"elapsed_seconds": 80.08159776800312,
|
|
"peak_allocated_bytes": 3137353216,
|
|
"peak_reserved_bytes": 3292528640
|
|
},
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/progress.json": {
|
|
"stage": "training",
|
|
"epoch": 6,
|
|
"parts_done": 49,
|
|
"parts_total": 145,
|
|
"updates": 916,
|
|
"elapsed_seconds": 79.6935267400113,
|
|
"peak_allocated_bytes": 3137353216,
|
|
"peak_reserved_bytes": 3271557120
|
|
},
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/progress.json": {
|
|
"stage": "training",
|
|
"epoch": 3,
|
|
"parts_done": 241,
|
|
"parts_total": 254,
|
|
"updates": 1001,
|
|
"elapsed_seconds": 77.72405460698064,
|
|
"peak_allocated_bytes": 3644658176,
|
|
"peak_reserved_bytes": 3797942272
|
|
}
|
|
},
|
|
"matrix_progress": {
|
|
"completed_blocks": 0,
|
|
"terminal_runs": 0,
|
|
"active_blocks": {
|
|
"0": 0,
|
|
"1": 1,
|
|
"2": 2,
|
|
"3": 3
|
|
},
|
|
"next_block": 4,
|
|
"settled_gpu_seconds": 0.0,
|
|
"active_gpu_seconds": 424.2205448030145,
|
|
"first_eight_passed": false,
|
|
"stop_reason": null,
|
|
"elapsed_seconds": 106.05700458900537
|
|
},
|
|
"idle_gate_receipts": {
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/idle-gates/before-run.json": {
|
|
"status": "IDLE_CONFIRMED",
|
|
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"polls": 3,
|
|
"consecutive_idle": 3,
|
|
"wall_seconds": 20.291602034994867,
|
|
"last": {
|
|
"unix": 1794706605.055837,
|
|
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"memory_MiB": 5,
|
|
"utilization_percent": 0,
|
|
"occupants": [],
|
|
"idle": true
|
|
}
|
|
},
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/idle-gates/before-run.json": {
|
|
"status": "IDLE_CONFIRMED",
|
|
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"polls": 3,
|
|
"consecutive_idle": 3,
|
|
"wall_seconds": 20.30101585999364,
|
|
"last": {
|
|
"unix": 1794706605.1271734,
|
|
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"memory_MiB": 5,
|
|
"utilization_percent": 0,
|
|
"occupants": [],
|
|
"idle": true
|
|
}
|
|
},
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/idle-gates/before-run.json": {
|
|
"status": "IDLE_CONFIRMED",
|
|
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
|
"polls": 3,
|
|
"consecutive_idle": 3,
|
|
"wall_seconds": 20.28960164700402,
|
|
"last": {
|
|
"unix": 1794706605.0470319,
|
|
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
|
"memory_MiB": 5,
|
|
"utilization_percent": 0,
|
|
"occupants": [],
|
|
"idle": true
|
|
}
|
|
},
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/idle-gates/before-run.json": {
|
|
"status": "IDLE_CONFIRMED",
|
|
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
|
"polls": 3,
|
|
"consecutive_idle": 3,
|
|
"wall_seconds": 20.291276325995568,
|
|
"last": {
|
|
"unix": 1794706605.1642594,
|
|
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
|
"memory_MiB": 5,
|
|
"utilization_percent": 0,
|
|
"occupants": [],
|
|
"idle": true
|
|
}
|
|
}
|
|
},
|
|
"target_access": {
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/target-access.json": {
|
|
"sha256": "ab03ec76b7e4170f30d192f3a2df70abd3d1286dac3d24e5d9452d4bb0f1c3e7",
|
|
"fold": 0,
|
|
"pool": 25,
|
|
"count": 256
|
|
},
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/target-access.json": {
|
|
"sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272",
|
|
"fold": 0,
|
|
"pool": 25,
|
|
"count": 256
|
|
},
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/target-access.json": {
|
|
"sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272",
|
|
"fold": 0,
|
|
"pool": 25,
|
|
"count": 256
|
|
},
|
|
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/target-access.json": {
|
|
"sha256": "d2f94029612562b1edea866c262dee8cc75205fa3a609ccb9df270dc7e724f37",
|
|
"fold": 0,
|
|
"pool": 50,
|
|
"count": 365
|
|
}
|
|
},
|
|
"source_pins_match": true,
|
|
"gpu_process_inventory": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 223127, 3528 MiB\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 223141, 3528 MiB\nGPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76, 223131, 3508 MiB\nGPU-8b71b032-1b75-2891-935c-ce106a53034a, 223145, 4010 MiB\n",
|
|
"failure_paths": [],
|
|
"r2_started": false,
|
|
"test_predictions": false,
|
|
"test_scoring": false
|
|
}
|