Files
MeltMind/artifacts/s2-main-matrix-2026-11-15/actual-start-audit.json
T

220 lines
6.9 KiB
JSON

{
"status": "FOUR_GPU_TRAINING_OBSERVED_NOT_COMPLETE",
"observed_unix": 1794706688.0787027,
"session_id": 92182,
"coordinator_pid": 223006,
"implementation_commit": "e3fa0af79a9bd4ecf74015e14a706fab0ea231a7",
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56",
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
"live_training_workers": [
{
"pid": 223145,
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"6"
],
"cpu_affinity": [
6,
7
],
"status": "running"
},
{
"pid": 223131,
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"4"
],
"cpu_affinity": [
4,
5
],
"status": "running"
},
{
"pid": 223141,
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"2"
],
"cpu_affinity": [
2,
3
],
"status": "running"
},
{
"pid": 223127,
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"0"
],
"cpu_affinity": [
0,
1
],
"status": "running"
}
],
"training_progress": {
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/progress.json": {
"stage": "training",
"epoch": 6,
"parts_done": 33,
"parts_total": 145,
"updates": 901,
"elapsed_seconds": 79.2007255940116,
"peak_allocated_bytes": 3137353216,
"peak_reserved_bytes": 3292528640
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/progress.json": {
"stage": "training",
"epoch": 6,
"parts_done": 49,
"parts_total": 145,
"updates": 917,
"elapsed_seconds": 80.08159776800312,
"peak_allocated_bytes": 3137353216,
"peak_reserved_bytes": 3292528640
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/progress.json": {
"stage": "training",
"epoch": 6,
"parts_done": 49,
"parts_total": 145,
"updates": 916,
"elapsed_seconds": 79.6935267400113,
"peak_allocated_bytes": 3137353216,
"peak_reserved_bytes": 3271557120
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/progress.json": {
"stage": "training",
"epoch": 3,
"parts_done": 241,
"parts_total": 254,
"updates": 1001,
"elapsed_seconds": 77.72405460698064,
"peak_allocated_bytes": 3644658176,
"peak_reserved_bytes": 3797942272
}
},
"matrix_progress": {
"completed_blocks": 0,
"terminal_runs": 0,
"active_blocks": {
"0": 0,
"1": 1,
"2": 2,
"3": 3
},
"next_block": 4,
"settled_gpu_seconds": 0.0,
"active_gpu_seconds": 424.2205448030145,
"first_eight_passed": false,
"stop_reason": null,
"elapsed_seconds": 106.05700458900537
},
"idle_gate_receipts": {
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/idle-gates/before-run.json": {
"status": "IDLE_CONFIRMED",
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"polls": 3,
"consecutive_idle": 3,
"wall_seconds": 20.291602034994867,
"last": {
"unix": 1794706605.055837,
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
}
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/idle-gates/before-run.json": {
"status": "IDLE_CONFIRMED",
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"polls": 3,
"consecutive_idle": 3,
"wall_seconds": 20.30101585999364,
"last": {
"unix": 1794706605.1271734,
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
}
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/idle-gates/before-run.json": {
"status": "IDLE_CONFIRMED",
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"polls": 3,
"consecutive_idle": 3,
"wall_seconds": 20.28960164700402,
"last": {
"unix": 1794706605.0470319,
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
}
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/idle-gates/before-run.json": {
"status": "IDLE_CONFIRMED",
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"polls": 3,
"consecutive_idle": 3,
"wall_seconds": 20.291276325995568,
"last": {
"unix": 1794706605.1642594,
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
}
}
},
"target_access": {
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/target-access.json": {
"sha256": "ab03ec76b7e4170f30d192f3a2df70abd3d1286dac3d24e5d9452d4bb0f1c3e7",
"fold": 0,
"pool": 25,
"count": 256
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/target-access.json": {
"sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272",
"fold": 0,
"pool": 25,
"count": 256
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/target-access.json": {
"sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272",
"fold": 0,
"pool": 25,
"count": 256
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/target-access.json": {
"sha256": "d2f94029612562b1edea866c262dee8cc75205fa3a609ccb9df270dc7e724f37",
"fold": 0,
"pool": 50,
"count": 365
}
},
"source_pins_match": true,
"gpu_process_inventory": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 223127, 3528 MiB\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 223141, 3528 MiB\nGPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76, 223131, 3508 MiB\nGPU-8b71b032-1b75-2891-935c-ce106a53034a, 223145, 4010 MiB\n",
"failure_paths": [],
"r2_started": false,
"test_predictions": false,
"test_scoring": false
}