176 lines
4.0 KiB
JSON
176 lines
4.0 KiB
JSON
{
|
|
"status": "ACTUAL_WORKERS_TRAINING",
|
|
"unix": 1795010701.9993742,
|
|
"coordinator": {
|
|
"pid": 1755713,
|
|
"created": 1795010602.48
|
|
},
|
|
"workers": [
|
|
{
|
|
"fold": 0,
|
|
"worker": {
|
|
"pid": 1755848,
|
|
"created": 1795010607.92
|
|
},
|
|
"guard": {
|
|
"pid": 1755765,
|
|
"created": 1795010605.64
|
|
},
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"worker_cpus": [
|
|
16
|
|
],
|
|
"stage": "TRAIN",
|
|
"epoch": 1,
|
|
"optimizer_steps": 265
|
|
},
|
|
{
|
|
"fold": 1,
|
|
"worker": {
|
|
"pid": 1755878,
|
|
"created": 1795010609.1
|
|
},
|
|
"guard": {
|
|
"pid": 1755824,
|
|
"created": 1795010607.05
|
|
},
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"worker_cpus": [
|
|
17
|
|
],
|
|
"stage": "TRAIN",
|
|
"epoch": 1,
|
|
"optimizer_steps": 261
|
|
},
|
|
{
|
|
"fold": 2,
|
|
"worker": {
|
|
"pid": 1755953,
|
|
"created": 1795010610.97
|
|
},
|
|
"guard": {
|
|
"pid": 1755862,
|
|
"created": 1795010608.29
|
|
},
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"worker_cpus": [
|
|
18
|
|
],
|
|
"stage": "TRAIN",
|
|
"epoch": 1,
|
|
"optimizer_steps": 237
|
|
},
|
|
{
|
|
"fold": 3,
|
|
"worker": {
|
|
"pid": 1756022,
|
|
"created": 1795010612.25
|
|
},
|
|
"guard": {
|
|
"pid": 1755893,
|
|
"created": 1795010609.69
|
|
},
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"worker_cpus": [
|
|
19
|
|
],
|
|
"stage": "TRAIN",
|
|
"epoch": 1,
|
|
"optimizer_steps": 232
|
|
}
|
|
],
|
|
"all_host_gpu_process_counts": {
|
|
"0": 1,
|
|
"1": 3,
|
|
"2": 3,
|
|
"3": 1
|
|
},
|
|
"snapshot": {
|
|
"devices": {
|
|
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c": {
|
|
"total": 17179869184,
|
|
"used": 1800404992
|
|
},
|
|
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36": {
|
|
"total": 17179869184,
|
|
"used": 5390729216
|
|
},
|
|
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76": {
|
|
"total": 17179869184,
|
|
"used": 5770313728
|
|
},
|
|
"GPU-8b71b032-1b75-2891-935c-ce106a53034a": {
|
|
"total": 17179869184,
|
|
"used": 1928331264
|
|
}
|
|
},
|
|
"processes": [
|
|
{
|
|
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
|
"pid": 1476707,
|
|
"vram": 1790967808
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"pid": 1476726,
|
|
"vram": 1793064960
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"pid": 1755848,
|
|
"vram": 1793064960
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"pid": 1755878,
|
|
"vram": 1795162112
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"pid": 1650207,
|
|
"vram": 1918894080
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"pid": 1756022,
|
|
"vram": 1920991232
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"pid": 1755953,
|
|
"vram": 1920991232
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
|
"pid": 1598144,
|
|
"vram": 1918894080
|
|
}
|
|
],
|
|
"ram_total": 67378212864,
|
|
"ram_available": 51414818816,
|
|
"cpu_percent": 28.7,
|
|
"old_pending": true,
|
|
"unix": 1795010701.9749477
|
|
},
|
|
"old": {
|
|
"phase3-seed1-v2-2026-11-18": {
|
|
"pins": "PASS",
|
|
"code_files": 51
|
|
},
|
|
"track-a-recovery-2026-11-18": {
|
|
"pins": "PASS",
|
|
"code_files": 27
|
|
}
|
|
},
|
|
"source_pins": "PASS",
|
|
"review_sha256": "e30612eabf43037f6258644945e9f486dc99426f08d1da160d14cfd6b3163632",
|
|
"implementation": "2ddecef3fac432ccc0267c468d5df4d3203e53ef",
|
|
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
|
"first_complete_epoch_gate_passed": false,
|
|
"pending": [
|
|
4
|
|
],
|
|
"no_retry": true,
|
|
"service_restart": "no"
|
|
}
|