292 lines
7.4 KiB
JSON
292 lines
7.4 KiB
JSON
{
|
|
"unix": 1795011943.081557,
|
|
"snapshot": {
|
|
"devices": {
|
|
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c": {
|
|
"total": 17179869184,
|
|
"used": 1928331264
|
|
},
|
|
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36": {
|
|
"total": 17179869184,
|
|
"used": 5390729216
|
|
},
|
|
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76": {
|
|
"total": 17179869184,
|
|
"used": 5770313728
|
|
},
|
|
"GPU-8b71b032-1b75-2891-935c-ce106a53034a": {
|
|
"total": 17179869184,
|
|
"used": 1928331264
|
|
}
|
|
},
|
|
"processes": [
|
|
{
|
|
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
|
"pid": 1765688,
|
|
"vram": 1918894080
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"pid": 1476726,
|
|
"vram": 1793064960
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"pid": 1755848,
|
|
"vram": 1793064960
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"pid": 1755878,
|
|
"vram": 1795162112
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"pid": 1650207,
|
|
"vram": 1918894080
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"pid": 1756022,
|
|
"vram": 1920991232
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"pid": 1755953,
|
|
"vram": 1920991232
|
|
},
|
|
{
|
|
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
|
"pid": 1598144,
|
|
"vram": 1918894080
|
|
}
|
|
],
|
|
"ram_total": 67378212864,
|
|
"ram_available": 51724812288,
|
|
"cpu_percent": 25.5,
|
|
"old_pending": true,
|
|
"unix": 1795011942.9723783
|
|
},
|
|
"leases": [
|
|
{
|
|
"token": "da7989f6fb454ec99e83ffa8092b056a",
|
|
"state": "GUARD_BOUND",
|
|
"root": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18",
|
|
"fold": 0,
|
|
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"worker_cpus": [
|
|
16
|
|
],
|
|
"owner": {
|
|
"pid": 1755765,
|
|
"created": 1795010605.64
|
|
},
|
|
"queue_owner": {
|
|
"pid": 1755713,
|
|
"created": 1795010602.48
|
|
},
|
|
"vram_reserve": 4294967296,
|
|
"rss_reserve": 4294967296,
|
|
"created_unix": 1795010605.6536286,
|
|
"observed_pids": [
|
|
1755765,
|
|
1755848
|
|
],
|
|
"observed_rss": 1805930496
|
|
},
|
|
{
|
|
"token": "a7e8a5931ff844faa5a61f420e155704",
|
|
"state": "GUARD_BOUND",
|
|
"root": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18",
|
|
"fold": 1,
|
|
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
|
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
|
"worker_cpus": [
|
|
17
|
|
],
|
|
"owner": {
|
|
"pid": 1755824,
|
|
"created": 1795010607.05
|
|
},
|
|
"queue_owner": {
|
|
"pid": 1755713,
|
|
"created": 1795010602.48
|
|
},
|
|
"vram_reserve": 4294967296,
|
|
"rss_reserve": 4294967296,
|
|
"created_unix": 1795010607.0678425,
|
|
"observed_pids": [
|
|
1755824,
|
|
1755878
|
|
],
|
|
"observed_rss": 1775656960
|
|
},
|
|
{
|
|
"token": "ded7a54470cb409b85604b8d7248abc3",
|
|
"state": "GUARD_BOUND",
|
|
"root": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18",
|
|
"fold": 2,
|
|
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"worker_cpus": [
|
|
18
|
|
],
|
|
"owner": {
|
|
"pid": 1755862,
|
|
"created": 1795010608.29
|
|
},
|
|
"queue_owner": {
|
|
"pid": 1755713,
|
|
"created": 1795010602.48
|
|
},
|
|
"vram_reserve": 4294967296,
|
|
"rss_reserve": 4294967296,
|
|
"created_unix": 1795010608.3074121,
|
|
"observed_pids": [
|
|
1755862,
|
|
1755953
|
|
],
|
|
"observed_rss": 1790693376
|
|
},
|
|
{
|
|
"token": "53f07e4a34204fd8a6641002c8d29f3e",
|
|
"state": "GUARD_BOUND",
|
|
"root": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18",
|
|
"fold": 3,
|
|
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"worker_cpus": [
|
|
19
|
|
],
|
|
"owner": {
|
|
"pid": 1755893,
|
|
"created": 1795010609.69
|
|
},
|
|
"queue_owner": {
|
|
"pid": 1755713,
|
|
"created": 1795010602.48
|
|
},
|
|
"vram_reserve": 4294967296,
|
|
"rss_reserve": 4294967296,
|
|
"created_unix": 1795010609.7058253,
|
|
"observed_pids": [
|
|
1755893,
|
|
1756022
|
|
],
|
|
"observed_rss": 1809506304
|
|
}
|
|
],
|
|
"admission": {
|
|
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c": [
|
|
false,
|
|
"GPU0_RESERVED_FOR_OLD_PENDING"
|
|
],
|
|
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36": [
|
|
false,
|
|
"GPU_SLOT_LIMIT"
|
|
],
|
|
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76": [
|
|
false,
|
|
"GPU_SLOT_LIMIT"
|
|
]
|
|
},
|
|
"free_worker_cpus": [
|
|
20,
|
|
21,
|
|
22,
|
|
23,
|
|
24,
|
|
25
|
|
],
|
|
"old_pins_and_progress": [
|
|
{
|
|
"root": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18",
|
|
"code_count": 52,
|
|
"progress": {
|
|
"unix": 1795011941.910367,
|
|
"stage": "ITEM_C_TRAINING",
|
|
"active": [
|
|
0,
|
|
1,
|
|
2,
|
|
3
|
|
],
|
|
"pending": [
|
|
4
|
|
],
|
|
"terminal": [],
|
|
"charged_training_gpu_seconds": 7582.2292879448505,
|
|
"postfit_started": false
|
|
},
|
|
"terminal": false
|
|
},
|
|
{
|
|
"root": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
|
"code_count": 51,
|
|
"progress": {
|
|
"stage": "SEED1_TRAINING",
|
|
"unix": 1795011942.2254956,
|
|
"active": [
|
|
{
|
|
"id": 1,
|
|
"guard_pid": 1476628
|
|
},
|
|
{
|
|
"id": 2,
|
|
"guard_pid": 1598086
|
|
},
|
|
{
|
|
"id": 3,
|
|
"guard_pid": 1765488
|
|
}
|
|
],
|
|
"terminal": [
|
|
{
|
|
"id": 0,
|
|
"status": "PASS",
|
|
"returncode": 0,
|
|
"gpu_seconds_charged": 18897.35776199703,
|
|
"guard_complete_sha256": "74043e45ac73a511be69bfe0d51c170752a7c8aecf4a46e12423c89baa174d70",
|
|
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
|
"assignment_sha256": "498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
|
|
}
|
|
],
|
|
"pending": [
|
|
4
|
|
],
|
|
"charged_training_gpu_seconds": 51869.11572699901,
|
|
"postfit_gpu_reserve": 7200
|
|
},
|
|
"terminal": false
|
|
},
|
|
{
|
|
"root": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
|
|
"code_count": 27,
|
|
"progress": {
|
|
"unix": 1795011942.1222477,
|
|
"active": [
|
|
{
|
|
"id": 4,
|
|
"guard_pid": 1650109
|
|
}
|
|
],
|
|
"pending": [
|
|
3
|
|
],
|
|
"terminal": [],
|
|
"old_gpu_seconds_once": 47943.94719455298,
|
|
"new_training_gpu_seconds": 15592.43198554893,
|
|
"cumulative_training_gpu_seconds": 63536.37919126195,
|
|
"postfit_started": false
|
|
},
|
|
"terminal": false
|
|
}
|
|
],
|
|
"duplicates": [],
|
|
"unit": "MainPID=0\nLoadState=not-found\nActiveState=inactive\n",
|
|
"entry_registered": false,
|
|
"started": false,
|
|
"free_disk": 237145300992
|
|
}
|