Files
MeltMind/artifacts/gpu-concurrency-assessment-2026-11-18/queues.json
T

428 lines
11 KiB
JSON

[
{
"root": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"code_pins": 51,
"pins": "PASS",
"budget": {
"gpu_cap": 158717.68659210298,
"train_gpu_cap": 151517.68659210298,
"postfit_gpu_reserve": 7200
},
"jobs": [
{
"id": 0,
"gpu_uuid": null,
"worker_cpus": [
0,
1
],
"guard_cpus": [
10
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
},
{
"id": 1,
"gpu_uuid": null,
"worker_cpus": [
2,
3
],
"guard_cpus": [
11
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
},
{
"id": 2,
"gpu_uuid": null,
"worker_cpus": [
4,
5
],
"guard_cpus": [
12
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
},
{
"id": 3,
"gpu_uuid": null,
"worker_cpus": [
6,
7
],
"guard_cpus": [
13
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
},
{
"id": 4,
"gpu_uuid": null,
"worker_cpus": [
8,
9
],
"guard_cpus": [
14
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 40000
}
],
"receipts": {
"live-progress.json": {
"stage": "SEED1_TRAINING",
"unix": 1795006694.1644843,
"active": [
{
"id": 0,
"guard_pid": 1476627
},
{
"id": 1,
"guard_pid": 1476628
},
{
"id": 2,
"guard_pid": 1598086
}
],
"terminal": [],
"pending": [
3,
4
],
"charged_training_gpu_seconds": 36065.127459958894,
"postfit_gpu_reserve": 7200
},
"latest-cost-projection.json": {
"status": "PASS",
"rows": [
{
"id": 0,
"projected_gpu_seconds": 20537.820046406356,
"actual_charged_gpu_seconds": 14611.722367653972,
"actual_within_runaway": true,
"basis": "measured_projection"
},
{
"id": 1,
"projected_gpu_seconds": 29538.486472418066,
"actual_charged_gpu_seconds": 14611.717376481974,
"actual_within_runaway": true,
"basis": "measured_projection"
},
{
"id": 2,
"projected_gpu_seconds": 25652.435660988966,
"actual_charged_gpu_seconds": 6834.808777031023,
"actual_within_runaway": true,
"basis": "measured_projection"
},
{
"id": 3,
"projected_gpu_seconds": 25652.435660988966,
"actual_charged_gpu_seconds": 0,
"actual_within_runaway": true,
"basis": "pending_median_reserve"
},
{
"id": 4,
"projected_gpu_seconds": 25652.435660988966,
"actual_charged_gpu_seconds": 0,
"actual_within_runaway": true,
"basis": "pending_median_reserve"
}
],
"projected_gpu_seconds": 134233.61350179132,
"phase3_prior_gpu_once": 80903.62940728385,
"projected_phase3_gpu_seconds": 215137.24290907517,
"postfit_reserve": 7200,
"pending_reserve_gpu_seconds": 25652.435660988966,
"actual_runaway_gpu_seconds": 40000,
"total_projection_within_cap": true,
"cell": {
"id": 2,
"projected_gpu_seconds": 25652.435660988966,
"validation_seconds_bound": 9.668979812064208,
"actual_charged_gpu_seconds": 6834.808727527969,
"actual_within_runaway": true,
"projected_above_runaway": false
},
"contract_sha256": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981",
"unix": 1795006691.8720207
}
},
"workers": [
{
"job": "0",
"progress": {
"fold": 0,
"stage": "TRAIN",
"epoch": 155,
"optimizer_steps": 49441,
"wall_seconds": 14543.364778421004,
"cpu_seconds": 14504.733303124001,
"case": "00000072_767e4372b5f94a88a7a17d90_step_013",
"epoch_parts": 161,
"epoch_nodes": 4444056,
"lr": 0.00014081666421679985,
"loss": 2.4992827093228698e-05,
"gradient_norm_before_clip": 0.000830289616715163,
"peak_allocated_bytes": 1334860288,
"peak_reserved_bytes": 1388314624
},
"completed_epochs": 154,
"recent10_epoch_seconds": [
91.79020690300968,
91.0233701330144,
91.02645130397286,
91.52249923499767,
95.11965373891871,
93.42180484591518,
89.95967863500118,
92.87886077200528,
91.14694070699625,
93.60838667897042
],
"median_recent10": 91.65635306900367,
"peak_allocated": 1334860288,
"peak_reserved": 1388314624
},
{
"job": "1",
"progress": {
"fold": 1,
"stage": "TRAIN",
"epoch": 119,
"optimizer_steps": 44474,
"wall_seconds": 14543.898690835922,
"cpu_seconds": 14503.983612828,
"case": "00000790_a25fb37392a44447b9684f16_step_015",
"epoch_parts": 342,
"epoch_nodes": 8888977,
"lr": 0.00039143515957685817,
"loss": 0.0005089675541967154,
"gradient_norm_before_clip": 0.01804092526435852,
"peak_allocated_bytes": 1334860288,
"peak_reserved_bytes": 1390411776
},
"completed_epochs": 118,
"recent10_epoch_seconds": [
104.03197531797923,
104.27232543297578,
102.27148887095973,
104.10447630204726,
103.55956940201577,
107.4991078999592,
105.82303702400532,
102.22429705096874,
106.02757873607334,
104.48560117301531
],
"median_recent10": 104.18840086751152,
"peak_allocated": 1334860288,
"peak_reserved": 1390411776
},
{
"job": "2",
"progress": {
"fold": 2,
"stage": "TRAIN",
"epoch": 65,
"optimizer_steps": 23435,
"wall_seconds": 6767.362545815995,
"cpu_seconds": 6766.308971986,
"case": "00001368_2e314c6e080b40fcb6a6db96_step_011",
"epoch_parts": 11,
"epoch_nodes": 263923,
"lr": 0.0008090352927813857,
"loss": 0.00021560114691965282,
"gradient_norm_before_clip": 0.030973436310887337,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
},
"completed_epochs": 64,
"recent10_epoch_seconds": [
103.85552064399235,
102.38861383905169,
103.54317934100982,
102.5205716479104,
107.35395550006069,
106.42893402301706,
102.09753456991166,
105.13476634700783,
104.87566378701013,
104.03085032501258
],
"median_recent10": 103.94318548450246,
"peak_allocated": 1461572608,
"peak_reserved": 1516240896
}
]
},
{
"root": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
"code_pins": 27,
"pins": "PASS",
"budget": {
"gpu_cap": 162000,
"train_gpu_cap": 106856.05280544702,
"postfit_gpu_reserve": 7200
},
"jobs": [
{
"id": 0,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"worker_cpus": [
8
],
"guard_cpus": [
28
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
},
{
"id": 1,
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"worker_cpus": [
10
],
"guard_cpus": [
29
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
},
{
"id": 2,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"worker_cpus": [
8
],
"guard_cpus": [
28
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
},
{
"id": 3,
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"worker_cpus": [
10
],
"guard_cpus": [
29
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
},
{
"id": 4,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"worker_cpus": [
8
],
"guard_cpus": [
28
],
"rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"gpu_seconds": 30960
}
],
"receipts": {
"live-progress.json": {
"unix": 1795006694.7851024,
"active": [
{
"id": 4,
"guard_pid": 1650109
}
],
"pending": [
3
],
"terminal": [],
"old_gpu_seconds_once": 47943.94719455298,
"new_training_gpu_seconds": 5097.757695970009,
"cumulative_training_gpu_seconds": 53041.704901624005,
"postfit_started": false
},
"latest-cost-projection.json": {
"status": "PASS",
"rows": [
{
"id": 3,
"projected_gpu_seconds": 30960,
"basis": "continuing_projection_or_pending_cap"
},
{
"id": 4,
"projected_gpu_seconds": 13693.198552801914,
"basis": "continuing_projection_or_pending_cap"
}
],
"new_projected_gpu_seconds": 51853.198552801914,
"old_gpu_seconds_once": 47943.94719455298,
"cumulative_projected_gpu_seconds": 99797.1457473549,
"postfit_reserve": 7200
}
},
"workers": [
{
"job": "4",
"progress": {
"fold": 4,
"stage": "TRAIN",
"epoch": 46,
"optimizer_steps": 14087,
"wall_seconds": 2476.8090013740584,
"cpu_seconds": 2475.898949847,
"case": "00000986_0dd1386597104265ab67e6cf_step_009",
"epoch_parts": 137,
"epoch_nodes": 3867916,
"lr": 0.00091486437141827,
"loss": 0.00040039376472122967,
"gradient_norm_before_clip": 0.020746927708387375,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
},
"completed_epochs": 45,
"recent10_epoch_seconds": [
56.72672600694932,
55.961798021919094,
54.08011715405155,
53.752001903019845,
52.82414131902624,
56.1399729748955,
55.80259197601117,
52.83759187697433,
55.623372485046275,
53.57573465502355
],
"median_recent10": 54.85174481954891,
"peak_allocated": 1461572608,
"peak_reserved": 1516240896
}
]
}
]