428 lines
11 KiB
JSON
428 lines
11 KiB
JSON
[
|
|
{
|
|
"root": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
|
"code_pins": 51,
|
|
"pins": "PASS",
|
|
"budget": {
|
|
"gpu_cap": 158717.68659210298,
|
|
"train_gpu_cap": 151517.68659210298,
|
|
"postfit_gpu_reserve": 7200
|
|
},
|
|
"jobs": [
|
|
{
|
|
"id": 0,
|
|
"gpu_uuid": null,
|
|
"worker_cpus": [
|
|
0,
|
|
1
|
|
],
|
|
"guard_cpus": [
|
|
10
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 40000
|
|
},
|
|
{
|
|
"id": 1,
|
|
"gpu_uuid": null,
|
|
"worker_cpus": [
|
|
2,
|
|
3
|
|
],
|
|
"guard_cpus": [
|
|
11
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 40000
|
|
},
|
|
{
|
|
"id": 2,
|
|
"gpu_uuid": null,
|
|
"worker_cpus": [
|
|
4,
|
|
5
|
|
],
|
|
"guard_cpus": [
|
|
12
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 40000
|
|
},
|
|
{
|
|
"id": 3,
|
|
"gpu_uuid": null,
|
|
"worker_cpus": [
|
|
6,
|
|
7
|
|
],
|
|
"guard_cpus": [
|
|
13
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 40000
|
|
},
|
|
{
|
|
"id": 4,
|
|
"gpu_uuid": null,
|
|
"worker_cpus": [
|
|
8,
|
|
9
|
|
],
|
|
"guard_cpus": [
|
|
14
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 40000
|
|
}
|
|
],
|
|
"receipts": {
|
|
"live-progress.json": {
|
|
"stage": "SEED1_TRAINING",
|
|
"unix": 1795006694.1644843,
|
|
"active": [
|
|
{
|
|
"id": 0,
|
|
"guard_pid": 1476627
|
|
},
|
|
{
|
|
"id": 1,
|
|
"guard_pid": 1476628
|
|
},
|
|
{
|
|
"id": 2,
|
|
"guard_pid": 1598086
|
|
}
|
|
],
|
|
"terminal": [],
|
|
"pending": [
|
|
3,
|
|
4
|
|
],
|
|
"charged_training_gpu_seconds": 36065.127459958894,
|
|
"postfit_gpu_reserve": 7200
|
|
},
|
|
"latest-cost-projection.json": {
|
|
"status": "PASS",
|
|
"rows": [
|
|
{
|
|
"id": 0,
|
|
"projected_gpu_seconds": 20537.820046406356,
|
|
"actual_charged_gpu_seconds": 14611.722367653972,
|
|
"actual_within_runaway": true,
|
|
"basis": "measured_projection"
|
|
},
|
|
{
|
|
"id": 1,
|
|
"projected_gpu_seconds": 29538.486472418066,
|
|
"actual_charged_gpu_seconds": 14611.717376481974,
|
|
"actual_within_runaway": true,
|
|
"basis": "measured_projection"
|
|
},
|
|
{
|
|
"id": 2,
|
|
"projected_gpu_seconds": 25652.435660988966,
|
|
"actual_charged_gpu_seconds": 6834.808777031023,
|
|
"actual_within_runaway": true,
|
|
"basis": "measured_projection"
|
|
},
|
|
{
|
|
"id": 3,
|
|
"projected_gpu_seconds": 25652.435660988966,
|
|
"actual_charged_gpu_seconds": 0,
|
|
"actual_within_runaway": true,
|
|
"basis": "pending_median_reserve"
|
|
},
|
|
{
|
|
"id": 4,
|
|
"projected_gpu_seconds": 25652.435660988966,
|
|
"actual_charged_gpu_seconds": 0,
|
|
"actual_within_runaway": true,
|
|
"basis": "pending_median_reserve"
|
|
}
|
|
],
|
|
"projected_gpu_seconds": 134233.61350179132,
|
|
"phase3_prior_gpu_once": 80903.62940728385,
|
|
"projected_phase3_gpu_seconds": 215137.24290907517,
|
|
"postfit_reserve": 7200,
|
|
"pending_reserve_gpu_seconds": 25652.435660988966,
|
|
"actual_runaway_gpu_seconds": 40000,
|
|
"total_projection_within_cap": true,
|
|
"cell": {
|
|
"id": 2,
|
|
"projected_gpu_seconds": 25652.435660988966,
|
|
"validation_seconds_bound": 9.668979812064208,
|
|
"actual_charged_gpu_seconds": 6834.808727527969,
|
|
"actual_within_runaway": true,
|
|
"projected_above_runaway": false
|
|
},
|
|
"contract_sha256": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981",
|
|
"unix": 1795006691.8720207
|
|
}
|
|
},
|
|
"workers": [
|
|
{
|
|
"job": "0",
|
|
"progress": {
|
|
"fold": 0,
|
|
"stage": "TRAIN",
|
|
"epoch": 155,
|
|
"optimizer_steps": 49441,
|
|
"wall_seconds": 14543.364778421004,
|
|
"cpu_seconds": 14504.733303124001,
|
|
"case": "00000072_767e4372b5f94a88a7a17d90_step_013",
|
|
"epoch_parts": 161,
|
|
"epoch_nodes": 4444056,
|
|
"lr": 0.00014081666421679985,
|
|
"loss": 2.4992827093228698e-05,
|
|
"gradient_norm_before_clip": 0.000830289616715163,
|
|
"peak_allocated_bytes": 1334860288,
|
|
"peak_reserved_bytes": 1388314624
|
|
},
|
|
"completed_epochs": 154,
|
|
"recent10_epoch_seconds": [
|
|
91.79020690300968,
|
|
91.0233701330144,
|
|
91.02645130397286,
|
|
91.52249923499767,
|
|
95.11965373891871,
|
|
93.42180484591518,
|
|
89.95967863500118,
|
|
92.87886077200528,
|
|
91.14694070699625,
|
|
93.60838667897042
|
|
],
|
|
"median_recent10": 91.65635306900367,
|
|
"peak_allocated": 1334860288,
|
|
"peak_reserved": 1388314624
|
|
},
|
|
{
|
|
"job": "1",
|
|
"progress": {
|
|
"fold": 1,
|
|
"stage": "TRAIN",
|
|
"epoch": 119,
|
|
"optimizer_steps": 44474,
|
|
"wall_seconds": 14543.898690835922,
|
|
"cpu_seconds": 14503.983612828,
|
|
"case": "00000790_a25fb37392a44447b9684f16_step_015",
|
|
"epoch_parts": 342,
|
|
"epoch_nodes": 8888977,
|
|
"lr": 0.00039143515957685817,
|
|
"loss": 0.0005089675541967154,
|
|
"gradient_norm_before_clip": 0.01804092526435852,
|
|
"peak_allocated_bytes": 1334860288,
|
|
"peak_reserved_bytes": 1390411776
|
|
},
|
|
"completed_epochs": 118,
|
|
"recent10_epoch_seconds": [
|
|
104.03197531797923,
|
|
104.27232543297578,
|
|
102.27148887095973,
|
|
104.10447630204726,
|
|
103.55956940201577,
|
|
107.4991078999592,
|
|
105.82303702400532,
|
|
102.22429705096874,
|
|
106.02757873607334,
|
|
104.48560117301531
|
|
],
|
|
"median_recent10": 104.18840086751152,
|
|
"peak_allocated": 1334860288,
|
|
"peak_reserved": 1390411776
|
|
},
|
|
{
|
|
"job": "2",
|
|
"progress": {
|
|
"fold": 2,
|
|
"stage": "TRAIN",
|
|
"epoch": 65,
|
|
"optimizer_steps": 23435,
|
|
"wall_seconds": 6767.362545815995,
|
|
"cpu_seconds": 6766.308971986,
|
|
"case": "00001368_2e314c6e080b40fcb6a6db96_step_011",
|
|
"epoch_parts": 11,
|
|
"epoch_nodes": 263923,
|
|
"lr": 0.0008090352927813857,
|
|
"loss": 0.00021560114691965282,
|
|
"gradient_norm_before_clip": 0.030973436310887337,
|
|
"peak_allocated_bytes": 1461572608,
|
|
"peak_reserved_bytes": 1516240896
|
|
},
|
|
"completed_epochs": 64,
|
|
"recent10_epoch_seconds": [
|
|
103.85552064399235,
|
|
102.38861383905169,
|
|
103.54317934100982,
|
|
102.5205716479104,
|
|
107.35395550006069,
|
|
106.42893402301706,
|
|
102.09753456991166,
|
|
105.13476634700783,
|
|
104.87566378701013,
|
|
104.03085032501258
|
|
],
|
|
"median_recent10": 103.94318548450246,
|
|
"peak_allocated": 1461572608,
|
|
"peak_reserved": 1516240896
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"root": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
|
|
"code_pins": 27,
|
|
"pins": "PASS",
|
|
"budget": {
|
|
"gpu_cap": 162000,
|
|
"train_gpu_cap": 106856.05280544702,
|
|
"postfit_gpu_reserve": 7200
|
|
},
|
|
"jobs": [
|
|
{
|
|
"id": 0,
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"worker_cpus": [
|
|
8
|
|
],
|
|
"guard_cpus": [
|
|
28
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 30960
|
|
},
|
|
{
|
|
"id": 1,
|
|
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
|
"worker_cpus": [
|
|
10
|
|
],
|
|
"guard_cpus": [
|
|
29
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 30960
|
|
},
|
|
{
|
|
"id": 2,
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"worker_cpus": [
|
|
8
|
|
],
|
|
"guard_cpus": [
|
|
28
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 30960
|
|
},
|
|
{
|
|
"id": 3,
|
|
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
|
"worker_cpus": [
|
|
10
|
|
],
|
|
"guard_cpus": [
|
|
29
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 30960
|
|
},
|
|
{
|
|
"id": 4,
|
|
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
|
"worker_cpus": [
|
|
8
|
|
],
|
|
"guard_cpus": [
|
|
28
|
|
],
|
|
"rss_bytes": 17179869184,
|
|
"vram_bytes": 16106127360,
|
|
"gpu_seconds": 30960
|
|
}
|
|
],
|
|
"receipts": {
|
|
"live-progress.json": {
|
|
"unix": 1795006694.7851024,
|
|
"active": [
|
|
{
|
|
"id": 4,
|
|
"guard_pid": 1650109
|
|
}
|
|
],
|
|
"pending": [
|
|
3
|
|
],
|
|
"terminal": [],
|
|
"old_gpu_seconds_once": 47943.94719455298,
|
|
"new_training_gpu_seconds": 5097.757695970009,
|
|
"cumulative_training_gpu_seconds": 53041.704901624005,
|
|
"postfit_started": false
|
|
},
|
|
"latest-cost-projection.json": {
|
|
"status": "PASS",
|
|
"rows": [
|
|
{
|
|
"id": 3,
|
|
"projected_gpu_seconds": 30960,
|
|
"basis": "continuing_projection_or_pending_cap"
|
|
},
|
|
{
|
|
"id": 4,
|
|
"projected_gpu_seconds": 13693.198552801914,
|
|
"basis": "continuing_projection_or_pending_cap"
|
|
}
|
|
],
|
|
"new_projected_gpu_seconds": 51853.198552801914,
|
|
"old_gpu_seconds_once": 47943.94719455298,
|
|
"cumulative_projected_gpu_seconds": 99797.1457473549,
|
|
"postfit_reserve": 7200
|
|
}
|
|
},
|
|
"workers": [
|
|
{
|
|
"job": "4",
|
|
"progress": {
|
|
"fold": 4,
|
|
"stage": "TRAIN",
|
|
"epoch": 46,
|
|
"optimizer_steps": 14087,
|
|
"wall_seconds": 2476.8090013740584,
|
|
"cpu_seconds": 2475.898949847,
|
|
"case": "00000986_0dd1386597104265ab67e6cf_step_009",
|
|
"epoch_parts": 137,
|
|
"epoch_nodes": 3867916,
|
|
"lr": 0.00091486437141827,
|
|
"loss": 0.00040039376472122967,
|
|
"gradient_norm_before_clip": 0.020746927708387375,
|
|
"peak_allocated_bytes": 1461572608,
|
|
"peak_reserved_bytes": 1516240896
|
|
},
|
|
"completed_epochs": 45,
|
|
"recent10_epoch_seconds": [
|
|
56.72672600694932,
|
|
55.961798021919094,
|
|
54.08011715405155,
|
|
53.752001903019845,
|
|
52.82414131902624,
|
|
56.1399729748955,
|
|
55.80259197601117,
|
|
52.83759187697433,
|
|
55.623372485046275,
|
|
53.57573465502355
|
|
],
|
|
"median_recent10": 54.85174481954891,
|
|
"peak_allocated": 1461572608,
|
|
"peak_reserved": 1516240896
|
|
}
|
|
]
|
|
}
|
|
]
|