Record independent research parallelism and withdrawn unstarted throughput draft
This commit is contained in:
1 parent
ff0033d733
commit
5b6a953fc5
10 files changed
+1812
No files matched your search
@@ -0,0 +1,427 @@
|
||||
[
|
||||
{
|
||||
"root": "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"code_pins": 51,
|
||||
"pins": "PASS",
|
||||
"budget": {
|
||||
"gpu_cap": 158717.68659210298,
|
||||
"train_gpu_cap": 151517.68659210298,
|
||||
"postfit_gpu_reserve": 7200
|
||||
},
|
||||
"jobs": [
|
||||
{
|
||||
"id": 0,
|
||||
"gpu_uuid": null,
|
||||
"worker_cpus": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"guard_cpus": [
|
||||
10
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 40000
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"gpu_uuid": null,
|
||||
"worker_cpus": [
|
||||
2,
|
||||
3
|
||||
],
|
||||
"guard_cpus": [
|
||||
11
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 40000
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"gpu_uuid": null,
|
||||
"worker_cpus": [
|
||||
4,
|
||||
5
|
||||
],
|
||||
"guard_cpus": [
|
||||
12
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 40000
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"gpu_uuid": null,
|
||||
"worker_cpus": [
|
||||
6,
|
||||
7
|
||||
],
|
||||
"guard_cpus": [
|
||||
13
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 40000
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"gpu_uuid": null,
|
||||
"worker_cpus": [
|
||||
8,
|
||||
9
|
||||
],
|
||||
"guard_cpus": [
|
||||
14
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 40000
|
||||
}
|
||||
],
|
||||
"receipts": {
|
||||
"live-progress.json": {
|
||||
"stage": "SEED1_TRAINING",
|
||||
"unix": 1795006694.1644843,
|
||||
"active": [
|
||||
{
|
||||
"id": 0,
|
||||
"guard_pid": 1476627
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"guard_pid": 1476628
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"guard_pid": 1598086
|
||||
}
|
||||
],
|
||||
"terminal": [],
|
||||
"pending": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"charged_training_gpu_seconds": 36065.127459958894,
|
||||
"postfit_gpu_reserve": 7200
|
||||
},
|
||||
"latest-cost-projection.json": {
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"id": 0,
|
||||
"projected_gpu_seconds": 20537.820046406356,
|
||||
"actual_charged_gpu_seconds": 14611.722367653972,
|
||||
"actual_within_runaway": true,
|
||||
"basis": "measured_projection"
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"projected_gpu_seconds": 29538.486472418066,
|
||||
"actual_charged_gpu_seconds": 14611.717376481974,
|
||||
"actual_within_runaway": true,
|
||||
"basis": "measured_projection"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"projected_gpu_seconds": 25652.435660988966,
|
||||
"actual_charged_gpu_seconds": 6834.808777031023,
|
||||
"actual_within_runaway": true,
|
||||
"basis": "measured_projection"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"projected_gpu_seconds": 25652.435660988966,
|
||||
"actual_charged_gpu_seconds": 0,
|
||||
"actual_within_runaway": true,
|
||||
"basis": "pending_median_reserve"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 25652.435660988966,
|
||||
"actual_charged_gpu_seconds": 0,
|
||||
"actual_within_runaway": true,
|
||||
"basis": "pending_median_reserve"
|
||||
}
|
||||
],
|
||||
"projected_gpu_seconds": 134233.61350179132,
|
||||
"phase3_prior_gpu_once": 80903.62940728385,
|
||||
"projected_phase3_gpu_seconds": 215137.24290907517,
|
||||
"postfit_reserve": 7200,
|
||||
"pending_reserve_gpu_seconds": 25652.435660988966,
|
||||
"actual_runaway_gpu_seconds": 40000,
|
||||
"total_projection_within_cap": true,
|
||||
"cell": {
|
||||
"id": 2,
|
||||
"projected_gpu_seconds": 25652.435660988966,
|
||||
"validation_seconds_bound": 9.668979812064208,
|
||||
"actual_charged_gpu_seconds": 6834.808727527969,
|
||||
"actual_within_runaway": true,
|
||||
"projected_above_runaway": false
|
||||
},
|
||||
"contract_sha256": "c8077c380802615f1caed9ca28448ad8b770e5dd83dc7edb4c0a618a3ad15981",
|
||||
"unix": 1795006691.8720207
|
||||
}
|
||||
},
|
||||
"workers": [
|
||||
{
|
||||
"job": "0",
|
||||
"progress": {
|
||||
"fold": 0,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 155,
|
||||
"optimizer_steps": 49441,
|
||||
"wall_seconds": 14543.364778421004,
|
||||
"cpu_seconds": 14504.733303124001,
|
||||
"case": "00000072_767e4372b5f94a88a7a17d90_step_013",
|
||||
"epoch_parts": 161,
|
||||
"epoch_nodes": 4444056,
|
||||
"lr": 0.00014081666421679985,
|
||||
"loss": 2.4992827093228698e-05,
|
||||
"gradient_norm_before_clip": 0.000830289616715163,
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
},
|
||||
"completed_epochs": 154,
|
||||
"recent10_epoch_seconds": [
|
||||
91.79020690300968,
|
||||
91.0233701330144,
|
||||
91.02645130397286,
|
||||
91.52249923499767,
|
||||
95.11965373891871,
|
||||
93.42180484591518,
|
||||
89.95967863500118,
|
||||
92.87886077200528,
|
||||
91.14694070699625,
|
||||
93.60838667897042
|
||||
],
|
||||
"median_recent10": 91.65635306900367,
|
||||
"peak_allocated": 1334860288,
|
||||
"peak_reserved": 1388314624
|
||||
},
|
||||
{
|
||||
"job": "1",
|
||||
"progress": {
|
||||
"fold": 1,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 119,
|
||||
"optimizer_steps": 44474,
|
||||
"wall_seconds": 14543.898690835922,
|
||||
"cpu_seconds": 14503.983612828,
|
||||
"case": "00000790_a25fb37392a44447b9684f16_step_015",
|
||||
"epoch_parts": 342,
|
||||
"epoch_nodes": 8888977,
|
||||
"lr": 0.00039143515957685817,
|
||||
"loss": 0.0005089675541967154,
|
||||
"gradient_norm_before_clip": 0.01804092526435852,
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1390411776
|
||||
},
|
||||
"completed_epochs": 118,
|
||||
"recent10_epoch_seconds": [
|
||||
104.03197531797923,
|
||||
104.27232543297578,
|
||||
102.27148887095973,
|
||||
104.10447630204726,
|
||||
103.55956940201577,
|
||||
107.4991078999592,
|
||||
105.82303702400532,
|
||||
102.22429705096874,
|
||||
106.02757873607334,
|
||||
104.48560117301531
|
||||
],
|
||||
"median_recent10": 104.18840086751152,
|
||||
"peak_allocated": 1334860288,
|
||||
"peak_reserved": 1390411776
|
||||
},
|
||||
{
|
||||
"job": "2",
|
||||
"progress": {
|
||||
"fold": 2,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 65,
|
||||
"optimizer_steps": 23435,
|
||||
"wall_seconds": 6767.362545815995,
|
||||
"cpu_seconds": 6766.308971986,
|
||||
"case": "00001368_2e314c6e080b40fcb6a6db96_step_011",
|
||||
"epoch_parts": 11,
|
||||
"epoch_nodes": 263923,
|
||||
"lr": 0.0008090352927813857,
|
||||
"loss": 0.00021560114691965282,
|
||||
"gradient_norm_before_clip": 0.030973436310887337,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
},
|
||||
"completed_epochs": 64,
|
||||
"recent10_epoch_seconds": [
|
||||
103.85552064399235,
|
||||
102.38861383905169,
|
||||
103.54317934100982,
|
||||
102.5205716479104,
|
||||
107.35395550006069,
|
||||
106.42893402301706,
|
||||
102.09753456991166,
|
||||
105.13476634700783,
|
||||
104.87566378701013,
|
||||
104.03085032501258
|
||||
],
|
||||
"median_recent10": 103.94318548450246,
|
||||
"peak_allocated": 1461572608,
|
||||
"peak_reserved": 1516240896
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"root": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
|
||||
"code_pins": 27,
|
||||
"pins": "PASS",
|
||||
"budget": {
|
||||
"gpu_cap": 162000,
|
||||
"train_gpu_cap": 106856.05280544702,
|
||||
"postfit_gpu_reserve": 7200
|
||||
},
|
||||
"jobs": [
|
||||
{
|
||||
"id": 0,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"worker_cpus": [
|
||||
8
|
||||
],
|
||||
"guard_cpus": [
|
||||
28
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 30960
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"worker_cpus": [
|
||||
10
|
||||
],
|
||||
"guard_cpus": [
|
||||
29
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 30960
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"worker_cpus": [
|
||||
8
|
||||
],
|
||||
"guard_cpus": [
|
||||
28
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 30960
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"worker_cpus": [
|
||||
10
|
||||
],
|
||||
"guard_cpus": [
|
||||
29
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 30960
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"worker_cpus": [
|
||||
8
|
||||
],
|
||||
"guard_cpus": [
|
||||
28
|
||||
],
|
||||
"rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"gpu_seconds": 30960
|
||||
}
|
||||
],
|
||||
"receipts": {
|
||||
"live-progress.json": {
|
||||
"unix": 1795006694.7851024,
|
||||
"active": [
|
||||
{
|
||||
"id": 4,
|
||||
"guard_pid": 1650109
|
||||
}
|
||||
],
|
||||
"pending": [
|
||||
3
|
||||
],
|
||||
"terminal": [],
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"new_training_gpu_seconds": 5097.757695970009,
|
||||
"cumulative_training_gpu_seconds": 53041.704901624005,
|
||||
"postfit_started": false
|
||||
},
|
||||
"latest-cost-projection.json": {
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"id": 3,
|
||||
"projected_gpu_seconds": 30960,
|
||||
"basis": "continuing_projection_or_pending_cap"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 13693.198552801914,
|
||||
"basis": "continuing_projection_or_pending_cap"
|
||||
}
|
||||
],
|
||||
"new_projected_gpu_seconds": 51853.198552801914,
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"cumulative_projected_gpu_seconds": 99797.1457473549,
|
||||
"postfit_reserve": 7200
|
||||
}
|
||||
},
|
||||
"workers": [
|
||||
{
|
||||
"job": "4",
|
||||
"progress": {
|
||||
"fold": 4,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 46,
|
||||
"optimizer_steps": 14087,
|
||||
"wall_seconds": 2476.8090013740584,
|
||||
"cpu_seconds": 2475.898949847,
|
||||
"case": "00000986_0dd1386597104265ab67e6cf_step_009",
|
||||
"epoch_parts": 137,
|
||||
"epoch_nodes": 3867916,
|
||||
"lr": 0.00091486437141827,
|
||||
"loss": 0.00040039376472122967,
|
||||
"gradient_norm_before_clip": 0.020746927708387375,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
},
|
||||
"completed_epochs": 45,
|
||||
"recent10_epoch_seconds": [
|
||||
56.72672600694932,
|
||||
55.961798021919094,
|
||||
54.08011715405155,
|
||||
53.752001903019845,
|
||||
52.82414131902624,
|
||||
56.1399729748955,
|
||||
55.80259197601117,
|
||||
52.83759187697433,
|
||||
55.623372485046275,
|
||||
53.57573465502355
|
||||
],
|
||||
"median_recent10": 54.85174481954891,
|
||||
"peak_allocated": 1461572608,
|
||||
"peak_reserved": 1516240896
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"start": 1795006695.2355924,
|
||||
"end": 1795006695.3000371,
|
||||
"returncode": 0,
|
||||
"stdout": "\n==============NVSMI LOG==============\n\nTimestamp : Wed Nov 18 20:58:15 2026\nDriver Version : 580.196.07\nCUDA Version : 13.0\n\nAttached GPUs : 4\nGPU 00000000:84:00.0\n Performance State : P0\n Clocks Event Reasons\n Idle : Not Active\n Applications Clocks Setting : Not Active\n SW Power Cap : Not Active\n HW Slowdown : Not Active\n HW Thermal Slowdown : Not Active\n HW Power Brake Slowdown : Not Active\n Sync Boost : Not Active\n SW Thermal Slowdown : Not Active\n Display Clock Setting : Not Active\n Clocks Event Reasons Counters\n SW Power Capping : 0 us\n Sync Boost : 0 us\n SW Thermal Slowdown : 0 us\n HW Thermal Slowdown : 0 us\n HW Power Braking : 0 us\n Sparse Operation Mode : N/A\n Temperature\n GPU Current Temp : 51 C\n GPU T.Limit Temp : N/A\n GPU Shutdown Temp : 90 C\n GPU Slowdown Temp : 87 C\n GPU Max Operating Temp : 83 C\n GPU Target Temperature : N/A\n Memory Current Temp : 50 C\n Memory Max Operating Temp : 85 C\n\nGPU 00000000:85:00.0\n Performance State : P0\n Clocks Event Reasons\n Idle : Not Active\n Applications Clocks Setting : Not Active\n SW Power Cap : Not Active\n HW Slowdown : Not Active\n HW Thermal Slowdown : Not Active\n HW Power Brake Slowdown : Not Active\n Sync Boost : Not Active\n SW Thermal Slowdown : Not Active\n Display Clock Setting : Not Active\n Clocks Event Reasons Counters\n SW Power Capping : 0 us\n Sync Boost : 0 us\n SW Thermal Slowdown : 0 us\n HW Thermal Slowdown : 0 us\n HW Power Braking : 0 us\n Sparse Operation Mode : N/A\n Temperature\n GPU Current Temp : 51 C\n GPU T.Limit Temp : N/A\n GPU Shutdown Temp : 90 C\n GPU Slowdown Temp : 87 C\n GPU Max Operating Temp : 83 C\n GPU Target Temperature : N/A\n Memory Current Temp : 50 C\n Memory Max Operating Temp : 85 C\n\nGPU 00000000:A4:00.0\n Performance State : P0\n Clocks Event Reasons\n Idle : Not Active\n Applications Clocks Setting : Not Active\n SW Power Cap : Not Active\n HW Slowdown : Not Active\n HW Thermal Slowdown : Not Active\n HW Power Brake Slowdown : Not Active\n Sync Boost : Not Active\n SW Thermal Slowdown : Not Active\n Display Clock Setting : Not Active\n Clocks Event Reasons Counters\n SW Power Capping : 0 us\n Sync Boost : 0 us\n SW Thermal Slowdown : 0 us\n HW Thermal Slowdown : 0 us\n HW Power Braking : 0 us\n Sparse Operation Mode : N/A\n Temperature\n GPU Current Temp : 55 C\n GPU T.Limit Temp : N/A\n GPU Shutdown Temp :Line truncated
|
||||
"stderr": ""
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
{
|
||||
"unix": 1795006914.731002,
|
||||
"benchmark_processes": [],
|
||||
"benchmark_root_exists": false,
|
||||
"existing_processes": [
|
||||
{
|
||||
"ppid": 2234,
|
||||
"pid": 1476593,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1476627,
|
||||
"pid": 1476707,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"0",
|
||||
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1476628,
|
||||
"pid": 1476726,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"1",
|
||||
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1598086,
|
||||
"pid": 1598144,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"2",
|
||||
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 2234,
|
||||
"pid": 1650006,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1650109,
|
||||
"pid": 1650207,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/track_a_recovery_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
|
||||
"4"
|
||||
]
|
||||
}
|
||||
],
|
||||
"queues": {
|
||||
"phase3-seed1-v2-2026-11-18": {
|
||||
"pins": "PASS",
|
||||
"live": {
|
||||
"stage": "SEED1_TRAINING",
|
||||
"unix": 1795006914.438547,
|
||||
"active": [
|
||||
{
|
||||
"id": 0,
|
||||
"guard_pid": 1476627
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"guard_pid": 1476628
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"guard_pid": 1598086
|
||||
}
|
||||
],
|
||||
"terminal": [],
|
||||
"pending": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"charged_training_gpu_seconds": 36725.94965413795,
|
||||
"postfit_gpu_reserve": 7200
|
||||
},
|
||||
"complete": false,
|
||||
"global_stop": false
|
||||
},
|
||||
"track-a-recovery-2026-11-18": {
|
||||
"pins": "PASS",
|
||||
"live": {
|
||||
"unix": 1795006913.7070353,
|
||||
"active": [
|
||||
{
|
||||
"id": 4,
|
||||
"guard_pid": 1650109
|
||||
}
|
||||
],
|
||||
"pending": [
|
||||
3
|
||||
],
|
||||
"terminal": [],
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"new_training_gpu_seconds": 5535.601560306037,
|
||||
"cumulative_training_gpu_seconds": 53479.548766289954,
|
||||
"postfit_started": false
|
||||
},
|
||||
"complete": false,
|
||||
"global_stop": false
|
||||
}
|
||||
},
|
||||
"benchmark_ever_launched": false
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
"""Independent H2 throughput experiment: learning-only, no scientific scoring."""
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import checked
|
||||
from phase2_home_freeze_io import verify
|
||||
|
||||
BASE = Path('/home/frank14f/optane/MeltMind')
|
||||
ROOT = BASE / 'artifacts/h2-concurrency-probe-2026-11-18'
|
||||
GPU1 = 'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36'
|
||||
|
||||
|
||||
def read(p):
|
||||
return json.loads(Path(p).read_text())
|
||||
|
||||
|
||||
def config(root):
|
||||
root = Path(root)
|
||||
if root.resolve() != ROOT:
|
||||
raise PermissionError('Sole labelled probe root')
|
||||
verify(root, 'implementation')
|
||||
c = read(root / 'contract.json')
|
||||
expected = dict(approval='1795006592.506519', gpu_uuid=GPU1,
|
||||
concurrency=[1, 2, 3], epochs=5, warmup_epochs=1,
|
||||
seed=0, gpu_seconds_cap=10800, no_retry=True,
|
||||
training_fold=0, test_reads=False, gpu3_limit=1)
|
||||
if any(c.get(k) != v for k, v in expected.items()):
|
||||
raise PermissionError('Frozen experiment scope changed')
|
||||
for p, h in {**c['code_sha256'], **c['source_sha256']}.items():
|
||||
checked(p, h)
|
||||
view = read(root / 'learning-view.json')
|
||||
f = view['folds'][0]
|
||||
if (len(view['folds']) != 1 or f['fold'] != 0 or f['validation'] or f['test']
|
||||
or set(view['parts']) != set(f['learning'])
|
||||
or set(view['geometry']) != set(f['learning'])):
|
||||
raise PermissionError('Learning-only view required')
|
||||
stats = read(c['standardization'])
|
||||
if stats['parts'] != f['learning'] or stats['nodes'] != f['learning_nodes']:
|
||||
raise ValueError('Normalization identity')
|
||||
return c, view, stats
|
||||
|
||||
|
||||
def charge(now, outside_seconds, jobs):
|
||||
"""Sum each slot's wall seconds, plus coordinator time without slots and60s."""
|
||||
values = [outside_seconds, now]
|
||||
amounts = []
|
||||
for j in jobs:
|
||||
end = j.get('end', now)
|
||||
values += [j['start'], end]
|
||||
if end < j['start']:
|
||||
raise ValueError('Nonmonotonic job clock')
|
||||
amounts.append(end - j['start'])
|
||||
if any(not math.isfinite(x) or x < 0 for x in values):
|
||||
raise ValueError('Invalid accounting input')
|
||||
return 60 + outside_seconds + sum(amounts)
|
||||
|
||||
|
||||
def throughput(rows):
|
||||
if not rows or any(r['completed_epochs'] != 5 for r in rows):
|
||||
raise ValueError('Five complete epochs per copy required')
|
||||
starts = [r['measure_start'] for r in rows]
|
||||
ends = [r['measure_end'] for r in rows]
|
||||
if any(not math.isfinite(x) for x in starts + ends) or any(b <= a for a, b in zip(starts, ends)):
|
||||
raise ValueError('Invalid measurement interval')
|
||||
return dict(total_epochs_per_hour=4 * len(rows) * 3600 / (max(ends) - min(starts)),
|
||||
per_copy_epochs_per_hour=[4 * 3600 / (b - a) for a, b in zip(starts, ends)])
|
||||
|
||||
|
||||
def choose(rows):
|
||||
"""A larger N requires every preceding step's >=1.3 gain and margin pass."""
|
||||
selected = None
|
||||
for i, row in enumerate(rows, 1):
|
||||
if row['concurrency'] != i:
|
||||
raise ValueError('Ordered contiguous measurements required')
|
||||
rate = row['total_epochs_per_hour']
|
||||
if not math.isfinite(rate) or rate <= 0:
|
||||
raise ValueError('Invalid throughput')
|
||||
if not row['margin_pass'] or (i > 1 and rate < 1.3 * rows[i-2]['total_epochs_per_hour']):
|
||||
break
|
||||
selected = i
|
||||
return selected
|
||||
|
||||
|
||||
def resource_reasons(sample, c):
|
||||
reasons = []
|
||||
if sample['device_used'] > .8 * sample['device_total']:
|
||||
reasons.append('DEVICE_20_PERCENT_MARGIN')
|
||||
if sample['host_available'] < .2 * sample['host_total']:
|
||||
reasons.append('HOST_RAM_20_PERCENT_MARGIN')
|
||||
if sample['host_cpu_percent'] > 80:
|
||||
reasons.append('HOST_CPU_20_PERCENT_MARGIN')
|
||||
if sample['temperature'] >= c['temperature_limit_c']:
|
||||
reasons.append('DEVICE_TEMPERATURE_LIMIT')
|
||||
if any(v > c['worker_rss_bytes'] for v in sample['rss']):
|
||||
reasons.append('WORKER_RSS_LIMIT')
|
||||
if sample['foreign_gpu_pids']:
|
||||
reasons.append('FOREIGN_GPU_PROCESS')
|
||||
return reasons
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
"""Fresh H2/seed0, original200-epoch LR prefix, five learning-only epochs."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
import psutil
|
||||
import torch
|
||||
from phase2_h1_worker_v2 import save
|
||||
from phase2_h2_data import load_part, to_device
|
||||
from phase2_h2_model import H2Geo, optimizer, lr_epoch
|
||||
from phase2_h2_pilot_worker import seed, train_step, peaks
|
||||
from h2_concurrency_probe_common import config
|
||||
|
||||
|
||||
def run(root, n, index):
|
||||
c, view, stats = config(root)
|
||||
if n not in (1, 2, 3) or index not in range(n):
|
||||
raise PermissionError('Unapproved copy')
|
||||
phase = root / ('n' + str(n)); out = phase / str(index)
|
||||
out.mkdir(exist_ok=False)
|
||||
os.sched_setaffinity(0, [c['worker_cpus'][index]])
|
||||
torch.set_num_threads(1); torch.set_num_interop_threads(1)
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES') != c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF') != 'expandable_segments:True':
|
||||
raise ValueError('GPU/allocator identity')
|
||||
start = time.monotonic(); process = psutil.Process(); epoch = steps = 0
|
||||
rows = []; measure_start = measure_end = None
|
||||
def barrier(name):
|
||||
while not (phase / name).exists():
|
||||
if time.monotonic() - start >= c['job_wall_seconds'] - 30:
|
||||
raise TimeoutError('Barrier exhausted own window')
|
||||
time.sleep(.1)
|
||||
try:
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.backends.cudnn.allow_tf32 = False
|
||||
torch.cuda.set_device(0); torch.cuda.init(); torch.cuda.reset_peak_memory_stats(); seed()
|
||||
model = H2Geo().cuda(); opt = optimizer(model)
|
||||
f = view['folds'][0]; parts = len(f['learning']); total = f['learning_nodes']
|
||||
save(out / 'ready.json', dict(pid=os.getpid(), unix=time.time(), **peaks()))
|
||||
barrier('go.json')
|
||||
for epoch in range(1, 6):
|
||||
if epoch == 2:
|
||||
save(out / 'warm.json', dict(epoch=1, pid=os.getpid()))
|
||||
barrier('measure-go.json'); measure_start = time.monotonic()
|
||||
begin = time.monotonic(); seen = nodes = 0; load_s = transfer_s = step_s = 0.
|
||||
io0 = process.io_counters(); cpu0 = time.process_time()
|
||||
if any(abs(g['lr'] - lr_epoch(epoch)) > 1e-15 for g in opt.param_groups):
|
||||
raise ValueError('Original learning-rate prefix changed')
|
||||
order = np.random.default_rng(np.random.SeedSequence([0, epoch])).permutation(parts)
|
||||
for k in order:
|
||||
case = f['learning'][int(k)]
|
||||
t = time.monotonic(); data = load_part(view, case, stats, 'learning', out)
|
||||
load_s += time.monotonic() - t
|
||||
t = time.monotonic(); data = to_device(data, 'cuda'); torch.cuda.synchronize()
|
||||
transfer_s += time.monotonic() - t
|
||||
t = time.monotonic(); result = train_step(model, opt, data, parts, total)
|
||||
step_s += time.monotonic() - t
|
||||
seen += 1; nodes += data['nodes']; steps += 1
|
||||
save(out / 'progress.json', dict(epoch=epoch, parts=seen, optimizer_steps=steps,
|
||||
rss=process.memory_info().rss, loss=result['loss'], **peaks()))
|
||||
del data
|
||||
if seen != parts or nodes != total:
|
||||
raise ValueError('Incomplete learning epoch')
|
||||
io1 = process.io_counters()
|
||||
row = dict(epoch=epoch, seconds=time.monotonic()-begin,
|
||||
load_prepare_seconds=load_s, transfer_seconds=transfer_s, train_step_seconds=step_s,
|
||||
cpu_seconds=time.process_time()-cpu0,
|
||||
io_delta={k: getattr(io1,k)-getattr(io0,k) for k in io0._fields}, **peaks())
|
||||
rows.append(row); save(out / 'epochs.json', rows)
|
||||
for group in opt.param_groups: group['lr'] = lr_epoch(epoch+1)
|
||||
measure_end = time.monotonic()
|
||||
save(out / 'result.json', dict(status='COMPLETE', completed_epochs=5, optimizer_steps=steps,
|
||||
measure_start=measure_start, measure_end=measure_end, epochs=rows, worker_wall=time.monotonic()-start,
|
||||
cpu_seconds=time.process_time(), test_reads=False, validation_reads=False,
|
||||
checkpoint_saved=False, role='throughput_only_not_scientific_model', **peaks()))
|
||||
except BaseException as exc:
|
||||
save(out / 'failure.json', dict(status='FAILED', type=type(exc).__name__, reason=str(exc),
|
||||
traceback=traceback.format_exc(), epoch=epoch, steps=steps,
|
||||
**(peaks() if torch.cuda.is_initialized() else {})))
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(Path(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]))
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"unix": 1795006860.9747784,
|
||||
"status": "WITHDRAWN_BEFORE_PREPARATION_TEST_OR_EXECUTION",
|
||||
"reason": "User corrected intent: concurrency of distinct research ideas, not same-workload throughput optimization. Claude leads revised work assignments.",
|
||||
"actual_work": [
|
||||
"six read-only resource samples over25.5685s",
|
||||
"two incomplete draft source files archived"
|
||||
],
|
||||
"gpu_workload_launched": false,
|
||||
"learning_or_test_targets_read": false,
|
||||
"tests_run": false,
|
||||
"implementation_frozen": false,
|
||||
"runtime_root_exists": false,
|
||||
"files": [
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/h2_concurrency_probe_common.py",
|
||||
"archive": "/home/frank14f/optane/MeltMind/artifacts/gpu-concurrency-assessment-2026-11-18/withdrawn-throughput-draft/h2_concurrency_probe_common.py",
|
||||
"sha256": "a80e431e3bf4d246b0f17ab66ca6c8b4f78a3db82b632fd1d601acb7f69bad87"
|
||||
},
|
||||
{
|
||||
"original": "/home/frank14f/optane/MeltMind/tools/h2_concurrency_probe_worker.py",
|
||||
"archive": "/home/frank14f/optane/MeltMind/artifacts/gpu-concurrency-assessment-2026-11-18/withdrawn-throughput-draft/h2_concurrency_probe_worker.py",
|
||||
"sha256": "1fb7aa10065977b55c71082adde6e15fc44e565353e11d3487dc29ec1a871072"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user