Record direct bundled human authorization and sole reviewed item c real-worker launch
This commit is contained in:
1 parent
38bb824de1
commit
2dcb690ac2
35 files changed
+926
No files matched your search
@@ -0,0 +1,175 @@
|
||||
{
|
||||
"status": "ACTUAL_WORKERS_TRAINING",
|
||||
"unix": 1795010701.9993742,
|
||||
"coordinator": {
|
||||
"pid": 1755713,
|
||||
"created": 1795010602.48
|
||||
},
|
||||
"workers": [
|
||||
{
|
||||
"fold": 0,
|
||||
"worker": {
|
||||
"pid": 1755848,
|
||||
"created": 1795010607.92
|
||||
},
|
||||
"guard": {
|
||||
"pid": 1755765,
|
||||
"created": 1795010605.64
|
||||
},
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"worker_cpus": [
|
||||
16
|
||||
],
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 265
|
||||
},
|
||||
{
|
||||
"fold": 1,
|
||||
"worker": {
|
||||
"pid": 1755878,
|
||||
"created": 1795010609.1
|
||||
},
|
||||
"guard": {
|
||||
"pid": 1755824,
|
||||
"created": 1795010607.05
|
||||
},
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"worker_cpus": [
|
||||
17
|
||||
],
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 261
|
||||
},
|
||||
{
|
||||
"fold": 2,
|
||||
"worker": {
|
||||
"pid": 1755953,
|
||||
"created": 1795010610.97
|
||||
},
|
||||
"guard": {
|
||||
"pid": 1755862,
|
||||
"created": 1795010608.29
|
||||
},
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"worker_cpus": [
|
||||
18
|
||||
],
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 237
|
||||
},
|
||||
{
|
||||
"fold": 3,
|
||||
"worker": {
|
||||
"pid": 1756022,
|
||||
"created": 1795010612.25
|
||||
},
|
||||
"guard": {
|
||||
"pid": 1755893,
|
||||
"created": 1795010609.69
|
||||
},
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"worker_cpus": [
|
||||
19
|
||||
],
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 232
|
||||
}
|
||||
],
|
||||
"all_host_gpu_process_counts": {
|
||||
"0": 1,
|
||||
"1": 3,
|
||||
"2": 3,
|
||||
"3": 1
|
||||
},
|
||||
"snapshot": {
|
||||
"devices": {
|
||||
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c": {
|
||||
"total": 17179869184,
|
||||
"used": 1800404992
|
||||
},
|
||||
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36": {
|
||||
"total": 17179869184,
|
||||
"used": 5390729216
|
||||
},
|
||||
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76": {
|
||||
"total": 17179869184,
|
||||
"used": 5770313728
|
||||
},
|
||||
"GPU-8b71b032-1b75-2891-935c-ce106a53034a": {
|
||||
"total": 17179869184,
|
||||
"used": 1928331264
|
||||
}
|
||||
},
|
||||
"processes": [
|
||||
{
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"pid": 1476707,
|
||||
"vram": 1790967808
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"pid": 1476726,
|
||||
"vram": 1793064960
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"pid": 1755848,
|
||||
"vram": 1793064960
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"pid": 1755878,
|
||||
"vram": 1795162112
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"pid": 1650207,
|
||||
"vram": 1918894080
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"pid": 1756022,
|
||||
"vram": 1920991232
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"pid": 1755953,
|
||||
"vram": 1920991232
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"pid": 1598144,
|
||||
"vram": 1918894080
|
||||
}
|
||||
],
|
||||
"ram_total": 67378212864,
|
||||
"ram_available": 51414818816,
|
||||
"cpu_percent": 28.7,
|
||||
"old_pending": true,
|
||||
"unix": 1795010701.9749477
|
||||
},
|
||||
"old": {
|
||||
"phase3-seed1-v2-2026-11-18": {
|
||||
"pins": "PASS",
|
||||
"code_files": 51
|
||||
},
|
||||
"track-a-recovery-2026-11-18": {
|
||||
"pins": "PASS",
|
||||
"code_files": 27
|
||||
}
|
||||
},
|
||||
"source_pins": "PASS",
|
||||
"review_sha256": "e30612eabf43037f6258644945e9f486dc99426f08d1da160d14cfd6b3163632",
|
||||
"implementation": "2ddecef3fac432ccc0267c468d5df4d3203e53ef",
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"first_complete_epoch_gate_passed": false,
|
||||
"pending": [
|
||||
4
|
||||
],
|
||||
"no_retry": true,
|
||||
"service_restart": "no"
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"status": "SAME_ACTION_ALLOWED_AFTER_NEW_DIRECT_HUMAN_APPROVAL",
|
||||
"prior_block": "1af5e773f6b56dff16e9645095831a7a474586e64e780e7734b2408bca5cbb57",
|
||||
"direct_user_approval": "84c315fda1d3c35c36603a7f92443af2c5b91cc03a6fe848edf59f18dc810a81",
|
||||
"launch_script_unchanged": true,
|
||||
"service_launch_sha256": "fefffcd75c4ed9f362d3279294b81f12637536a063d3a357bf417a7b60a1b5ad",
|
||||
"no_alternate_route": true
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"source": "direct_human_reply_in_current_MeltAstra_conversation",
|
||||
"question_item_id": "[\"request_user_input_async\",\"call_0a83681b57bf4449b502c1e8446ce092\",0]",
|
||||
"question": "是否授权团队按各自通过审阅的固定方案,并行执行 phase 3 研究任务:c(末填位置损失,≤57 GPU 小时)、1b(缩短训练日程,≤25 GPU 小时)、b(区域上下文,≤50 GPU 小时)及 Track B(GINO 基准复现),在 GPU0–2 的共享容量内运行、GPU3 保持单任务,并遵守 phase 3 累计 200 GPU 小时上限?当前仅 c 已完成入口审阅和启动审计,其余任务仍须各自完成合同、预算分配及审阅后才能启动。",
|
||||
"answer": "批准按上述范围执行",
|
||||
"received_unix": 1795010594.815036,
|
||||
"scope": "bundled phase3 compute authorization; each entry keeps own reviewed contract and assigned budget",
|
||||
"same_rejected_action_reassessment": true,
|
||||
"prior_rejection_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"fold": 0,
|
||||
"stage": "EPOCH_COMPLETE",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 320,
|
||||
"wall_seconds": 105.47476764209569,
|
||||
"cpu_seconds": 105.271192156,
|
||||
"train_mean_weighted_loss": 0.005440240334805011,
|
||||
"train_epoch_seconds": 86.92266950709745,
|
||||
"epoch_parts": 320,
|
||||
"epoch_nodes": 8742060,
|
||||
"lr": 0.0001,
|
||||
"peak_allocated_bytes": 1335593984,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"projection": {
|
||||
"status": "PASS",
|
||||
"projected_gpu_seconds": 191957.48510470253,
|
||||
"actual_training_and_preparation_gpu_seconds": 1455.3412984377937,
|
||||
"folds": {
|
||||
"0": {
|
||||
"charged": 171.0682568339398,
|
||||
"projected": 26237.48510470253
|
||||
},
|
||||
"1": {
|
||||
"charged": 171.06825940893032,
|
||||
"projected": 39480.0
|
||||
},
|
||||
"2": {
|
||||
"charged": 171.06826009997167,
|
||||
"projected": 39480.0
|
||||
},
|
||||
"3": {
|
||||
"charged": 171.06826073199045,
|
||||
"projected": 39480.0
|
||||
},
|
||||
"4": {
|
||||
"charged": 171.0682613629615,
|
||||
"projected": 39480.0
|
||||
}
|
||||
},
|
||||
"postfit_gpu_reserve": 7200,
|
||||
"safety_factor": 1.5
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"fold": 1,
|
||||
"stage": "EPOCH_COMPLETE",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 374,
|
||||
"wall_seconds": 119.86219473194797,
|
||||
"cpu_seconds": 119.676679298,
|
||||
"train_mean_weighted_loss": 0.005402988824570518,
|
||||
"train_epoch_seconds": 99.55817544599995,
|
||||
"epoch_parts": 374,
|
||||
"epoch_nodes": 9731293,
|
||||
"lr": 0.0001,
|
||||
"peak_allocated_bytes": 1335593984,
|
||||
"peak_reserved_bytes": 1390411776
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"projection": {
|
||||
"status": "PASS",
|
||||
"projected_gpu_seconds": 169040.0875815915,
|
||||
"actual_training_and_preparation_gpu_seconds": 1535.3012667269213,
|
||||
"folds": {
|
||||
"0": {
|
||||
"charged": 187.0602509570308,
|
||||
"projected": 26253.47709882562
|
||||
},
|
||||
"1": {
|
||||
"charged": 187.06025304098148,
|
||||
"projected": 30025.175623671967
|
||||
},
|
||||
"2": {
|
||||
"charged": 187.0602537220111,
|
||||
"projected": 39480.0
|
||||
},
|
||||
"3": {
|
||||
"charged": 187.06025424296968,
|
||||
"projected": 26001.43485909392
|
||||
},
|
||||
"4": {
|
||||
"charged": 187.06025476392824,
|
||||
"projected": 39480.0
|
||||
}
|
||||
},
|
||||
"postfit_gpu_reserve": 7200,
|
||||
"safety_factor": 1.5
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"fold": 2,
|
||||
"stage": "EPOCH_COMPLETE",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 366,
|
||||
"wall_seconds": 126.36284828197677,
|
||||
"cpu_seconds": 126.308897311,
|
||||
"train_mean_weighted_loss": 0.005475391645391959,
|
||||
"train_epoch_seconds": 106.40938244294375,
|
||||
"epoch_parts": 366,
|
||||
"epoch_nodes": 9875407,
|
||||
"lr": 0.0001,
|
||||
"peak_allocated_bytes": 1462367232,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"projection": {
|
||||
"status": "PASS",
|
||||
"projected_gpu_seconds": 161662.477024262,
|
||||
"actual_training_and_preparation_gpu_seconds": 1575.4619296207093,
|
||||
"folds": {
|
||||
"0": {
|
||||
"charged": 195.09238332894165,
|
||||
"projected": 26261.50923119753
|
||||
},
|
||||
"1": {
|
||||
"charged": 195.09238517296035,
|
||||
"projected": 30033.207755803945
|
||||
},
|
||||
"2": {
|
||||
"charged": 195.09238589392044,
|
||||
"projected": 32078.29304511263
|
||||
},
|
||||
"3": {
|
||||
"charged": 195.0923872969579,
|
||||
"projected": 26009.466992147907
|
||||
},
|
||||
"4": {
|
||||
"charged": 195.09238792792894,
|
||||
"projected": 39480.0
|
||||
}
|
||||
},
|
||||
"postfit_gpu_reserve": 7200,
|
||||
"safety_factor": 1.5
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"fold": 3,
|
||||
"stage": "EPOCH_COMPLETE",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 291,
|
||||
"wall_seconds": 103.46997219300829,
|
||||
"cpu_seconds": 103.338361796,
|
||||
"train_mean_weighted_loss": 0.005969444835758766,
|
||||
"train_epoch_seconds": 86.07830688392278,
|
||||
"epoch_parts": 291,
|
||||
"epoch_nodes": 8118914,
|
||||
"lr": 0.0001,
|
||||
"peak_allocated_bytes": 1462367232,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"projection": {
|
||||
"status": "PASS",
|
||||
"projected_gpu_seconds": 178467.96709384752,
|
||||
"actual_training_and_preparation_gpu_seconds": 1467.9391069828998,
|
||||
"folds": {
|
||||
"0": {
|
||||
"charged": 173.58781869499944,
|
||||
"projected": 26240.00466656359
|
||||
},
|
||||
"1": {
|
||||
"charged": 173.5878210000228,
|
||||
"projected": 39480.0
|
||||
},
|
||||
"2": {
|
||||
"charged": 173.5878217909485,
|
||||
"projected": 39480.0
|
||||
},
|
||||
"3": {
|
||||
"charged": 173.587822432979,
|
||||
"projected": 25987.96242728393
|
||||
},
|
||||
"4": {
|
||||
"charged": 173.58782306395005,
|
||||
"projected": 39480.0
|
||||
}
|
||||
},
|
||||
"postfit_gpu_reserve": 7200,
|
||||
"safety_factor": 1.5
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"unix": 1795010992.7387826,
|
||||
"first_epoch_rows": [
|
||||
{
|
||||
"fold": 0,
|
||||
"epoch_seconds": 86.92266950709745,
|
||||
"gate": "PASS",
|
||||
"gate_sha256": "88cafd96714b21b7f965a4121e0b93b8f9bb9d307b51fde73c48cb143522a808"
|
||||
},
|
||||
{
|
||||
"fold": 1,
|
||||
"epoch_seconds": 99.55817544599995,
|
||||
"gate": "PASS",
|
||||
"gate_sha256": "0c999842b9c2b4bd8881f8b8a68ac965e06b4432f557098615d937ca545d3ad6"
|
||||
},
|
||||
{
|
||||
"fold": 2,
|
||||
"epoch_seconds": 106.40938244294375,
|
||||
"gate": "PASS",
|
||||
"gate_sha256": "3f0768a181796f344d5c9eba1afbe593dcd19bf61abd84b1330183904b3b515f"
|
||||
},
|
||||
{
|
||||
"fold": 3,
|
||||
"epoch_seconds": 86.07830688392278,
|
||||
"gate": "PASS",
|
||||
"gate_sha256": "1cdd1c46ffe51813b88ff7b12eb8c00a6042c700e9965eb84498440278c8d221"
|
||||
}
|
||||
],
|
||||
"current_projection": {
|
||||
"status": "PASS",
|
||||
"projected_gpu_seconds": 164700.81824973988,
|
||||
"actual_training_and_preparation_gpu_seconds": 2836.2468068549642,
|
||||
"folds": {
|
||||
"0": {
|
||||
"charged": 447.2493583749747,
|
||||
"projected": 27237.05170475261
|
||||
},
|
||||
"1": {
|
||||
"charged": 447.2493612009566,
|
||||
"projected": 30753.302897236892
|
||||
},
|
||||
"2": {
|
||||
"charged": 447.24936192203313,
|
||||
"projected": 32853.22009066754
|
||||
},
|
||||
"3": {
|
||||
"charged": 447.24936243298,
|
||||
"projected": 26577.24355708284
|
||||
},
|
||||
"4": {
|
||||
"charged": 447.2493629240198,
|
||||
"projected": 39480.0
|
||||
}
|
||||
},
|
||||
"postfit_gpu_reserve": 7200,
|
||||
"safety_factor": 1.5
|
||||
},
|
||||
"global_stop": false,
|
||||
"terminal": false
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"id": 0,
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"worker_cpus": [
|
||||
16
|
||||
],
|
||||
"token": "da7989f6fb454ec99e83ffa8092b056a",
|
||||
"queue_identity": {
|
||||
"pid": 1755713,
|
||||
"created": 1795010602.48
|
||||
},
|
||||
"pre_dispatch_wait_seconds": 0.19124818302225322
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"token": "da7989f6fb454ec99e83ffa8092b056a",
|
||||
"owner": {
|
||||
"pid": 1755765,
|
||||
"created": 1795010605.64
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"fold": 0,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 265,
|
||||
"wall_seconds": 90.80718446907122,
|
||||
"cpu_seconds": 90.608083871,
|
||||
"case": "00000104_52f7124261ae4c558f800775_step_001",
|
||||
"epoch_parts": 265,
|
||||
"epoch_nodes": 7216569,
|
||||
"lr": 0.0001,
|
||||
"loss": 0.0024264770141769016,
|
||||
"regression_loss": 0.001842887606471777,
|
||||
"location_loss": 0.5835894077051246,
|
||||
"gradient_norm_before_clip": 0.15353010594844818,
|
||||
"peak_allocated_bytes": 1335593984,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"pid": 1755848,
|
||||
"created": 1795010607.92
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"id": 1,
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"worker_cpus": [
|
||||
17
|
||||
],
|
||||
"token": "a7e8a5931ff844faa5a61f420e155704",
|
||||
"queue_identity": {
|
||||
"pid": 1755713,
|
||||
"created": 1795010602.48
|
||||
},
|
||||
"pre_dispatch_wait_seconds": 1.6059088689507917
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"token": "a7e8a5931ff844faa5a61f420e155704",
|
||||
"owner": {
|
||||
"pid": 1755824,
|
||||
"created": 1795010607.05
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"fold": 1,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 261,
|
||||
"wall_seconds": 89.70508777303621,
|
||||
"cpu_seconds": 89.534064026,
|
||||
"case": "00000890_e2d3a4dc5f7c438bb240e9d1_step_001",
|
||||
"epoch_parts": 261,
|
||||
"epoch_nodes": 6618267,
|
||||
"lr": 0.0001,
|
||||
"loss": 0.0018121458841109577,
|
||||
"regression_loss": 0.00025266691227443516,
|
||||
"location_loss": 1.5594789718365225,
|
||||
"gradient_norm_before_clip": 0.037657082080841064,
|
||||
"peak_allocated_bytes": 1335593984,
|
||||
"peak_reserved_bytes": 1390411776
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"pid": 1755878,
|
||||
"created": 1795010609.1
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"id": 2,
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"worker_cpus": [
|
||||
18
|
||||
],
|
||||
"token": "ded7a54470cb409b85604b8d7248abc3",
|
||||
"queue_identity": {
|
||||
"pid": 1755713,
|
||||
"created": 1795010602.48
|
||||
},
|
||||
"pre_dispatch_wait_seconds": 2.845533316023648
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"token": "ded7a54470cb409b85604b8d7248abc3",
|
||||
"owner": {
|
||||
"pid": 1755862,
|
||||
"created": 1795010608.29
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"fold": 2,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 237,
|
||||
"wall_seconds": 88.10464875295293,
|
||||
"cpu_seconds": 88.05315865700001,
|
||||
"case": "00000890_e2d3a4dc5f7c438bb240e9d1_step_001",
|
||||
"epoch_parts": 237,
|
||||
"epoch_nodes": 6122901,
|
||||
"lr": 0.0001,
|
||||
"loss": 0.0017308261971914383,
|
||||
"regression_loss": 0.00019051120034419,
|
||||
"location_loss": 1.5403149968472483,
|
||||
"gradient_norm_before_clip": 0.029645292088389397,
|
||||
"peak_allocated_bytes": 1462367232,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"pid": 1755953,
|
||||
"created": 1795010610.97
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"id": 3,
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"worker_cpus": [
|
||||
19
|
||||
],
|
||||
"token": "53f07e4a34204fd8a6641002c8d29f3e",
|
||||
"queue_identity": {
|
||||
"pid": 1755713,
|
||||
"created": 1795010602.48
|
||||
},
|
||||
"pre_dispatch_wait_seconds": 4.24427183100488
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"token": "53f07e4a34204fd8a6641002c8d29f3e",
|
||||
"owner": {
|
||||
"pid": 1755893,
|
||||
"created": 1795010609.69
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"fold": 3,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 232,
|
||||
"wall_seconds": 86.94565156602766,
|
||||
"cpu_seconds": 86.816446004,
|
||||
"case": "00000068_767e4372b5f94a88a7a17d90_step_009",
|
||||
"epoch_parts": 232,
|
||||
"epoch_nodes": 6521921,
|
||||
"lr": 0.0001,
|
||||
"loss": 0.0018344386192713845,
|
||||
"regression_loss": 0.001042529591359198,
|
||||
"location_loss": 0.7919090279121863,
|
||||
"gradient_norm_before_clip": 0.027779340744018555,
|
||||
"peak_allocated_bytes": 1459044864,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"pid": 1756022,
|
||||
"created": 1795010612.25
|
||||
}
|
||||
@@ -768,3 +768,14 @@ Assist1795010229实际PASS e30612eabf43037f6258644945e9f486dc99426f08d1da160d14c
|
||||
调用唯一启动命令时自动审批在执行前拒绝。原文理由为五折GPU训练成本和主机负载状态变更,具体启动裁量只出现在untrusted Slack/assistant context,可信指令仍要求新增科学计算的explicit scope authorization/review。launch-attempt/service-launch/started/complete和shared registry均不存在;没有新训练失败、没有运行,也不能称首轮成本门通过。完整方案、脚本、原拒绝在artifacts/phase3-item-c-launch-2026-11-18。
|
||||
|
||||
按Claude0251已向用户发出一次合并计算授权问题:c≤57GPUh/1b≤25/b≤50/Track B,GPU0–2共享、GPU3单任务、phase3累计≤200GPUh;其它入口仍须各自合同、预算分配及实际审阅。等待直接用户回复或可验证原始证据,不重复sharing/既有review、不代启、不绕过拒绝或重复重试。1795010488已告知Claude/TRON。旧BLOCKED/失败/反例继续完整保留;下方未采用PASS/待Assist条目为历史。
|
||||
|
||||
|
||||
## 最新 item c 已获直接用户授权并实际运行(2026-11-18 22:03)
|
||||
|
||||
用户在本Melt会话对一次合并phase3计算问题直接答“批准按上述范围执行”;原问题/答复存artifacts/phase3-item-c-launch-2026-11-18/direct-user-authorization.json。范围c≤57h、1b≤25h、b≤50h及Track B,GPU0–2共享/GPU3单任务、phase3累计≤200GPUh;各入口仍需自己的固定合同、预算分配和实际审阅。后续不再逐项重复问用户。原自动审批拒绝保留,同一未改脚本携新直接证据重审允许,无旁路。
|
||||
|
||||
sole service meltmind-phase3-item-c-20261118.service,Restart=no,coordinator1755713,started1795010605.4638069。实际启动审计1795010701:f0 worker1755848/guard1755765、f1 worker1755878/guard1755824在GPU1;f2 worker1755953/guard1755862、f3 worker1756022/guard1755893在GPU2;均TRAIN epoch1且真实optimizersteps>0,f4待容量。PID创建时间/父子链/NVML匹配,所有lease GUARD_BOUND,整机GPU0/1/2/3进程数1/3/3/1。当前以真实live/terminal为准,不能从此历史快照重启。
|
||||
|
||||
固定2ddecef3/0e853319、实际PASS e30612ea、review freeze0b17ba94不变;新52/旧78pins核匹配。总205200含post7200/CPU3600,phase3总720000,Track A独立、旧失败账不重复;不重试/不改active pins/不自动postfit。实际启动已在1795010745报Claude/TRON。首轮后续检查另见artifacts/phase3-item-c-launch-2026-11-18/first-epochs-live-audit.json,不把启动当科学结果。
|
||||
|
||||
Claude1795010507/0666已实读:现在准备1b独立入口,原spec55ffc6f9全文归档并核SHA;五折可先训练100完整轮,fold1 validation筛选改为判决门,等seed1同域参照;只有筛选PASS才允许独立事前审阅后的test postfit,失败则五模型归档不读test。c/shared原运行pins不动。1b目前仅实现准备,不是已审/已启动。下方c“等待人类计算授权/未启动”均为历史。
|
||||
@@ -1,3 +1,15 @@
|
||||
## 最新 item c 已获直接用户授权并实际运行(2026-11-18 22:03)
|
||||
|
||||
用户在本Melt会话对一次合并phase3计算问题直接答“批准按上述范围执行”;原问题/答复存artifacts/phase3-item-c-launch-2026-11-18/direct-user-authorization.json。范围c≤57h、1b≤25h、b≤50h及Track B,GPU0–2共享/GPU3单任务、phase3累计≤200GPUh;各入口仍需自己的固定合同、预算分配和实际审阅。后续不再逐项重复问用户。原自动审批拒绝保留,同一未改脚本携新直接证据重审允许,无旁路。
|
||||
|
||||
sole service meltmind-phase3-item-c-20261118.service,Restart=no,coordinator1755713,started1795010605.4638069。实际启动审计1795010701:f0 worker1755848/guard1755765、f1 worker1755878/guard1755824在GPU1;f2 worker1755953/guard1755862、f3 worker1756022/guard1755893在GPU2;均TRAIN epoch1且真实optimizersteps>0,f4待容量。PID创建时间/父子链/NVML匹配,所有lease GUARD_BOUND,整机GPU0/1/2/3进程数1/3/3/1。当前以真实live/terminal为准,不能从此历史快照重启。
|
||||
|
||||
固定2ddecef3/0e853319、实际PASS e30612ea、review freeze0b17ba94不变;新52/旧78pins核匹配。总205200含post7200/CPU3600,phase3总720000,Track A独立、旧失败账不重复;不重试/不改active pins/不自动postfit。实际启动已在1795010745报Claude/TRON。首轮后续检查另见artifacts/phase3-item-c-launch-2026-11-18/first-epochs-live-audit.json,不把启动当科学结果。
|
||||
|
||||
Claude1795010507/0666已实读:现在准备1b独立入口,原spec55ffc6f9全文归档并核SHA;五折可先训练100完整轮,fold1 validation筛选改为判决门,等seed1同域参照;只有筛选PASS才允许独立事前审阅后的test postfit,失败则五模型归档不读test。c/shared原运行pins不动。1b目前仅实现准备,不是已审/已启动。下方c“等待人类计算授权/未启动”均为历史。
|
||||
|
||||
---
|
||||
|
||||
## 最新 item c 实际PASS已采用,启动被自动审批拦截(2026-11-18)
|
||||
|
||||
Assist1795010229实际PASS e30612eabf43037f6258644945e9f486dc99426f08d1da160d14cfd6b3163632已逐字核SHA与52code完整五字段采用,review freeze0b17ba94d52066debb79702104683c95ae3ba0b2,validate_review/config PASS。implementation仍2ddecef3/contract0e853319/proofb3287e99/comparisona9dac0a4;Claude1795010251明确接受,要求审计后GPU1/2启动,总205200不变。
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
{
|
||||
"status": "ACTUAL_WORKERS_TRAINING",
|
||||
"unix": 1795010701.9993742,
|
||||
"coordinator": {
|
||||
"pid": 1755713,
|
||||
"created": 1795010602.48
|
||||
},
|
||||
"workers": [
|
||||
{
|
||||
"fold": 0,
|
||||
"worker": {
|
||||
"pid": 1755848,
|
||||
"created": 1795010607.92
|
||||
},
|
||||
"guard": {
|
||||
"pid": 1755765,
|
||||
"created": 1795010605.64
|
||||
},
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"worker_cpus": [
|
||||
16
|
||||
],
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 265
|
||||
},
|
||||
{
|
||||
"fold": 1,
|
||||
"worker": {
|
||||
"pid": 1755878,
|
||||
"created": 1795010609.1
|
||||
},
|
||||
"guard": {
|
||||
"pid": 1755824,
|
||||
"created": 1795010607.05
|
||||
},
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"worker_cpus": [
|
||||
17
|
||||
],
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 261
|
||||
},
|
||||
{
|
||||
"fold": 2,
|
||||
"worker": {
|
||||
"pid": 1755953,
|
||||
"created": 1795010610.97
|
||||
},
|
||||
"guard": {
|
||||
"pid": 1755862,
|
||||
"created": 1795010608.29
|
||||
},
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"worker_cpus": [
|
||||
18
|
||||
],
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 237
|
||||
},
|
||||
{
|
||||
"fold": 3,
|
||||
"worker": {
|
||||
"pid": 1756022,
|
||||
"created": 1795010612.25
|
||||
},
|
||||
"guard": {
|
||||
"pid": 1755893,
|
||||
"created": 1795010609.69
|
||||
},
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"worker_cpus": [
|
||||
19
|
||||
],
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 232
|
||||
}
|
||||
],
|
||||
"all_host_gpu_process_counts": {
|
||||
"0": 1,
|
||||
"1": 3,
|
||||
"2": 3,
|
||||
"3": 1
|
||||
},
|
||||
"snapshot": {
|
||||
"devices": {
|
||||
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c": {
|
||||
"total": 17179869184,
|
||||
"used": 1800404992
|
||||
},
|
||||
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36": {
|
||||
"total": 17179869184,
|
||||
"used": 5390729216
|
||||
},
|
||||
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76": {
|
||||
"total": 17179869184,
|
||||
"used": 5770313728
|
||||
},
|
||||
"GPU-8b71b032-1b75-2891-935c-ce106a53034a": {
|
||||
"total": 17179869184,
|
||||
"used": 1928331264
|
||||
}
|
||||
},
|
||||
"processes": [
|
||||
{
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"pid": 1476707,
|
||||
"vram": 1790967808
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"pid": 1476726,
|
||||
"vram": 1793064960
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"pid": 1755848,
|
||||
"vram": 1793064960
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"pid": 1755878,
|
||||
"vram": 1795162112
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"pid": 1650207,
|
||||
"vram": 1918894080
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"pid": 1756022,
|
||||
"vram": 1920991232
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"pid": 1755953,
|
||||
"vram": 1920991232
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"pid": 1598144,
|
||||
"vram": 1918894080
|
||||
}
|
||||
],
|
||||
"ram_total": 67378212864,
|
||||
"ram_available": 51414818816,
|
||||
"cpu_percent": 28.7,
|
||||
"old_pending": true,
|
||||
"unix": 1795010701.9749477
|
||||
},
|
||||
"old": {
|
||||
"phase3-seed1-v2-2026-11-18": {
|
||||
"pins": "PASS",
|
||||
"code_files": 51
|
||||
},
|
||||
"track-a-recovery-2026-11-18": {
|
||||
"pins": "PASS",
|
||||
"code_files": 27
|
||||
}
|
||||
},
|
||||
"source_pins": "PASS",
|
||||
"review_sha256": "e30612eabf43037f6258644945e9f486dc99426f08d1da160d14cfd6b3163632",
|
||||
"implementation": "2ddecef3fac432ccc0267c468d5df4d3203e53ef",
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"first_complete_epoch_gate_passed": false,
|
||||
"pending": [
|
||||
4
|
||||
],
|
||||
"no_retry": true,
|
||||
"service_restart": "no"
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"unix": 1795010602.4873137,
|
||||
"snapshot": {
|
||||
"devices": {
|
||||
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c": {
|
||||
"total": 17179869184,
|
||||
"used": 1800404992
|
||||
},
|
||||
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36": {
|
||||
"total": 17179869184,
|
||||
"used": 1802502144
|
||||
},
|
||||
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76": {
|
||||
"total": 17179869184,
|
||||
"used": 1928331264
|
||||
},
|
||||
"GPU-8b71b032-1b75-2891-935c-ce106a53034a": {
|
||||
"total": 17179869184,
|
||||
"used": 1928331264
|
||||
}
|
||||
},
|
||||
"processes": [
|
||||
{
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"pid": 1476707,
|
||||
"vram": 1790967808
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"pid": 1476726,
|
||||
"vram": 1793064960
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"pid": 1650207,
|
||||
"vram": 1918894080
|
||||
},
|
||||
{
|
||||
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"pid": 1598144,
|
||||
"vram": 1918894080
|
||||
}
|
||||
],
|
||||
"ram_total": 67378212864,
|
||||
"ram_available": 57046564864,
|
||||
"cpu_percent": 15.0,
|
||||
"old_pending": true,
|
||||
"unix": 1795010602.477901
|
||||
},
|
||||
"admission": {
|
||||
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36": [
|
||||
true,
|
||||
"ADMITTED"
|
||||
],
|
||||
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76": [
|
||||
true,
|
||||
"ADMITTED"
|
||||
]
|
||||
},
|
||||
"old_pins": "78PASS",
|
||||
"duplicate_count": 0
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"unix": 1795010602.4887905,
|
||||
"command": [
|
||||
"systemd-run",
|
||||
"--user",
|
||||
"--collect",
|
||||
"--unit=meltmind-phase3-item-c-20261118.service",
|
||||
"--working-directory=/home/frank14f/optane/MeltMind",
|
||||
"--property=Restart=no",
|
||||
"--property=KillMode=control-group",
|
||||
"--property=TimeoutStopSec=60",
|
||||
"--property=StandardOutput=append:/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/service.log",
|
||||
"--property=StandardError=inherit",
|
||||
"--setenv=OPENBLAS_NUM_THREADS=1",
|
||||
"--setenv=OMP_NUM_THREADS=1",
|
||||
"--setenv=MKL_NUM_THREADS=1",
|
||||
"--setenv=NUMEXPR_NUM_THREADS=1",
|
||||
"--setenv=PYTHONPATH=tools",
|
||||
"--setenv=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
||||
"--setenv=PYTHONUNBUFFERED=1",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py"
|
||||
],
|
||||
"claude_ruling": "1795010251.158839",
|
||||
"review_sha256": "e30612eabf43037f6258644945e9f486dc99426f08d1da160d14cfd6b3163632",
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465",
|
||||
"new_fits": 5,
|
||||
"seed": 0,
|
||||
"epochs": 200,
|
||||
"gpu_cap": 205200,
|
||||
"postfit_reserve": 7200,
|
||||
"phase3_cap": 720000,
|
||||
"no_retry": true
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"returncode": 0,
|
||||
"stdout": "",
|
||||
"stderr": "Running as unit: meltmind-phase3-item-c-20261118.service\n",
|
||||
"unit": "meltmind-phase3-item-c-20261118.service",
|
||||
"unix": 1795010602.5146396
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"unix": 1795010605.4638069,
|
||||
"identity": {
|
||||
"pid": 1755713,
|
||||
"created": 1795010602.48
|
||||
},
|
||||
"contract_sha256": "0e8533194b9987d234a8ea419a2d4e1a3993ab18c41400d5a89d0beaacf6a465"
|
||||
}
|
||||
Reference in new issue
Block a user