Publish GPU1 recovery worker and first epoch evidence
This commit is contained in:
1 parent
4c19d90e52
commit
3d28cbca84
10 files changed
+127
-1
No files matched your search
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"status": "ACTUAL_TRAINING_STARTED_FIRST_EPOCH_PASS",
|
||||
"audited_utc": "2026-11-19T06:18:29.134722+00:00",
|
||||
"ruling": "h-9f0a081e",
|
||||
"adopted_review_commit": "5763a008a5c089743b7bb89386318c064ae25218",
|
||||
"coordinator_pid": 2516974,
|
||||
"worker_pid": 2517067,
|
||||
"fixed_gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"observation": "MainPID=2516974\nActiveState=active\nSubState=running\n PID PPID ELAPSED STAT COMMAND\n2516974 2234 58 Rsl python\n2517067 2516974 34 Rsl python\nGPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 2512405, 820 MiB\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 2517067, 1708 MiB\n",
|
||||
"idle_wall_seconds": 20.293668510974385,
|
||||
"first_epoch": {
|
||||
"status": "PASS",
|
||||
"contract_sha256": "31ba0ab50f063730f1be7728b05910ffe092b0e3654c32a32dcf49156430eeb0",
|
||||
"measured_epoch_seconds": 59.25489787396509,
|
||||
"projected_train_gpu_seconds": 10240.75894455763
|
||||
},
|
||||
"latest_progress_at_snapshot": {
|
||||
"fold": 0,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 2,
|
||||
"optimizer_steps": 468,
|
||||
"wall_seconds": 90.04963541508187,
|
||||
"cpu_seconds": 90.00858470700001,
|
||||
"case": "00001061_284effca5a19488fa8a228e2_step_002",
|
||||
"epoch_parts": 148,
|
||||
"epoch_nodes": 4169025,
|
||||
"lr": 0.000325,
|
||||
"loss": 6.1020014982204884e-05,
|
||||
"gradient_norm_before_clip": 0.015706609934568405,
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
},
|
||||
"clean_from_epoch0": true,
|
||||
"old_infrastructure_gpu_seconds_counted_once": 741.1597241259879,
|
||||
"prior_item_gpu_seconds": 20003.217328388826,
|
||||
"new_gpu_seconds_cap": 13180,
|
||||
"test_targets_read": false,
|
||||
"final_screen_started": false,
|
||||
"labelled_unique_attempt": true
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"contract_sha256": "31ba0ab50f063730f1be7728b05910ffe092b0e3654c32a32dcf49156430eeb0",
|
||||
"measured_epoch_seconds": 59.25489787396509,
|
||||
"projected_train_gpu_seconds": 10240.75894455763
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"status": "IDLE_CONFIRMED",
|
||||
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"polls": 3,
|
||||
"consecutive_idle": 3,
|
||||
"wall_seconds": 20.293668510974385,
|
||||
"last": {
|
||||
"unix": 1795069015.0045886,
|
||||
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"memory_MiB": 5,
|
||||
"utilization_percent": 0,
|
||||
"occupants": [],
|
||||
"idle": true
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"fold": 0,
|
||||
"stage": "EPOCH_COMPLETE",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 320,
|
||||
"wall_seconds": 63.700445002061315,
|
||||
"cpu_seconds": 63.664391539,
|
||||
"train_mean_weighted_loss": 0.004886068566793256,
|
||||
"train_epoch_seconds": 59.25489787396509,
|
||||
"epoch_parts": 320,
|
||||
"epoch_nodes": 8742060,
|
||||
"lr": 0.0001,
|
||||
"peak_allocated_bytes": 1334860288,
|
||||
"peak_reserved_bytes": 1388314624
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"pid": 2517067,
|
||||
"created": 1795069015.16
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"observed_unix": 1795068991.0385342,
|
||||
"gpu": {
|
||||
"unix": 1795068991.0382462,
|
||||
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"memory_MiB": 5,
|
||||
"utilization_percent": 0,
|
||||
"occupants": [],
|
||||
"idle": true
|
||||
},
|
||||
"exact_review": "PASS",
|
||||
"old_service_restarted": false
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"unit": "meltmind-1b-fold0-labelled-recovery-20261119.service",
|
||||
"contract_sha256": "31ba0ab50f063730f1be7728b05910ffe092b0e3654c32a32dcf49156430eeb0",
|
||||
"review_sha256": "b3ca737e971dcc0dfe95619139a1460e3ee61b73a49b3a44e8cd61deddeb16a7",
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"unix": 1795068994.5451603,
|
||||
"labelled_once": true
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"coordinator": {
|
||||
"pid": 2516974,
|
||||
"created": 1795068991.6
|
||||
},
|
||||
"contract_sha256": "31ba0ab50f063730f1be7728b05910ffe092b0e3654c32a32dcf49156430eeb0",
|
||||
"unix": 1795068994.677433
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_fold0_labelled_recovery.py",
|
||||
"train",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-labelled-recovery-2026-11-19"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 30000,
|
||||
"job_cpu_seconds": 12817,
|
||||
"job_wall_seconds": 12806.913619801053,
|
||||
"tree_rss_bytes": 4294967296,
|
||||
"disk_bytes": 8589934592,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1795069015.1793163
|
||||
}
|
||||
@@ -15,4 +15,4 @@ BDF SHA-256 `e9c4f0f3241df7e62870dcb31469e2065d4078bfedcbb68d9bfa03f92377e62b`
|
||||
|
||||
**实机回执。** 记录产品、版本和 build、许可类型、可见的脚本或命令行入口、求解在本地还是云端及预计 credits。许可截图遮住账号、序列号等凭据。打开 study 后验证浇口、材料和工艺可查看并编辑。分别记 setup、solve、export 人工分钟和求解墙钟时间。保留 solver log、短射或失败原文。若云端会扣费或没有数值导出入口,记录具体阻塞。
|
||||
|
||||
**数字导出。** 保留同一 study 的 node ID、XYZ 和坐标单位,并导出 Fill time 的 node ID、值、时间单位和有效性。分别导出时按唯一 node ID 拼接,核对重复与覆盖。若重编号或重划,保留两套网格及映射。回执通过条件为 study 可打开,设置可查看和编辑,能导出逐节点结果,MeltSol 能将结果拼回网格。相对 L2 与秩相关只作本件描述。 [Autodesk 数值 XML 导出](https://help.autodesk.com/cloudhelp/2023/ENU/MoldflowInsight-CLC-Results/files/About-viewing-results/Result-plots/Plot-data/MoldflowInsight_CLC_Results_About_viewing_results_Result_plots_Plot_data_Saving_plot_results_in_XML_format_html.html)
|
||||
**数字导出。** 保留同一 study 的 node ID、XYZ 和坐标单位,并导出 Fill time 的 node ID、值、时间单位和有效性。分别导出时按唯一 node ID 拼接,核对重复与覆盖。若重编号或重划,保留两套网格及映射。回执通过条件为 study 可打开,设置可查看和编辑,能导出逐节点结果,MeltSol 能将结果拼回网格。相对 L2 与秩相关只作本件描述。 [Autodesk 数值 XML 导出](https://help.autodesk.com/cloudhelp/2023/ENU/MoldflowInsight-CLC-Results/files/About-viewing-results/Result-plots/Plot-data/MoldflowInsight_CLC_Results_About_viewing_results_Result_plots_Plot_data_Saving_plot_results_in_XML_html.html)
|
||||
Reference in new issue
Block a user