Publish GPU1 recovery worker and first epoch evidence

This commit is contained in:
NewMeltAssist committed 2026-11-19 14:18:51 +08:00
1 parent 4c19d90e52
commit 3d28cbca84
10 files changed
+127 -1

No files matched your search

@@ -0,0 +1,40 @@
{
"status": "ACTUAL_TRAINING_STARTED_FIRST_EPOCH_PASS",
"audited_utc": "2026-11-19T06:18:29.134722+00:00",
"ruling": "h-9f0a081e",
"adopted_review_commit": "5763a008a5c089743b7bb89386318c064ae25218",
"coordinator_pid": 2516974,
"worker_pid": 2517067,
"fixed_gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"observation": "MainPID=2516974\nActiveState=active\nSubState=running\n PID PPID ELAPSED STAT COMMAND\n2516974 2234 58 Rsl python\n2517067 2516974 34 Rsl python\nGPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 2512405, 820 MiB\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 2517067, 1708 MiB\n",
"idle_wall_seconds": 20.293668510974385,
"first_epoch": {
"status": "PASS",
"contract_sha256": "31ba0ab50f063730f1be7728b05910ffe092b0e3654c32a32dcf49156430eeb0",
"measured_epoch_seconds": 59.25489787396509,
"projected_train_gpu_seconds": 10240.75894455763
},
"latest_progress_at_snapshot": {
"fold": 0,
"stage": "TRAIN",
"epoch": 2,
"optimizer_steps": 468,
"wall_seconds": 90.04963541508187,
"cpu_seconds": 90.00858470700001,
"case": "00001061_284effca5a19488fa8a228e2_step_002",
"epoch_parts": 148,
"epoch_nodes": 4169025,
"lr": 0.000325,
"loss": 6.1020014982204884e-05,
"gradient_norm_before_clip": 0.015706609934568405,
"peak_allocated_bytes": 1334860288,
"peak_reserved_bytes": 1388314624
},
"clean_from_epoch0": true,
"old_infrastructure_gpu_seconds_counted_once": 741.1597241259879,
"prior_item_gpu_seconds": 20003.217328388826,
"new_gpu_seconds_cap": 13180,
"test_targets_read": false,
"final_screen_started": false,
"labelled_unique_attempt": true
}
@@ -0,0 +1,6 @@
{
"status": "PASS",
"contract_sha256": "31ba0ab50f063730f1be7728b05910ffe092b0e3654c32a32dcf49156430eeb0",
"measured_epoch_seconds": 59.25489787396509,
"projected_train_gpu_seconds": 10240.75894455763
}
@@ -0,0 +1,15 @@
{
"status": "IDLE_CONFIRMED",
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"polls": 3,
"consecutive_idle": 3,
"wall_seconds": 20.293668510974385,
"last": {
"unix": 1795069015.0045886,
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
}
}
@@ -0,0 +1,15 @@
{
"fold": 0,
"stage": "EPOCH_COMPLETE",
"epoch": 1,
"optimizer_steps": 320,
"wall_seconds": 63.700445002061315,
"cpu_seconds": 63.664391539,
"train_mean_weighted_loss": 0.004886068566793256,
"train_epoch_seconds": 59.25489787396509,
"epoch_parts": 320,
"epoch_nodes": 8742060,
"lr": 0.0001,
"peak_allocated_bytes": 1334860288,
"peak_reserved_bytes": 1388314624
}
@@ -0,0 +1,4 @@
{
"pid": 2517067,
"created": 1795069015.16
}
@@ -0,0 +1,13 @@
{
"observed_unix": 1795068991.0385342,
"gpu": {
"unix": 1795068991.0382462,
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
},
"exact_review": "PASS",
"old_service_restarted": false
}
@@ -0,0 +1,8 @@
{
"unit": "meltmind-1b-fold0-labelled-recovery-20261119.service",
"contract_sha256": "31ba0ab50f063730f1be7728b05910ffe092b0e3654c32a32dcf49156430eeb0",
"review_sha256": "b3ca737e971dcc0dfe95619139a1460e3ee61b73a49b3a44e8cd61deddeb16a7",
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"unix": 1795068994.5451603,
"labelled_once": true
}
@@ -0,0 +1,8 @@
{
"coordinator": {
"pid": 2516974,
"created": 1795068991.6
},
"contract_sha256": "31ba0ab50f063730f1be7728b05910ffe092b0e3654c32a32dcf49156430eeb0",
"unix": 1795068994.677433
}
@@ -0,0 +1,17 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_fold0_labelled_recovery.py",
"train",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item1b-fold0-labelled-recovery-2026-11-19"
],
"limits": {
"total_cpu_seconds": 30000,
"job_cpu_seconds": 12817,
"job_wall_seconds": 12806.913619801053,
"tree_rss_bytes": 4294967296,
"disk_bytes": 8589934592,
"max_processes": 1
},
"started_unix": 1795069015.1793163
}
@@ -15,4 +15,4 @@ BDF SHA-256 `e9c4f0f3241df7e62870dcb31469e2065d4078bfedcbb68d9bfa03f92377e62b`
**实机回执。** 记录产品、版本和 build、许可类型、可见的脚本或命令行入口、求解在本地还是云端及预计 credits。许可截图遮住账号、序列号等凭据。打开 study 后验证浇口、材料和工艺可查看并编辑。分别记 setup、solve、export 人工分钟和求解墙钟时间。保留 solver log、短射或失败原文。若云端会扣费或没有数值导出入口,记录具体阻塞。
**数字导出。** 保留同一 study 的 node ID、XYZ 和坐标单位,并导出 Fill time 的 node ID、值、时间单位和有效性。分别导出时按唯一 node ID 拼接,核对重复与覆盖。若重编号或重划,保留两套网格及映射。回执通过条件为 study 可打开,设置可查看和编辑,能导出逐节点结果,MeltSol 能将结果拼回网格。相对 L2 与秩相关只作本件描述。 [Autodesk 数值 XML 导出](https://help.autodesk.com/cloudhelp/2023/ENU/MoldflowInsight-CLC-Results/files/About-viewing-results/Result-plots/Plot-data/MoldflowInsight_CLC_Results_About_viewing_results_Result_plots_Plot_data_Saving_plot_results_in_XML_format_html.html)
**数字导出。** 保留同一 study 的 node ID、XYZ 和坐标单位,并导出 Fill time 的 node ID、值、时间单位和有效性。分别导出时按唯一 node ID 拼接,核对重复与覆盖。若重编号或重划,保留两套网格及映射。回执通过条件为 study 可打开,设置可查看和编辑,能导出逐节点结果,MeltSol 能将结果拼回网格。相对 L2 与秩相关只作本件描述。 [Autodesk 数值 XML 导出](https://help.autodesk.com/cloudhelp/2023/ENU/MoldflowInsight-CLC-Results/files/About-viewing-results/Result-plots/Plot-data/MoldflowInsight_CLC_Results_About_viewing_results_Result_plots_Plot_data_Saving_plot_results_in_XML_html.html)