Record authorized recovery actual start and first fold cost gate

This commit is contained in:
MeltAstra committed 2026-11-18 20:19:00 +08:00
1 parent bfb4b7c84d
commit ff0033d733
25 files changed
+905

No files matched your search

@@ -0,0 +1,18 @@
{
"unix": 1795004292.8713422,
"status": "SAME_ACTION_REVIEW_ALLOWED_AND_WORKER_OBSERVED",
"previous_block": "approval-block.json",
"authorization_relay": "user-confirmation-relay.json",
"review_provenance": "Relayed human question and assent submitted transparently to the same automatic approval workflow; original user item not independently returned by history tool.",
"service_launch": {
"unit": "meltmind-track-a-recovery-20261118.service",
"returncode": 0,
"stdout": "",
"stderr": "Running as unit: meltmind-track-a-recovery-20261118.service\n",
"unix": 1795004202.9873908,
"submitted_only": true
},
"actual_start_audit": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/actual-start-audit.json",
"original_rejection_preserved": true,
"alternate_execution_route_used": false
}
@@ -0,0 +1,24 @@
{
"received_unix": 1795004170.198358,
"source": "native codex_delegation",
"source_thread_id": "6afb1249-81d4-8191-b190-6fa32c3880de",
"question": "是否允许在 seed 1 继续运行时利用空闲 GPU 一次性恢复 Track A,复用三折、重训中断两折、总预算 162000 GPU 秒含已有成本,不再自动追加恢复。",
"answer": "我同意",
"attribution": "Yanqi, relayed by TRON",
"verification": "Exact question and reply supplied in incoming native relay. read_thread for source returned in-progress turn with no items; direct original user message was not independently retrieved. Submit this provenance to the same automatic approval reviewer; do not bypass any renewed rejection.",
"scope": {
"reused_folds": [
0,
1,
2
],
"fresh_folds": [
3,
4
],
"total_gpu_seconds": 162000,
"old_gpu_seconds_once": 47943.94719455298,
"seed1_unchanged": true,
"no_further_recovery": true
}
}
@@ -0,0 +1,55 @@
{
"unix": 1795004330.5901194,
"status": "FIRST_DISPATCHED_FOLD_COST_PASS",
"fold": 4,
"first_epoch_seconds": 57.02044904301874,
"own_projection": 14437.42601964093,
"cumulative_projection": 100541.37321419391,
"original_old_cost_once": 47943.94719455298,
"postfit_reserve": 7200,
"pending_fold3_unmeasured": true,
"worker_progress": {
"fold": 4,
"stage": "TRAIN",
"epoch": 2,
"optimizer_steps": 600,
"wall_seconds": 112.14072392100934,
"cpu_seconds": 111.596696969,
"case": "00000787_a25fb37392a44447b9684f16_step_012",
"epoch_parts": 290,
"epoch_nodes": 8188943,
"lr": 0.00019999999999999998,
"loss": 0.0017632129602134228,
"gradient_norm_before_clip": 0.06885933130979538,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
},
"files": [
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/first-epoch-projection.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/first-epoch-projection.json",
"sha256": "e524294579063ab269f9996ee6025dad198372b0eeb3d642f4a838e279e99b56"
},
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/history/001.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/history/001.json",
"sha256": "9538696224a700851e7868f6d2b36c4512e4b2eedd79725af6b05596c849497e"
},
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/progress.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/progress.json",
"sha256": "bd39f8aa3916e7d7eea424c35b5581312abd7ac67231ca70325f4002b3092f82"
},
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/live-progress.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/live-progress.json",
"sha256": "b577571afa3aab4c0b83dc4c79a3ea6ca7be8aa00f49f65f0840b4f89b9d742e"
},
{
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/dispatch-idle.json",
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/3/dispatch-idle.json",
"sha256": "d0e38b8d59b1dffcc02640e3d506e4ced0238115c2fdc0d42ab820f7599ec116"
}
],
"no_relaunch": true
}
@@ -0,0 +1,18 @@
{
"start_unix": 1795004329.9132068,
"end_unix": 1795004329.9554157,
"valid": true,
"memory_bytes": 1928331264,
"utilization": 20,
"monitor_policy": {
"rule": "three_consecutive_failed_samples_per_operation",
"failure_limit": 3,
"failed": false,
"errors": 0,
"streaks": {
"gpu_usage": 0
},
"max_streaks": {}
},
"charged_gpu_seconds": 184.04915182094555
}
@@ -0,0 +1,29 @@
{
"status": "PASS",
"cell": {
"id": 4,
"projected_gpu_seconds": 14437.42601964093,
"within_cap": true
},
"total": {
"status": "PASS",
"rows": [
{
"id": 3,
"projected_gpu_seconds": 30960,
"basis": "continuing_projection_or_pending_cap"
},
{
"id": 4,
"projected_gpu_seconds": 14437.42601964093,
"basis": "continuing_projection_or_pending_cap"
}
],
"new_projected_gpu_seconds": 52597.42601964093,
"old_gpu_seconds_once": 47943.94719455298,
"cumulative_projected_gpu_seconds": 100541.37321419391,
"postfit_reserve": 7200
},
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"unix": 1795004279.4981134
}
@@ -0,0 +1,15 @@
{
"fold": 4,
"stage": "EPOCH_COMPLETE",
"epoch": 1,
"optimizer_steps": 310,
"wall_seconds": 60.80659728299361,
"cpu_seconds": 60.766106956,
"train_mean_weighted_loss": 0.005705016106361244,
"train_epoch_seconds": 57.02044904301874,
"epoch_parts": 310,
"epoch_nodes": 8642607,
"lr": 0.0001,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
}
@@ -0,0 +1,16 @@
{
"fold": 4,
"stage": "TRAIN",
"epoch": 2,
"optimizer_steps": 600,
"wall_seconds": 112.14072392100934,
"cpu_seconds": 111.596696969,
"case": "00000787_a25fb37392a44447b9684f16_step_012",
"epoch_parts": 290,
"epoch_nodes": 8188943,
"lr": 0.00019999999999999998,
"loss": 0.0017632129602134228,
"gradient_norm_before_clip": 0.06885933130979538,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
}
@@ -0,0 +1,17 @@
{
"unix": 1795004329.9561625,
"active": [
{
"id": 4,
"guard_pid": 1650109
}
],
"pending": [
3
],
"terminal": [],
"old_gpu_seconds_once": 47943.94719455298,
"new_training_gpu_seconds": 368.09981042100117,
"cumulative_training_gpu_seconds": 48312.04701779806,
"postfit_started": false
}
+18
View File
@@ -711,3 +711,21 @@ Assist1795003274实际PASS d66e5ca1/evidence867e20da/validation50cf4ba6已逐字
调用启动命令时自动审批拒绝,命令根本未执行、未创建launch-attempt/service-launch/started;当前不是运行中,也不是新的训练失败。拒绝理由:当前可信用户指令未明确覆盖与另一队列并行的新GPU计算及资源使用。已核现有Claude/TRON裁量与项目记录,未找到更明确的原始人类计算授权;不以共享许可替代。不换执行途径,等待直接用户授权或可验证原始授权。准备范围/命令与原始拒绝见artifacts/track-a-recovery-launch-approval-2026-11-18/launch-plan.json、approval-block.json;seed1不动。
## 最新 Track A 唯一恢复实际运行:fold4训练,fold3计费等待(2026-11-18 20:16:46 CST)
TRON原生转达用户对“seed1继续运行时利用空闲GPU一次恢复、复用三折/重训两折、总162000含旧账、无进一步恢复”的回答“我同意”。已原样记录问题/回复及来源限制:历史工具未独立返回用户原条,不冒称直接实读。携此新证据经过同一自动审批重审允许,原拒绝证据不改。没有换途径或代启。
实际PASS d66e5ca1原样采用/冻结309dadfb/validate保持。唯一service meltmind-track-a-recovery-20261118.service(Restart=no),coord1650006、PPID2234;fold4 guard1650109/worker1650207在GPU2实际TRAIN epoch1,启动审计实见123steps。三idle polls各5MiB/0%;fold3仍pending,GPU3占用1928331264B(seed1),没有assignment/worker,等待按原单折30960计费。首轮成本门尚待;启动不代表全折/成本门通过。当前以实际live/terminal回执为准。
source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,旧三PASS hash复用。原旧47943.94719455298只计一次,新train106856.05280544702/post7200,总162000;单折30960含等待/setup/失败,无进一步恢复。seed1原队列1476593及三worker1476707/1476726/1598144保持,51pins PASS。内存58.36GB/需36.51GB、磁盘237.47GB/需100GiB启动复查通过。
原root全部旧失败/不完整状态保留,不重启旧服务。不复制新队列、不改固定源码/预算。全五成功冻结/退出后才做identity-only postfit新绑定及Assist窄delta,再读test;若恢复折再失败,只报告完成折。实际审计root/actual-start-audit.json、started、host-precheck-user-confirmed、launch-attempt/service-launch及artifacts/track-a-recovery-launch-approval-2026-11-18。下方“启动拦截/等待用户”均为历史。
## 最新恢复fold4首轮成本门PASS,fold3仍待派(2026-11-18)
原1650006/1650109/1650207队列和worker,fold4首轮57.020449043秒,单折投影14437.426020≤30960;pending fold3仍保留原30960整帽,含post7200及old47943.947195一次,累计投影100541.373214≤162000。fold4已真实进入epoch2;fold3尚未实测,不称全两折成本PASS。快照/首轮原history/门在artifacts/track-a-recovery-start-evidence-2026-11-18。无重复启动/改源/新计算范围,当前状态看live/terminal。
+14
View File
@@ -1,3 +1,17 @@
## 最新恢复fold4首轮成本门PASS,fold3仍待派(2026-11-18)
原1650006/1650109/1650207队列和worker,fold4首轮57.020449043秒,单折投影14437.426020≤30960;pending fold3仍保留原30960整帽,含post7200及old47943.947195一次,累计投影100541.373214≤162000。fold4已真实进入epoch2;fold3尚未实测,不称全两折成本PASS。快照/首轮原history/门在artifacts/track-a-recovery-start-evidence-2026-11-18。无重复启动/改源/新计算范围,当前状态看live/terminal。
## 最新 Track A 唯一恢复实际运行:fold4训练,fold3计费等待(2026-11-18 20:16:46 CST)
TRON原生转达用户对“seed1继续运行时利用空闲GPU一次恢复、复用三折/重训两折、总162000含旧账、无进一步恢复”的回答“我同意”。已原样记录问题/回复及来源限制:历史工具未独立返回用户原条,不冒称直接实读。携此新证据经过同一自动审批重审允许,原拒绝证据不改。没有换途径或代启。
实际PASS d66e5ca1原样采用/冻结309dadfb/validate保持。唯一service meltmind-track-a-recovery-20261118.service(Restart=no),coord1650006、PPID2234;fold4 guard1650109/worker1650207在GPU2实际TRAIN epoch1,启动审计实见123steps。三idle polls各5MiB/0%;fold3仍pending,GPU3占用1928331264B(seed1),没有assignment/worker,等待按原单折30960计费。首轮成本门尚待;启动不代表全折/成本门通过。当前以实际live/terminal回执为准。
source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,旧三PASS hash复用。原旧47943.94719455298只计一次,新train106856.05280544702/post7200,总162000;单折30960含等待/setup/失败,无进一步恢复。seed1原队列1476593及三worker1476707/1476726/1598144保持,51pins PASS。内存58.36GB/需36.51GB、磁盘237.47GB/需100GiB启动复查通过。
原root全部旧失败/不完整状态保留,不重启旧服务。不复制新队列、不改固定源码/预算。全五成功冻结/退出后才做identity-only postfit新绑定及Assist窄delta,再读test;若恢复折再失败,只报告完成折。实际审计root/actual-start-audit.json、started、host-precheck-user-confirmed、launch-attempt/service-launch及artifacts/track-a-recovery-launch-approval-2026-11-18。下方“启动拦截/等待用户”均为历史。
## 最新 Track A recovery 实际PASS已采用;启动被自动审批拦截,未运行(2026-11-18)
Assist1795003274实际PASS d66e5ca1/evidence867e20da/validation50cf4ba6已逐字核SHA与五字段、采用并冻结309dadfb;validate_review/config/reuse_identity PASS。implementation8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27map不变。Claude1795003292明确接受并要求启动,原件已实读。
@@ -0,0 +1,16 @@
## 最新 Track A 唯一恢复实际运行:fold4训练,fold3计费等待(2026-11-18 20:16:46 CST)
TRON原生转达用户对“seed1继续运行时利用空闲GPU一次恢复、复用三折/重训两折、总162000含旧账、无进一步恢复”的回答“我同意”。已原样记录问题/回复及来源限制:历史工具未独立返回用户原条,不冒称直接实读。携此新证据经过同一自动审批重审允许,原拒绝证据不改。没有换途径或代启。
实际PASS d66e5ca1原样采用/冻结309dadfb/validate保持。唯一service meltmind-track-a-recovery-20261118.service(Restart=no),coord1650006、PPID2234;fold4 guard1650109/worker1650207在GPU2实际TRAIN epoch1,启动审计实见123steps。三idle polls各5MiB/0%;fold3仍pending,GPU3占用1928331264B(seed1),没有assignment/worker,等待按原单折30960计费。首轮成本门尚待;启动不代表全折/成本门通过。当前以实际live/terminal回执为准。
source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,旧三PASS hash复用。原旧47943.94719455298只计一次,新train106856.05280544702/post7200,总162000;单折30960含等待/setup/失败,无进一步恢复。seed1原队列1476593及三worker1476707/1476726/1598144保持,51pins PASS。内存58.36GB/需36.51GB、磁盘237.47GB/需100GiB启动复查通过。
原root全部旧失败/不完整状态保留,不重启旧服务。不复制新队列、不改固定源码/预算。全五成功冻结/退出后才做identity-only postfit新绑定及Assist窄delta,再读test;若恢复折再失败,只报告完成折。实际审计root/actual-start-audit.json、started、host-precheck-user-confirmed、launch-attempt/service-launch及artifacts/track-a-recovery-launch-approval-2026-11-18。下方“启动拦截/等待用户”均为历史。
## 最新恢复fold4首轮成本门PASS,fold3仍待派(2026-11-18)
原1650006/1650109/1650207队列和worker,fold4首轮57.020449043秒,单折投影14437.426020≤30960;pending fold3仍保留原30960整帽,含post7200及old47943.947195一次,累计投影100541.373214≤162000。fold4已真实进入epoch2;fold3尚未实测,不称全两折成本PASS。快照/首轮原history/门在artifacts/track-a-recovery-start-evidence-2026-11-18。无重复启动/改源/新计算范围,当前状态看live/terminal。
@@ -0,0 +1,312 @@
{
"unix": 1795004245.9109008,
"implementation": "8a8ebd1f9f29f73b26cc0cacde27b9d1d8ca333b",
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"review_sha256": "d66e5ca101348ea8ccd7c167c8428af21eee362c3d74de356be3203fb944870b",
"service": {
"returncode": 0,
"stdout": "Restart=no\nMainPID=1650006\nExecMainStatus=0\nActiveState=active\nSubState=running\n",
"stderr": ""
},
"track_a_processes": [
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py",
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18"
],
"create_time": 1795004202.95,
"pid": 1650006,
"ppid": 2234
},
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/track_a_recovery_guard.py",
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
"4"
],
"create_time": 1795004207.83,
"pid": 1650109,
"ppid": 1650006
},
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/track_a_recovery_worker.py",
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
"4"
],
"create_time": 1795004214.51,
"pid": 1650207,
"ppid": 1650109
}
],
"seed1_processes": [
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18"
],
"create_time": 1794992135.84,
"pid": 1476593,
"ppid": 2234
},
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_guard.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"0",
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
],
"create_time": 1794992140.13,
"pid": 1476627,
"ppid": 1476593
},
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_guard.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"1",
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
],
"create_time": 1794992140.14,
"pid": 1476628,
"ppid": 1476593
},
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_worker.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"0",
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
],
"create_time": 1794992146.66,
"pid": 1476707,
"ppid": 1476627
},
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_worker.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"1",
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
],
"create_time": 1794992147.88,
"pid": 1476726,
"ppid": 1476628
},
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_guard.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"2",
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
],
"create_time": 1794999917.04,
"pid": 1598086,
"ppid": 1476593
},
{
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_worker.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"2",
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
],
"create_time": 1794999924.07,
"pid": 1598144,
"ppid": 1598086
}
],
"seed1_source_pins": "PASS",
"original_and_recovery_source_pins": "PASS",
"receipts": {
"started.json": {
"sha256": "db31964ac4f97fd8029f63ce38eb5c591d3270ca7e62979b41dda62f23ca283b",
"data": {
"pid": 1650006,
"unix": 1795004206.9920263,
"new_fits": 2,
"reused_folds": [
0,
1,
2
],
"fresh_folds": [
3,
4
],
"old_gpu_seconds_once": 47943.94719455298,
"new_train_cap": 106856.05280544702,
"cumulative_gpu_cap": 162000,
"test_targets_read": false,
"postfit_started": false
}
},
"live-progress.json": {
"sha256": "b1697659ed723860b20dd4d3bd5f55c9693e7337906a20a19a56de636d6239fe",
"data": {
"unix": 1795004245.3020048,
"active": [
{
"id": 4,
"guard_pid": 1650109
}
],
"pending": [
3
],
"terminal": [],
"old_gpu_seconds_once": 47943.94719455298,
"new_training_gpu_seconds": 198.791494303965,
"cumulative_training_gpu_seconds": 48142.73869843397,
"postfit_started": false
}
}
},
"folds": {
"3": {
"dispatch-idle.json": {
"sha256": "38f390cc75a8134e9325508596a63359dd54d07df21775eae8926514b7c961a7",
"data": {
"start_unix": 1795004245.250895,
"end_unix": 1795004245.301113,
"valid": true,
"memory_bytes": 1928331264,
"utilization": 0,
"monitor_policy": {
"rule": "three_consecutive_failed_samples_per_operation",
"failure_limit": 3,
"failed": false,
"errors": 0,
"streaks": {
"gpu_usage": 0
},
"max_streaks": {}
},
"charged_gpu_seconds": 99.39484800293576
}
}
},
"4": {
"dispatch-idle.json": {
"sha256": "1d57b0e8f92ed643e1d176c638d55550bcb10ffb010f17f6f4668f31c9f1ecf4",
"data": {
"start_unix": 1795004207.795226,
"end_unix": 1795004207.8412235,
"valid": true,
"memory_bytes": 5242880,
"utilization": 0,
"monitor_policy": {
"rule": "three_consecutive_failed_samples_per_operation",
"failure_limit": 3,
"failed": false,
"errors": 0,
"streaks": {
"gpu_usage": 0
},
"max_streaks": {}
},
"charged_gpu_seconds": 61.93496167601552
}
},
"assignment.json": {
"sha256": "03a7d5a0353fe1ad530e1e82facad8be8bdecc3a7376741e89fa37d897baf18b",
"data": {
"id": 4,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"pre_dispatch_wait_seconds": 1.9354809819487855,
"unix": 1795004207.844153
}
},
"dispatch.json": {
"sha256": "1cb1902c5e200498512c441039acab4bfabe15a767c87a89d79d06e2805c9ab9",
"data": {
"pid": 1650109,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"unix": 1795004207.8453767,
"pre_dispatch_wait_seconds": 1.9354809819487855
}
},
"started.json": {
"sha256": "9e82d5bbb0a062d4251d2ebe995e34cbd3ee3630eb66741cdfde3953c8ae7d05",
"data": {
"pid": 1650109,
"unix": 1795004210.3711073,
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"pre_dispatch_wait_seconds": 1.9354809819487855,
"remaining_gpu_seconds": 30958.06451901805,
"fresh_seed": 0
}
},
"idle-polls.json": {
"sha256": "80c6fd271989b8206ebea7beaf41abde2226f4084b40502fc284eb1127cbf1ff",
"data": [
{
"unix": 1795004210.4212985,
"memory_bytes": 5242880,
"utilization": 0
},
{
"unix": 1795004212.47237,
"memory_bytes": 5242880,
"utilization": 0
},
{
"unix": 1795004214.5197601,
"memory_bytes": 5242880,
"utilization": 0
}
]
},
"worker/environment.json": {
"sha256": "4009fb617556b981a3c8be4519c4125388d7f45b918954a49cbdfe8ff78f6f18",
"data": {
"torch": "2.5.1+cu124",
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"allocator_settings": "{'PYTORCH_CUDA_ALLOC_CONF': 'expandable_segments:True', 'max_split_size': -1, 'garbage_collection_threshold': 0.0, 'expandable_segments': True, 'pinned_num_register_threads': 1, 'release_lock_on_cudamalloc': False, 'pinned_use_cuda_host_register': False, 'roundup_power2_divisions': {'1': 0, '2': 0, '4': 0, '8': 0, '16': 0, '32': 0, '64': 0, '128': 0, '256': 0, '512': 0, '1024': 0, '2048': 0, '4096': 0, '8192': 0, '16384': 0, '32768': 0}}",
"model_dtype": "FP32",
"autocast": false,
"grad_scaler": false,
"fresh_seed": 0,
"pilot_weights_reused": false,
"parameter_count": 195077
}
},
"worker/progress.json": {
"sha256": "98a090b9a4b8bcd1078687f5ce6e9cc7f75e859fe546bb4a3ab0c20144c5f567",
"data": {
"fold": 4,
"stage": "TRAIN",
"epoch": 1,
"optimizer_steps": 123,
"wall_seconds": 27.234459250001237,
"cpu_seconds": 27.20321656,
"case": "00001243_96503b6a26c64074ad598927_step_005",
"epoch_parts": 123,
"epoch_nodes": 3337956,
"lr": 0.0001,
"loss": 0.00012004363816231489,
"gradient_norm_before_clip": 0.014971739612519741,
"peak_allocated_bytes": 1461572608,
"peak_reserved_bytes": 1516240896
}
}
}
},
"status": "ACTUAL_WORKER_TRAINING_OBSERVED",
"actual_worker_pids": [
1650207
],
"scientific_result_complete": false
}
@@ -0,0 +1,151 @@
{
"unix": 1795004174.3436382,
"review_sha256": "d66e5ca101348ea8ccd7c167c8428af21eee362c3d74de356be3203fb944870b",
"no_existing_track_a_process": true,
"existing_track_a": [],
"seed1_processes": [
{
"ppid": 2234,
"pid": 1476593,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18"
]
},
{
"ppid": 1476593,
"pid": 1476627,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_guard.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"0",
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
]
},
{
"ppid": 1476593,
"pid": 1476628,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_guard.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"1",
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
]
},
{
"ppid": 1476627,
"pid": 1476707,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_worker.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"0",
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
]
},
{
"ppid": 1476628,
"pid": 1476726,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_worker.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"1",
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
]
},
{
"ppid": 1476593,
"pid": 1598086,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_guard.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"2",
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
]
},
{
"ppid": 1598086,
"pid": 1598144,
"cmdline": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/phase3_seed1_v2_worker.py",
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
"2",
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
]
}
],
"seed1_source_pins": "PASS",
"gpu": {
"command": [
"nvidia-smi",
"--query-gpu=index,uuid,memory.used,utilization.gpu",
"--format=csv,noheader,nounits"
],
"returncode": 0,
"stdout": "0, GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 1717, 7\n1, GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 1719, 21\n2, GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76, 9, 0\n3, GPU-8b71b032-1b75-2891-935c-ce106a53034a, 1839, 23\n",
"stderr": ""
},
"gpu_processes": {
"command": [
"nvidia-smi",
"--query-compute-apps=gpu_uuid,pid,process_name,used_memory",
"--format=csv,noheader,nounits"
],
"returncode": 0,
"stdout": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 1476707, /home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python, 1708\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 1476726, /home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python, 1710\nGPU-8b71b032-1b75-2891-935c-ce106a53034a, 1598144, /home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python, 1830\n",
"stderr": ""
},
"observations": [
{
"index": 0,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"memory_bytes": 1800404992,
"utilization": 7,
"idle": false
},
{
"index": 1,
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"memory_bytes": 1802502144,
"utilization": 21,
"idle": false
},
{
"index": 2,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"memory_bytes": 9437184,
"utilization": 0,
"idle": true
},
{
"index": 3,
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"memory_bytes": 1928331264,
"utilization": 23,
"idle": false
}
],
"idle_folds_snapshot": [
4
],
"memory_available_bytes": 58359533568,
"required_available_memory_bytes": 36507222016,
"disk_free_bytes": 237470846976,
"required_disk_bytes": 107374182400,
"load_average": [
4.39599609375,
4.25830078125,
4.39111328125
],
"logical_cpus": 32,
"cpu_percent": 15.6,
"later_dispatch": "Pinned queue checks fixed GPU idle; waiting charged per fold; guard requires three polls; no source changes",
"scientific_execution": false,
"status": "PASS"
}
@@ -0,0 +1,7 @@
{
"id": 4,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"pre_dispatch_wait_seconds": 1.9354809819487855,
"unix": 1795004207.844153
}
@@ -0,0 +1,6 @@
{
"pid": 1650109,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"unix": 1795004207.8453767,
"pre_dispatch_wait_seconds": 1.9354809819487855
}
@@ -0,0 +1,29 @@
{
"status": "PASS",
"cell": {
"id": 4,
"projected_gpu_seconds": 14437.42601964093,
"within_cap": true
},
"total": {
"status": "PASS",
"rows": [
{
"id": 3,
"projected_gpu_seconds": 30960,
"basis": "continuing_projection_or_pending_cap"
},
{
"id": 4,
"projected_gpu_seconds": 14437.42601964093,
"basis": "continuing_projection_or_pending_cap"
}
],
"new_projected_gpu_seconds": 52597.42601964093,
"old_gpu_seconds_once": 47943.94719455298,
"cumulative_projected_gpu_seconds": 100541.37321419391,
"postfit_reserve": 7200
},
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"unix": 1795004279.4981134
}
@@ -0,0 +1,17 @@
[
{
"unix": 1795004210.4212985,
"memory_bytes": 5242880,
"utilization": 0
},
{
"unix": 1795004212.47237,
"memory_bytes": 5242880,
"utilization": 0
},
{
"unix": 1795004214.5197601,
"memory_bytes": 5242880,
"utilization": 0
}
]
@@ -0,0 +1,9 @@
{
"pid": 1650109,
"unix": 1795004210.3711073,
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"pre_dispatch_wait_seconds": 1.9354809819487855,
"remaining_gpu_seconds": 30958.06451901805,
"fresh_seed": 0
}
@@ -0,0 +1,11 @@
{
"torch": "2.5.1+cu124",
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"allocator_settings": "{'PYTORCH_CUDA_ALLOC_CONF': 'expandable_segments:True', 'max_split_size': -1, 'garbage_collection_threshold': 0.0, 'expandable_segments': True, 'pinned_num_register_threads': 1, 'release_lock_on_cudamalloc': False, 'pinned_use_cuda_host_register': False, 'roundup_power2_divisions': {'1': 0, '2': 0, '4': 0, '8': 0, '16': 0, '32': 0, '64': 0, '128': 0, '256': 0, '512': 0, '1024': 0, '2048': 0, '4096': 0, '8192': 0, '16384': 0, '32768': 0}}",
"model_dtype": "FP32",
"autocast": false,
"grad_scaler": false,
"fresh_seed": 0,
"pilot_weights_reused": false,
"parameter_count": 195077
}
@@ -0,0 +1,32 @@
{
"unix": 1795004202.9514024,
"unit": "meltmind-track-a-recovery-20261118.service",
"command": [
"systemd-run",
"--user",
"--unit=meltmind-track-a-recovery-20261118.service",
"--property=Restart=no",
"--property=KillMode=control-group",
"--working-directory=/home/frank14f/optane/MeltMind",
"--setenv=OPENBLAS_NUM_THREADS=1",
"--setenv=OMP_NUM_THREADS=1",
"--setenv=MKL_NUM_THREADS=1",
"--setenv=PYTHONPATH=/home/frank14f/optane/MeltMind/tools",
"--property=StandardOutput=append:/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/service.log",
"--property=StandardError=append:/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/service.log",
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py",
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18"
],
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"review_sha256": "d66e5ca101348ea8ccd7c167c8428af21eee362c3d74de356be3203fb944870b",
"no_retry": true,
"claude_ruling": "1795003292.974599",
"user_confirmation_relay": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-launch-approval-2026-11-18/user-confirmation-relay.json",
"user_confirmation_relay_sha256": "97c6bfcff06a66816c66be4666426c30d41faa0213ec2025ff02e8585790bbff",
"prelaunch_service_check": {
"stdout": "MainPID=0\nLoadState=not-found\nActiveState=inactive\n",
"stderr": "",
"returncode": 0
}
}
@@ -0,0 +1,4 @@
{
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"proof_sha256": "87d4a56bb8a748ce0f823a49a9355d05af617b60ed413685b8663bc259accafc"
}
@@ -0,0 +1,25 @@
{
"commit": "fac3543c0001f6cb4786e6fb1f5f36294da403f2",
"files": {
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/reused-models.json": "ce371fa5b1eb559c8edfe31fc1f064a1b253f0186c048c14f066acc26437159f",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt": "767c186bdd0c99d9bcd195cadd5e7b15374c4c07bcb53aa03c157219c74b326d",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt": "9b1356403017f5c351729ce3d33494c48033b0addff4b7b9ca6c1aad6bf90b93",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt": "0f71453359e8c6df759434b00759b5721e0cd3131cfb474a17a3b47f0c3c2e18",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json": "6fab06dd34d30d10409198bc177627293d4d8da559be3b33585c410964e6a15c",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json": "6734f28dc7ca6ffd3a39a43ad2964ffb747c77e55d24d0705c3b5b41f56f397b",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json": "e1f6c14617ed019379998142d49bf9359474db41e1614e2b4e20834ccc37b09d"
},
"git_paths": {
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "home-runs/track-a-recovery-2026-11-18/contract.json",
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/reused-models.json": "home-runs/track-a-recovery-2026-11-18/reused-models.json",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json",
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json"
},
"remote_published": false,
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
}
@@ -0,0 +1,35 @@
{
"models": [
{
"job": 0,
"path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt",
"sha256": "767c186bdd0c99d9bcd195cadd5e7b15374c4c07bcb53aa03c157219c74b326d",
"standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json",
"standardization_sha256": "6fab06dd34d30d10409198bc177627293d4d8da559be3b33585c410964e6a15c",
"origin": "ORIGINAL_HASH_REUSE",
"training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
"guard_complete_sha256": "024bb110576da7c351ad8e98227cace2f40f4247a8f349c01ea4ebfde954891c"
},
{
"job": 1,
"path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt",
"sha256": "9b1356403017f5c351729ce3d33494c48033b0addff4b7b9ca6c1aad6bf90b93",
"standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json",
"standardization_sha256": "6734f28dc7ca6ffd3a39a43ad2964ffb747c77e55d24d0705c3b5b41f56f397b",
"origin": "ORIGINAL_HASH_REUSE",
"training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
"guard_complete_sha256": "9259261f44a5813451e9f6f7878fd69d65db6e509889f7c78adcb6f417e00b3d"
},
{
"job": 2,
"path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt",
"sha256": "0f71453359e8c6df759434b00759b5721e0cd3131cfb474a17a3b47f0c3c2e18",
"standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json",
"standardization_sha256": "e1f6c14617ed019379998142d49bf9359474db41e1614e2b4e20834ccc37b09d",
"origin": "ORIGINAL_HASH_REUSE",
"training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
"guard_complete_sha256": "b60b2575b9efdbb28eabcc2682dc04ce3519c9e7b2f6954cc4dcea0e29b1ab2f"
}
],
"old_gpu_seconds_once": 47943.94719455298
}
@@ -0,0 +1,8 @@
{
"unit": "meltmind-track-a-recovery-20261118.service",
"returncode": 0,
"stdout": "",
"stderr": "Running as unit: meltmind-track-a-recovery-20261118.service\n",
"unix": 1795004202.9873908,
"submitted_only": true
}
@@ -0,0 +1,19 @@
{
"pid": 1650006,
"unix": 1795004206.9920263,
"new_fits": 2,
"reused_folds": [
0,
1,
2
],
"fresh_folds": [
3,
4
],
"old_gpu_seconds_once": 47943.94719455298,
"new_train_cap": 106856.05280544702,
"cumulative_gpu_cap": 162000,
"test_targets_read": false,
"postfit_started": false
}