Record authorized recovery actual start and first fold cost gate
This commit is contained in:
1 parent
bfb4b7c84d
commit
ff0033d733
25 files changed
+905
No files matched your search
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"unix": 1795004292.8713422,
|
||||
"status": "SAME_ACTION_REVIEW_ALLOWED_AND_WORKER_OBSERVED",
|
||||
"previous_block": "approval-block.json",
|
||||
"authorization_relay": "user-confirmation-relay.json",
|
||||
"review_provenance": "Relayed human question and assent submitted transparently to the same automatic approval workflow; original user item not independently returned by history tool.",
|
||||
"service_launch": {
|
||||
"unit": "meltmind-track-a-recovery-20261118.service",
|
||||
"returncode": 0,
|
||||
"stdout": "",
|
||||
"stderr": "Running as unit: meltmind-track-a-recovery-20261118.service\n",
|
||||
"unix": 1795004202.9873908,
|
||||
"submitted_only": true
|
||||
},
|
||||
"actual_start_audit": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/actual-start-audit.json",
|
||||
"original_rejection_preserved": true,
|
||||
"alternate_execution_route_used": false
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"received_unix": 1795004170.198358,
|
||||
"source": "native codex_delegation",
|
||||
"source_thread_id": "6afb1249-81d4-8191-b190-6fa32c3880de",
|
||||
"question": "是否允许在 seed 1 继续运行时利用空闲 GPU 一次性恢复 Track A,复用三折、重训中断两折、总预算 162000 GPU 秒含已有成本,不再自动追加恢复。",
|
||||
"answer": "我同意",
|
||||
"attribution": "Yanqi, relayed by TRON",
|
||||
"verification": "Exact question and reply supplied in incoming native relay. read_thread for source returned in-progress turn with no items; direct original user message was not independently retrieved. Submit this provenance to the same automatic approval reviewer; do not bypass any renewed rejection.",
|
||||
"scope": {
|
||||
"reused_folds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"fresh_folds": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"total_gpu_seconds": 162000,
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"seed1_unchanged": true,
|
||||
"no_further_recovery": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"unix": 1795004330.5901194,
|
||||
"status": "FIRST_DISPATCHED_FOLD_COST_PASS",
|
||||
"fold": 4,
|
||||
"first_epoch_seconds": 57.02044904301874,
|
||||
"own_projection": 14437.42601964093,
|
||||
"cumulative_projection": 100541.37321419391,
|
||||
"original_old_cost_once": 47943.94719455298,
|
||||
"postfit_reserve": 7200,
|
||||
"pending_fold3_unmeasured": true,
|
||||
"worker_progress": {
|
||||
"fold": 4,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 2,
|
||||
"optimizer_steps": 600,
|
||||
"wall_seconds": 112.14072392100934,
|
||||
"cpu_seconds": 111.596696969,
|
||||
"case": "00000787_a25fb37392a44447b9684f16_step_012",
|
||||
"epoch_parts": 290,
|
||||
"epoch_nodes": 8188943,
|
||||
"lr": 0.00019999999999999998,
|
||||
"loss": 0.0017632129602134228,
|
||||
"gradient_norm_before_clip": 0.06885933130979538,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
},
|
||||
"files": [
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/first-epoch-projection.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/first-epoch-projection.json",
|
||||
"sha256": "e524294579063ab269f9996ee6025dad198372b0eeb3d642f4a838e279e99b56"
|
||||
},
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/history/001.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/history/001.json",
|
||||
"sha256": "9538696224a700851e7868f6d2b36c4512e4b2eedd79725af6b05596c849497e"
|
||||
},
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/progress.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/progress.json",
|
||||
"sha256": "bd39f8aa3916e7d7eea424c35b5581312abd7ac67231ca70325f4002b3092f82"
|
||||
},
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/live-progress.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/live-progress.json",
|
||||
"sha256": "b577571afa3aab4c0b83dc4c79a3ea6ca7be8aa00f49f65f0840b4f89b9d742e"
|
||||
},
|
||||
{
|
||||
"source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/dispatch-idle.json",
|
||||
"snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/3/dispatch-idle.json",
|
||||
"sha256": "d0e38b8d59b1dffcc02640e3d506e4ced0238115c2fdc0d42ab820f7599ec116"
|
||||
}
|
||||
],
|
||||
"no_relaunch": true
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"start_unix": 1795004329.9132068,
|
||||
"end_unix": 1795004329.9554157,
|
||||
"valid": true,
|
||||
"memory_bytes": 1928331264,
|
||||
"utilization": 20,
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"gpu_usage": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
},
|
||||
"charged_gpu_seconds": 184.04915182094555
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"cell": {
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 14437.42601964093,
|
||||
"within_cap": true
|
||||
},
|
||||
"total": {
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"id": 3,
|
||||
"projected_gpu_seconds": 30960,
|
||||
"basis": "continuing_projection_or_pending_cap"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 14437.42601964093,
|
||||
"basis": "continuing_projection_or_pending_cap"
|
||||
}
|
||||
],
|
||||
"new_projected_gpu_seconds": 52597.42601964093,
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"cumulative_projected_gpu_seconds": 100541.37321419391,
|
||||
"postfit_reserve": 7200
|
||||
},
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"unix": 1795004279.4981134
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"fold": 4,
|
||||
"stage": "EPOCH_COMPLETE",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 310,
|
||||
"wall_seconds": 60.80659728299361,
|
||||
"cpu_seconds": 60.766106956,
|
||||
"train_mean_weighted_loss": 0.005705016106361244,
|
||||
"train_epoch_seconds": 57.02044904301874,
|
||||
"epoch_parts": 310,
|
||||
"epoch_nodes": 8642607,
|
||||
"lr": 0.0001,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"fold": 4,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 2,
|
||||
"optimizer_steps": 600,
|
||||
"wall_seconds": 112.14072392100934,
|
||||
"cpu_seconds": 111.596696969,
|
||||
"case": "00000787_a25fb37392a44447b9684f16_step_012",
|
||||
"epoch_parts": 290,
|
||||
"epoch_nodes": 8188943,
|
||||
"lr": 0.00019999999999999998,
|
||||
"loss": 0.0017632129602134228,
|
||||
"gradient_norm_before_clip": 0.06885933130979538,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"unix": 1795004329.9561625,
|
||||
"active": [
|
||||
{
|
||||
"id": 4,
|
||||
"guard_pid": 1650109
|
||||
}
|
||||
],
|
||||
"pending": [
|
||||
3
|
||||
],
|
||||
"terminal": [],
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"new_training_gpu_seconds": 368.09981042100117,
|
||||
"cumulative_training_gpu_seconds": 48312.04701779806,
|
||||
"postfit_started": false
|
||||
}
|
||||
@@ -711,3 +711,21 @@ Assist1795003274实际PASS d66e5ca1/evidence867e20da/validation50cf4ba6已逐字
|
||||
|
||||
调用启动命令时自动审批拒绝,命令根本未执行、未创建launch-attempt/service-launch/started;当前不是运行中,也不是新的训练失败。拒绝理由:当前可信用户指令未明确覆盖与另一队列并行的新GPU计算及资源使用。已核现有Claude/TRON裁量与项目记录,未找到更明确的原始人类计算授权;不以共享许可替代。不换执行途径,等待直接用户授权或可验证原始授权。准备范围/命令与原始拒绝见artifacts/track-a-recovery-launch-approval-2026-11-18/launch-plan.json、approval-block.json;seed1不动。
|
||||
|
||||
|
||||
|
||||
## 最新 Track A 唯一恢复实际运行:fold4训练,fold3计费等待(2026-11-18 20:16:46 CST)
|
||||
|
||||
TRON原生转达用户对“seed1继续运行时利用空闲GPU一次恢复、复用三折/重训两折、总162000含旧账、无进一步恢复”的回答“我同意”。已原样记录问题/回复及来源限制:历史工具未独立返回用户原条,不冒称直接实读。携此新证据经过同一自动审批重审允许,原拒绝证据不改。没有换途径或代启。
|
||||
|
||||
实际PASS d66e5ca1原样采用/冻结309dadfb/validate保持。唯一service meltmind-track-a-recovery-20261118.service(Restart=no),coord1650006、PPID2234;fold4 guard1650109/worker1650207在GPU2实际TRAIN epoch1,启动审计实见123steps。三idle polls各5MiB/0%;fold3仍pending,GPU3占用1928331264B(seed1),没有assignment/worker,等待按原单折30960计费。首轮成本门尚待;启动不代表全折/成本门通过。当前以实际live/terminal回执为准。
|
||||
|
||||
source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,旧三PASS hash复用。原旧47943.94719455298只计一次,新train106856.05280544702/post7200,总162000;单折30960含等待/setup/失败,无进一步恢复。seed1原队列1476593及三worker1476707/1476726/1598144保持,51pins PASS。内存58.36GB/需36.51GB、磁盘237.47GB/需100GiB启动复查通过。
|
||||
|
||||
原root全部旧失败/不完整状态保留,不重启旧服务。不复制新队列、不改固定源码/预算。全五成功冻结/退出后才做identity-only postfit新绑定及Assist窄delta,再读test;若恢复折再失败,只报告完成折。实际审计root/actual-start-audit.json、started、host-precheck-user-confirmed、launch-attempt/service-launch及artifacts/track-a-recovery-launch-approval-2026-11-18。下方“启动拦截/等待用户”均为历史。
|
||||
|
||||
|
||||
|
||||
## 最新恢复fold4首轮成本门PASS,fold3仍待派(2026-11-18)
|
||||
|
||||
原1650006/1650109/1650207队列和worker,fold4首轮57.020449043秒,单折投影14437.426020≤30960;pending fold3仍保留原30960整帽,含post7200及old47943.947195一次,累计投影100541.373214≤162000。fold4已真实进入epoch2;fold3尚未实测,不称全两折成本PASS。快照/首轮原history/门在artifacts/track-a-recovery-start-evidence-2026-11-18。无重复启动/改源/新计算范围,当前状态看live/terminal。
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
## 最新恢复fold4首轮成本门PASS,fold3仍待派(2026-11-18)
|
||||
|
||||
原1650006/1650109/1650207队列和worker,fold4首轮57.020449043秒,单折投影14437.426020≤30960;pending fold3仍保留原30960整帽,含post7200及old47943.947195一次,累计投影100541.373214≤162000。fold4已真实进入epoch2;fold3尚未实测,不称全两折成本PASS。快照/首轮原history/门在artifacts/track-a-recovery-start-evidence-2026-11-18。无重复启动/改源/新计算范围,当前状态看live/terminal。
|
||||
|
||||
## 最新 Track A 唯一恢复实际运行:fold4训练,fold3计费等待(2026-11-18 20:16:46 CST)
|
||||
|
||||
TRON原生转达用户对“seed1继续运行时利用空闲GPU一次恢复、复用三折/重训两折、总162000含旧账、无进一步恢复”的回答“我同意”。已原样记录问题/回复及来源限制:历史工具未独立返回用户原条,不冒称直接实读。携此新证据经过同一自动审批重审允许,原拒绝证据不改。没有换途径或代启。
|
||||
|
||||
实际PASS d66e5ca1原样采用/冻结309dadfb/validate保持。唯一service meltmind-track-a-recovery-20261118.service(Restart=no),coord1650006、PPID2234;fold4 guard1650109/worker1650207在GPU2实际TRAIN epoch1,启动审计实见123steps。三idle polls各5MiB/0%;fold3仍pending,GPU3占用1928331264B(seed1),没有assignment/worker,等待按原单折30960计费。首轮成本门尚待;启动不代表全折/成本门通过。当前以实际live/terminal回执为准。
|
||||
|
||||
source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,旧三PASS hash复用。原旧47943.94719455298只计一次,新train106856.05280544702/post7200,总162000;单折30960含等待/setup/失败,无进一步恢复。seed1原队列1476593及三worker1476707/1476726/1598144保持,51pins PASS。内存58.36GB/需36.51GB、磁盘237.47GB/需100GiB启动复查通过。
|
||||
|
||||
原root全部旧失败/不完整状态保留,不重启旧服务。不复制新队列、不改固定源码/预算。全五成功冻结/退出后才做identity-only postfit新绑定及Assist窄delta,再读test;若恢复折再失败,只报告完成折。实际审计root/actual-start-audit.json、started、host-precheck-user-confirmed、launch-attempt/service-launch及artifacts/track-a-recovery-launch-approval-2026-11-18。下方“启动拦截/等待用户”均为历史。
|
||||
|
||||
## 最新 Track A recovery 实际PASS已采用;启动被自动审批拦截,未运行(2026-11-18)
|
||||
|
||||
Assist1795003274实际PASS d66e5ca1/evidence867e20da/validation50cf4ba6已逐字核SHA与五字段、采用并冻结309dadfb;validate_review/config/reuse_identity PASS。implementation8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27map不变。Claude1795003292明确接受并要求启动,原件已实读。
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
## 最新 Track A 唯一恢复实际运行:fold4训练,fold3计费等待(2026-11-18 20:16:46 CST)
|
||||
|
||||
TRON原生转达用户对“seed1继续运行时利用空闲GPU一次恢复、复用三折/重训两折、总162000含旧账、无进一步恢复”的回答“我同意”。已原样记录问题/回复及来源限制:历史工具未独立返回用户原条,不冒称直接实读。携此新证据经过同一自动审批重审允许,原拒绝证据不改。没有换途径或代启。
|
||||
|
||||
实际PASS d66e5ca1原样采用/冻结309dadfb/validate保持。唯一service meltmind-track-a-recovery-20261118.service(Restart=no),coord1650006、PPID2234;fold4 guard1650109/worker1650207在GPU2实际TRAIN epoch1,启动审计实见123steps。三idle polls各5MiB/0%;fold3仍pending,GPU3占用1928331264B(seed1),没有assignment/worker,等待按原单折30960计费。首轮成本门尚待;启动不代表全折/成本门通过。当前以实际live/terminal回执为准。
|
||||
|
||||
source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,旧三PASS hash复用。原旧47943.94719455298只计一次,新train106856.05280544702/post7200,总162000;单折30960含等待/setup/失败,无进一步恢复。seed1原队列1476593及三worker1476707/1476726/1598144保持,51pins PASS。内存58.36GB/需36.51GB、磁盘237.47GB/需100GiB启动复查通过。
|
||||
|
||||
原root全部旧失败/不完整状态保留,不重启旧服务。不复制新队列、不改固定源码/预算。全五成功冻结/退出后才做identity-only postfit新绑定及Assist窄delta,再读test;若恢复折再失败,只报告完成折。实际审计root/actual-start-audit.json、started、host-precheck-user-confirmed、launch-attempt/service-launch及artifacts/track-a-recovery-launch-approval-2026-11-18。下方“启动拦截/等待用户”均为历史。
|
||||
|
||||
|
||||
|
||||
## 最新恢复fold4首轮成本门PASS,fold3仍待派(2026-11-18)
|
||||
|
||||
原1650006/1650109/1650207队列和worker,fold4首轮57.020449043秒,单折投影14437.426020≤30960;pending fold3仍保留原30960整帽,含post7200及old47943.947195一次,累计投影100541.373214≤162000。fold4已真实进入epoch2;fold3尚未实测,不称全两折成本PASS。快照/首轮原history/门在artifacts/track-a-recovery-start-evidence-2026-11-18。无重复启动/改源/新计算范围,当前状态看live/terminal。
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
{
|
||||
"unix": 1795004245.9109008,
|
||||
"implementation": "8a8ebd1f9f29f73b26cc0cacde27b9d1d8ca333b",
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"review_sha256": "d66e5ca101348ea8ccd7c167c8428af21eee362c3d74de356be3203fb944870b",
|
||||
"service": {
|
||||
"returncode": 0,
|
||||
"stdout": "Restart=no\nMainPID=1650006\nExecMainStatus=0\nActiveState=active\nSubState=running\n",
|
||||
"stderr": ""
|
||||
},
|
||||
"track_a_processes": [
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18"
|
||||
],
|
||||
"create_time": 1795004202.95,
|
||||
"pid": 1650006,
|
||||
"ppid": 2234
|
||||
},
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/track_a_recovery_guard.py",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
|
||||
"4"
|
||||
],
|
||||
"create_time": 1795004207.83,
|
||||
"pid": 1650109,
|
||||
"ppid": 1650006
|
||||
},
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/track_a_recovery_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18",
|
||||
"4"
|
||||
],
|
||||
"create_time": 1795004214.51,
|
||||
"pid": 1650207,
|
||||
"ppid": 1650109
|
||||
}
|
||||
],
|
||||
"seed1_processes": [
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18"
|
||||
],
|
||||
"create_time": 1794992135.84,
|
||||
"pid": 1476593,
|
||||
"ppid": 2234
|
||||
},
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_guard.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"0",
|
||||
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
|
||||
],
|
||||
"create_time": 1794992140.13,
|
||||
"pid": 1476627,
|
||||
"ppid": 1476593
|
||||
},
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_guard.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"1",
|
||||
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
|
||||
],
|
||||
"create_time": 1794992140.14,
|
||||
"pid": 1476628,
|
||||
"ppid": 1476593
|
||||
},
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"0",
|
||||
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
|
||||
],
|
||||
"create_time": 1794992146.66,
|
||||
"pid": 1476707,
|
||||
"ppid": 1476627
|
||||
},
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"1",
|
||||
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
|
||||
],
|
||||
"create_time": 1794992147.88,
|
||||
"pid": 1476726,
|
||||
"ppid": 1476628
|
||||
},
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_guard.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"2",
|
||||
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
|
||||
],
|
||||
"create_time": 1794999917.04,
|
||||
"pid": 1598086,
|
||||
"ppid": 1476593
|
||||
},
|
||||
{
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"2",
|
||||
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
|
||||
],
|
||||
"create_time": 1794999924.07,
|
||||
"pid": 1598144,
|
||||
"ppid": 1598086
|
||||
}
|
||||
],
|
||||
"seed1_source_pins": "PASS",
|
||||
"original_and_recovery_source_pins": "PASS",
|
||||
"receipts": {
|
||||
"started.json": {
|
||||
"sha256": "db31964ac4f97fd8029f63ce38eb5c591d3270ca7e62979b41dda62f23ca283b",
|
||||
"data": {
|
||||
"pid": 1650006,
|
||||
"unix": 1795004206.9920263,
|
||||
"new_fits": 2,
|
||||
"reused_folds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"fresh_folds": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"new_train_cap": 106856.05280544702,
|
||||
"cumulative_gpu_cap": 162000,
|
||||
"test_targets_read": false,
|
||||
"postfit_started": false
|
||||
}
|
||||
},
|
||||
"live-progress.json": {
|
||||
"sha256": "b1697659ed723860b20dd4d3bd5f55c9693e7337906a20a19a56de636d6239fe",
|
||||
"data": {
|
||||
"unix": 1795004245.3020048,
|
||||
"active": [
|
||||
{
|
||||
"id": 4,
|
||||
"guard_pid": 1650109
|
||||
}
|
||||
],
|
||||
"pending": [
|
||||
3
|
||||
],
|
||||
"terminal": [],
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"new_training_gpu_seconds": 198.791494303965,
|
||||
"cumulative_training_gpu_seconds": 48142.73869843397,
|
||||
"postfit_started": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"folds": {
|
||||
"3": {
|
||||
"dispatch-idle.json": {
|
||||
"sha256": "38f390cc75a8134e9325508596a63359dd54d07df21775eae8926514b7c961a7",
|
||||
"data": {
|
||||
"start_unix": 1795004245.250895,
|
||||
"end_unix": 1795004245.301113,
|
||||
"valid": true,
|
||||
"memory_bytes": 1928331264,
|
||||
"utilization": 0,
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"gpu_usage": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
},
|
||||
"charged_gpu_seconds": 99.39484800293576
|
||||
}
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"dispatch-idle.json": {
|
||||
"sha256": "1d57b0e8f92ed643e1d176c638d55550bcb10ffb010f17f6f4668f31c9f1ecf4",
|
||||
"data": {
|
||||
"start_unix": 1795004207.795226,
|
||||
"end_unix": 1795004207.8412235,
|
||||
"valid": true,
|
||||
"memory_bytes": 5242880,
|
||||
"utilization": 0,
|
||||
"monitor_policy": {
|
||||
"rule": "three_consecutive_failed_samples_per_operation",
|
||||
"failure_limit": 3,
|
||||
"failed": false,
|
||||
"errors": 0,
|
||||
"streaks": {
|
||||
"gpu_usage": 0
|
||||
},
|
||||
"max_streaks": {}
|
||||
},
|
||||
"charged_gpu_seconds": 61.93496167601552
|
||||
}
|
||||
},
|
||||
"assignment.json": {
|
||||
"sha256": "03a7d5a0353fe1ad530e1e82facad8be8bdecc3a7376741e89fa37d897baf18b",
|
||||
"data": {
|
||||
"id": 4,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"pre_dispatch_wait_seconds": 1.9354809819487855,
|
||||
"unix": 1795004207.844153
|
||||
}
|
||||
},
|
||||
"dispatch.json": {
|
||||
"sha256": "1cb1902c5e200498512c441039acab4bfabe15a767c87a89d79d06e2805c9ab9",
|
||||
"data": {
|
||||
"pid": 1650109,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"unix": 1795004207.8453767,
|
||||
"pre_dispatch_wait_seconds": 1.9354809819487855
|
||||
}
|
||||
},
|
||||
"started.json": {
|
||||
"sha256": "9e82d5bbb0a062d4251d2ebe995e34cbd3ee3630eb66741cdfde3953c8ae7d05",
|
||||
"data": {
|
||||
"pid": 1650109,
|
||||
"unix": 1795004210.3711073,
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"pre_dispatch_wait_seconds": 1.9354809819487855,
|
||||
"remaining_gpu_seconds": 30958.06451901805,
|
||||
"fresh_seed": 0
|
||||
}
|
||||
},
|
||||
"idle-polls.json": {
|
||||
"sha256": "80c6fd271989b8206ebea7beaf41abde2226f4084b40502fc284eb1127cbf1ff",
|
||||
"data": [
|
||||
{
|
||||
"unix": 1795004210.4212985,
|
||||
"memory_bytes": 5242880,
|
||||
"utilization": 0
|
||||
},
|
||||
{
|
||||
"unix": 1795004212.47237,
|
||||
"memory_bytes": 5242880,
|
||||
"utilization": 0
|
||||
},
|
||||
{
|
||||
"unix": 1795004214.5197601,
|
||||
"memory_bytes": 5242880,
|
||||
"utilization": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"worker/environment.json": {
|
||||
"sha256": "4009fb617556b981a3c8be4519c4125388d7f45b918954a49cbdfe8ff78f6f18",
|
||||
"data": {
|
||||
"torch": "2.5.1+cu124",
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"allocator_settings": "{'PYTORCH_CUDA_ALLOC_CONF': 'expandable_segments:True', 'max_split_size': -1, 'garbage_collection_threshold': 0.0, 'expandable_segments': True, 'pinned_num_register_threads': 1, 'release_lock_on_cudamalloc': False, 'pinned_use_cuda_host_register': False, 'roundup_power2_divisions': {'1': 0, '2': 0, '4': 0, '8': 0, '16': 0, '32': 0, '64': 0, '128': 0, '256': 0, '512': 0, '1024': 0, '2048': 0, '4096': 0, '8192': 0, '16384': 0, '32768': 0}}",
|
||||
"model_dtype": "FP32",
|
||||
"autocast": false,
|
||||
"grad_scaler": false,
|
||||
"fresh_seed": 0,
|
||||
"pilot_weights_reused": false,
|
||||
"parameter_count": 195077
|
||||
}
|
||||
},
|
||||
"worker/progress.json": {
|
||||
"sha256": "98a090b9a4b8bcd1078687f5ce6e9cc7f75e859fe546bb4a3ab0c20144c5f567",
|
||||
"data": {
|
||||
"fold": 4,
|
||||
"stage": "TRAIN",
|
||||
"epoch": 1,
|
||||
"optimizer_steps": 123,
|
||||
"wall_seconds": 27.234459250001237,
|
||||
"cpu_seconds": 27.20321656,
|
||||
"case": "00001243_96503b6a26c64074ad598927_step_005",
|
||||
"epoch_parts": 123,
|
||||
"epoch_nodes": 3337956,
|
||||
"lr": 0.0001,
|
||||
"loss": 0.00012004363816231489,
|
||||
"gradient_norm_before_clip": 0.014971739612519741,
|
||||
"peak_allocated_bytes": 1461572608,
|
||||
"peak_reserved_bytes": 1516240896
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"status": "ACTUAL_WORKER_TRAINING_OBSERVED",
|
||||
"actual_worker_pids": [
|
||||
1650207
|
||||
],
|
||||
"scientific_result_complete": false
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
{
|
||||
"unix": 1795004174.3436382,
|
||||
"review_sha256": "d66e5ca101348ea8ccd7c167c8428af21eee362c3d74de356be3203fb944870b",
|
||||
"no_existing_track_a_process": true,
|
||||
"existing_track_a": [],
|
||||
"seed1_processes": [
|
||||
{
|
||||
"ppid": 2234,
|
||||
"pid": 1476593,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1476593,
|
||||
"pid": 1476627,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_guard.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"0",
|
||||
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1476593,
|
||||
"pid": 1476628,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_guard.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"1",
|
||||
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1476627,
|
||||
"pid": 1476707,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"0",
|
||||
"498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1476628,
|
||||
"pid": 1476726,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"1",
|
||||
"980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1476593,
|
||||
"pid": 1598086,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_guard.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"2",
|
||||
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ppid": 1598086,
|
||||
"pid": 1598144,
|
||||
"cmdline": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/phase3_seed1_v2_worker.py",
|
||||
"/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18",
|
||||
"2",
|
||||
"e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34"
|
||||
]
|
||||
}
|
||||
],
|
||||
"seed1_source_pins": "PASS",
|
||||
"gpu": {
|
||||
"command": [
|
||||
"nvidia-smi",
|
||||
"--query-gpu=index,uuid,memory.used,utilization.gpu",
|
||||
"--format=csv,noheader,nounits"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stdout": "0, GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 1717, 7\n1, GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 1719, 21\n2, GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76, 9, 0\n3, GPU-8b71b032-1b75-2891-935c-ce106a53034a, 1839, 23\n",
|
||||
"stderr": ""
|
||||
},
|
||||
"gpu_processes": {
|
||||
"command": [
|
||||
"nvidia-smi",
|
||||
"--query-compute-apps=gpu_uuid,pid,process_name,used_memory",
|
||||
"--format=csv,noheader,nounits"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stdout": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 1476707, /home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python, 1708\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 1476726, /home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python, 1710\nGPU-8b71b032-1b75-2891-935c-ce106a53034a, 1598144, /home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python, 1830\n",
|
||||
"stderr": ""
|
||||
},
|
||||
"observations": [
|
||||
{
|
||||
"index": 0,
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"memory_bytes": 1800404992,
|
||||
"utilization": 7,
|
||||
"idle": false
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"memory_bytes": 1802502144,
|
||||
"utilization": 21,
|
||||
"idle": false
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"memory_bytes": 9437184,
|
||||
"utilization": 0,
|
||||
"idle": true
|
||||
},
|
||||
{
|
||||
"index": 3,
|
||||
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"memory_bytes": 1928331264,
|
||||
"utilization": 23,
|
||||
"idle": false
|
||||
}
|
||||
],
|
||||
"idle_folds_snapshot": [
|
||||
4
|
||||
],
|
||||
"memory_available_bytes": 58359533568,
|
||||
"required_available_memory_bytes": 36507222016,
|
||||
"disk_free_bytes": 237470846976,
|
||||
"required_disk_bytes": 107374182400,
|
||||
"load_average": [
|
||||
4.39599609375,
|
||||
4.25830078125,
|
||||
4.39111328125
|
||||
],
|
||||
"logical_cpus": 32,
|
||||
"cpu_percent": 15.6,
|
||||
"later_dispatch": "Pinned queue checks fixed GPU idle; waiting charged per fold; guard requires three polls; no source changes",
|
||||
"scientific_execution": false,
|
||||
"status": "PASS"
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"id": 4,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"pre_dispatch_wait_seconds": 1.9354809819487855,
|
||||
"unix": 1795004207.844153
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"pid": 1650109,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"unix": 1795004207.8453767,
|
||||
"pre_dispatch_wait_seconds": 1.9354809819487855
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"cell": {
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 14437.42601964093,
|
||||
"within_cap": true
|
||||
},
|
||||
"total": {
|
||||
"status": "PASS",
|
||||
"rows": [
|
||||
{
|
||||
"id": 3,
|
||||
"projected_gpu_seconds": 30960,
|
||||
"basis": "continuing_projection_or_pending_cap"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"projected_gpu_seconds": 14437.42601964093,
|
||||
"basis": "continuing_projection_or_pending_cap"
|
||||
}
|
||||
],
|
||||
"new_projected_gpu_seconds": 52597.42601964093,
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"cumulative_projected_gpu_seconds": 100541.37321419391,
|
||||
"postfit_reserve": 7200
|
||||
},
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"unix": 1795004279.4981134
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
[
|
||||
{
|
||||
"unix": 1795004210.4212985,
|
||||
"memory_bytes": 5242880,
|
||||
"utilization": 0
|
||||
},
|
||||
{
|
||||
"unix": 1795004212.47237,
|
||||
"memory_bytes": 5242880,
|
||||
"utilization": 0
|
||||
},
|
||||
{
|
||||
"unix": 1795004214.5197601,
|
||||
"memory_bytes": 5242880,
|
||||
"utilization": 0
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"pid": 1650109,
|
||||
"unix": 1795004210.3711073,
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"pre_dispatch_wait_seconds": 1.9354809819487855,
|
||||
"remaining_gpu_seconds": 30958.06451901805,
|
||||
"fresh_seed": 0
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"torch": "2.5.1+cu124",
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"allocator_settings": "{'PYTORCH_CUDA_ALLOC_CONF': 'expandable_segments:True', 'max_split_size': -1, 'garbage_collection_threshold': 0.0, 'expandable_segments': True, 'pinned_num_register_threads': 1, 'release_lock_on_cudamalloc': False, 'pinned_use_cuda_host_register': False, 'roundup_power2_divisions': {'1': 0, '2': 0, '4': 0, '8': 0, '16': 0, '32': 0, '64': 0, '128': 0, '256': 0, '512': 0, '1024': 0, '2048': 0, '4096': 0, '8192': 0, '16384': 0, '32768': 0}}",
|
||||
"model_dtype": "FP32",
|
||||
"autocast": false,
|
||||
"grad_scaler": false,
|
||||
"fresh_seed": 0,
|
||||
"pilot_weights_reused": false,
|
||||
"parameter_count": 195077
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"unix": 1795004202.9514024,
|
||||
"unit": "meltmind-track-a-recovery-20261118.service",
|
||||
"command": [
|
||||
"systemd-run",
|
||||
"--user",
|
||||
"--unit=meltmind-track-a-recovery-20261118.service",
|
||||
"--property=Restart=no",
|
||||
"--property=KillMode=control-group",
|
||||
"--working-directory=/home/frank14f/optane/MeltMind",
|
||||
"--setenv=OPENBLAS_NUM_THREADS=1",
|
||||
"--setenv=OMP_NUM_THREADS=1",
|
||||
"--setenv=MKL_NUM_THREADS=1",
|
||||
"--setenv=PYTHONPATH=/home/frank14f/optane/MeltMind/tools",
|
||||
"--property=StandardOutput=append:/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/service.log",
|
||||
"--property=StandardError=append:/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/service.log",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18"
|
||||
],
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"review_sha256": "d66e5ca101348ea8ccd7c167c8428af21eee362c3d74de356be3203fb944870b",
|
||||
"no_retry": true,
|
||||
"claude_ruling": "1795003292.974599",
|
||||
"user_confirmation_relay": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-launch-approval-2026-11-18/user-confirmation-relay.json",
|
||||
"user_confirmation_relay_sha256": "97c6bfcff06a66816c66be4666426c30d41faa0213ec2025ff02e8585790bbff",
|
||||
"prelaunch_service_check": {
|
||||
"stdout": "MainPID=0\nLoadState=not-found\nActiveState=inactive\n",
|
||||
"stderr": "",
|
||||
"returncode": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"proof_sha256": "87d4a56bb8a748ce0f823a49a9355d05af617b60ed413685b8663bc259accafc"
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"commit": "fac3543c0001f6cb4786e6fb1f5f36294da403f2",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/reused-models.json": "ce371fa5b1eb559c8edfe31fc1f064a1b253f0186c048c14f066acc26437159f",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt": "767c186bdd0c99d9bcd195cadd5e7b15374c4c07bcb53aa03c157219c74b326d",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt": "9b1356403017f5c351729ce3d33494c48033b0addff4b7b9ca6c1aad6bf90b93",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt": "0f71453359e8c6df759434b00759b5721e0cd3131cfb474a17a3b47f0c3c2e18",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json": "6fab06dd34d30d10409198bc177627293d4d8da559be3b33585c410964e6a15c",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json": "6734f28dc7ca6ffd3a39a43ad2964ffb747c77e55d24d0705c3b5b41f56f397b",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json": "e1f6c14617ed019379998142d49bf9359474db41e1614e2b4e20834ccc37b09d"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "home-runs/track-a-recovery-2026-11-18/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/reused-models.json": "home-runs/track-a-recovery-2026-11-18/reused-models.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"job": 0,
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt",
|
||||
"sha256": "767c186bdd0c99d9bcd195cadd5e7b15374c4c07bcb53aa03c157219c74b326d",
|
||||
"standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json",
|
||||
"standardization_sha256": "6fab06dd34d30d10409198bc177627293d4d8da559be3b33585c410964e6a15c",
|
||||
"origin": "ORIGINAL_HASH_REUSE",
|
||||
"training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"guard_complete_sha256": "024bb110576da7c351ad8e98227cace2f40f4247a8f349c01ea4ebfde954891c"
|
||||
},
|
||||
{
|
||||
"job": 1,
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt",
|
||||
"sha256": "9b1356403017f5c351729ce3d33494c48033b0addff4b7b9ca6c1aad6bf90b93",
|
||||
"standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json",
|
||||
"standardization_sha256": "6734f28dc7ca6ffd3a39a43ad2964ffb747c77e55d24d0705c3b5b41f56f397b",
|
||||
"origin": "ORIGINAL_HASH_REUSE",
|
||||
"training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"guard_complete_sha256": "9259261f44a5813451e9f6f7878fd69d65db6e509889f7c78adcb6f417e00b3d"
|
||||
},
|
||||
{
|
||||
"job": 2,
|
||||
"path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt",
|
||||
"sha256": "0f71453359e8c6df759434b00759b5721e0cd3131cfb474a17a3b47f0c3c2e18",
|
||||
"standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json",
|
||||
"standardization_sha256": "e1f6c14617ed019379998142d49bf9359474db41e1614e2b4e20834ccc37b09d",
|
||||
"origin": "ORIGINAL_HASH_REUSE",
|
||||
"training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7",
|
||||
"guard_complete_sha256": "b60b2575b9efdbb28eabcc2682dc04ce3519c9e7b2f6954cc4dcea0e29b1ab2f"
|
||||
}
|
||||
],
|
||||
"old_gpu_seconds_once": 47943.94719455298
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"unit": "meltmind-track-a-recovery-20261118.service",
|
||||
"returncode": 0,
|
||||
"stdout": "",
|
||||
"stderr": "Running as unit: meltmind-track-a-recovery-20261118.service\n",
|
||||
"unix": 1795004202.9873908,
|
||||
"submitted_only": true
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"pid": 1650006,
|
||||
"unix": 1795004206.9920263,
|
||||
"new_fits": 2,
|
||||
"reused_folds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"fresh_folds": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"old_gpu_seconds_once": 47943.94719455298,
|
||||
"new_train_cap": 106856.05280544702,
|
||||
"cumulative_gpu_cap": 162000,
|
||||
"test_targets_read": false,
|
||||
"postfit_started": false
|
||||
}
|
||||
Reference in new issue
Block a user