diff --git a/artifacts/track-a-recovery-launch-approval-2026-11-18/approval-resolution.json b/artifacts/track-a-recovery-launch-approval-2026-11-18/approval-resolution.json new file mode 100644 index 000000000..9f2a88a15 --- /dev/null +++ b/artifacts/track-a-recovery-launch-approval-2026-11-18/approval-resolution.json @@ -0,0 +1,18 @@ +{ + "unix": 1795004292.8713422, + "status": "SAME_ACTION_REVIEW_ALLOWED_AND_WORKER_OBSERVED", + "previous_block": "approval-block.json", + "authorization_relay": "user-confirmation-relay.json", + "review_provenance": "Relayed human question and assent submitted transparently to the same automatic approval workflow; original user item not independently returned by history tool.", + "service_launch": { + "unit": "meltmind-track-a-recovery-20261118.service", + "returncode": 0, + "stdout": "", + "stderr": "Running as unit: meltmind-track-a-recovery-20261118.service\n", + "unix": 1795004202.9873908, + "submitted_only": true + }, + "actual_start_audit": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/actual-start-audit.json", + "original_rejection_preserved": true, + "alternate_execution_route_used": false +} diff --git a/artifacts/track-a-recovery-launch-approval-2026-11-18/user-confirmation-relay.json b/artifacts/track-a-recovery-launch-approval-2026-11-18/user-confirmation-relay.json new file mode 100644 index 000000000..522850753 --- /dev/null +++ b/artifacts/track-a-recovery-launch-approval-2026-11-18/user-confirmation-relay.json @@ -0,0 +1,24 @@ +{ + "received_unix": 1795004170.198358, + "source": "native codex_delegation", + "source_thread_id": "6afb1249-81d4-8191-b190-6fa32c3880de", + "question": "是否允许在 seed 1 继续运行时利用空闲 GPU 一次性恢复 Track A,复用三折、重训中断两折、总预算 162000 GPU 秒含已有成本,不再自动追加恢复。", + "answer": "我同意", + "attribution": "Yanqi, relayed by TRON", + "verification": "Exact question and reply supplied in incoming native relay. read_thread for source returned in-progress turn with no items; direct original user message was not independently retrieved. Submit this provenance to the same automatic approval reviewer; do not bypass any renewed rejection.", + "scope": { + "reused_folds": [ + 0, + 1, + 2 + ], + "fresh_folds": [ + 3, + 4 + ], + "total_gpu_seconds": 162000, + "old_gpu_seconds_once": 47943.94719455298, + "seed1_unchanged": true, + "no_further_recovery": true + } +} diff --git a/artifacts/track-a-recovery-start-evidence-2026-11-18/first-epoch-audit.json b/artifacts/track-a-recovery-start-evidence-2026-11-18/first-epoch-audit.json new file mode 100644 index 000000000..0d28f3aa9 --- /dev/null +++ b/artifacts/track-a-recovery-start-evidence-2026-11-18/first-epoch-audit.json @@ -0,0 +1,55 @@ +{ + "unix": 1795004330.5901194, + "status": "FIRST_DISPATCHED_FOLD_COST_PASS", + "fold": 4, + "first_epoch_seconds": 57.02044904301874, + "own_projection": 14437.42601964093, + "cumulative_projection": 100541.37321419391, + "original_old_cost_once": 47943.94719455298, + "postfit_reserve": 7200, + "pending_fold3_unmeasured": true, + "worker_progress": { + "fold": 4, + "stage": "TRAIN", + "epoch": 2, + "optimizer_steps": 600, + "wall_seconds": 112.14072392100934, + "cpu_seconds": 111.596696969, + "case": "00000787_a25fb37392a44447b9684f16_step_012", + "epoch_parts": 290, + "epoch_nodes": 8188943, + "lr": 0.00019999999999999998, + "loss": 0.0017632129602134228, + "gradient_norm_before_clip": 0.06885933130979538, + "peak_allocated_bytes": 1461572608, + "peak_reserved_bytes": 1516240896 + }, + "files": [ + { + "source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/first-epoch-projection.json", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/first-epoch-projection.json", + "sha256": "e524294579063ab269f9996ee6025dad198372b0eeb3d642f4a838e279e99b56" + }, + { + "source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/history/001.json", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/history/001.json", + "sha256": "9538696224a700851e7868f6d2b36c4512e4b2eedd79725af6b05596c849497e" + }, + { + "source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/4/worker/progress.json", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/progress.json", + "sha256": "bd39f8aa3916e7d7eea424c35b5581312abd7ac67231ca70325f4002b3092f82" + }, + { + "source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/live-progress.json", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/live-progress.json", + "sha256": "b577571afa3aab4c0b83dc4c79a3ea6ca7be8aa00f49f65f0840b4f89b9d742e" + }, + { + "source": "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/jobs/3/dispatch-idle.json", + "snapshot": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/3/dispatch-idle.json", + "sha256": "d0e38b8d59b1dffcc02640e3d506e4ced0238115c2fdc0d42ab820f7599ec116" + } + ], + "no_relaunch": true +} diff --git a/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/3/dispatch-idle.json b/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/3/dispatch-idle.json new file mode 100644 index 000000000..8e51e3a16 --- /dev/null +++ b/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/3/dispatch-idle.json @@ -0,0 +1,18 @@ +{ + "start_unix": 1795004329.9132068, + "end_unix": 1795004329.9554157, + "valid": true, + "memory_bytes": 1928331264, + "utilization": 20, + "monitor_policy": { + "rule": "three_consecutive_failed_samples_per_operation", + "failure_limit": 3, + "failed": false, + "errors": 0, + "streaks": { + "gpu_usage": 0 + }, + "max_streaks": {} + }, + "charged_gpu_seconds": 184.04915182094555 +} diff --git a/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/first-epoch-projection.json b/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/first-epoch-projection.json new file mode 100644 index 000000000..0f53e2692 --- /dev/null +++ b/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/first-epoch-projection.json @@ -0,0 +1,29 @@ +{ + "status": "PASS", + "cell": { + "id": 4, + "projected_gpu_seconds": 14437.42601964093, + "within_cap": true + }, + "total": { + "status": "PASS", + "rows": [ + { + "id": 3, + "projected_gpu_seconds": 30960, + "basis": "continuing_projection_or_pending_cap" + }, + { + "id": 4, + "projected_gpu_seconds": 14437.42601964093, + "basis": "continuing_projection_or_pending_cap" + } + ], + "new_projected_gpu_seconds": 52597.42601964093, + "old_gpu_seconds_once": 47943.94719455298, + "cumulative_projected_gpu_seconds": 100541.37321419391, + "postfit_reserve": 7200 + }, + "contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "unix": 1795004279.4981134 +} diff --git a/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/history/001.json b/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/history/001.json new file mode 100644 index 000000000..94eb35dff --- /dev/null +++ b/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/history/001.json @@ -0,0 +1,15 @@ +{ + "fold": 4, + "stage": "EPOCH_COMPLETE", + "epoch": 1, + "optimizer_steps": 310, + "wall_seconds": 60.80659728299361, + "cpu_seconds": 60.766106956, + "train_mean_weighted_loss": 0.005705016106361244, + "train_epoch_seconds": 57.02044904301874, + "epoch_parts": 310, + "epoch_nodes": 8642607, + "lr": 0.0001, + "peak_allocated_bytes": 1461572608, + "peak_reserved_bytes": 1516240896 +} diff --git a/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/progress.json b/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/progress.json new file mode 100644 index 000000000..142fb4cad --- /dev/null +++ b/artifacts/track-a-recovery-start-evidence-2026-11-18/jobs/4/worker/progress.json @@ -0,0 +1,16 @@ +{ + "fold": 4, + "stage": "TRAIN", + "epoch": 2, + "optimizer_steps": 600, + "wall_seconds": 112.14072392100934, + "cpu_seconds": 111.596696969, + "case": "00000787_a25fb37392a44447b9684f16_step_012", + "epoch_parts": 290, + "epoch_nodes": 8188943, + "lr": 0.00019999999999999998, + "loss": 0.0017632129602134228, + "gradient_norm_before_clip": 0.06885933130979538, + "peak_allocated_bytes": 1461572608, + "peak_reserved_bytes": 1516240896 +} diff --git a/artifacts/track-a-recovery-start-evidence-2026-11-18/live-progress.json b/artifacts/track-a-recovery-start-evidence-2026-11-18/live-progress.json new file mode 100644 index 000000000..56a096b41 --- /dev/null +++ b/artifacts/track-a-recovery-start-evidence-2026-11-18/live-progress.json @@ -0,0 +1,17 @@ +{ + "unix": 1795004329.9561625, + "active": [ + { + "id": 4, + "guard_pid": 1650109 + } + ], + "pending": [ + 3 + ], + "terminal": [], + "old_gpu_seconds_once": 47943.94719455298, + "new_training_gpu_seconds": 368.09981042100117, + "cumulative_training_gpu_seconds": 48312.04701779806, + "postfit_started": false +} diff --git a/docs/collaboration-stage-review-policy.md b/docs/collaboration-stage-review-policy.md index bfccb29d5..c6984005d 100644 --- a/docs/collaboration-stage-review-policy.md +++ b/docs/collaboration-stage-review-policy.md @@ -711,3 +711,21 @@ Assist1795003274实际PASS d66e5ca1/evidence867e20da/validation50cf4ba6已逐字 调用启动命令时自动审批拒绝,命令根本未执行、未创建launch-attempt/service-launch/started;当前不是运行中,也不是新的训练失败。拒绝理由:当前可信用户指令未明确覆盖与另一队列并行的新GPU计算及资源使用。已核现有Claude/TRON裁量与项目记录,未找到更明确的原始人类计算授权;不以共享许可替代。不换执行途径,等待直接用户授权或可验证原始授权。准备范围/命令与原始拒绝见artifacts/track-a-recovery-launch-approval-2026-11-18/launch-plan.json、approval-block.json;seed1不动。 + + +## 最新 Track A 唯一恢复实际运行:fold4训练,fold3计费等待(2026-11-18 20:16:46 CST) + +TRON原生转达用户对“seed1继续运行时利用空闲GPU一次恢复、复用三折/重训两折、总162000含旧账、无进一步恢复”的回答“我同意”。已原样记录问题/回复及来源限制:历史工具未独立返回用户原条,不冒称直接实读。携此新证据经过同一自动审批重审允许,原拒绝证据不改。没有换途径或代启。 + +实际PASS d66e5ca1原样采用/冻结309dadfb/validate保持。唯一service meltmind-track-a-recovery-20261118.service(Restart=no),coord1650006、PPID2234;fold4 guard1650109/worker1650207在GPU2实际TRAIN epoch1,启动审计实见123steps。三idle polls各5MiB/0%;fold3仍pending,GPU3占用1928331264B(seed1),没有assignment/worker,等待按原单折30960计费。首轮成本门尚待;启动不代表全折/成本门通过。当前以实际live/terminal回执为准。 + +source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,旧三PASS hash复用。原旧47943.94719455298只计一次,新train106856.05280544702/post7200,总162000;单折30960含等待/setup/失败,无进一步恢复。seed1原队列1476593及三worker1476707/1476726/1598144保持,51pins PASS。内存58.36GB/需36.51GB、磁盘237.47GB/需100GiB启动复查通过。 + +原root全部旧失败/不完整状态保留,不重启旧服务。不复制新队列、不改固定源码/预算。全五成功冻结/退出后才做identity-only postfit新绑定及Assist窄delta,再读test;若恢复折再失败,只报告完成折。实际审计root/actual-start-audit.json、started、host-precheck-user-confirmed、launch-attempt/service-launch及artifacts/track-a-recovery-launch-approval-2026-11-18。下方“启动拦截/等待用户”均为历史。 + + + +## 最新恢复fold4首轮成本门PASS,fold3仍待派(2026-11-18) + +原1650006/1650109/1650207队列和worker,fold4首轮57.020449043秒,单折投影14437.426020≤30960;pending fold3仍保留原30960整帽,含post7200及old47943.947195一次,累计投影100541.373214≤162000。fold4已真实进入epoch2;fold3尚未实测,不称全两折成本PASS。快照/首轮原history/门在artifacts/track-a-recovery-start-evidence-2026-11-18。无重复启动/改源/新计算范围,当前状态看live/terminal。 + diff --git a/docs/phase2-live-handoff-2026-11-17.md b/docs/phase2-live-handoff-2026-11-17.md index 70208f259..552c67380 100644 --- a/docs/phase2-live-handoff-2026-11-17.md +++ b/docs/phase2-live-handoff-2026-11-17.md @@ -1,3 +1,17 @@ +## 最新恢复fold4首轮成本门PASS,fold3仍待派(2026-11-18) + +原1650006/1650109/1650207队列和worker,fold4首轮57.020449043秒,单折投影14437.426020≤30960;pending fold3仍保留原30960整帽,含post7200及old47943.947195一次,累计投影100541.373214≤162000。fold4已真实进入epoch2;fold3尚未实测,不称全两折成本PASS。快照/首轮原history/门在artifacts/track-a-recovery-start-evidence-2026-11-18。无重复启动/改源/新计算范围,当前状态看live/terminal。 + +## 最新 Track A 唯一恢复实际运行:fold4训练,fold3计费等待(2026-11-18 20:16:46 CST) + +TRON原生转达用户对“seed1继续运行时利用空闲GPU一次恢复、复用三折/重训两折、总162000含旧账、无进一步恢复”的回答“我同意”。已原样记录问题/回复及来源限制:历史工具未独立返回用户原条,不冒称直接实读。携此新证据经过同一自动审批重审允许,原拒绝证据不改。没有换途径或代启。 + +实际PASS d66e5ca1原样采用/冻结309dadfb/validate保持。唯一service meltmind-track-a-recovery-20261118.service(Restart=no),coord1650006、PPID2234;fold4 guard1650109/worker1650207在GPU2实际TRAIN epoch1,启动审计实见123steps。三idle polls各5MiB/0%;fold3仍pending,GPU3占用1928331264B(seed1),没有assignment/worker,等待按原单折30960计费。首轮成本门尚待;启动不代表全折/成本门通过。当前以实际live/terminal回执为准。 + +source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,旧三PASS hash复用。原旧47943.94719455298只计一次,新train106856.05280544702/post7200,总162000;单折30960含等待/setup/失败,无进一步恢复。seed1原队列1476593及三worker1476707/1476726/1598144保持,51pins PASS。内存58.36GB/需36.51GB、磁盘237.47GB/需100GiB启动复查通过。 + +原root全部旧失败/不完整状态保留,不重启旧服务。不复制新队列、不改固定源码/预算。全五成功冻结/退出后才做identity-only postfit新绑定及Assist窄delta,再读test;若恢复折再失败,只报告完成折。实际审计root/actual-start-audit.json、started、host-precheck-user-confirmed、launch-attempt/service-launch及artifacts/track-a-recovery-launch-approval-2026-11-18。下方“启动拦截/等待用户”均为历史。 + ## 最新 Track A recovery 实际PASS已采用;启动被自动审批拦截,未运行(2026-11-18) Assist1795003274实际PASS d66e5ca1/evidence867e20da/validation50cf4ba6已逐字核SHA与五字段、采用并冻结309dadfb;validate_review/config/reuse_identity PASS。implementation8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27map不变。Claude1795003292明确接受并要求启动,原件已实读。 diff --git a/docs/track-a-recovery-launch-2026-11-18.md b/docs/track-a-recovery-launch-2026-11-18.md new file mode 100644 index 000000000..9735258ff --- /dev/null +++ b/docs/track-a-recovery-launch-2026-11-18.md @@ -0,0 +1,16 @@ +## 最新 Track A 唯一恢复实际运行:fold4训练,fold3计费等待(2026-11-18 20:16:46 CST) + +TRON原生转达用户对“seed1继续运行时利用空闲GPU一次恢复、复用三折/重训两折、总162000含旧账、无进一步恢复”的回答“我同意”。已原样记录问题/回复及来源限制:历史工具未独立返回用户原条,不冒称直接实读。携此新证据经过同一自动审批重审允许,原拒绝证据不改。没有换途径或代启。 + +实际PASS d66e5ca1原样采用/冻结309dadfb/validate保持。唯一service meltmind-track-a-recovery-20261118.service(Restart=no),coord1650006、PPID2234;fold4 guard1650109/worker1650207在GPU2实际TRAIN epoch1,启动审计实见123steps。三idle polls各5MiB/0%;fold3仍pending,GPU3占用1928331264B(seed1),没有assignment/worker,等待按原单折30960计费。首轮成本门尚待;启动不代表全折/成本门通过。当前以实际live/terminal回执为准。 + +source8a8ebd1f/contract4b21dedd/proof8baec54f/comparison828530e4/27pins不变,旧三PASS hash复用。原旧47943.94719455298只计一次,新train106856.05280544702/post7200,总162000;单折30960含等待/setup/失败,无进一步恢复。seed1原队列1476593及三worker1476707/1476726/1598144保持,51pins PASS。内存58.36GB/需36.51GB、磁盘237.47GB/需100GiB启动复查通过。 + +原root全部旧失败/不完整状态保留,不重启旧服务。不复制新队列、不改固定源码/预算。全五成功冻结/退出后才做identity-only postfit新绑定及Assist窄delta,再读test;若恢复折再失败,只报告完成折。实际审计root/actual-start-audit.json、started、host-precheck-user-confirmed、launch-attempt/service-launch及artifacts/track-a-recovery-launch-approval-2026-11-18。下方“启动拦截/等待用户”均为历史。 + + + +## 最新恢复fold4首轮成本门PASS,fold3仍待派(2026-11-18) + +原1650006/1650109/1650207队列和worker,fold4首轮57.020449043秒,单折投影14437.426020≤30960;pending fold3仍保留原30960整帽,含post7200及old47943.947195一次,累计投影100541.373214≤162000。fold4已真实进入epoch2;fold3尚未实测,不称全两折成本PASS。快照/首轮原history/门在artifacts/track-a-recovery-start-evidence-2026-11-18。无重复启动/改源/新计算范围,当前状态看live/terminal。 + diff --git a/home-runs/track-a-recovery-2026-11-18/actual-start-audit.json b/home-runs/track-a-recovery-2026-11-18/actual-start-audit.json new file mode 100644 index 000000000..a942a1299 --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/actual-start-audit.json @@ -0,0 +1,312 @@ +{ + "unix": 1795004245.9109008, + "implementation": "8a8ebd1f9f29f73b26cc0cacde27b9d1d8ca333b", + "contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "review_sha256": "d66e5ca101348ea8ccd7c167c8428af21eee362c3d74de356be3203fb944870b", + "service": { + "returncode": 0, + "stdout": "Restart=no\nMainPID=1650006\nExecMainStatus=0\nActiveState=active\nSubState=running\n", + "stderr": "" + }, + "track_a_processes": [ + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py", + "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18" + ], + "create_time": 1795004202.95, + "pid": 1650006, + "ppid": 2234 + }, + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/track_a_recovery_guard.py", + "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18", + "4" + ], + "create_time": 1795004207.83, + "pid": 1650109, + "ppid": 1650006 + }, + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/track_a_recovery_worker.py", + "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18", + "4" + ], + "create_time": 1795004214.51, + "pid": 1650207, + "ppid": 1650109 + } + ], + "seed1_processes": [ + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18" + ], + "create_time": 1794992135.84, + "pid": 1476593, + "ppid": 2234 + }, + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_guard.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "0", + "498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134" + ], + "create_time": 1794992140.13, + "pid": 1476627, + "ppid": 1476593 + }, + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_guard.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "1", + "980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4" + ], + "create_time": 1794992140.14, + "pid": 1476628, + "ppid": 1476593 + }, + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_worker.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "0", + "498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134" + ], + "create_time": 1794992146.66, + "pid": 1476707, + "ppid": 1476627 + }, + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_worker.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "1", + "980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4" + ], + "create_time": 1794992147.88, + "pid": 1476726, + "ppid": 1476628 + }, + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_guard.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "2", + "e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34" + ], + "create_time": 1794999917.04, + "pid": 1598086, + "ppid": 1476593 + }, + { + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_worker.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "2", + "e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34" + ], + "create_time": 1794999924.07, + "pid": 1598144, + "ppid": 1598086 + } + ], + "seed1_source_pins": "PASS", + "original_and_recovery_source_pins": "PASS", + "receipts": { + "started.json": { + "sha256": "db31964ac4f97fd8029f63ce38eb5c591d3270ca7e62979b41dda62f23ca283b", + "data": { + "pid": 1650006, + "unix": 1795004206.9920263, + "new_fits": 2, + "reused_folds": [ + 0, + 1, + 2 + ], + "fresh_folds": [ + 3, + 4 + ], + "old_gpu_seconds_once": 47943.94719455298, + "new_train_cap": 106856.05280544702, + "cumulative_gpu_cap": 162000, + "test_targets_read": false, + "postfit_started": false + } + }, + "live-progress.json": { + "sha256": "b1697659ed723860b20dd4d3bd5f55c9693e7337906a20a19a56de636d6239fe", + "data": { + "unix": 1795004245.3020048, + "active": [ + { + "id": 4, + "guard_pid": 1650109 + } + ], + "pending": [ + 3 + ], + "terminal": [], + "old_gpu_seconds_once": 47943.94719455298, + "new_training_gpu_seconds": 198.791494303965, + "cumulative_training_gpu_seconds": 48142.73869843397, + "postfit_started": false + } + } + }, + "folds": { + "3": { + "dispatch-idle.json": { + "sha256": "38f390cc75a8134e9325508596a63359dd54d07df21775eae8926514b7c961a7", + "data": { + "start_unix": 1795004245.250895, + "end_unix": 1795004245.301113, + "valid": true, + "memory_bytes": 1928331264, + "utilization": 0, + "monitor_policy": { + "rule": "three_consecutive_failed_samples_per_operation", + "failure_limit": 3, + "failed": false, + "errors": 0, + "streaks": { + "gpu_usage": 0 + }, + "max_streaks": {} + }, + "charged_gpu_seconds": 99.39484800293576 + } + } + }, + "4": { + "dispatch-idle.json": { + "sha256": "1d57b0e8f92ed643e1d176c638d55550bcb10ffb010f17f6f4668f31c9f1ecf4", + "data": { + "start_unix": 1795004207.795226, + "end_unix": 1795004207.8412235, + "valid": true, + "memory_bytes": 5242880, + "utilization": 0, + "monitor_policy": { + "rule": "three_consecutive_failed_samples_per_operation", + "failure_limit": 3, + "failed": false, + "errors": 0, + "streaks": { + "gpu_usage": 0 + }, + "max_streaks": {} + }, + "charged_gpu_seconds": 61.93496167601552 + } + }, + "assignment.json": { + "sha256": "03a7d5a0353fe1ad530e1e82facad8be8bdecc3a7376741e89fa37d897baf18b", + "data": { + "id": 4, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "pre_dispatch_wait_seconds": 1.9354809819487855, + "unix": 1795004207.844153 + } + }, + "dispatch.json": { + "sha256": "1cb1902c5e200498512c441039acab4bfabe15a767c87a89d79d06e2805c9ab9", + "data": { + "pid": 1650109, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "unix": 1795004207.8453767, + "pre_dispatch_wait_seconds": 1.9354809819487855 + } + }, + "started.json": { + "sha256": "9e82d5bbb0a062d4251d2ebe995e34cbd3ee3630eb66741cdfde3953c8ae7d05", + "data": { + "pid": 1650109, + "unix": 1795004210.3711073, + "contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "pre_dispatch_wait_seconds": 1.9354809819487855, + "remaining_gpu_seconds": 30958.06451901805, + "fresh_seed": 0 + } + }, + "idle-polls.json": { + "sha256": "80c6fd271989b8206ebea7beaf41abde2226f4084b40502fc284eb1127cbf1ff", + "data": [ + { + "unix": 1795004210.4212985, + "memory_bytes": 5242880, + "utilization": 0 + }, + { + "unix": 1795004212.47237, + "memory_bytes": 5242880, + "utilization": 0 + }, + { + "unix": 1795004214.5197601, + "memory_bytes": 5242880, + "utilization": 0 + } + ] + }, + "worker/environment.json": { + "sha256": "4009fb617556b981a3c8be4519c4125388d7f45b918954a49cbdfe8ff78f6f18", + "data": { + "torch": "2.5.1+cu124", + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "allocator_settings": "{'PYTORCH_CUDA_ALLOC_CONF': 'expandable_segments:True', 'max_split_size': -1, 'garbage_collection_threshold': 0.0, 'expandable_segments': True, 'pinned_num_register_threads': 1, 'release_lock_on_cudamalloc': False, 'pinned_use_cuda_host_register': False, 'roundup_power2_divisions': {'1': 0, '2': 0, '4': 0, '8': 0, '16': 0, '32': 0, '64': 0, '128': 0, '256': 0, '512': 0, '1024': 0, '2048': 0, '4096': 0, '8192': 0, '16384': 0, '32768': 0}}", + "model_dtype": "FP32", + "autocast": false, + "grad_scaler": false, + "fresh_seed": 0, + "pilot_weights_reused": false, + "parameter_count": 195077 + } + }, + "worker/progress.json": { + "sha256": "98a090b9a4b8bcd1078687f5ce6e9cc7f75e859fe546bb4a3ab0c20144c5f567", + "data": { + "fold": 4, + "stage": "TRAIN", + "epoch": 1, + "optimizer_steps": 123, + "wall_seconds": 27.234459250001237, + "cpu_seconds": 27.20321656, + "case": "00001243_96503b6a26c64074ad598927_step_005", + "epoch_parts": 123, + "epoch_nodes": 3337956, + "lr": 0.0001, + "loss": 0.00012004363816231489, + "gradient_norm_before_clip": 0.014971739612519741, + "peak_allocated_bytes": 1461572608, + "peak_reserved_bytes": 1516240896 + } + } + } + }, + "status": "ACTUAL_WORKER_TRAINING_OBSERVED", + "actual_worker_pids": [ + 1650207 + ], + "scientific_result_complete": false +} diff --git a/home-runs/track-a-recovery-2026-11-18/host-precheck-user-confirmed.json b/home-runs/track-a-recovery-2026-11-18/host-precheck-user-confirmed.json new file mode 100644 index 000000000..cc067c529 --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/host-precheck-user-confirmed.json @@ -0,0 +1,151 @@ +{ + "unix": 1795004174.3436382, + "review_sha256": "d66e5ca101348ea8ccd7c167c8428af21eee362c3d74de356be3203fb944870b", + "no_existing_track_a_process": true, + "existing_track_a": [], + "seed1_processes": [ + { + "ppid": 2234, + "pid": 1476593, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_queue.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18" + ] + }, + { + "ppid": 1476593, + "pid": 1476627, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_guard.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "0", + "498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134" + ] + }, + { + "ppid": 1476593, + "pid": 1476628, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_guard.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "1", + "980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4" + ] + }, + { + "ppid": 1476627, + "pid": 1476707, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_worker.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "0", + "498011b74b369fb6c4144659d9ea7449863205446329dc36b08e60a78e0d8134" + ] + }, + { + "ppid": 1476628, + "pid": 1476726, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_worker.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "1", + "980c49414d8a7b27c63ddb4776606a39fb1654c2d3841bf658cad3f101142dc4" + ] + }, + { + "ppid": 1476593, + "pid": 1598086, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_guard.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "2", + "e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34" + ] + }, + { + "ppid": 1598086, + "pid": 1598144, + "cmdline": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/phase3_seed1_v2_worker.py", + "/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18", + "2", + "e0e4f94ba36fc799534a4d16bbf88edae4e4d647babb188824d197deef1bec34" + ] + } + ], + "seed1_source_pins": "PASS", + "gpu": { + "command": [ + "nvidia-smi", + "--query-gpu=index,uuid,memory.used,utilization.gpu", + "--format=csv,noheader,nounits" + ], + "returncode": 0, + "stdout": "0, GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 1717, 7\n1, GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 1719, 21\n2, GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76, 9, 0\n3, GPU-8b71b032-1b75-2891-935c-ce106a53034a, 1839, 23\n", + "stderr": "" + }, + "gpu_processes": { + "command": [ + "nvidia-smi", + "--query-compute-apps=gpu_uuid,pid,process_name,used_memory", + "--format=csv,noheader,nounits" + ], + "returncode": 0, + "stdout": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 1476707, /home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python, 1708\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 1476726, /home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python, 1710\nGPU-8b71b032-1b75-2891-935c-ce106a53034a, 1598144, /home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python, 1830\n", + "stderr": "" + }, + "observations": [ + { + "index": 0, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "memory_bytes": 1800404992, + "utilization": 7, + "idle": false + }, + { + "index": 1, + "gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36", + "memory_bytes": 1802502144, + "utilization": 21, + "idle": false + }, + { + "index": 2, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "memory_bytes": 9437184, + "utilization": 0, + "idle": true + }, + { + "index": 3, + "gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "memory_bytes": 1928331264, + "utilization": 23, + "idle": false + } + ], + "idle_folds_snapshot": [ + 4 + ], + "memory_available_bytes": 58359533568, + "required_available_memory_bytes": 36507222016, + "disk_free_bytes": 237470846976, + "required_disk_bytes": 107374182400, + "load_average": [ + 4.39599609375, + 4.25830078125, + 4.39111328125 + ], + "logical_cpus": 32, + "cpu_percent": 15.6, + "later_dispatch": "Pinned queue checks fixed GPU idle; waiting charged per fold; guard requires three polls; no source changes", + "scientific_execution": false, + "status": "PASS" +} diff --git a/home-runs/track-a-recovery-2026-11-18/jobs/4/assignment.json b/home-runs/track-a-recovery-2026-11-18/jobs/4/assignment.json new file mode 100644 index 000000000..bc414b6f8 --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/jobs/4/assignment.json @@ -0,0 +1,7 @@ +{ + "id": 4, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "pre_dispatch_wait_seconds": 1.9354809819487855, + "unix": 1795004207.844153 +} diff --git a/home-runs/track-a-recovery-2026-11-18/jobs/4/dispatch.json b/home-runs/track-a-recovery-2026-11-18/jobs/4/dispatch.json new file mode 100644 index 000000000..1a88ef90b --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/jobs/4/dispatch.json @@ -0,0 +1,6 @@ +{ + "pid": 1650109, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "unix": 1795004207.8453767, + "pre_dispatch_wait_seconds": 1.9354809819487855 +} diff --git a/home-runs/track-a-recovery-2026-11-18/jobs/4/first-epoch-projection.json b/home-runs/track-a-recovery-2026-11-18/jobs/4/first-epoch-projection.json new file mode 100644 index 000000000..0f53e2692 --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/jobs/4/first-epoch-projection.json @@ -0,0 +1,29 @@ +{ + "status": "PASS", + "cell": { + "id": 4, + "projected_gpu_seconds": 14437.42601964093, + "within_cap": true + }, + "total": { + "status": "PASS", + "rows": [ + { + "id": 3, + "projected_gpu_seconds": 30960, + "basis": "continuing_projection_or_pending_cap" + }, + { + "id": 4, + "projected_gpu_seconds": 14437.42601964093, + "basis": "continuing_projection_or_pending_cap" + } + ], + "new_projected_gpu_seconds": 52597.42601964093, + "old_gpu_seconds_once": 47943.94719455298, + "cumulative_projected_gpu_seconds": 100541.37321419391, + "postfit_reserve": 7200 + }, + "contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "unix": 1795004279.4981134 +} diff --git a/home-runs/track-a-recovery-2026-11-18/jobs/4/idle-polls.json b/home-runs/track-a-recovery-2026-11-18/jobs/4/idle-polls.json new file mode 100644 index 000000000..cc3bb84e8 --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/jobs/4/idle-polls.json @@ -0,0 +1,17 @@ +[ + { + "unix": 1795004210.4212985, + "memory_bytes": 5242880, + "utilization": 0 + }, + { + "unix": 1795004212.47237, + "memory_bytes": 5242880, + "utilization": 0 + }, + { + "unix": 1795004214.5197601, + "memory_bytes": 5242880, + "utilization": 0 + } +] diff --git a/home-runs/track-a-recovery-2026-11-18/jobs/4/started.json b/home-runs/track-a-recovery-2026-11-18/jobs/4/started.json new file mode 100644 index 000000000..a7512da74 --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/jobs/4/started.json @@ -0,0 +1,9 @@ +{ + "pid": 1650109, + "unix": 1795004210.3711073, + "contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "pre_dispatch_wait_seconds": 1.9354809819487855, + "remaining_gpu_seconds": 30958.06451901805, + "fresh_seed": 0 +} diff --git a/home-runs/track-a-recovery-2026-11-18/jobs/4/worker/environment.json b/home-runs/track-a-recovery-2026-11-18/jobs/4/worker/environment.json new file mode 100644 index 000000000..68f713d67 --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/jobs/4/worker/environment.json @@ -0,0 +1,11 @@ +{ + "torch": "2.5.1+cu124", + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "allocator_settings": "{'PYTORCH_CUDA_ALLOC_CONF': 'expandable_segments:True', 'max_split_size': -1, 'garbage_collection_threshold': 0.0, 'expandable_segments': True, 'pinned_num_register_threads': 1, 'release_lock_on_cudamalloc': False, 'pinned_use_cuda_host_register': False, 'roundup_power2_divisions': {'1': 0, '2': 0, '4': 0, '8': 0, '16': 0, '32': 0, '64': 0, '128': 0, '256': 0, '512': 0, '1024': 0, '2048': 0, '4096': 0, '8192': 0, '16384': 0, '32768': 0}}", + "model_dtype": "FP32", + "autocast": false, + "grad_scaler": false, + "fresh_seed": 0, + "pilot_weights_reused": false, + "parameter_count": 195077 +} diff --git a/home-runs/track-a-recovery-2026-11-18/launch-attempt.json b/home-runs/track-a-recovery-2026-11-18/launch-attempt.json new file mode 100644 index 000000000..15de89a6b --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/launch-attempt.json @@ -0,0 +1,32 @@ +{ + "unix": 1795004202.9514024, + "unit": "meltmind-track-a-recovery-20261118.service", + "command": [ + "systemd-run", + "--user", + "--unit=meltmind-track-a-recovery-20261118.service", + "--property=Restart=no", + "--property=KillMode=control-group", + "--working-directory=/home/frank14f/optane/MeltMind", + "--setenv=OPENBLAS_NUM_THREADS=1", + "--setenv=OMP_NUM_THREADS=1", + "--setenv=MKL_NUM_THREADS=1", + "--setenv=PYTHONPATH=/home/frank14f/optane/MeltMind/tools", + "--property=StandardOutput=append:/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/service.log", + "--property=StandardError=append:/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/service.log", + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/track_a_recovery_queue.py", + "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18" + ], + "contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "review_sha256": "d66e5ca101348ea8ccd7c167c8428af21eee362c3d74de356be3203fb944870b", + "no_retry": true, + "claude_ruling": "1795003292.974599", + "user_confirmation_relay": "/home/frank14f/optane/MeltMind/artifacts/track-a-recovery-launch-approval-2026-11-18/user-confirmation-relay.json", + "user_confirmation_relay_sha256": "97c6bfcff06a66816c66be4666426c30d41faa0213ec2025ff02e8585790bbff", + "prelaunch_service_check": { + "stdout": "MainPID=0\nLoadState=not-found\nActiveState=inactive\n", + "stderr": "", + "returncode": 0 + } +} diff --git a/home-runs/track-a-recovery-2026-11-18/reuse-authorization.json b/home-runs/track-a-recovery-2026-11-18/reuse-authorization.json new file mode 100644 index 000000000..b916e7c1e --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/reuse-authorization.json @@ -0,0 +1,4 @@ +{ + "contract_sha256": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "proof_sha256": "87d4a56bb8a748ce0f823a49a9355d05af617b60ed413685b8663bc259accafc" +} diff --git a/home-runs/track-a-recovery-2026-11-18/reuse-git-freeze.json b/home-runs/track-a-recovery-2026-11-18/reuse-git-freeze.json new file mode 100644 index 000000000..3b5aee260 --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/reuse-git-freeze.json @@ -0,0 +1,25 @@ +{ + "commit": "fac3543c0001f6cb4786e6fb1f5f36294da403f2", + "files": { + "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "4b21dedde07534bbe059e9886d7d5fbd7f6d3c33a5b08f4564c9bb360ef511aa", + "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/reused-models.json": "ce371fa5b1eb559c8edfe31fc1f064a1b253f0186c048c14f066acc26437159f", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt": "767c186bdd0c99d9bcd195cadd5e7b15374c4c07bcb53aa03c157219c74b326d", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt": "9b1356403017f5c351729ce3d33494c48033b0addff4b7b9ca6c1aad6bf90b93", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt": "0f71453359e8c6df759434b00759b5721e0cd3131cfb474a17a3b47f0c3c2e18", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json": "6fab06dd34d30d10409198bc177627293d4d8da559be3b33585c410964e6a15c", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json": "6734f28dc7ca6ffd3a39a43ad2964ffb747c77e55d24d0705c3b5b41f56f397b", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json": "e1f6c14617ed019379998142d49bf9359474db41e1614e2b4e20834ccc37b09d" + }, + "git_paths": { + "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/contract.json": "home-runs/track-a-recovery-2026-11-18/contract.json", + "/home/frank14f/MeltMind-runs/track-a-recovery-2026-11-18/reused-models.json": "home-runs/track-a-recovery-2026-11-18/reused-models.json", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt": "home-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json", + "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json": "home-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json" + }, + "remote_published": false, + "local_repository": "/tmp/meltmind-phase2-local-fixed-20261117" +} diff --git a/home-runs/track-a-recovery-2026-11-18/reused-models.json b/home-runs/track-a-recovery-2026-11-18/reused-models.json new file mode 100644 index 000000000..bafd576d8 --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/reused-models.json @@ -0,0 +1,35 @@ +{ + "models": [ + { + "job": 0, + "path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/last-epoch200.pt", + "sha256": "767c186bdd0c99d9bcd195cadd5e7b15374c4c07bcb53aa03c157219c74b326d", + "standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/0/worker/standardization.json", + "standardization_sha256": "6fab06dd34d30d10409198bc177627293d4d8da559be3b33585c410964e6a15c", + "origin": "ORIGINAL_HASH_REUSE", + "training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7", + "guard_complete_sha256": "024bb110576da7c351ad8e98227cace2f40f4247a8f349c01ea4ebfde954891c" + }, + { + "job": 1, + "path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/last-epoch200.pt", + "sha256": "9b1356403017f5c351729ce3d33494c48033b0addff4b7b9ca6c1aad6bf90b93", + "standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/1/worker/standardization.json", + "standardization_sha256": "6734f28dc7ca6ffd3a39a43ad2964ffb747c77e55d24d0705c3b5b41f56f397b", + "origin": "ORIGINAL_HASH_REUSE", + "training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7", + "guard_complete_sha256": "9259261f44a5813451e9f6f7878fd69d65db6e509889f7c78adcb6f417e00b3d" + }, + { + "job": 2, + "path": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/last-epoch200.pt", + "sha256": "0f71453359e8c6df759434b00759b5721e0cd3131cfb474a17a3b47f0c3c2e18", + "standardization": "/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18/jobs/2/worker/standardization.json", + "standardization_sha256": "e1f6c14617ed019379998142d49bf9359474db41e1614e2b4e20834ccc37b09d", + "origin": "ORIGINAL_HASH_REUSE", + "training_contract_sha256": "be4ac1f6d5a5d6c0183aa50d2e046b22d7e5fa77c7714fef7eab6d616e68bee7", + "guard_complete_sha256": "b60b2575b9efdbb28eabcc2682dc04ce3519c9e7b2f6954cc4dcea0e29b1ab2f" + } + ], + "old_gpu_seconds_once": 47943.94719455298 +} diff --git a/home-runs/track-a-recovery-2026-11-18/service-launch.json b/home-runs/track-a-recovery-2026-11-18/service-launch.json new file mode 100644 index 000000000..a1720dc3f --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/service-launch.json @@ -0,0 +1,8 @@ +{ + "unit": "meltmind-track-a-recovery-20261118.service", + "returncode": 0, + "stdout": "", + "stderr": "Running as unit: meltmind-track-a-recovery-20261118.service\n", + "unix": 1795004202.9873908, + "submitted_only": true +} diff --git a/home-runs/track-a-recovery-2026-11-18/started.json b/home-runs/track-a-recovery-2026-11-18/started.json new file mode 100644 index 000000000..f65695aed --- /dev/null +++ b/home-runs/track-a-recovery-2026-11-18/started.json @@ -0,0 +1,19 @@ +{ + "pid": 1650006, + "unix": 1795004206.9920263, + "new_fits": 2, + "reused_folds": [ + 0, + 1, + 2 + ], + "fresh_folds": [ + 3, + 4 + ], + "old_gpu_seconds_once": 47943.94719455298, + "new_train_cap": 106856.05280544702, + "cumulative_gpu_cap": 162000, + "test_targets_read": false, + "postfit_started": false +}