Record live recovery launch and retire stale preparation handoff
This commit is contained in:
1 parent
a5c6c0444e
commit
1fa41c8e90
8 files changed
+154
No files matched your search
@@ -12,3 +12,5 @@
|
||||
- Post-witness update (2026-11-14 21:12) supersedes treating the preceding E1-only scope as the entire remaining project scope. Follow the latest section of `docs/collaboration-stage-review-policy.md` and Claude Slack `1794661944.308269`: custody join, label-free copy calibration, k=3 per-route candidate count, bounded registration (up to 4000 pairs / 100 worker CPU-h / 16 workers), band review, component freeze, then coordinated S2 continuation. No repeated approval gates or reruns of completed review; preserve concrete execution limits and distinguish future stages from completed work. User preference: tens of CPU-hours alone are not a reason to block progress. No additional scientific task is implied by this preference.
|
||||
|
||||
- S2 execution update (2026-11-15): components/splits and the fixed ten-part measurement are complete; ten-part 10/10 PASS is in `docs/s2-ten-part-results-2026-11-15.md`. The 499 non-test preparation queue has started. Read `docs/s2-pilot-live-handoff-2026-11-15.md` and existing receipts before any launch; never restart from chat idle. Claude accepted implementation fixes at `1794683016.134929`, accepted the frozen ID-order dispatch at `1794683049.057259`, and raised the first-50 continuation ruling to 80 worker CPU-h / 100 GiB at `1794682712.505709`. The running original supervisor still enforces its pinned 60-hour cap; any required continuation may contain only never-attempted parts and must preserve cumulative accounting. After all 499 terminal attempts and required evidence, the authorized single-GPU pilot includes an E256 timing probe and four runs within 2 GPU-hours total, per-node train/validation extrapolation and a frozen common epoch limit. Keep 125 test labels sealed; no repeated user-go or main-review round is required for these accepted fixes. This does not authorize the full training matrix.
|
||||
|
||||
- Infrastructure-recovery update (2026-11-15 05:16 UTC+8) supersedes the original live sessions above: original preparation16682 exited INFRA_ABORTED and idle waiter13519 is retired; never restart either or manufacture original batch completion. Active recovery82149 and chain42022 use `artifacts/s2-pilot-recovery-inputs-2026-11-15` and `artifacts/s2-pilot-recovery-continuation-2026-11-15`. Read the leading section of `docs/s2-pilot-live-handoff-2026-11-15.md`. Fixed134 =16 explicitly authorized infrastructure recoveries +118 never attempted; original362 PASS and3 real timeouts are retained. Cumulative60 worker CPU-h/100GiB includes old aborted CPU once; no80h extension, no new first50 barrier or automatic retry. Preserve active source pins. Three-source verification precedes the existing bounded probe/four-run pilot; no main matrix.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"action": "EXPLICIT_RETIREMENT_OF_IDLE_OLD_WAITER",
|
||||
"pid": 193438,
|
||||
"session": 13519,
|
||||
"before": {
|
||||
"stage": "WAITING_FOR_EXISTING_PREPARATION",
|
||||
"completed": 365,
|
||||
"waiting_seconds": 6388.06448772602,
|
||||
"GPU_started": false,
|
||||
"original_preparation_not_restarted": true
|
||||
},
|
||||
"children_before": [],
|
||||
"GPU_started": false,
|
||||
"ruling": "1794690635.595169",
|
||||
"reason": "Old waiter cannot consume the new infrastructure-incident three-source manifest; replaced by a separately pinned recovery chain",
|
||||
"retired_unix": 1794690935.999284,
|
||||
"signal": "SIGTERM",
|
||||
"observed_exit_status": null,
|
||||
"confirmed_exited": true
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"stage": "WAITING_FOR_AUTHORIZED_RECOVERY",
|
||||
"completed": 16,
|
||||
"recovery_stop_reason": null,
|
||||
"waiting_seconds": 270.39477991103195,
|
||||
"GPU_started": false
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"started_unix": 1794690993.3521092,
|
||||
"pid": 204555,
|
||||
"recovery_root": "artifacts/s2-pilot-recovery-inputs-2026-11-15",
|
||||
"recovery_contract_sha256": "a19cf8e50ef03e1c096e2aa2b962ededf7d09d4ad71737ddf38f66d18a86db7b",
|
||||
"incident_ledger_sha256": "29f1d00586ba2b33b0f637b6ac6c5c5b3451f77591ba941b51f3f33a075b5a93",
|
||||
"code_sha256": {
|
||||
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"tools/prepare_s2_incident_recovery.py": "4f772573dedc2b7921986e25eaedb14f91d98b235421829c284f3b40b85710aa",
|
||||
"tools/prepare_s2_pilot_inputs.py": "8a04afe82c0add692c7cd5182b3fd70919ed9979f417af7223fba64753ca3144",
|
||||
"tools/prepare_s2_pilot_training.py": "6eccaae3c08ef43207ce67f772e7db0671c74da9c9f85dfd49e2fe6790050423",
|
||||
"tools/prepare_s2_recovered_pilot_training.py": "ab5a4274363598ff933ff873df2b00497f069b1fb45756a1d1dba8d723294f68",
|
||||
"tools/run_s2_pilot.py": "b0a3e786e17a1c0cdcc37d037ff5f03cbb7cf481a996a241d0b12c7f704e7585",
|
||||
"tools/run_s2_pilot_preparation.py": "08f07593a5f5f434ff1bcfdf952b9079500e2378ea3afb2e4d7e152db1a58def",
|
||||
"tools/run_s2_pilot_probe.py": "2360ab7b44b179144a5ecd2d691e33cd36a6ec4bfeac71452bf062e7f42f81eb",
|
||||
"tools/run_s2_pilot_training.py": "d5e543478543243fe1db3475a4d4cd135fbdcf631d3bce48c4b5af8b2edafb22",
|
||||
"tools/run_s2_pilot_when_ready.py": "03dca86c6aaa6a856a69836e2e913bf3f478ac1eb0d0d64adc083f4692b7d8cc",
|
||||
"tools/run_s2_recovered_pilot.py": "56962009bf5eaaee599ffea6a3d454372e0aea2842fa79eac609fd4a24ecbb39",
|
||||
"tools/run_s2_recovery_chain.py": "76d0808bdb2d624cfbaadf5a3c5e7e9a0c2be5d5ac41d4ac3fd9ccd48743933c",
|
||||
"tools/run_s2_recovery_preparation.py": "c135a8f718468b5b1286924d45b83597bf6cf8785af53c70c53fc074787730f1",
|
||||
"tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02",
|
||||
"tools/s2_incident_recovery_manifest.py": "8cbb2e1067c2cc22762d8ae3cf192aa2c5fa7c7a199a22e7a39a25570ee1c3b1",
|
||||
"tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97",
|
||||
"tools/s2_pilot_gpu_guard.py": "62dc35d71166d51c732d6cec2b3900ca1e88adecb9638a3b96bdaa37517f0b40",
|
||||
"tools/s2_pilot_prepare.py": "0987a5fe49816557d2cd8ccaeaf01de5effcb5d080221f2c93eb834719d413d2",
|
||||
"tools/s2_pilot_terminal.py": "89dcdf837df09d343c44b60678f857391f24382b609f4446c9391dc73abfd4d9",
|
||||
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"tools/s2_preprocess_pilot.py": "46cec13e13c0b94e5a51444b5d5ee78579dec9a3aa42ed24294e0a0deec55833",
|
||||
"tools/s2_recovered_pilot_gpu_guard.py": "9a27f47918c865ce37e46b0be54144dc3f9ab1058e4eddc9f6fbc771f4959eb0",
|
||||
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"tools/summarize_s2_pilot_checkpoint.py": "89610840c8ae9be90174c7469cebce5b894c3522f96917339316d9a9a3b7ff86",
|
||||
"tools/test_s2_incident_recovery.py": "007975a4842b79a9774b2508362801eb9ce202f09c72eb967a0ffeb726f6d418",
|
||||
"tools/test_s2_pilot_data.py": "cd9a5f69a5207d0ea62e8c150df737992ab640a08d82b182d504c9559e729562",
|
||||
"tools/test_s2_pilot_entry.py": "13c446642d679e86c0c0e744835a669e7f04b544384f8f5b9f82a9c5fe757b4b",
|
||||
"tools/test_s2_pilot_gpu_guard.py": "e335d24ba4745dc409b4d37aa8705c0e2f2b7e665c299469b154cfbfb3e84644",
|
||||
"tools/test_s2_pilot_orchestration.py": "8769109f973d211dc5490db22025b3fa4f5547b9f86dee1f2a7cda3e122d1371",
|
||||
"tools/test_s2_pilot_preparation.py": "71667a38b352ad64e588653b7d3545e71a473fb151de775e515e174b5358403c",
|
||||
"tools/test_s2_pilot_train.py": "7bebfa2cd9744d4186b0cf6acfc182d7023e6903eb3e779c3785506cefef01e7",
|
||||
"tools/test_s2_pilot_waiter.py": "9cb7096eedc0494e4bbe6faf015c8b16ca319d9d38b9cd6461724f77e9196aec",
|
||||
"tools/test_s2_recovered_gpu_guard.py": "f5ec28233a20b098c3004785fc620eee3bd8b1b2249ba90d8e0763908c28c0d4",
|
||||
"tools/test_s2_recovery_infrastructure.py": "3263b2c143d1bac908093acff15a77ffe5914edd40654297b69e8ad47e2fc829"
|
||||
},
|
||||
"pilot_root": "artifacts/s2-pilot-training-2026-11-15",
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"ruling": "1794689807.792819",
|
||||
"waiting_uses_GPU": false
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"started_unix": 1794690959.2146914,
|
||||
"pid": 2,
|
||||
"contract_sha256": "a19cf8e50ef03e1c096e2aa2b962ededf7d09d4ad71737ddf38f66d18a86db7b"
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"observed_unix": 1794691270.2063215,
|
||||
"preparation_session": 82149,
|
||||
"chain_session": 42022,
|
||||
"old_preparation_session": 16682,
|
||||
"old_preparation_exit": 1,
|
||||
"old_waiter_session": 13519,
|
||||
"old_waiter_exit": 143,
|
||||
"recovery_guard_receipts": 16,
|
||||
"recovery_pass": 16,
|
||||
"recovery_failures": [],
|
||||
"code_pins_match": true,
|
||||
"original_batch_complete_exists": false,
|
||||
"training_root_exists": false,
|
||||
"snapshots_sha256": {
|
||||
"artifacts/s2-pilot-recovery-inputs-2026-11-15/public/recovery-launch-progress.json": "cb6b18491a0796f8f9592af0fd353c42859475e4f96aca25b3ed6f10ea6923b5",
|
||||
"artifacts/s2-pilot-recovery-continuation-2026-11-15/launch-progress.json": "e2b6588a84259259f50815cf41fa2c6c7e05da20678bb60f886d022415084d0d"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"completed": 16,
|
||||
"active": [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
18,
|
||||
19,
|
||||
23,
|
||||
24,
|
||||
26,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31
|
||||
],
|
||||
"next_index": 32,
|
||||
"stop_reason": null,
|
||||
"prior_worker_cpu_seconds": 85611.3830659999,
|
||||
"recovery_worker_cpu_seconds": 1806.87858,
|
||||
"cumulative_settled_worker_cpu_seconds": 87418.2616459999,
|
||||
"live_guard_and_worker_cpu_seconds": 3170.51,
|
||||
"active_tree_rss_bytes": 4030201856,
|
||||
"disk_bytes": 4893450240,
|
||||
"elapsed_seconds": 310.8656697840197,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"fresh_accounting": true,
|
||||
"no_new_first50_barrier": true
|
||||
}
|
||||
@@ -1,5 +1,16 @@
|
||||
# S2 当前执行接续记录
|
||||
|
||||
## 当前状态:基础设施恢复队列已启动(2026-11-15 05:16,UTC+8)
|
||||
|
||||
本节取代下文旧运行入口的“当前”描述,下文保留为历史。原准备 session16682 因扫描瞬时临时文件的基础设施竞态退出1;原等待器 session13519 已在空闲、无子进程、GPU未启动时明确退休(退出143)。两者均不得重启,也不得补造原 batch-complete.json。
|
||||
|
||||
基础设施修复固定于 f9eeb1262f4593a3a83e0c8a483210a10c26e479,134件恢复契约及三来源合并入口固定于 a5c6c0444e2a813d7b9d68d56413b88ead56bb4e;裁量为 Claude 1794689807.792819、1794690635.595169。原362 PASS、3件真实 wall_limit 和16件 INFRA_ABORTED 全部保留。16件部分产物已按逐文件hash隔离,不复用;恢复队列仅16件显式一次恢复加118件从未尝试项。
|
||||
|
||||
现存准备 session82149,根 artifacts/s2-pilot-recovery-inputs-2026-11-15;接续 session42022,根 artifacts/s2-pilot-recovery-continuation-2026-11-15。以各自 progress.json、回执、启动锁查实时状态,禁止因聊天空闲重复启动。准备累计60 worker CPU-h /100GiB、16 workers、每件1800s/1.25GiB、零自动重试,无新首50屏障。既有23.78093974 worker CPU-h已经包含16件中止耗时,不能重复加计;80h扩展本次未使用。启动投影累计34.80316722 worker CPU-h、6.75622347GiB。
|
||||
|
||||
接续入口只等新恢复队列完整结束,再核验原批次INFRA_ABORTED快照、恢复批次、事故账本三个来源及hash,生成499个唯一终态;随后既有E256 probe和四run仍合计≤7200s,实际GPU启动前查空闲,125件测试封存。等待不使用GPU。训练根 artifacts/s2-pilot-training-2026-11-15 由完成核验后的入口创建,勿提前创建。不得修改当前入口 started.json / contract.json 所固定的源文件。启动时健康快照见恢复根 public/recovery-launch-audit.json;它不是整批完成证明。
|
||||
|
||||
|
||||
十件原队列已完整结束:10/10 PASS,正式交付见s2-ten-part-results-2026-11-15.md。代码/证据固定提交cdb52988c3bc915257478ff49eb131d4f49032c4已推送。Claude 1794683016.134929抽查认可修正;不另设主审/用户审批轮。
|
||||
|
||||
499非测试准备已于2026-11-15 03:02:50(Asia/Shanghai)启动;exec session16682。根目录artifacts/s2-pilot-inputs-2026-11-15。只通过现存progress.json、jobs回执、batch-complete.json及锁核状态,不因聊天idle或其他PID命名空间看不到进程而重启。已启动标记和契约不可覆盖。
|
||||
|
||||
Reference in new issue
Block a user