Record live recovery launch and retire stale preparation handoff
This commit is contained in:
1 parent
a5c6c0444e
commit
1fa41c8e90
8 files changed
+154
No files matched your search
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"action": "EXPLICIT_RETIREMENT_OF_IDLE_OLD_WAITER",
|
||||
"pid": 193438,
|
||||
"session": 13519,
|
||||
"before": {
|
||||
"stage": "WAITING_FOR_EXISTING_PREPARATION",
|
||||
"completed": 365,
|
||||
"waiting_seconds": 6388.06448772602,
|
||||
"GPU_started": false,
|
||||
"original_preparation_not_restarted": true
|
||||
},
|
||||
"children_before": [],
|
||||
"GPU_started": false,
|
||||
"ruling": "1794690635.595169",
|
||||
"reason": "Old waiter cannot consume the new infrastructure-incident three-source manifest; replaced by a separately pinned recovery chain",
|
||||
"retired_unix": 1794690935.999284,
|
||||
"signal": "SIGTERM",
|
||||
"observed_exit_status": null,
|
||||
"confirmed_exited": true
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"stage": "WAITING_FOR_AUTHORIZED_RECOVERY",
|
||||
"completed": 16,
|
||||
"recovery_stop_reason": null,
|
||||
"waiting_seconds": 270.39477991103195,
|
||||
"GPU_started": false
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"started_unix": 1794690993.3521092,
|
||||
"pid": 204555,
|
||||
"recovery_root": "artifacts/s2-pilot-recovery-inputs-2026-11-15",
|
||||
"recovery_contract_sha256": "a19cf8e50ef03e1c096e2aa2b962ededf7d09d4ad71737ddf38f66d18a86db7b",
|
||||
"incident_ledger_sha256": "29f1d00586ba2b33b0f637b6ac6c5c5b3451f77591ba941b51f3f33a075b5a93",
|
||||
"code_sha256": {
|
||||
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"tools/prepare_s2_incident_recovery.py": "4f772573dedc2b7921986e25eaedb14f91d98b235421829c284f3b40b85710aa",
|
||||
"tools/prepare_s2_pilot_inputs.py": "8a04afe82c0add692c7cd5182b3fd70919ed9979f417af7223fba64753ca3144",
|
||||
"tools/prepare_s2_pilot_training.py": "6eccaae3c08ef43207ce67f772e7db0671c74da9c9f85dfd49e2fe6790050423",
|
||||
"tools/prepare_s2_recovered_pilot_training.py": "ab5a4274363598ff933ff873df2b00497f069b1fb45756a1d1dba8d723294f68",
|
||||
"tools/run_s2_pilot.py": "b0a3e786e17a1c0cdcc37d037ff5f03cbb7cf481a996a241d0b12c7f704e7585",
|
||||
"tools/run_s2_pilot_preparation.py": "08f07593a5f5f434ff1bcfdf952b9079500e2378ea3afb2e4d7e152db1a58def",
|
||||
"tools/run_s2_pilot_probe.py": "2360ab7b44b179144a5ecd2d691e33cd36a6ec4bfeac71452bf062e7f42f81eb",
|
||||
"tools/run_s2_pilot_training.py": "d5e543478543243fe1db3475a4d4cd135fbdcf631d3bce48c4b5af8b2edafb22",
|
||||
"tools/run_s2_pilot_when_ready.py": "03dca86c6aaa6a856a69836e2e913bf3f478ac1eb0d0d64adc083f4692b7d8cc",
|
||||
"tools/run_s2_recovered_pilot.py": "56962009bf5eaaee599ffea6a3d454372e0aea2842fa79eac609fd4a24ecbb39",
|
||||
"tools/run_s2_recovery_chain.py": "76d0808bdb2d624cfbaadf5a3c5e7e9a0c2be5d5ac41d4ac3fd9ccd48743933c",
|
||||
"tools/run_s2_recovery_preparation.py": "c135a8f718468b5b1286924d45b83597bf6cf8785af53c70c53fc074787730f1",
|
||||
"tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02",
|
||||
"tools/s2_incident_recovery_manifest.py": "8cbb2e1067c2cc22762d8ae3cf192aa2c5fa7c7a199a22e7a39a25570ee1c3b1",
|
||||
"tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97",
|
||||
"tools/s2_pilot_gpu_guard.py": "62dc35d71166d51c732d6cec2b3900ca1e88adecb9638a3b96bdaa37517f0b40",
|
||||
"tools/s2_pilot_prepare.py": "0987a5fe49816557d2cd8ccaeaf01de5effcb5d080221f2c93eb834719d413d2",
|
||||
"tools/s2_pilot_terminal.py": "89dcdf837df09d343c44b60678f857391f24382b609f4446c9391dc73abfd4d9",
|
||||
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"tools/s2_preprocess_pilot.py": "46cec13e13c0b94e5a51444b5d5ee78579dec9a3aa42ed24294e0a0deec55833",
|
||||
"tools/s2_recovered_pilot_gpu_guard.py": "9a27f47918c865ce37e46b0be54144dc3f9ab1058e4eddc9f6fbc771f4959eb0",
|
||||
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"tools/summarize_s2_pilot_checkpoint.py": "89610840c8ae9be90174c7469cebce5b894c3522f96917339316d9a9a3b7ff86",
|
||||
"tools/test_s2_incident_recovery.py": "007975a4842b79a9774b2508362801eb9ce202f09c72eb967a0ffeb726f6d418",
|
||||
"tools/test_s2_pilot_data.py": "cd9a5f69a5207d0ea62e8c150df737992ab640a08d82b182d504c9559e729562",
|
||||
"tools/test_s2_pilot_entry.py": "13c446642d679e86c0c0e744835a669e7f04b544384f8f5b9f82a9c5fe757b4b",
|
||||
"tools/test_s2_pilot_gpu_guard.py": "e335d24ba4745dc409b4d37aa8705c0e2f2b7e665c299469b154cfbfb3e84644",
|
||||
"tools/test_s2_pilot_orchestration.py": "8769109f973d211dc5490db22025b3fa4f5547b9f86dee1f2a7cda3e122d1371",
|
||||
"tools/test_s2_pilot_preparation.py": "71667a38b352ad64e588653b7d3545e71a473fb151de775e515e174b5358403c",
|
||||
"tools/test_s2_pilot_train.py": "7bebfa2cd9744d4186b0cf6acfc182d7023e6903eb3e779c3785506cefef01e7",
|
||||
"tools/test_s2_pilot_waiter.py": "9cb7096eedc0494e4bbe6faf015c8b16ca319d9d38b9cd6461724f77e9196aec",
|
||||
"tools/test_s2_recovered_gpu_guard.py": "f5ec28233a20b098c3004785fc620eee3bd8b1b2249ba90d8e0763908c28c0d4",
|
||||
"tools/test_s2_recovery_infrastructure.py": "3263b2c143d1bac908093acff15a77ffe5914edd40654297b69e8ad47e2fc829"
|
||||
},
|
||||
"pilot_root": "artifacts/s2-pilot-training-2026-11-15",
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"ruling": "1794689807.792819",
|
||||
"waiting_uses_GPU": false
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"started_unix": 1794690959.2146914,
|
||||
"pid": 2,
|
||||
"contract_sha256": "a19cf8e50ef03e1c096e2aa2b962ededf7d09d4ad71737ddf38f66d18a86db7b"
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"observed_unix": 1794691270.2063215,
|
||||
"preparation_session": 82149,
|
||||
"chain_session": 42022,
|
||||
"old_preparation_session": 16682,
|
||||
"old_preparation_exit": 1,
|
||||
"old_waiter_session": 13519,
|
||||
"old_waiter_exit": 143,
|
||||
"recovery_guard_receipts": 16,
|
||||
"recovery_pass": 16,
|
||||
"recovery_failures": [],
|
||||
"code_pins_match": true,
|
||||
"original_batch_complete_exists": false,
|
||||
"training_root_exists": false,
|
||||
"snapshots_sha256": {
|
||||
"artifacts/s2-pilot-recovery-inputs-2026-11-15/public/recovery-launch-progress.json": "cb6b18491a0796f8f9592af0fd353c42859475e4f96aca25b3ed6f10ea6923b5",
|
||||
"artifacts/s2-pilot-recovery-continuation-2026-11-15/launch-progress.json": "e2b6588a84259259f50815cf41fa2c6c7e05da20678bb60f886d022415084d0d"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"completed": 16,
|
||||
"active": [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
18,
|
||||
19,
|
||||
23,
|
||||
24,
|
||||
26,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31
|
||||
],
|
||||
"next_index": 32,
|
||||
"stop_reason": null,
|
||||
"prior_worker_cpu_seconds": 85611.3830659999,
|
||||
"recovery_worker_cpu_seconds": 1806.87858,
|
||||
"cumulative_settled_worker_cpu_seconds": 87418.2616459999,
|
||||
"live_guard_and_worker_cpu_seconds": 3170.51,
|
||||
"active_tree_rss_bytes": 4030201856,
|
||||
"disk_bytes": 4893450240,
|
||||
"elapsed_seconds": 310.8656697840197,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"fresh_accounting": true,
|
||||
"no_new_first50_barrier": true
|
||||
}
|
||||
Reference in new issue
Block a user