From a15f2882ee2e8c3392c6b7c80aa521d7ba2ad58e Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Sun, 15 Nov 2026 06:30:46 +0800 Subject: [PATCH] Record v2 probe pass and pre-training busy-gate stop; correct launch status --- AGENTS.md | 2 + .../s2-pilot-v2-2026-11-15/guards/ledger.json | 10 + .../guards/timing-probe.log | 1 + .../guards/timing-probe.receipt.json | 24 ++ .../guards/timing-probe.started.json | 16 + .../orchestrator-complete.json | 7 + .../orchestrator-started.json | 4 + .../pilot-complete.json | 12 + .../s2-pilot-v2-2026-11-15/pilot-started.json | 4 + .../public/probe-launch-audit.json | 49 +++ .../timing-probe-guard-receipt.json | 33 ++ .../timing-probe/coverage-addendum.json | 190 ++++++++++ .../timing-probe/progress.json | 232 ++++++++++++ .../timing-probe/result.json | 249 +++++++++++++ .../training-contract.json | 335 ++++++++++++++++++ docs/s2-pilot-live-handoff-2026-11-15.md | 6 + 16 files changed, 1174 insertions(+) create mode 100644 artifacts/s2-pilot-v2-2026-11-15/guards/ledger.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.log create mode 100644 artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.receipt.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.started.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/orchestrator-complete.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/orchestrator-started.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/pilot-complete.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/pilot-started.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/public/probe-launch-audit.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/timing-probe-guard-receipt.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/timing-probe/coverage-addendum.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/timing-probe/progress.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/timing-probe/result.json create mode 100644 artifacts/s2-pilot-v2-2026-11-15/training-contract.json diff --git a/AGENTS.md b/AGENTS.md index da7a6e483..b2cfcc4e6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,3 +16,5 @@ - Infrastructure-recovery update (2026-11-15 05:16 UTC+8) supersedes the original live sessions above: original preparation16682 exited INFRA_ABORTED and idle waiter13519 is retired; never restart either or manufacture original batch completion. Active recovery82149 and chain42022 use `artifacts/s2-pilot-recovery-inputs-2026-11-15` and `artifacts/s2-pilot-recovery-continuation-2026-11-15`. Read the leading section of `docs/s2-pilot-live-handoff-2026-11-15.md`. Fixed134 =16 explicitly authorized infrastructure recoveries +118 never attempted; original362 PASS and3 real timeouts are retained. Cumulative60 worker CPU-h/100GiB includes old aborted CPU once; no80h extension, no new first50 barrier or automatic retry. Preserve active source pins. Three-source verification precedes the existing bounded probe/four-run pilot; no main matrix. - Terminal update (2026-11-15 06:14 UTC+8): recovery82149 and chain42022 have finished. Recovery134=131 PASS+3 wall_limit; canonical499=493 PASS+6 wall_limit. E256 probe passed; N128-seed0 failed at epoch10 with nonfinite encoder output. Pilot stopped without retry/width selection after296.023387s of7200s; other three runs never started. Read `docs/s2-recovery-pilot-stage-result-2026-11-15.md`; preserve all evidence,125-test seal,and source pins. Do not restart from stale progress. Claude received stop notice1794694508.930959; next changes require the existing collaboration review, not a repeat sharing approval. + +- Precision-v2 terminal correction (2026-11-15 06:30 UTC+8): diagnostic41846 completed and reproduced fp16 slice overflow; fixed8a548cc preserves original sources and implements the approved shared FP32 aggregation. New v2 session26792 has already exited: probe PASS at187345 nodes, VRAM12586057728B, E_max42, but SELECTED_GPU_BUSY_BEFORE_RUN stopped before any training run (runs=[]), total11.143491s. Earlier inference from pilot-started to N128 running was incorrect and retracted. Do not restart or rerun probe without the engineering continuation ruling; read latest handoff and raw complete receipts.125 tests remain sealed. diff --git a/artifacts/s2-pilot-v2-2026-11-15/guards/ledger.json b/artifacts/s2-pilot-v2-2026-11-15/guards/ledger.json new file mode 100644 index 000000000..5812f786b --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/guards/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 11.103604, + "attempts": [ + { + "name": "timing-probe", + "status": "PASS", + "cpu_seconds": 11.103604 + } + ] +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.log b/artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.log new file mode 100644 index 000000000..0a01342c1 --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.log @@ -0,0 +1 @@ +{"status": "COMPLETE", "E_max": 42, "cosine_T_max": 42, "projected_epoch_seconds": 35.118895331723465, "projected_epochs_seconds": 1474.9936039323854, "training_seconds_per_node": 2.5702122098117325e-06, "inference_seconds_per_node": 1.389278743139353e-06, "learning_nodes": 11863762, "validation_nodes": 3330152, "probe_nodes": 700086, "amp": {"attempted_steps": 20, "skipped_steps": 0, "successful_updates": 20, "skipped_fraction": 0.0, "consecutive_skipped": 0}, "standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03", "synthetic_zero_targets": true, "labels_read": false, "metrics_evaluated": false, "model_discarded": true, "wall_seconds": 7.467638947011437, "peak_allocated_bytes": 7524594176, "peak_reserved_bytes": 12169773056} diff --git a/artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.receipt.json b/artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.receipt.json new file mode 100644 index 000000000..1d4f1f97f --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.receipt.json @@ -0,0 +1,24 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_pilot_probe_v2.py", + "artifacts/s2-pilot-v2-2026-11-15" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 11.103604, + "wall_seconds": 11.052519564982504, + "peak_tree_rss_bytes": 1396215808, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 8192, + "max_poll_gap_seconds": 0.38153137400513515, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.started.json b/artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.started.json new file mode 100644 index 000000000..8d81be310 --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.started.json @@ -0,0 +1,16 @@ +{ + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_pilot_probe_v2.py", + "artifacts/s2-pilot-v2-2026-11-15" + ], + "limits": { + "total_cpu_seconds": 7200, + "job_cpu_seconds": 300, + "job_wall_seconds": 297.45584029197926, + "tree_rss_bytes": 17179869184, + "disk_bytes": 5368709120, + "max_processes": 2 + }, + "started_unix": 1794695302.3197525 +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/orchestrator-complete.json b/artifacts/s2-pilot-v2-2026-11-15/orchestrator-complete.json new file mode 100644 index 000000000..22d734c74 --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/orchestrator-complete.json @@ -0,0 +1,7 @@ +{ + "status": "PILOT_INCOMPLETE", + "total_wall_seconds": 11.143491394002922, + "selected_global_width": null, + "test_labels_read": false, + "probe_included_in_7200_seconds": true +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/orchestrator-started.json b/artifacts/s2-pilot-v2-2026-11-15/orchestrator-started.json new file mode 100644 index 000000000..8aff6cda4 --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/orchestrator-started.json @@ -0,0 +1,4 @@ +{ + "started_unix": 1794695302.273772, + "plan_sha256": "5a127d5ddfab366da49475867c9f14f2854f5000a733c7a3211df61bd7acecae" +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/pilot-complete.json b/artifacts/s2-pilot-v2-2026-11-15/pilot-complete.json new file mode 100644 index 000000000..2551f63a6 --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/pilot-complete.json @@ -0,0 +1,12 @@ +{ + "status": "PILOT_INCOMPLETE", + "reason": "SELECTED_GPU_BUSY_BEFORE_RUN", + "runs": [], + "selected_global_width": null, + "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", + "pilot_wall_seconds": 11.140450741047971, + "probe_and_setup_wall_seconds": 11.102904540020972, + "test_labels_read": false, + "test_predictions": false, + "retries": 0 +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/pilot-started.json b/artifacts/s2-pilot-v2-2026-11-15/pilot-started.json new file mode 100644 index 000000000..04201fe5d --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/pilot-started.json @@ -0,0 +1,4 @@ +{ + "started_unix": 1794695313.379208, + "contract_sha256": "0218b74f8d86ae4190e0b814f7c5c8a7224a616b94ad7587120156e755de2101" +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/public/probe-launch-audit.json b/artifacts/s2-pilot-v2-2026-11-15/public/probe-launch-audit.json new file mode 100644 index 000000000..9603ceeea --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/public/probe-launch-audit.json @@ -0,0 +1,49 @@ +{ + "status": "PROBE_PASS_ZERO_TRAINING_RUNS_BUSY_GATE_STOP", + "observed_unix": 1794695358.164342, + "session": 26792, + "source_commit": "8a548cc", + "original_source_pins_unchanged": true, + "v2_source_pins_match": true, + "sealed_tests": 125, + "test_manifest_overlap": 0, + "test_probe_overlap": 0, + "test_labels_read": false, + "training_contract_sha256": "0218b74f8d86ae4190e0b814f7c5c8a7224a616b94ad7587120156e755de2101", + "probe_cases": 20, + "actual_probe_max_nodes": 187345, + "probe_peak_device_memory_bytes": 12586057728, + "probe_total_guard_wall_seconds": 11.099626450042706, + "E_max": 42, + "cosine_T_max": 42, + "projected_epoch_seconds": 35.118895331723465, + "total_pilot_wall_limit_seconds": 7200, + "remaining_runs_not_yet_claimed_complete": true, + "artifact_sha256": { + "artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.log": "8ecc6051dc1aea4f9aaea94440d22552ba1e4064f8e7ba7d55b6e6fabd91de97", + "artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.receipt.json": "6e3aa7fd31220d9d55573f6b2b09202d7d58a746cdf1bd59f8f73034e19c37f9", + "artifacts/s2-pilot-v2-2026-11-15/guards/timing-probe.started.json": "c4f051d9fed9b8692d847e89d28c6b6c08158d45926ee4acb98ee644e9f22ba9", + "artifacts/s2-pilot-v2-2026-11-15/orchestrator-started.json": "bdbb5cd4a4a82b14256bea30b0cc64b8ef37e9f4097df118a902a6c18ae30316", + "artifacts/s2-pilot-v2-2026-11-15/pilot-started.json": "fc68105a4e52551f8e3b6c1ce5341fffa137bb17d19766fb341cf9b39ee72a9b", + "artifacts/s2-pilot-v2-2026-11-15/timing-probe/coverage-addendum.json": "0480fea8bfc6ef21fb7681dd28faec2212a6d02367972d32d332139af0503eeb", + "artifacts/s2-pilot-v2-2026-11-15/timing-probe/result.json": "96b95ec8dd0e37c38096dda9ec06257eb02662f8fd62cbe11b9ad58887d8376b", + "artifacts/s2-pilot-v2-2026-11-15/timing-probe-guard-receipt.json": "45e9fe236d31dd7eabbbd46edce4fc7a80914c5a12088d1977de3633d79d6c2a", + "artifacts/s2-pilot-v2-2026-11-15/training-contract.json": "0218b74f8d86ae4190e0b814f7c5c8a7224a616b94ad7587120156e755de2101", + "artifacts/s2-pilot-v2-2026-11-15/orchestrator-complete.json": "e2794244d0664cc10aa3707a8da4f2d078946f985a1095f82a2f12b5310128a9", + "artifacts/s2-pilot-v2-2026-11-15/pilot-complete.json": "337ea9552569db135e22b93f3e0b1557e03c5b56669cdb4d1bbcd391e6f67982" + }, + "superseded_incorrect_status": "PROBE_PASS_N128_STARTED", + "correction": "pilot-started marks entry into run orchestration, not a spawned training run; actual runs=[]", + "corrected_unix": 1794695437.6724546, + "training_runs_started": 0, + "pilot_status": "PILOT_INCOMPLETE", + "pilot_stop_reason": "SELECTED_GPU_BUSY_BEFORE_RUN", + "pilot_total_wall_seconds": 11.143491394002922, + "post_stop_GPU_snapshot": { + "memory_MiB": 5, + "utilization_percent": 0, + "compute_processes": [] + }, + "busy_gate_sample_counters": "NOT_RECORDED_BY_ORIGINAL_GUARD", + "current_processes": [] +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/timing-probe-guard-receipt.json b/artifacts/s2-pilot-v2-2026-11-15/timing-probe-guard-receipt.json new file mode 100644 index 000000000..d5061681b --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/timing-probe-guard-receipt.json @@ -0,0 +1,33 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_pilot_probe_v2.py", + "artifacts/s2-pilot-v2-2026-11-15" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 11.103604, + "wall_seconds": 11.052519564982504, + "peak_tree_rss_bytes": 1396215808, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 8192, + "max_poll_gap_seconds": 0.38153137400513515, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null, + "device_monitor": { + "peak_device_memory_bytes": 12586057728, + "peak_whole_run_tree_rss_bytes": 1770561536, + "max_poll_gap_seconds": 0.8863910030340776, + "reason": null, + "samples": 19 + }, + "total_run_wall_seconds": 11.099626450042706, + "result_sha256": "96b95ec8dd0e37c38096dda9ec06257eb02662f8fd62cbe11b9ad58887d8376b" +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/timing-probe/coverage-addendum.json b/artifacts/s2-pilot-v2-2026-11-15/timing-probe/coverage-addendum.json new file mode 100644 index 000000000..c5c24ea4c --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/timing-probe/coverage-addendum.json @@ -0,0 +1,190 @@ +{ + "planned_max_learning_nodes": 187345, + "max_successful_learning_nodes": 187345, + "failed_parts": [ + { + "case": "00000575_5a4cef68211d4706b1ec8586_step_004", + "role": "validation", + "nodes": 75046, + "component_id": "component-3a3e340f4dfac391", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-inputs-2026-11-15", + "181" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.766116, + "wall_seconds": 1800.400393687014, + "peak_tree_rss_bytes": 602218496, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.128913332009688, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false + }, + "failure": null + }, + { + "case": "00000614_c710d414bbbc4340b682c58a_step_000", + "role": "learning", + "nodes": 145860, + "component_id": "component-4bd089778094e746", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-inputs-2026-11-15", + "201" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.796751, + "wall_seconds": 1800.0567526929663, + "peak_tree_rss_bytes": 629305344, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.1576595069491304, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false + }, + "failure": null + }, + { + "case": "00000625_6af98f5458b347bb843f0562_step_000", + "role": "learning", + "nodes": 145707, + "component_id": "component-4bd089778094e746", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-inputs-2026-11-15", + "211" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.8864629999998, + "wall_seconds": 1800.119237862993, + "peak_tree_rss_bytes": 647880704, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.159920622012578, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false + }, + "failure": null + }, + { + "case": "00001466_a45f6c99e2154f3b94955175_step_050", + "role": "learning", + "nodes": 77138, + "component_id": "component-7fe5173e77c86f9d", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-recovery-inputs-2026-11-15", + "83" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.926194, + "wall_seconds": 1800.0714844280155, + "peak_tree_rss_bytes": 383070208, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.0109735240112059, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00001470_a45f6c99e2154f3b94955175_step_054", + "role": "learning", + "nodes": 124817, + "component_id": "component-7fe5173e77c86f9d", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-recovery-inputs-2026-11-15", + "87" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.664371, + "wall_seconds": 1800.0443400070071, + "peak_tree_rss_bytes": 578732032, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.007104828953743, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00001473_a45f6c99e2154f3b94955175_step_057", + "role": "learning", + "nodes": 69428, + "component_id": "component-7fe5173e77c86f9d", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-recovery-inputs-2026-11-15", + "90" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.751432, + "wall_seconds": 1800.2498029720155, + "peak_tree_rss_bytes": 438853632, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.8929723129840568, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + } + ], + "statement": "VRAM probe covers only the maximum successful learning part included; not failed/larger unprobed parts", + "actual_probe_max_nodes": 187345, + "probe_receipt_sha256": "96b95ec8dd0e37c38096dda9ec06257eb02662f8fd62cbe11b9ad58887d8376b", + "probe_guard_sha256": "45e9fe236d31dd7eabbbd46edce4fc7a80914c5a12088d1977de3633d79d6c2a" +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/timing-probe/progress.json b/artifacts/s2-pilot-v2-2026-11-15/timing-probe/progress.json new file mode 100644 index 000000000..2cf67b058 --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/timing-probe/progress.json @@ -0,0 +1,232 @@ +{ + "rows": [ + { + "case": "00001260_a4715d363f88400c8f0238d1_step_003", + "nodes": 4607, + "training_seconds": 0.4922556870151311, + "inference_seconds": 0.011500941007398069, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000290_58f8ff46b7564e989f9695b7_step_003", + "nodes": 9658, + "training_seconds": 0.043944053002633154, + "inference_seconds": 0.012096819991711527, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001557_16c191aef5034779aaab6e3f_step_000", + "nodes": 13474, + "training_seconds": 0.042322815977968276, + "inference_seconds": 0.015360463003162295, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000293_58f8ff46b7564e989f9695b7_step_006", + "nodes": 15129, + "training_seconds": 0.04943311697570607, + "inference_seconds": 0.015177619003225118, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001193_46b17dd88ed24f55bca2eb69_step_000", + "nodes": 17335, + "training_seconds": 0.044143727980554104, + "inference_seconds": 0.01700160698965192, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000105_52f7124261ae4c558f800775_step_002", + "nodes": 18966, + "training_seconds": 0.045052865985780954, + "inference_seconds": 0.018405335023999214, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001592_35f4abaacc9447e4ac0b964f_step_001", + "nodes": 19904, + "training_seconds": 0.046116926008835435, + "inference_seconds": 0.019136257003992796, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000140_2fc54fcd110d4f49969163c4_step_003", + "nodes": 21696, + "training_seconds": 0.045592980983201414, + "inference_seconds": 0.021079389029182494, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000166_2246cab58dbe4a2fb00fed50_step_000", + "nodes": 23389, + "training_seconds": 0.05165633402066305, + "inference_seconds": 0.022340489027556032, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001321_7cef9b79a6b04db894fd74e5_step_000", + "nodes": 24986, + "training_seconds": 0.051258848048746586, + "inference_seconds": 0.02330216602422297, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000462_516b1784884d4facbc4c0d74_step_010", + "nodes": 27004, + "training_seconds": 0.053460714989341795, + "inference_seconds": 0.025028460018802434, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000470_11951e51b89249dba38ded3e_step_005", + "nodes": 29053, + "training_seconds": 0.05698901502182707, + "inference_seconds": 0.02694380801403895, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "nodes": 31350, + "training_seconds": 0.057809877034742385, + "inference_seconds": 0.029152337985578924, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001484_a45f6c99e2154f3b94955175_step_068", + "nodes": 33144, + "training_seconds": 0.06063747999724001, + "inference_seconds": 0.030325794010423124, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000784_a25fb37392a44447b9684f16_step_009", + "nodes": 35825, + "training_seconds": 0.0670759990462102, + "inference_seconds": 0.03271694597788155, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000314_3062bccff48e47a2b9de05e3_step_009", + "nodes": 39329, + "training_seconds": 0.0710820949752815, + "inference_seconds": 0.035635549982544035, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001186_507b60b57791425bbb73273e_step_001", + "nodes": 43341, + "training_seconds": 0.07648064900422469, + "inference_seconds": 0.03943444904871285, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001569_6fc579f3224d4d53a1b60710_step_000", + "nodes": 48491, + "training_seconds": 0.08332318800967187, + "inference_seconds": 0.37380995601415634, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000617_c710d414bbbc4340b682c58a_step_003", + "nodes": 56060, + "training_seconds": 0.08700873103225604, + "inference_seconds": 0.049461876042187214, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000666_6f67a59c16b949a4b66dcfe1_step_000", + "nodes": 187345, + "training_seconds": 0.2737244800082408, + "inference_seconds": 0.1547043349710293, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + } + ], + "amp": { + "attempted_steps": 20, + "skipped_steps": 0, + "successful_updates": 20, + "skipped_fraction": 0.0, + "consecutive_skipped": 0 + }, + "elapsed_seconds": 7.466452167020179 +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/timing-probe/result.json b/artifacts/s2-pilot-v2-2026-11-15/timing-probe/result.json new file mode 100644 index 000000000..8acf4593b --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/timing-probe/result.json @@ -0,0 +1,249 @@ +{ + "status": "COMPLETE", + "E_max": 42, + "cosine_T_max": 42, + "projected_epoch_seconds": 35.118895331723465, + "projected_epochs_seconds": 1474.9936039323854, + "training_seconds_per_node": 2.5702122098117325e-06, + "inference_seconds_per_node": 1.389278743139353e-06, + "learning_nodes": 11863762, + "validation_nodes": 3330152, + "rows": [ + { + "case": "00001260_a4715d363f88400c8f0238d1_step_003", + "nodes": 4607, + "training_seconds": 0.4922556870151311, + "inference_seconds": 0.011500941007398069, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000290_58f8ff46b7564e989f9695b7_step_003", + "nodes": 9658, + "training_seconds": 0.043944053002633154, + "inference_seconds": 0.012096819991711527, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001557_16c191aef5034779aaab6e3f_step_000", + "nodes": 13474, + "training_seconds": 0.042322815977968276, + "inference_seconds": 0.015360463003162295, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000293_58f8ff46b7564e989f9695b7_step_006", + "nodes": 15129, + "training_seconds": 0.04943311697570607, + "inference_seconds": 0.015177619003225118, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001193_46b17dd88ed24f55bca2eb69_step_000", + "nodes": 17335, + "training_seconds": 0.044143727980554104, + "inference_seconds": 0.01700160698965192, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000105_52f7124261ae4c558f800775_step_002", + "nodes": 18966, + "training_seconds": 0.045052865985780954, + "inference_seconds": 0.018405335023999214, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001592_35f4abaacc9447e4ac0b964f_step_001", + "nodes": 19904, + "training_seconds": 0.046116926008835435, + "inference_seconds": 0.019136257003992796, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000140_2fc54fcd110d4f49969163c4_step_003", + "nodes": 21696, + "training_seconds": 0.045592980983201414, + "inference_seconds": 0.021079389029182494, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000166_2246cab58dbe4a2fb00fed50_step_000", + "nodes": 23389, + "training_seconds": 0.05165633402066305, + "inference_seconds": 0.022340489027556032, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001321_7cef9b79a6b04db894fd74e5_step_000", + "nodes": 24986, + "training_seconds": 0.051258848048746586, + "inference_seconds": 0.02330216602422297, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000462_516b1784884d4facbc4c0d74_step_010", + "nodes": 27004, + "training_seconds": 0.053460714989341795, + "inference_seconds": 0.025028460018802434, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000470_11951e51b89249dba38ded3e_step_005", + "nodes": 29053, + "training_seconds": 0.05698901502182707, + "inference_seconds": 0.02694380801403895, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "nodes": 31350, + "training_seconds": 0.057809877034742385, + "inference_seconds": 0.029152337985578924, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001484_a45f6c99e2154f3b94955175_step_068", + "nodes": 33144, + "training_seconds": 0.06063747999724001, + "inference_seconds": 0.030325794010423124, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000784_a25fb37392a44447b9684f16_step_009", + "nodes": 35825, + "training_seconds": 0.0670759990462102, + "inference_seconds": 0.03271694597788155, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000314_3062bccff48e47a2b9de05e3_step_009", + "nodes": 39329, + "training_seconds": 0.0710820949752815, + "inference_seconds": 0.035635549982544035, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001186_507b60b57791425bbb73273e_step_001", + "nodes": 43341, + "training_seconds": 0.07648064900422469, + "inference_seconds": 0.03943444904871285, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00001569_6fc579f3224d4d53a1b60710_step_000", + "nodes": 48491, + "training_seconds": 0.08332318800967187, + "inference_seconds": 0.37380995601415634, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000617_c710d414bbbc4340b682c58a_step_003", + "nodes": 56060, + "training_seconds": 0.08700873103225604, + "inference_seconds": 0.049461876042187214, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + }, + { + "case": "00000666_6f67a59c16b949a4b66dcfe1_step_000", + "nodes": 187345, + "training_seconds": 0.2737244800082408, + "inference_seconds": 0.1547043349710293, + "amp": { + "step_skipped": false, + "scale_before": 65536.0, + "scale_after": 65536.0 + } + } + ], + "probe_nodes": 700086, + "amp": { + "attempted_steps": 20, + "skipped_steps": 0, + "successful_updates": 20, + "skipped_fraction": 0.0, + "consecutive_skipped": 0 + }, + "standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03", + "synthetic_zero_targets": true, + "labels_read": false, + "metrics_evaluated": false, + "model_discarded": true, + "wall_seconds": 7.467638947011437, + "peak_allocated_bytes": 7524594176, + "peak_reserved_bytes": 12169773056 +} diff --git a/artifacts/s2-pilot-v2-2026-11-15/training-contract.json b/artifacts/s2-pilot-v2-2026-11-15/training-contract.json new file mode 100644 index 000000000..106731734 --- /dev/null +++ b/artifacts/s2-pilot-v2-2026-11-15/training-contract.json @@ -0,0 +1,335 @@ +{ + "fold": 0, + "pool": 100, + "seed": 0, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "code_sha256": { + "tools/prepare_s2_recovered_pilot_training.py": "ab5a4274363598ff933ff873df2b00497f069b1fb45756a1d1dba8d723294f68", + "tools/run_s2_recovered_pilot.py": "56962009bf5eaaee599ffea6a3d454372e0aea2842fa79eac609fd4a24ecbb39", + "tools/run_s2_pilot_probe.py": "2360ab7b44b179144a5ecd2d691e33cd36a6ec4bfeac71452bf062e7f42f81eb", + "tools/run_s2_pilot_training.py": "d5e543478543243fe1db3475a4d4cd135fbdcf631d3bce48c4b5af8b2edafb22", + "tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "tools/s2_recovered_pilot_gpu_guard.py": "9a27f47918c865ce37e46b0be54144dc3f9ab1058e4eddc9f6fbc771f4959eb0", + "tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02", + "tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97", + "tools/s2_incident_recovery_manifest.py": "8cbb2e1067c2cc22762d8ae3cf192aa2c5fa7c7a199a22e7a39a25570ee1c3b1", + "tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "tools/prepare_s2_incident_recovery.py": "4f772573dedc2b7921986e25eaedb14f91d98b235421829c284f3b40b85710aa", + "tools/run_s2_recovery_chain.py": "76d0808bdb2d624cfbaadf5a3c5e7e9a0c2be5d5ac41d4ac3fd9ccd48743933c", + "tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609", + "tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61", + "tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75", + "tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4", + "tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0", + "tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1", + "tools/prepare_s2_precision_pilot.py": "c3a71b0ff58bcea4433ff93e18ef14d3a00521438a76baaa69830a5ee8ac098a", + "tools/run_s2_precision_pilot.py": "ac8347be1ca11f48cb67abbf5e5c78b39af46ed3a7c6f89b49098fd5e77a6904", + "tools/s2_precision_pilot_gpu_guard.py": "5cf0f41de34b28394dbee4672b5bd34eb9c4fe2344822aa3d648f20561d96328", + "tools/run_s2_pilot_probe_v2.py": "61c08499346d50405d0c254de54deea24b3522256bf1c8b0aec3e9f4ddab6390", + "tools/run_s2_pilot_training_v2.py": "f461cb43bc602a32677d7dcfb8a322e206531ad152fa33bb4e36ea8546a1c0f2", + "tools/run_s2_encoder_diagnostic.py": "8f31d312fb53bfdb9b1804af064e463d3989f1b33c4aa7279a967ca82dd2c0a4", + "tools/s2_models_v2/__init__.py": "0302d333bcd8b33e85aaa6a4e7822e7e1ccfc66343136107fea6f4ca283ba1ec", + "tools/s2_models_v2/attention.py": "3829e34ca30942c863699c4d43fb6fa104452c6c04c4d606c11019d3dde16ffa", + "tools/s2_models_v2/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75", + "tools/s2_models_v2/network.py": "a443470caeecdf529504bffab3ca8cc0b70ce44b505bad60f9c63db9709dbb83" + }, + "role_manifest": "artifacts/s2-pilot-inputs-2026-11-15/role-manifest.json", + "role_manifest_sha256": "a6dce3ebf875dbbe2df349f06e728960140f131d4d55ebd00af03945a46ac35c", + "success_manifest": "artifacts/s2-pilot-v2-2026-11-15/success-manifest.json", + "success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3", + "probe_cases": [ + "00001260_a4715d363f88400c8f0238d1_step_003", + "00000290_58f8ff46b7564e989f9695b7_step_003", + "00001557_16c191aef5034779aaab6e3f_step_000", + "00000293_58f8ff46b7564e989f9695b7_step_006", + "00001193_46b17dd88ed24f55bca2eb69_step_000", + "00000105_52f7124261ae4c558f800775_step_002", + "00001592_35f4abaacc9447e4ac0b964f_step_001", + "00000140_2fc54fcd110d4f49969163c4_step_003", + "00000166_2246cab58dbe4a2fb00fed50_step_000", + "00001321_7cef9b79a6b04db894fd74e5_step_000", + "00000462_516b1784884d4facbc4c0d74_step_010", + "00000470_11951e51b89249dba38ded3e_step_005", + "00001495_6c8ef6d2f5564d139a5077f8_step_006", + "00001484_a45f6c99e2154f3b94955175_step_068", + "00000784_a25fb37392a44447b9684f16_step_009", + "00000314_3062bccff48e47a2b9de05e3_step_009", + "00001186_507b60b57791425bbb73273e_step_001", + "00001569_6fc579f3224d4d53a1b60710_step_000", + "00000617_c710d414bbbc4340b682c58a_step_003", + "00000666_6f67a59c16b949a4b66dcfe1_step_000" + ], + "probe_selection": "20 equally spaced ranks in successful learning node-count order; includes min/max; tie by case", + "learning_nodes": 11863762, + "validation_nodes": 3330152, + "successful": { + "learning": { + "parts": 382, + "nodes": 11300812 + }, + "validation": { + "parts": 111, + "nodes": 3255106 + } + }, + "extrapolation": "Train rate * all 387 planned learning nodes + FP32 inference rate * all 112 planned validation nodes; conservative if failed parts dropped", + "runs": [ + { + "name": "N128-seed0", + "arm": "N", + "width": 128, + "seed": 0 + }, + { + "name": "E128-seed0", + "arm": "E", + "width": 128, + "seed": 0 + }, + { + "name": "N256-seed0", + "arm": "N", + "width": 256, + "seed": 0 + }, + { + "name": "E256-seed0", + "arm": "E", + "width": 256, + "seed": 0 + } + ], + "limits": { + "total_gpu_wall_seconds": 7200, + "probe_wall_seconds": 300, + "run_wall_seconds": 1800, + "host_rss_bytes": 17179869184, + "vram_bytes": 16106127360, + "disk_bytes": 5368709120 + }, + "coverage_boundary": { + "planned_max_learning_nodes": 187345, + "max_successful_learning_nodes": 187345, + "failed_parts": [ + { + "case": "00000575_5a4cef68211d4706b1ec8586_step_004", + "role": "validation", + "nodes": 75046, + "component_id": "component-3a3e340f4dfac391", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-inputs-2026-11-15", + "181" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.766116, + "wall_seconds": 1800.400393687014, + "peak_tree_rss_bytes": 602218496, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.128913332009688, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false + }, + "failure": null + }, + { + "case": "00000614_c710d414bbbc4340b682c58a_step_000", + "role": "learning", + "nodes": 145860, + "component_id": "component-4bd089778094e746", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-inputs-2026-11-15", + "201" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.796751, + "wall_seconds": 1800.0567526929663, + "peak_tree_rss_bytes": 629305344, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.1576595069491304, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false + }, + "failure": null + }, + { + "case": "00000625_6af98f5458b347bb843f0562_step_000", + "role": "learning", + "nodes": 145707, + "component_id": "component-4bd089778094e746", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-inputs-2026-11-15", + "211" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.8864629999998, + "wall_seconds": 1800.119237862993, + "peak_tree_rss_bytes": 647880704, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.159920622012578, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false + }, + "failure": null + }, + { + "case": "00001466_a45f6c99e2154f3b94955175_step_050", + "role": "learning", + "nodes": 77138, + "component_id": "component-7fe5173e77c86f9d", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-recovery-inputs-2026-11-15", + "83" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.926194, + "wall_seconds": 1800.0714844280155, + "peak_tree_rss_bytes": 383070208, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.0109735240112059, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00001470_a45f6c99e2154f3b94955175_step_054", + "role": "learning", + "nodes": 124817, + "component_id": "component-7fe5173e77c86f9d", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-recovery-inputs-2026-11-15", + "87" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.664371, + "wall_seconds": 1800.0443400070071, + "peak_tree_rss_bytes": 578732032, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 1.007104828953743, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00001473_a45f6c99e2154f3b94955175_step_057", + "role": "learning", + "nodes": 69428, + "component_id": "component-7fe5173e77c86f9d", + "guard": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/s2_pilot_prepare.py", + "artifacts/s2-pilot-recovery-inputs-2026-11-15", + "90" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1799.751432, + "wall_seconds": 1800.2498029720155, + "peak_tree_rss_bytes": 438853632, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.8929723129840568, + "poll_target_seconds": 0.25, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + } + ], + "statement": "VRAM probe covers only the maximum successful learning part included; not failed/larger unprobed parts" + }, + "authorizing_rulings": [ + "1794680129.303719", + "1794681576.845789", + "1794681807.376669", + "1794689807.792819", + "1794694544.819729", + "1794694646.032819" + ], + "test_labels_read": false, + "precision_variant": { + "version": "S2-pilot-v2-slice-FP32", + "changed": "Only shared N/E slice aggregation numerator, denominator and normalization in FP32 with autocast disabled", + "original_pilot": "artifacts/s2-pilot-training-2026-11-15", + "original_contract_sha256": "f6987431f478275485d18f62240144c2626cab3fbeb75c3e130b68b5d57a1fd2", + "original_status": "PILOT_INCOMPLETE", + "diagnostic_root": "artifacts/s2-encoder-diagnostic-2026-11-15", + "diagnostic_result_sha256": "69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41", + "diagnostic_guard_sha256": "ec7b55e2cbca465c29aec84630fc02c9cfd9fed71e98226a90eb5b3fb2b45537", + "from_scratch": true, + "original_weights_reused": false, + "preparation_repeated": false, + "reprobe_max_nodes": 187345, + "VRAM_limit_failure_no_batch_or_width_fallback": true + }, + "E_max": 42, + "cosine_T_max": 42, + "projected_epoch_seconds": 35.118895331723465, + "projected_epochs_seconds": 1474.9936039323854, + "training_seconds_per_node": 2.5702122098117325e-06, + "inference_seconds_per_node": 1.389278743139353e-06, + "timing_probe_receipt": "artifacts/s2-pilot-v2-2026-11-15/timing-probe/result.json", + "timing_probe_sha256": "96b95ec8dd0e37c38096dda9ec06257eb02662f8fd62cbe11b9ad58887d8376b", + "timing_probe_guard_sha256": "45e9fe236d31dd7eabbbd46edce4fc7a80914c5a12088d1977de3633d79d6c2a", + "probe_guard_wall_seconds": 11.099626450042706, + "standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03" +} diff --git a/docs/s2-pilot-live-handoff-2026-11-15.md b/docs/s2-pilot-live-handoff-2026-11-15.md index fcfe1a8f8..e6064efbc 100644 --- a/docs/s2-pilot-live-handoff-2026-11-15.md +++ b/docs/s2-pilot-live-handoff-2026-11-15.md @@ -1,5 +1,11 @@ # S2 当前执行接续记录 +## 最新终态更正:v2 probe通过,GPU空闲门停止,零训练run + +session26792已结束。v2 probe PASS、187345节点/峰值12586057728B、E_max42均为实测;但probe后首run前GPU空闲门返回SELECTED_GPU_BUSY_BEFORE_RUN,pilot-complete的runs=[],四个训练run均未启动,总wall11.143491秒。此前将pilot-started入口标记误写为N128启动,现明确撤回;不得从该标记推断训练已运行。原始回执不改。 + +当前主机GPU快照5MiB/0%/无计算进程,触发时门控计数未被原guard记录,不能确定是异步释放/利用率窗口还是其他瞬时占用。已报Claude/TRON1794695396.091489,待工程接续裁量;不重启、不重probe、不覆盖旧run或新建第二队列。原pilot失败和诊断仍按各自账本保留。125测试封存。 + ## 最新终态:恢复完成,pilot因首run数值失败停止(06:14核查) 本节取代下文“当前运行”描述。134恢复终态131 PASS/3 wall_limit,三来源499终态493成功/6超时,E256 probe PASS;N128-seed0在epoch10报nonfinite encoder output,pilot已无重试停止,总GPU阶段wall296.023387s/7200s,未选宽。82149/42022及子进程均结束;勿重启,勿把旧progress当存活证据。125测试封存,详情和实际文件见 `s2-recovery-pilot-stage-result-2026-11-15.md`。等待Claude对失败证据的主审裁量,不启动第二队列或静默改阈值。