{ "status": "TEN_STOPPED", "stop_reason": "NUMERICAL_FAILURE", "selected_solver": null, "candidates": [ { "method": "splu", "status": "STOPPED", "reason": "RESOURCE_STOP", "classification": { "category": "RESOURCE_STOP", "solver_attributable": true, "reason": "wall_limit", "last_phase": "PRESSURE_FACTORIZE" }, "attempts": [ { "method": "splu", "index": 0, "case": "00001130_c2db313c2ce749d784c610d2_step_002", "worker": { "status": "PASS", "command": [ "taskset", "-c", "16", "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", "tools/run_s3b_v04_part.py", "artifacts/s3b-a-v04-ten-2026-11-15", "0", "splu" ], "returncode": 0, "stop_reason": null, "cpu_seconds": 36.975542, "wall_seconds": 37.031975959951524, "peak_tree_rss_bytes": 135684096, "peak_process_count": 1, "output_disk_bytes_before_receipt": 5173248, "max_poll_gap_seconds": 0.3977732859784737, "poll_target_seconds": 0.2, "reaped_orphan_cpu_seconds": 0.0, "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", "gpu_visible": false, "retry": false, "monitor_errors": 0, "last_monitor_error": null }, "classification": { "category": "PASS", "solver_attributable": false, "reason": null }, "files": { "result": { "path": "artifacts/s3b-a-v04-ten-2026-11-15/candidates/splu/parts/part-00/output/result.json", "sha256": "654860d69a59944af8ca850772710be2a3fe7ae702cd03b703e598a82d56ae1d" }, "progress": { "path": "artifacts/s3b-a-v04-ten-2026-11-15/candidates/splu/parts/part-00/output/progress.json", "sha256": "4ac0eca43d81d9f439da8071e2b29e72aff203dee26559e036c7145001feb528" } }, "reused_original": true, "original_guard_result": "artifacts/s3b-a-v04-ten-2026-11-15/candidates/splu/parts/part-00/guard-result.json", "original_guard_result_sha256": "9ca50e2510f578c9745df7a04720d474b85a9c1f594c16a15774b99f1e26e7fe", "pressure_calls": 468, "early_resolves": 386, "dead_end_share": 0.8247863247863247 }, { "method": "splu", "index": 1, "case": "00001324_5bf4a74af99f405392406e40_step_000", "worker": { "status": "PASS", "command": [ "taskset", "-c", "16", "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", "tools/run_s3b_v04_part.py", "artifacts/s3b-a-v04-ten-2026-11-15", "1", "splu" ], "returncode": 0, "stop_reason": null, "cpu_seconds": 360.905106, "wall_seconds": 361.0114358169958, "peak_tree_rss_bytes": 148307968, "peak_process_count": 1, "output_disk_bytes_before_receipt": 7524352, "max_poll_gap_seconds": 0.7691794019774534, "poll_target_seconds": 0.2, "reaped_orphan_cpu_seconds": 0.0, "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", "gpu_visible": false, "retry": false, "monitor_errors": 0, "last_monitor_error": null }, "classification": { "category": "PASS", "solver_attributable": false, "reason": null }, "files": { "result": { "path": "artifacts/s3b-a-v04-ten-2026-11-15/candidates/splu/parts/part-01/output/result.json", "sha256": "601778b817235773f539f3985cd4cc846206dd040ba5c348c41343da99bf489f" }, "progress": { "path": "artifacts/s3b-a-v04-ten-2026-11-15/candidates/splu/parts/part-01/output/progress.json", "sha256": "7bfd7626508b07fd39648992df13ec9d51410f9aa258a91a1ee5c0e716f20300" } }, "reused_original": true, "original_guard_result": "artifacts/s3b-a-v04-ten-2026-11-15/candidates/splu/parts/part-01/guard-result.json", "original_guard_result_sha256": "13a96b0072e4713cca94a9f7c7d6e89c53e044d1b5f0108a8de8d32750ccff21", "pressure_calls": 2808, "early_resolves": 2807, "dead_end_share": 0.9996438746438746 }, { "method": "splu", "index": 2, "case": "00000842_2c1efcc32d094e9fb9e1a88e_step_002", "worker": { "status": "PASS", "command": [ "taskset", "-c", "16", "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", "tools/run_s3b_v04_part.py", "artifacts/s3b-a-v04-ten-2026-11-15", "2", "splu" ], "returncode": 0, "stop_reason": null, "cpu_seconds": 565.748434, "wall_seconds": 565.9334301319905, "peak_tree_rss_bytes": 171499520, "peak_process_count": 1, "output_disk_bytes_before_receipt": 7901184, "max_poll_gap_seconds": 1.3971312350477092, "poll_target_seconds": 0.2, "reaped_orphan_cpu_seconds": 0.0, "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", "gpu_visible": false, "retry": false, "monitor_errors": 0, "last_monitor_error": null }, "classification": { "category": "PASS", "solver_attributable": false, "reason": null }, "files": { "result": { "path": "artifacts/s3b-a-v04-ten-2026-11-15/candidates/splu/parts/part-02/output/result.json", "sha256": "6f972eaba4f27eb4bd7f24941a4ec55419691fef1e460376c68343e5b130edb2" }, "progress": { "path": "artifacts/s3b-a-v04-ten-2026-11-15/candidates/splu/parts/part-02/output/progress.json", "sha256": "f754cfa618e96d901333a33a4f1dec9a1d7c154dad4fd5e836ed41a1fc0642e3" } }, "reused_original": true, "original_guard_result": "artifacts/s3b-a-v04-ten-2026-11-15/candidates/splu/parts/part-02/guard-result.json", "original_guard_result_sha256": "928aa79b59c6ac3b5b0dfe8b2747eed3e3c8b10b03f9ed2588edc67b9f97a204", "pressure_calls": 2332, "early_resolves": 2330, "dead_end_share": 0.9991423670668954 }, { "method": "splu", "index": 3, "case": "00000367_23c957e8e7b2428282a13ae2_step_000", "worker": { "status": "STOPPED_OR_FAILED", "command": [ "taskset", "-c", "16", "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", "tools/run_s3b_v04_part.py", "artifacts/s3b-a-v04-continuation-2026-11-15", "3", "splu" ], "returncode": -9, "stop_reason": "wall_limit", "cpu_seconds": 1799.533932, "wall_seconds": 1800.0501605010359, "peak_tree_rss_bytes": 317460480, "peak_process_count": 1, "output_disk_bytes_before_receipt": 15327232, "max_poll_gap_seconds": 0.9448504350148141, "poll_target_seconds": 0.2, "reaped_orphan_cpu_seconds": 0.0, "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", "gpu_visible": false, "retry": false, "monitor_errors": 0, "last_monitor_error": null }, "classification": { "category": "RESOURCE_STOP", "solver_attributable": true, "reason": "wall_limit", "last_phase": "PRESSURE_FACTORIZE" }, "pressure_calls": null, "early_resolves": null, "dead_end_share": null, "files": { "progress": { "path": "artifacts/s3b-a-v04-continuation-2026-11-15/candidates/splu/parts/part-03/output/progress.json", "sha256": "fd1527b1a0d4cd24ab7a876ffa1458eb2a016493a41b179eb8faf8efac186a74" } } } ], "not_run_cases": [ "00000746_6cb3ddbb49974d7db0e97135_step_001", "00000777_a25fb37392a44447b9684f16_step_002", "00001552_b0a9c4d836294846891615fe_step_000", "00001379_5b1d6e2599ea47d498a04256_step_000", "00000610_04e799936ae446b9a3d8d2b8_step_001", "00000666_6f67a59c16b949a4b66dcfe1_step_000" ] }, { "method": "gmres_ilu", "status": "STOPPED", "reason": "NUMERICAL_FAILURE", "classification": { "category": "NUMERICAL_FAILURE", "solver_attributable": true, "reason": "Pressure flux imbalance=1.0133935246514625e-08", "exception_type": "SolverFailure" }, "attempts": [ { "method": "gmres_ilu", "index": 0, "case": "00001130_c2db313c2ce749d784c610d2_step_002", "worker": { "status": "PASS", "command": [ "taskset", "-c", "16", "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", "tools/run_s3b_v04_part.py", "artifacts/s3b-a-v04-continuation-2026-11-15", "0", "gmres_ilu" ], "returncode": 0, "stop_reason": null, "cpu_seconds": 38.340387, "wall_seconds": 38.541681631002575, "peak_tree_rss_bytes": 117825536, "peak_process_count": 1, "output_disk_bytes_before_receipt": 5185536, "max_poll_gap_seconds": 0.7111166219692677, "poll_target_seconds": 0.2, "reaped_orphan_cpu_seconds": 0.0, "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", "gpu_visible": false, "retry": false, "monitor_errors": 0, "last_monitor_error": null }, "classification": { "category": "PASS", "solver_attributable": false, "reason": null }, "pressure_calls": 462, "early_resolves": 385, "dead_end_share": 0.8333333333333334, "files": { "result": { "path": "artifacts/s3b-a-v04-continuation-2026-11-15/candidates/gmres_ilu/parts/part-00/output/result.json", "sha256": "3121b815a6f1427d1087bce9f53265501adc49b8f99c6465b7e01e808a4a684e" }, "progress": { "path": "artifacts/s3b-a-v04-continuation-2026-11-15/candidates/gmres_ilu/parts/part-00/output/progress.json", "sha256": "0cc62e986635d4a7d269a9d65e0f87c6e2a570ef1d9fdfd457cd6031f5b5e0a7" } } }, { "method": "gmres_ilu", "index": 1, "case": "00001324_5bf4a74af99f405392406e40_step_000", "worker": { "status": "PASS", "command": [ "taskset", "-c", "16", "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", "tools/run_s3b_v04_part.py", "artifacts/s3b-a-v04-continuation-2026-11-15", "1", "gmres_ilu" ], "returncode": 0, "stop_reason": null, "cpu_seconds": 539.1514619999999, "wall_seconds": 539.4747563850251, "peak_tree_rss_bytes": 126115840, "peak_process_count": 1, "output_disk_bytes_before_receipt": 7581696, "max_poll_gap_seconds": 0.7155927050043829, "poll_target_seconds": 0.2, "reaped_orphan_cpu_seconds": 0.0, "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", "gpu_visible": false, "retry": false, "monitor_errors": 0, "last_monitor_error": null }, "classification": { "category": "PASS", "solver_attributable": false, "reason": null }, "pressure_calls": 2808, "early_resolves": 2807, "dead_end_share": 0.9996438746438746, "files": { "result": { "path": "artifacts/s3b-a-v04-continuation-2026-11-15/candidates/gmres_ilu/parts/part-01/output/result.json", "sha256": "b03105b1f2f71de55ee370520476a321eb06368bfef497ff14a21437c93c3f34" }, "progress": { "path": "artifacts/s3b-a-v04-continuation-2026-11-15/candidates/gmres_ilu/parts/part-01/output/progress.json", "sha256": "0e3f66a7643bcb1b454912cb298c1bd229c51b8ae650076b7b96058c335f52a8" } } }, { "method": "gmres_ilu", "index": 2, "case": "00000842_2c1efcc32d094e9fb9e1a88e_step_002", "worker": { "status": "STOPPED_OR_FAILED", "command": [ "taskset", "-c", "16", "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", "tools/run_s3b_v04_part.py", "artifacts/s3b-a-v04-continuation-2026-11-15", "2", "gmres_ilu" ], "returncode": 1, "stop_reason": null, "cpu_seconds": 304.09186, "wall_seconds": 304.32680653996067, "peak_tree_rss_bytes": 129392640, "peak_process_count": 1, "output_disk_bytes_before_receipt": 6287360, "max_poll_gap_seconds": 1.5154092180309817, "poll_target_seconds": 0.2, "reaped_orphan_cpu_seconds": 0.0, "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", "gpu_visible": false, "retry": false, "monitor_errors": 0, "last_monitor_error": null }, "classification": { "category": "NUMERICAL_FAILURE", "solver_attributable": true, "reason": "Pressure flux imbalance=1.0133935246514625e-08", "exception_type": "SolverFailure" }, "pressure_calls": null, "early_resolves": null, "dead_end_share": null, "files": { "failure": { "path": "artifacts/s3b-a-v04-continuation-2026-11-15/candidates/gmres_ilu/parts/part-02/output/failure.json", "sha256": "97b5e80cb685e82940085e9db515e3072834dfd3edbe39108e96a442d8c21aeb" }, "progress": { "path": "artifacts/s3b-a-v04-continuation-2026-11-15/candidates/gmres_ilu/parts/part-02/output/progress.json", "sha256": "2890cc47b92f7e1837e2f380b89c499c5db30f46eef5fecdbe446c01dcb02411" } } } ], "not_run_cases": [ "00000367_23c957e8e7b2428282a13ae2_step_000", "00000746_6cb3ddbb49974d7db0e97135_step_001", "00000777_a25fb37392a44447b9684f16_step_002", "00001552_b0a9c4d836294846891615fe_step_000", "00001379_5b1d6e2599ea47d498a04256_step_000", "00000610_04e799936ae446b9a3d8d2b8_step_001", "00000666_6f67a59c16b949a4b66dcfe1_step_000" ] } ], "attempts": 7, "new_worker_CPU_s": 3650.991436, "old_worker_CPU_s": 3239.962177, "cumulative_worker_CPU_s": 6890.953613, "postfit_reserve_CPU_s": 7200, "remaining_compute_CPU_s": 57909.046387, "coordinator_CPU_s": 210.73065, "wall_s": 2682.4470656110207, "account_complete": true, "no_silent_retries": true, "labels_read": false, "reused_original_PASS": 3, "new_attempts": 4, "candidate_CPU_s": { "splu": 2769.407727, "gmres_ilu": 881.5837089999999 }, "note": "Reference and full-count cost gates still required; no full617 launch by this coordinator." }