diff --git a/artifacts/combined-status-2026-11-16/status-20261116-193644.json b/artifacts/combined-status-2026-11-16/status-20261116-193644.json new file mode 100644 index 000000000..357eba316 --- /dev/null +++ b/artifacts/combined-status-2026-11-16/status-20261116-193644.json @@ -0,0 +1,1020 @@ +{ + "checked_at": "2026-11-16T19:36:43.892855+08:00", + "scope": "read-only existing processes and receipts; no scientific workload or restart", + "r4a": { + "root": "artifacts/r4a-execution-2026-11-16", + "progress": { + "completed": 240, + "attempted": 244, + "total": 617, + "passed": 240, + "active": { + "2": 11, + "3": 211, + "0": 239, + "1": 243 + }, + "settled_worker_cpu_seconds": 1050.2671650000004, + "reserved_active_worker_cpu_seconds": 7208, + "stop_reason": null, + "disk_bytes": 745791488, + "wall_seconds": 807.7830596329877 + }, + "coordinator": 515150, + "processes": [ + { + "pid": 515150, + "ppid": 2234, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "-u", + "tools/run_r4a.py", + "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16" + ], + "affinity": [ + 25 + ], + "cpu_seconds": 106.74, + "rss": 124624896 + }, + { + "pid": 515190, + "ppid": 515150, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_r4a.py", + "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16", + "--guard", + "11", + "--slot", + "2" + ], + "affinity": [ + 28 + ], + "cpu_seconds": 41.33, + "rss": 120741888 + }, + { + "pid": 516256, + "ppid": 515150, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_r4a.py", + "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16", + "--guard", + "211", + "--slot", + "3" + ], + "affinity": [ + 29 + ], + "cpu_seconds": 19.69, + "rss": 120213504 + }, + { + "pid": 516332, + "ppid": 515150, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_r4a.py", + "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16", + "--guard", + "239", + "--slot", + "0" + ], + "affinity": [ + 24 + ], + "cpu_seconds": 14.93, + "rss": 120492032 + }, + { + "pid": 516343, + "ppid": 515150, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_r4a.py", + "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16", + "--guard", + "243", + "--slot", + "1" + ], + "affinity": [ + 26 + ], + "cpu_seconds": 11.899999999999999, + "rss": 120709120 + }, + { + "pid": 516344, + "ppid": 516343, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/r4a_part.py", + "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16", + "243" + ], + "affinity": [ + 26 + ], + "cpu_seconds": 231.39000000000001, + "rss": 1439735808 + }, + { + "pid": 516334, + "ppid": 516332, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/r4a_part.py", + "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16", + "239" + ], + "affinity": [ + 24 + ], + "cpu_seconds": 257.91, + "rss": 745721856 + }, + { + "pid": 516259, + "ppid": 516256, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/r4a_part.py", + "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16", + "211" + ], + "affinity": [ + 29 + ], + "cpu_seconds": 345.65000000000003, + "rss": 1011757056 + }, + { + "pid": 515193, + "ppid": 515190, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/r4a_part.py", + "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16", + "11" + ], + "affinity": [ + 28 + ], + "cpu_seconds": 749.8599999999999, + "rss": 1885536256 + } + ], + "active_stages": [ + { + "index": 11, + "case": "00000033_ad34a3f60c4a4caa99646600_step_013", + "progress": { + "stage": "DISTANCE_ONLY_DF", + "case": "00000033_ad34a3f60c4a4caa99646600_step_013", + "pid": 515193, + "unix": 1794828881.640506, + "cpu_seconds": 632.273861933, + "wall_seconds": 667.0197777189896 + } + }, + { + "index": 211, + "case": "00000589_2417b410de78416ebe50d460_step_002", + "progress": { + "stage": "DISTANCE_ONLY_DF", + "case": "00000589_2417b410de78416ebe50d460_step_002", + "pid": 516259, + "unix": 1794828816.7441971, + "cpu_seconds": 167.040195631, + "wall_seconds": 175.48960097297095 + } + }, + { + "index": 239, + "case": "00000644_761a451dfea94bfa93e23cfe_step_000", + "progress": { + "stage": "DISTANCE_ONLY_DW", + "case": "00000644_761a451dfea94bfa93e23cfe_step_000", + "pid": 516334, + "unix": 1794828737.2965713, + "cpu_seconds": 3.578652186, + "wall_seconds": 3.780294761992991 + } + }, + { + "index": 243, + "case": "00000666_6f67a59c16b949a4b66dcfe1_step_000", + "progress": { + "stage": "DISTANCE_ONLY_DW", + "case": "00000666_6f67a59c16b949a4b66dcfe1_step_000", + "pid": 516344, + "unix": 1794828771.7815719, + "cpu_seconds": 8.577485997, + "wall_seconds": 9.032716328976676 + } + } + ], + "statuses": { + "PASS": 240 + }, + "failures": [], + "terminal_worker_cpu_seconds": 989.0595709999996, + "monitor_errors": 0, + "code_pins": { + "tools/gre24_features_v2.py": true, + "tools/gre24_prepare_v2.py": true, + "tools/run_gre24_protocol_preparation.py": true, + "tools/boundary_distance.py": true, + "tools/audit_samples.py": true, + "tools/s2_recovery_supervisor.py": true, + "tools/s2_recovery_monitoring.py": true, + "tools/r4a_core.py": true, + "tools/r4a_part.py": true, + "tools/r4a_score.py": true, + "tools/run_r4a.py": true, + "tools/prepare_r4a.py": true, + "tools/test_r4a.py": true, + "tools/s2_custody_common.py": true, + "tools/s2_custody_statistics.py": true, + "tools/s2_models/features.py": true, + "tools/s2_models/objectives.py": true, + "tools/freeze_g1_components.py": true + }, + "complete": null + }, + "v4": { + "root": "artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "progress": { + "completed": 403, + "attempted": 411, + "total": 629, + "active": { + "7": 386, + "2": 390, + "3": 400, + "4": 406, + "6": 407, + "5": 408, + "0": 409, + "1": 410 + }, + "settled_worker_cpu_seconds": 90126.96605799993, + "reserved_active_worker_cpu_seconds": 14416, + "stop_reason": null, + "first_part_checked": true, + "elapsed_seconds": 12321.98791659699 + }, + "coordinator": 510346, + "processes": [ + { + "pid": 510346, + "ppid": 2234, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "-u", + "tools/run_gre24_protocol_preparation.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol" + ], + "affinity": [ + 30, + 31 + ], + "cpu_seconds": 17.88, + "rss": 117022720 + }, + { + "pid": 515553, + "ppid": 510346, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_gre24_protocol_preparation.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "--guard", + "386", + "--slot", + "7" + ], + "affinity": [ + 22 + ], + "cpu_seconds": 36.44, + "rss": 113238016 + }, + { + "pid": 516110, + "ppid": 510346, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_gre24_protocol_preparation.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "--guard", + "390", + "--slot", + "2" + ], + "affinity": [ + 12 + ], + "cpu_seconds": 27.03, + "rss": 113090560 + }, + { + "pid": 516351, + "ppid": 510346, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_gre24_protocol_preparation.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "--guard", + "400", + "--slot", + "3" + ], + "affinity": [ + 14 + ], + "cpu_seconds": 13.25, + "rss": 112951296 + }, + { + "pid": 516386, + "ppid": 510346, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_gre24_protocol_preparation.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "--guard", + "406", + "--slot", + "4" + ], + "affinity": [ + 16 + ], + "cpu_seconds": 6.0, + "rss": 113184768 + }, + { + "pid": 516391, + "ppid": 510346, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_gre24_protocol_preparation.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "--guard", + "407", + "--slot", + "6" + ], + "affinity": [ + 20 + ], + "cpu_seconds": 3.78, + "rss": 112967680 + }, + { + "pid": 516448, + "ppid": 510346, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_gre24_protocol_preparation.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "--guard", + "408", + "--slot", + "5" + ], + "affinity": [ + 18 + ], + "cpu_seconds": 1.52, + "rss": 112873472 + }, + { + "pid": 516450, + "ppid": 510346, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_gre24_protocol_preparation.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "--guard", + "409", + "--slot", + "0" + ], + "affinity": [ + 8 + ], + "cpu_seconds": 1.14, + "rss": 112930816 + }, + { + "pid": 516452, + "ppid": 510346, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "/home/frank14f/optane/MeltMind/tools/run_gre24_protocol_preparation.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "--guard", + "410", + "--slot", + "1" + ], + "affinity": [ + 10 + ], + "cpu_seconds": 1.1099999999999999, + "rss": 112963584 + }, + { + "pid": 516461, + "ppid": 516452, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "410" + ], + "affinity": [ + 10 + ], + "cpu_seconds": 3.86, + "rss": 290848768 + }, + { + "pid": 516451, + "ppid": 516450, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "409" + ], + "affinity": [ + 8 + ], + "cpu_seconds": 5.88, + "rss": 322801664 + }, + { + "pid": 516449, + "ppid": 516448, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "408" + ], + "affinity": [ + 18 + ], + "cpu_seconds": 6.7299999999999995, + "rss": 256827392 + }, + { + "pid": 516392, + "ppid": 516391, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "407" + ], + "affinity": [ + 20 + ], + "cpu_seconds": 57.62, + "rss": 475856896 + }, + { + "pid": 516387, + "ppid": 516386, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "406" + ], + "affinity": [ + 16 + ], + "cpu_seconds": 87.87, + "rss": 220418048 + }, + { + "pid": 516352, + "ppid": 516351, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "400" + ], + "affinity": [ + 14 + ], + "cpu_seconds": 188.03, + "rss": 237416448 + }, + { + "pid": 516127, + "ppid": 516110, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "390" + ], + "affinity": [ + 12 + ], + "cpu_seconds": 437.69, + "rss": 243695616 + }, + { + "pid": 515555, + "ppid": 515553, + "cmd": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "386" + ], + "affinity": [ + 22 + ], + "cpu_seconds": 607.41, + "rss": 351612928 + } + ], + "active_stages": [ + { + "index": 386, + "case": "00000995_ac4ba032d4b5431ca04f542c_step_002", + "progress": { + "stage": "SPATIAL_MOMENT", + "case": "00000995_ac4ba032d4b5431ca04f542c_step_002", + "pid": 515555, + "unix": 1794828998.3618073, + "CPU_s": 601.309847356, + "wall_s": 636.0037915790454, + "completed": 8192, + "total": 71208 + } + }, + { + "index": 390, + "case": "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "progress": { + "stage": "SPATIAL_MOMENT", + "case": "00001002_6962bc8aad2c4e27b046d5b4_step_004", + "pid": 516127, + "unix": 1794828986.2077768, + "CPU_s": 419.766119638, + "wall_s": 444.36912770802155, + "completed": 28672, + "total": 54335 + } + }, + { + "index": 400, + "case": "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "progress": { + "stage": "SPATIAL_MOMENT", + "case": "00001015_6962bc8aad2c4e27b046d5b4_step_017", + "pid": 516352, + "unix": 1794828995.465062, + "CPU_s": 179.180740834, + "wall_s": 190.45949850301258, + "completed": 20480, + "total": 36000 + } + }, + { + "index": 406, + "case": "00001026_0f133d8e7fd448708a554e28_step_000", + "progress": { + "stage": "SPATIAL_MOMENT", + "case": "00001026_0f133d8e7fd448708a554e28_step_000", + "pid": 516387, + "unix": 1794828999.333138, + "CPU_s": 81.99525405300001, + "wall_s": 86.6175122190034, + "completed": 16384, + "total": 22139 + } + }, + { + "index": 407, + "case": "00001027_0f133d8e7fd448708a554e28_step_001", + "progress": { + "stage": "EIKONAL_DW", + "case": "00001027_0f133d8e7fd448708a554e28_step_001", + "pid": 516392, + "unix": 1794828946.3616507, + "CPU_s": 1.6946328840000002, + "wall_s": 1.7943739010370336 + } + }, + { + "index": 408, + "case": "00001029_cde47df6e28543d1b434f206_step_001", + "progress": { + "stage": "EIKONAL_DW", + "case": "00001029_cde47df6e28543d1b434f206_step_001", + "pid": 516449, + "unix": 1794828998.81889, + "CPU_s": 0.878861081, + "wall_s": 0.929792957031168 + } + }, + { + "index": 409, + "case": "00001030_cde47df6e28543d1b434f206_step_002", + "progress": { + "stage": "EIKONAL_DW", + "case": "00001030_cde47df6e28543d1b434f206_step_002", + "pid": 516451, + "unix": 1794829000.205969, + "CPU_s": 1.224686784, + "wall_s": 1.3043186760041863 + } + }, + { + "index": 410, + "case": "00001031_cde47df6e28543d1b434f206_step_003", + "progress": { + "stage": "EIKONAL_DW", + "case": "00001031_cde47df6e28543d1b434f206_step_003", + "pid": 516461, + "unix": 1794829002.7779684, + "CPU_s": 1.20619512, + "wall_s": 1.2851224210462533 + } + } + ], + "statuses": { + "PASS": 392, + "STOPPED_OR_FAILED": 11 + }, + "failures": [ + { + "case": "00000954_a741c7966d2c40a8a511263e_step_000", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "371" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 48.101245, + "wall_seconds": 51.47026148799341, + "peak_tree_rss_bytes": 378437632, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 3076096, + "max_poll_gap_seconds": 1.1429865359677933, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": { + "status": "FAILED", + "case": "00000954_a741c7966d2c40a8a511263e_step_000", + "type": "ValueError", + "message": "Invalid fill time on full tetrahedral domain", + "traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_prepare_v2.py\", line 125, in worker\n if not np.isfinite(y).all() or np.any(y<0):raise ValueError('Invalid fill time on full tetrahedral domain')\nValueError: Invalid fill time on full tetrahedral domain\n", + "worker_cpu_seconds": 47.135229131, + "wall_seconds": 49.850583860999905, + "automatic_retry": false + } + }, + { + "case": "00000628_60653f03c5044ef6a9b2e188_step_000", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "243" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1713.5742189999999, + "wall_seconds": 1800.1707399750012, + "peak_tree_rss_bytes": 3516559360, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 1.1422064260113984, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00000027_ad34a3f60c4a4caa99646600_step_007", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "10" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 228.527997, + "wall_seconds": 242.16197866998846, + "peak_tree_rss_bytes": 710733824, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 6512640, + "max_poll_gap_seconds": 1.401320293021854, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": { + "status": "FAILED", + "case": "00000027_ad34a3f60c4a4caa99646600_step_007", + "type": "ValueError", + "message": "Invalid fill time on full tetrahedral domain", + "traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_prepare_v2.py\", line 125, in worker\n if not np.isfinite(y).all() or np.any(y<0):raise ValueError('Invalid fill time on full tetrahedral domain')\nValueError: Invalid fill time on full tetrahedral domain\n", + "worker_cpu_seconds": 227.528839686, + "wall_seconds": 240.97960974898888, + "automatic_retry": false + } + }, + { + "case": "00000614_c710d414bbbc4340b682c58a_step_000", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "230" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1708.278232, + "wall_seconds": 1800.111270613037, + "peak_tree_rss_bytes": 3260960768, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 1.2739749389584176, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00000666_6f67a59c16b949a4b66dcfe1_step_000", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "252" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1713.8793269999999, + "wall_seconds": 1800.114591200021, + "peak_tree_rss_bytes": 3618865152, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 1.2054457549820654, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00000216_5324c79b172f41e8a49afc6f_step_003", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "76" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1710.2251780000001, + "wall_seconds": 1800.1801030740025, + "peak_tree_rss_bytes": 3159498752, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 1.316871771006845, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00000033_ad34a3f60c4a4caa99646600_step_013", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "13" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1703.8102840000001, + "wall_seconds": 1800.1185044309823, + "peak_tree_rss_bytes": 2882441216, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 2.0484302809927613, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00000575_5a4cef68211d4706b1ec8586_step_004", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "210" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1696.9662660000001, + "wall_seconds": 1800.1311887949705, + "peak_tree_rss_bytes": 2502758400, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 1.4239858739892952, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00000026_ad34a3f60c4a4caa99646600_step_006", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "9" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 389.07906299999996, + "wall_seconds": 411.82913551601814, + "peak_tree_rss_bytes": 834813952, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 7962624, + "max_poll_gap_seconds": 1.4062682209769264, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": { + "status": "FAILED", + "case": "00000026_ad34a3f60c4a4caa99646600_step_006", + "type": "ValueError", + "message": "Invalid fill time on full tetrahedral domain", + "traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_prepare_v2.py\", line 125, in worker\n if not np.isfinite(y).all() or np.any(y<0):raise ValueError('Invalid fill time on full tetrahedral domain')\nValueError: Invalid fill time on full tetrahedral domain\n", + "worker_cpu_seconds": 387.90751829699997, + "wall_seconds": 410.55529744998785, + "automatic_retry": false + } + }, + { + "case": "00000589_2417b410de78416ebe50d460_step_002", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "217" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1700.627005, + "wall_seconds": 1800.0867463279865, + "peak_tree_rss_bytes": 1376378880, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 1.1986027770326473, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + }, + { + "case": "00000625_6af98f5458b347bb843f0562_step_000", + "worker": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_prepare_v2.py", + "/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol", + "240" + ], + "returncode": -9, + "stop_reason": "wall_limit", + "cpu_seconds": 1705.598771, + "wall_seconds": 1800.3204157999717, + "peak_tree_rss_bytes": 3253915648, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 1.2696590220439248, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "failure": null + } + ], + "terminal_worker_cpu_seconds": 90066.96605800003, + "monitor_errors": 0, + "code_pins": { + "tools/gre24_features_v2.py": true, + "tools/gre24_prepare_v2.py": true, + "tools/run_gre24_protocol_preparation.py": true, + "tools/boundary_distance.py": true, + "tools/audit_samples.py": true, + "tools/s2_recovery_supervisor.py": true, + "tools/s2_recovery_monitoring.py": true + }, + "complete": null, + "model_stage": { + "stage": "WAIT_PREPARATION", + "unix": 1794828995.0993218 + } + } +}