Record R4 distance-only completions and v4 progress delta at 19:46

This commit is contained in:
Frank14f committed 2026-11-16 19:46:38 +08:00
1 parent 408a1d5502
commit 3914fc193f
1 file changed
+599
@@ -0,0 +1,599 @@
{
"checked_at": "2026-11-16T19:46:23.322530+08:00",
"host_service_observation": "2026-11-16 19:45:57 Asia/Shanghai: R4 PID515150, v4 prep510346/models510350 all active/running; no fit/score processes",
"scope": "read-only delta from 19:36",
"r4a": {
"progress": {
"completed": 354,
"attempted": 358,
"total": 617,
"passed": 354,
"active": {
"3": 211,
"1": 243,
"0": 356,
"2": 357
},
"settled_worker_cpu_seconds": 2913.787307000002,
"reserved_active_worker_cpu_seconds": 7208,
"stop_reason": null,
"disk_bytes": 1074114560,
"wall_seconds": 1387.2311872979626
},
"statuses": {
"PASS": 354
},
"terminal_worker_cpu_seconds": 2852.579712999998,
"monitor_errors": 0,
"failures": [],
"active": [
{
"case": "00000589_2417b410de78416ebe50d460_step_002",
"stage": {
"stage": "DISTANCE_ONLY_DF",
"case": "00000589_2417b410de78416ebe50d460_step_002",
"pid": 516259,
"unix": 1794828816.7441971,
"cpu_seconds": 167.040195631,
"wall_seconds": 175.48960097297095
}
},
{
"case": "00000666_6f67a59c16b949a4b66dcfe1_step_000",
"stage": {
"stage": "DISTANCE_ONLY_DW",
"case": "00000666_6f67a59c16b949a4b66dcfe1_step_000",
"pid": 516344,
"unix": 1794828771.7815719,
"cpu_seconds": 8.577485997,
"wall_seconds": 9.032716328976676
}
},
{
"case": "00000928_cdcebf32644443b9812097ba_step_012",
"stage": null
},
{
"case": "00000938_4f351b53bfd84bb4a2f7809c_step_003",
"stage": null
}
],
"source_pins": true,
"complete_exists": false,
"distance_only_terminals": [
{
"index": 239,
"case": "00000644_761a451dfea94bfa93e23cfe_step_000",
"worker": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/r4a_part.py",
"/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16",
"239"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 577.5051960000001,
"wall_seconds": 613.2635964300134,
"peak_tree_rss_bytes": 1524932608,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 8327168,
"max_poll_gap_seconds": 1.6022033719928004,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"result_path": "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16/parts/00000644_761a451dfea94bfa93e23cfe_step_000/output/result.json",
"result_sha256": "4e5bf80213fb4616f241c450f3b882edb9cd43dc2cde4e930284f15e896a27ee",
"guard_cpu_seconds": 36.347293
},
{
"index": 11,
"case": "00000033_ad34a3f60c4a4caa99646600_step_013",
"worker": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/r4a_part.py",
"/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16",
"11"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 849.8406729999999,
"wall_seconds": 896.5682624609908,
"peak_tree_rss_bytes": 2561675264,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 14487552,
"max_poll_gap_seconds": 1.967857150011696,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"result_path": "/home/frank14f/optane/MeltMind/artifacts/r4a-execution-2026-11-16/parts/00000033_ad34a3f60c4a4caa99646600_step_013/output/result.json",
"result_sha256": "adbeff3d0c9326fc96ef0b068ca8405df02837a76caacc86d009dc5c8286aa89",
"guard_cpu_seconds": 46.987772
}
]
},
"v4": {
"progress": {
"completed": 424,
"attempted": 432,
"total": 629,
"active": {
"7": 386,
"4": 411,
"3": 412,
"0": 423,
"6": 424,
"1": 428,
"2": 430,
"5": 431
},
"settled_worker_cpu_seconds": 93283.53410799992,
"reserved_active_worker_cpu_seconds": 14416,
"stop_reason": null,
"first_part_checked": true,
"elapsed_seconds": 12901.672486757976
},
"statuses": {
"PASS": 413,
"STOPPED_OR_FAILED": 11
},
"terminal_worker_cpu_seconds": 93223.53410800005,
"monitor_errors": 0,
"failures": [
{
"case": "00000954_a741c7966d2c40a8a511263e_step_000",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"371"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 48.101245,
"wall_seconds": 51.47026148799341,
"peak_tree_rss_bytes": 378437632,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 3076096,
"max_poll_gap_seconds": 1.1429865359677933,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": {
"status": "FAILED",
"case": "00000954_a741c7966d2c40a8a511263e_step_000",
"type": "ValueError",
"message": "Invalid fill time on full tetrahedral domain",
"traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_prepare_v2.py\", line 125, in worker\n if not np.isfinite(y).all() or np.any(y<0):raise ValueError('Invalid fill time on full tetrahedral domain')\nValueError: Invalid fill time on full tetrahedral domain\n",
"worker_cpu_seconds": 47.135229131,
"wall_seconds": 49.850583860999905,
"automatic_retry": false
}
},
{
"case": "00000628_60653f03c5044ef6a9b2e188_step_000",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"243"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1713.5742189999999,
"wall_seconds": 1800.1707399750012,
"peak_tree_rss_bytes": 3516559360,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 1.1422064260113984,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
},
{
"case": "00000027_ad34a3f60c4a4caa99646600_step_007",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"10"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 228.527997,
"wall_seconds": 242.16197866998846,
"peak_tree_rss_bytes": 710733824,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 6512640,
"max_poll_gap_seconds": 1.401320293021854,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": {
"status": "FAILED",
"case": "00000027_ad34a3f60c4a4caa99646600_step_007",
"type": "ValueError",
"message": "Invalid fill time on full tetrahedral domain",
"traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_prepare_v2.py\", line 125, in worker\n if not np.isfinite(y).all() or np.any(y<0):raise ValueError('Invalid fill time on full tetrahedral domain')\nValueError: Invalid fill time on full tetrahedral domain\n",
"worker_cpu_seconds": 227.528839686,
"wall_seconds": 240.97960974898888,
"automatic_retry": false
}
},
{
"case": "00000614_c710d414bbbc4340b682c58a_step_000",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"230"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1708.278232,
"wall_seconds": 1800.111270613037,
"peak_tree_rss_bytes": 3260960768,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 1.2739749389584176,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
},
{
"case": "00000666_6f67a59c16b949a4b66dcfe1_step_000",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"252"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1713.8793269999999,
"wall_seconds": 1800.114591200021,
"peak_tree_rss_bytes": 3618865152,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 1.2054457549820654,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
},
{
"case": "00000216_5324c79b172f41e8a49afc6f_step_003",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"76"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1710.2251780000001,
"wall_seconds": 1800.1801030740025,
"peak_tree_rss_bytes": 3159498752,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 1.316871771006845,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
},
{
"case": "00000033_ad34a3f60c4a4caa99646600_step_013",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"13"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1703.8102840000001,
"wall_seconds": 1800.1185044309823,
"peak_tree_rss_bytes": 2882441216,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 2.0484302809927613,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
},
{
"case": "00000575_5a4cef68211d4706b1ec8586_step_004",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"210"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1696.9662660000001,
"wall_seconds": 1800.1311887949705,
"peak_tree_rss_bytes": 2502758400,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 1.4239858739892952,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
},
{
"case": "00000026_ad34a3f60c4a4caa99646600_step_006",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"9"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 389.07906299999996,
"wall_seconds": 411.82913551601814,
"peak_tree_rss_bytes": 834813952,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 7962624,
"max_poll_gap_seconds": 1.4062682209769264,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": {
"status": "FAILED",
"case": "00000026_ad34a3f60c4a4caa99646600_step_006",
"type": "ValueError",
"message": "Invalid fill time on full tetrahedral domain",
"traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_prepare_v2.py\", line 125, in worker\n if not np.isfinite(y).all() or np.any(y<0):raise ValueError('Invalid fill time on full tetrahedral domain')\nValueError: Invalid fill time on full tetrahedral domain\n",
"worker_cpu_seconds": 387.90751829699997,
"wall_seconds": 410.55529744998785,
"automatic_retry": false
}
},
{
"case": "00000589_2417b410de78416ebe50d460_step_002",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"217"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1700.627005,
"wall_seconds": 1800.0867463279865,
"peak_tree_rss_bytes": 1376378880,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 1.1986027770326473,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
},
{
"case": "00000625_6af98f5458b347bb843f0562_step_000",
"worker": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_prepare_v2.py",
"/home/frank14f/optane/MeltMind/artifacts/gre24-v4-reproduction-2026-11-16/protocol",
"240"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1705.598771,
"wall_seconds": 1800.3204157999717,
"peak_tree_rss_bytes": 3253915648,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 1.2696590220439248,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
}
],
"active": [
{
"case": "00000995_ac4ba032d4b5431ca04f542c_step_002",
"stage": {
"stage": "SPATIAL_MOMENT",
"case": "00000995_ac4ba032d4b5431ca04f542c_step_002",
"pid": 515555,
"unix": 1794829512.790551,
"CPU_s": 1084.56625472,
"wall_s": 1150.4325324300444,
"completed": 45056,
"total": 71208
}
},
{
"case": "00001033_18151464e6434759b7c2ece7_step_000",
"stage": {
"stage": "SPATIAL_MOMENT",
"case": "00001033_18151464e6434759b7c2ece7_step_000",
"pid": 516467,
"unix": 1794829548.8055508,
"CPU_s": 505.81389160500004,
"wall_s": 537.1125250970363,
"completed": 24576,
"total": 71478
}
},
{
"case": "00001034_085828cd621d404a90ccd883_step_000",
"stage": {
"stage": "SPATIAL_MOMENT",
"case": "00001034_085828cd621d404a90ccd883_step_000",
"pid": 516614,
"unix": 1794829576.8869166,
"CPU_s": 484.285290868,
"wall_s": 512.7607590300031,
"completed": 49152,
"total": 77128
}
},
{
"case": "00001062_c04956ddff6c4324b5cad67c_step_000",
"stage": {
"stage": "SPATIAL_MOMENT",
"case": "00001062_c04956ddff6c4324b5cad67c_step_000",
"pid": 516847,
"unix": 1794829575.763751,
"CPU_s": 133.75944645799999,
"wall_s": 141.28714136203052,
"completed": 0,
"total": 44878
}
},
{
"case": "00001063_c6fda10dc58448d1951c5cc3_step_000",
"stage": {
"stage": "EIKONAL_DF",
"case": "00001063_c6fda10dc58448d1951c5cc3_step_000",
"pid": 516864,
"unix": 1794829553.9446516,
"CPU_s": 98.693033607,
"wall_s": 104.45657724502962
}
},
{
"case": "00001069_8a2edcf36bcd435092f4c8ae_step_003",
"stage": {
"stage": "SPATIAL_MOMENT",
"case": "00001069_8a2edcf36bcd435092f4c8ae_step_003",
"pid": 516935,
"unix": 1794829577.5080228,
"CPU_s": 59.750058657,
"wall_s": 63.207584253977984,
"completed": 0,
"total": 23912
}
},
{
"case": "00001097_256ee7ec3a4b4972a8b87c8d_step_001",
"stage": {
"stage": "EIKONAL_DW",
"case": "00001097_256ee7ec3a4b4972a8b87c8d_step_001",
"pid": 516999,
"unix": 1794829555.2450433,
"CPU_s": 1.249096701,
"wall_s": 1.3144874810241163
}
},
{
"case": "00001099_256ee7ec3a4b4972a8b87c8d_step_003",
"stage": {
"stage": "EIKONAL_DW",
"case": "00001099_256ee7ec3a4b4972a8b87c8d_step_003",
"pid": 517017,
"unix": 1794829558.6935182,
"CPU_s": 1.074551749,
"wall_s": 1.1390120180440135
}
}
],
"source_pins": true,
"complete_exists": false,
"model_stage": {
"stage": "WAIT_PREPARATION",
"unix": 1794829577.1623201
}
}
}