Freeze Gre24 model chain and retain synthetic acceptance evidence

This commit is contained in:
Frank14f committed 2026-11-15 23:54:50 +08:00
1 parent 8409910b1e
commit eb9ed0c762
124 files changed
+5993

No files matched your search

@@ -0,0 +1,32 @@
{
"status": "COMPLETE",
"accounting_complete": true,
"records": [
{
"case": "learn",
"worker": {
"status": "PASS"
},
"result_path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn/result.json",
"result_sha256": "9110e0f3a43d41602c1b17281544163455d2ae18b4f3509fca72aae063253751"
},
{
"case": "validation",
"worker": {
"status": "PASS"
},
"result_path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation/result.json",
"result_sha256": "2e7bf2831b9d0136b848c76aa3ed7e974f0dcbd26e97369b288b18bda9d57031"
},
{
"case": "test",
"worker": {
"status": "PASS"
},
"result_path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test/result.json",
"result_sha256": "7c8a234f50a9ab9424b78226cfd4d3709b140ff63cef2138aa19e2eb20e22c86"
}
],
"worker_cpu_seconds": 0.0,
"initial_cpu_reserve_seconds": 0.0
}
@@ -0,0 +1,7 @@
{
"parts": [
"learn",
"validation",
"test"
]
}
@@ -0,0 +1,16 @@
{
"status": "PASS",
"case": "learn",
"geometry": {
"tet_nodes": 128,
"outside_tet_nodes": 0
},
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
}
@@ -0,0 +1,16 @@
{
"status": "PASS",
"case": "learn",
"geometry": {
"tet_nodes": 128,
"outside_tet_nodes": 0
},
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
}
@@ -0,0 +1,6 @@
{
"pid": 480565,
"unix": 1794757290.256113,
"plan_sha256": "6e410499203d3e13862b271ad29b31f62cab20b2fc5530aa79b30156124c83d3",
"stage": "WAIT_PREPARATION"
}
@@ -0,0 +1,50 @@
{
"fits": {
"0/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"1/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"2/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"3/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"4/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"0/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"1/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"2/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"3/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"4/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
}
},
"charged_worker_cpu_seconds": 0.0,
"charged_gpu_seconds": 0.0,
"uuid": null,
"training_contract_sha256": "c37fbd2bcb5fbd4b12a9bab5cb4aa0b06f9ca5fab638fd46e09e03a7bcae32f9",
"all_fits_terminal": true,
"test_scores_read": false
}
@@ -0,0 +1,21 @@
{
"code_sha256": {
"tools/run_gre24_models.py": "2a5393a5009a2eb1bdd470370a9183b80aa7029d6b69a545831706a0139eb774",
"tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688",
"tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0",
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa"
},
"total_worker_cpu_seconds": 1000,
"postfit_cpu_reserve_seconds": 998,
"total_gpu_seconds": 60,
"initial_gpu_seconds": 0,
"dataset_version": "synthetic_budget_only",
"protocol": "synthetic",
"xgb_threads": 1,
"fnn_cpu_threads": 1,
"xgb_per_fit_cpu_seconds": 10,
"fnn_per_fit_cpu_seconds": 10,
"gpu_reserved_byte_cap": 16106127360
}
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 1.566765,
"attempts": [
{
"name": "worker",
"status": "PASS",
"cpu_seconds": 1.566765
}
]
}
@@ -0,0 +1,26 @@
{
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
"predict"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.566765,
"wall_seconds": 1.4599632910103537,
"peak_tree_rss_bytes": 110755840,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.363760320993606,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 4,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,17 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
"predict"
],
"limits": {
"total_cpu_seconds": 1000.0,
"job_cpu_seconds": 1000.0,
"job_wall_seconds": 7200,
"tree_rss_bytes": 25769803776,
"disk_bytes": 3221225472,
"max_processes": 1
},
"started_unix": 1794757290.2597783
}
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 1.299077,
"attempts": [
{
"name": "worker",
"status": "PASS",
"cpu_seconds": 1.299077
}
]
}
@@ -0,0 +1,26 @@
{
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
"score"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.299077,
"wall_seconds": 1.1043277920107357,
"peak_tree_rss_bytes": 111403008,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.21797074796631932,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 4,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,17 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
"score"
],
"limits": {
"total_cpu_seconds": 998.433235,
"job_cpu_seconds": 998.433235,
"job_wall_seconds": 7200,
"tree_rss_bytes": 25769803776,
"disk_bytes": 3221225472,
"max_processes": 1
},
"started_unix": 1794757291.7210498
}
@@ -0,0 +1,4 @@
{
"test_case_visits": 5,
"worker_CPU_s": 0.006219989999999953
}
@@ -0,0 +1,79 @@
{
"models_freeze_sha256": "d2c1c2d58053e3e42902604bbfdcd0fe6d07ec1d8893f548207abc01f44ec563",
"rows": [
{
"split": 0,
"case": "test",
"status": "PREDICTED",
"predictions": {
"naive": {
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-0/naive/test.npy",
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
"nodes": 128,
"no_clipping": true
}
},
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
},
{
"split": 1,
"case": "test",
"status": "PREDICTED",
"predictions": {
"naive": {
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-1/naive/test.npy",
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
"nodes": 128,
"no_clipping": true
}
},
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
},
{
"split": 2,
"case": "test",
"status": "PREDICTED",
"predictions": {
"naive": {
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-2/naive/test.npy",
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
"nodes": 128,
"no_clipping": true
}
},
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
},
{
"split": 3,
"case": "test",
"status": "PREDICTED",
"predictions": {
"naive": {
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-3/naive/test.npy",
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
"nodes": 128,
"no_clipping": true
}
},
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
},
{
"split": 4,
"case": "test",
"status": "PREDICTED",
"predictions": {
"naive": {
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-4/naive/test.npy",
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
"nodes": 128,
"no_clipping": true
}
},
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
}
],
"worker_CPU_s": 0.0065045639999998794,
"test_targets_read": false,
"device": "CPU",
"prediction_units": "seconds"
}
@@ -0,0 +1,50 @@
{
"successful": {
"learn": {
"output": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn",
"nodes": 128,
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
},
"result_sha256": "9110e0f3a43d41602c1b17281544163455d2ae18b4f3509fca72aae063253751",
"outside_tet_nodes": 0
},
"validation": {
"output": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation",
"nodes": 128,
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
},
"result_sha256": "2e7bf2831b9d0136b848c76aa3ed7e974f0dcbd26e97369b288b18bda9d57031",
"outside_tet_nodes": 0
},
"test": {
"output": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test",
"nodes": 128,
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
},
"result_sha256": "7c8a234f50a9ab9424b78226cfd4d3709b140ff63cef2138aa19e2eb20e22c86",
"outside_tet_nodes": 0
}
},
"failed": [],
"requested": 3,
"preparation_complete_sha256": "84d3c0674e4f128f7ac2bb88b09d6376976211efabde3e8746f239b68c5b835a",
"replacement": false,
"cumulative_initial_worker_cpu_seconds": 0.0
}
@@ -0,0 +1,111 @@
{
"status": "SCORED",
"fits": {
"0/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"1/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"2/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"3/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"4/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"0/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"1/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"2/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"3/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"4/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
}
},
"postfit": [
{
"phase": "predict",
"receipt": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
"predict"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.566765,
"wall_seconds": 1.4599632910103537,
"peak_tree_rss_bytes": 110755840,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.363760320993606,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 4,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
},
{
"phase": "score",
"receipt": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
"score"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.299077,
"wall_seconds": 1.1043277920107357,
"peak_tree_rss_bytes": 111403008,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.21797074796631932,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 4,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
}
],
"charged_worker_cpu_seconds": 2.865842,
"charged_gpu_seconds": 0.0,
"total_worker_cpu_cap": 1000,
"total_gpu_cap": 60,
"dataset_version": "synthetic_budget_only",
"paper_numbers_diagnostic_only": true
}
@@ -0,0 +1,519 @@
{
"dataset_version": "synthetic_budget_only",
"protocol": "synthetic",
"status": "SCORED_WITH_EXPLICIT_COVERAGE",
"summary": {
"naive": {
"rrmse_rms": {
"per_split": [
{
"split": 0,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.1607788938732118,
"median": 1.1607788938732118,
"max": 1.1607788938732118
},
{
"split": 1,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.1607788938732118,
"median": 1.1607788938732118,
"max": 1.1607788938732118
},
{
"split": 2,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.1607788938732118,
"median": 1.1607788938732118,
"max": 1.1607788938732118
},
{
"split": 3,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.1607788938732118,
"median": 1.1607788938732118,
"max": 1.1607788938732118
},
{
"split": 4,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.1607788938732118,
"median": 1.1607788938732118,
"max": 1.1607788938732118
}
],
"mean_of_five_split_summaries": {
"min": 1.1607788938732118,
"median": 1.1607788938732118,
"max": 1.1607788938732118
},
"full_requested_coverage": true
},
"rrmse_maximum": {
"per_split": [
{
"split": 0,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 0.9117259819436264,
"median": 0.9117259819436264,
"max": 0.9117259819436264
},
{
"split": 1,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 0.9117259819436264,
"median": 0.9117259819436264,
"max": 0.9117259819436264
},
{
"split": 2,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 0.9117259819436264,
"median": 0.9117259819436264,
"max": 0.9117259819436264
},
{
"split": 3,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 0.9117259819436264,
"median": 0.9117259819436264,
"max": 0.9117259819436264
},
{
"split": 4,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 0.9117259819436264,
"median": 0.9117259819436264,
"max": 0.9117259819436264
}
],
"mean_of_five_split_summaries": {
"min": 0.9117259819436263,
"median": 0.9117259819436263,
"max": 0.9117259819436263
},
"full_requested_coverage": true
},
"rrmse_mean": {
"per_split": [
{
"split": 0,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.173094770814564,
"median": 1.173094770814564,
"max": 1.173094770814564
},
{
"split": 1,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.173094770814564,
"median": 1.173094770814564,
"max": 1.173094770814564
},
{
"split": 2,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.173094770814564,
"median": 1.173094770814564,
"max": 1.173094770814564
},
{
"split": 3,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.173094770814564,
"median": 1.173094770814564,
"max": 1.173094770814564
},
{
"split": 4,
"requested_test_geometries": 1,
"scored": 1,
"missing": 0,
"min": 1.173094770814564,
"median": 1.173094770814564,
"max": 1.173094770814564
}
],
"mean_of_five_split_summaries": {
"min": 1.173094770814564,
"median": 1.173094770814564,
"max": 1.173094770814564
},
"full_requested_coverage": true
}
},
"GBRT": {
"rrmse_rms": {
"per_split": [
{
"split": 0,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 1,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 2,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 3,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 4,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
}
],
"mean_of_five_split_summaries": {
"min": null,
"median": null,
"max": null
},
"full_requested_coverage": false
},
"rrmse_maximum": {
"per_split": [
{
"split": 0,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 1,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 2,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 3,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 4,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
}
],
"mean_of_five_split_summaries": {
"min": null,
"median": null,
"max": null
},
"full_requested_coverage": false
},
"rrmse_mean": {
"per_split": [
{
"split": 0,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 1,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 2,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 3,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 4,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
}
],
"mean_of_five_split_summaries": {
"min": null,
"median": null,
"max": null
},
"full_requested_coverage": false
}
},
"FNN": {
"rrmse_rms": {
"per_split": [
{
"split": 0,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 1,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 2,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 3,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 4,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
}
],
"mean_of_five_split_summaries": {
"min": null,
"median": null,
"max": null
},
"full_requested_coverage": false
},
"rrmse_maximum": {
"per_split": [
{
"split": 0,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 1,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 2,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 3,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 4,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
}
],
"mean_of_five_split_summaries": {
"min": null,
"median": null,
"max": null
},
"full_requested_coverage": false
},
"rrmse_mean": {
"per_split": [
{
"split": 0,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 1,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 2,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 3,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
},
{
"split": 4,
"requested_test_geometries": 1,
"scored": 0,
"missing": 1,
"min": null,
"median": null,
"max": null
}
],
"mean_of_five_split_summaries": {
"min": null,
"median": null,
"max": null
},
"full_requested_coverage": false
}
}
},
"worker_CPU_s": 0.004958694000000152,
"scores_sha256": "0cb25cf6f6224e7cf2a5707bf99d85acada792d9fbf80166cfd4cf7ab3e0784e",
"paper_numbers_diagnostic_only": true,
"limitations": [
"v5 data instead of paper v4",
"R1 discretisation choices are explicit supplements",
"t_set substitutes unavailable applied Q",
"FNN fill-only and early stopping are declared choices",
"Random geometry splits may leak near-duplicate geometry families",
"Failed preparations not replaced; coverage is explicit"
]
}
@@ -0,0 +1,105 @@
{
"rows": [
{
"split": 0,
"case": "test",
"status": "SCORED",
"scores": {
"naive": {
"nodes": 128,
"rmse": 0.7029024782641866,
"denominators": {
"rms": 0.6055438137049401,
"maximum": 0.770958042421619,
"mean": 0.5991864389405733
},
"undefined_metrics": [],
"rrmse_rms": 1.1607788938732118,
"rrmse_maximum": 0.9117259819436264,
"rrmse_mean": 1.173094770814564
}
}
},
{
"split": 1,
"case": "test",
"status": "SCORED",
"scores": {
"naive": {
"nodes": 128,
"rmse": 0.7029024782641866,
"denominators": {
"rms": 0.6055438137049401,
"maximum": 0.770958042421619,
"mean": 0.5991864389405733
},
"undefined_metrics": [],
"rrmse_rms": 1.1607788938732118,
"rrmse_maximum": 0.9117259819436264,
"rrmse_mean": 1.173094770814564
}
}
},
{
"split": 2,
"case": "test",
"status": "SCORED",
"scores": {
"naive": {
"nodes": 128,
"rmse": 0.7029024782641866,
"denominators": {
"rms": 0.6055438137049401,
"maximum": 0.770958042421619,
"mean": 0.5991864389405733
},
"undefined_metrics": [],
"rrmse_rms": 1.1607788938732118,
"rrmse_maximum": 0.9117259819436264,
"rrmse_mean": 1.173094770814564
}
}
},
{
"split": 3,
"case": "test",
"status": "SCORED",
"scores": {
"naive": {
"nodes": 128,
"rmse": 0.7029024782641866,
"denominators": {
"rms": 0.6055438137049401,
"maximum": 0.770958042421619,
"mean": 0.5991864389405733
},
"undefined_metrics": [],
"rrmse_rms": 1.1607788938732118,
"rrmse_maximum": 0.9117259819436264,
"rrmse_mean": 1.173094770814564
}
}
},
{
"split": 4,
"case": "test",
"status": "SCORED",
"scores": {
"naive": {
"nodes": 128,
"rmse": 0.7029024782641866,
"denominators": {
"rms": 0.6055438137049401,
"maximum": 0.770958042421619,
"mean": 0.5991864389405733
},
"undefined_metrics": [],
"rrmse_rms": 1.1607788938732118,
"rrmse_maximum": 0.9117259819436264,
"rrmse_mean": 1.173094770814564
}
}
}
],
"predictions_freeze_sha256": "5cc930becfef50f13d4c724fd884f8de4cf95e47afd09d0250c3bae7a2f1b868"
}
@@ -0,0 +1,87 @@
[
{
"index": 0,
"seed": 91,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 1,
"seed": 92,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 2,
"seed": 93,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 3,
"seed": 94,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 4,
"seed": 95,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
}
]
@@ -0,0 +1,121 @@
{
"status": "PASS",
"checks": [
"CPU reserve prevents all fits",
"no GPU starts",
"terminal model freeze exists before prediction",
"naive score survives missing models",
"actual postfit CPU charged"
],
"result": {
"status": "SCORED",
"fits": {
"0/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"1/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"2/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"3/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"4/GBRT": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"0/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"1/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"2/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"3/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
},
"4/FNN": {
"status": "NOT_ATTEMPTED",
"reason": "SHARED_CPU_CAP_RESERVE"
}
},
"postfit": [
{
"phase": "predict",
"receipt": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
"predict"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.566765,
"wall_seconds": 1.4599632910103537,
"peak_tree_rss_bytes": 110755840,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.363760320993606,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 4,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
},
{
"phase": "score",
"receipt": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
"score"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.299077,
"wall_seconds": 1.1043277920107357,
"peak_tree_rss_bytes": 111403008,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.21797074796631932,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 4,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
}
],
"charged_worker_cpu_seconds": 2.865842,
"charged_gpu_seconds": 0.0,
"total_worker_cpu_cap": 1000,
"total_gpu_cap": 60,
"dataset_version": "synthetic_budget_only",
"paper_numbers_diagnostic_only": true
}
}
@@ -0,0 +1,15 @@
{
"status": "PASS",
"case": "test",
"geometry": {
"tet_nodes": 128,
"outside_tet_nodes": 0
},
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
}
@@ -0,0 +1,15 @@
{
"status": "PASS",
"case": "test",
"geometry": {
"tet_nodes": 128,
"outside_tet_nodes": 0
},
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
}
@@ -0,0 +1,23 @@
{
"code_sha256": {
"tools/run_gre24_models.py": "2a5393a5009a2eb1bdd470370a9183b80aa7029d6b69a545831706a0139eb774",
"tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688",
"tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0",
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa"
},
"total_worker_cpu_seconds": 1000,
"postfit_cpu_reserve_seconds": 998,
"total_gpu_seconds": 60,
"initial_gpu_seconds": 0,
"dataset_version": "synthetic_budget_only",
"protocol": "synthetic",
"xgb_threads": 1,
"fnn_cpu_threads": 1,
"xgb_per_fit_cpu_seconds": 10,
"fnn_per_fit_cpu_seconds": 10,
"gpu_reserved_byte_cap": 16106127360,
"preparation_freeze_sha256": "b2095d362c98db182025439f05891a766c72fda41bc8a392337a0a99926ad7f0",
"splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809"
}
@@ -0,0 +1,16 @@
{
"status": "PASS",
"case": "validation",
"geometry": {
"tet_nodes": 128,
"outside_tet_nodes": 0
},
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
}
@@ -0,0 +1,16 @@
{
"status": "PASS",
"case": "validation",
"geometry": {
"tet_nodes": 128,
"outside_tet_nodes": 0
},
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
}
@@ -0,0 +1,27 @@
{
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-01",
"0",
"GBRT"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 1.5006439999999999,
"wall_seconds": 1.512771409994457,
"peak_tree_rss_bytes": 148819968,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 225280,
"max_poll_gap_seconds": 0.11697613802971318,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,33 @@
{
"status": "FAILED",
"receipts": {
"GBRT": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-01",
"0",
"GBRT"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 1.5006439999999999,
"wall_seconds": 1.512771409994457,
"peak_tree_rss_bytes": 148819968,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 225280,
"max_poll_gap_seconds": 0.11697613802971318,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
},
"GPU_seconds": 0
}
@@ -0,0 +1,8 @@
{
"type": "ValueError",
"message": "Out of range float values are not JSON compliant: nan",
"traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_fit.py\", line 113, in fit\n save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,\n File \"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py\", line 23, in save\n tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\\n')\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/__init__.py\", line 238, in dumps\n **kw).encode(obj)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 201, in encode\n chunks = list(chunks)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 431, in _iterencode\n yield from _iterencode_dict(o, _current_indent_level)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 405, in _iterencode_dict\n yield from chunks\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 397, in _iterencode_dict\n yield _floatstr(value)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 239, in floatstr\n raise ValueError(\nValueError: Out of range float values are not JSON compliant: nan\n",
"CPU_s": 0.490331745,
"wall_s": 0.4909153020125814,
"automatic_retry": false
}
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 1.5006439999999999,
"attempts": [
{
"name": "worker",
"status": "STOPPED_OR_FAILED",
"cpu_seconds": 1.5006439999999999
}
]
}
@@ -0,0 +1,11 @@
{
"stage": "FIT",
"pid": 480171,
"unix": 1794757056.8670084,
"split": 0,
"model": "GBRT",
"CPU_s": 0.03734539199999998,
"wall_s": 0.03740089898929,
"nodes": 256,
"geometries": 2
}
@@ -0,0 +1,8 @@
{
"pid": 480171,
"unix": 1794757056.82963,
"split": 0,
"model": "GBRT",
"seed": 91,
"training_contract_sha256": "4180d820c98034ab333cf402e03de38f5979af34d3b6078b1c098ea40727e658"
}
@@ -0,0 +1,18 @@
[
{
"case": "learn",
"role": "train",
"nodes": 128,
"source": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/learn",
"feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
"target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc"
},
{
"case": "validation",
"role": "train",
"nodes": 128,
"source": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/validation",
"feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
"target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3"
}
]
@@ -0,0 +1,20 @@
Traceback (most recent call last):
File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 133, in <module>
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 113, in fit
save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,
File "/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py", line 23, in save
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/__init__.py", line 238, in dumps
**kw).encode(obj)
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 201, in encode
chunks = list(chunks)
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 431, in _iterencode
yield from _iterencode_dict(o, _current_indent_level)
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 405, in _iterencode_dict
yield from chunks
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 397, in _iterencode_dict
yield _floatstr(value)
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 239, in floatstr
raise ValueError(
ValueError: Out of range float values are not JSON compliant: nan
@@ -0,0 +1,27 @@
{
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-01",
"0",
"GBRT"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 1.5006439999999999,
"wall_seconds": 1.512771409994457,
"peak_tree_rss_bytes": 148819968,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 225280,
"max_poll_gap_seconds": 0.11697613802971318,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,18 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-01",
"0",
"GBRT"
],
"limits": {
"total_cpu_seconds": 120,
"job_cpu_seconds": 120,
"job_wall_seconds": 120,
"tree_rss_bytes": 4294967296,
"disk_bytes": 268435456,
"max_processes": 1
},
"started_unix": 1794757055.9604247
}
@@ -0,0 +1 @@
{"learner":{"generic_param":{"device":"cpu","fail_on_invalid_gpu_id":"0","n_jobs":"1","nthread":"1","random_state":"91","seed":"91","seed_per_iteration":"0","validate_parameters":"1"},"gradient_booster":{"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"100"},"gbtree_train_param":{"process_type":"default","tree_method":"auto","updater":"grow_quantile_histmaker","updater_seq":"grow_quantile_histmaker"},"name":"gbtree","specified_updater":false,"tree_train_param":{"alpha":"0","cache_opt":"1","colsample_bylevel":"1","colsample_bynode":"1","colsample_bytree":"1","eta":"0.100000001","gamma":"0","grow_policy":"depthwise","interaction_constraints":"","lambda":"1","learning_rate":"0.100000001","max_bin":"256","max_cat_threshold":"64","max_cat_to_onehot":"4","max_delta_step":"0","max_depth":"8","max_leaves":"0","min_child_weight":"1","min_split_loss":"0","monotone_constraints":"()","refresh_leaf":"1","reg_alpha":"0","reg_lambda":"1","sampling_method":"uniform","sketch_ratio":"2","sparse_threshold":"0.20000000000000001","subsample":"1"},"updater":[{"hist_train_param":{"debug_synchronize":"0","max_cached_hist_node":"65536"},"name":"grow_quantile_histmaker"}]},"learner_model_param":{"base_score":"3.0167848E-1","boost_from_average":"1","num_class":"0","num_feature":"19","num_target":"1"},"learner_train_param":{"booster":"gbtree","disable_default_eval_metric":"0","multi_strategy":"one_output_per_tree","objective":"reg:squarederror"},"metrics":[{"name":"rmse"}],"objective":{"name":"reg:squarederror","reg_loss_param":{"scale_pos_weight":"1"}}},"version":[2,1,4]}
@@ -0,0 +1,40 @@
{
"successful": {
"learn": {
"output": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/learn",
"nodes": 128,
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
},
"validation": {
"output": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/validation",
"nodes": 128,
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
},
"test": {
"output": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/test",
"nodes": 128,
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
}
},
"failed": []
}
@@ -0,0 +1,133 @@
"""One fixed split/model fit; reads training-side targets only, no test scores."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
import numpy as np
from gre24_prepare_v2 import sha
from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update
from s2_recovery_supervisor import save
def contract(root):
c=json.loads((root/'training-contract.json').read_text())
for p,h in c['code_sha256'].items():
if sha(p)!=h:raise ValueError('Pinned training source changed: '+p)
if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']:
raise ValueError('Preparation freeze changed')
if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed')
return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text())
def training_arrays(root,freeze,split,role,read_log):
selected=[case for case in split[role] if case in freeze['successful']]
n=sum(freeze['successful'][case]['nodes'] for case in selected)
if not n:raise ValueError('No successful training rows')
x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0
for case in selected:
require_training_role(case,split,role)
r=freeze['successful'][case];source=Path(r['output'])
for name in ['features.npy','target.npy']:
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed')
a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False)
count=len(b)
if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch')
x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count
read_log.append(dict(case=case,role=role,nodes=count,source=str(source),
feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy']))
if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input')
return x,y
def fit(root,index,model_name):
c,f,splits=contract(root);split=splits[index]
dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True)
if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite')
start=time.monotonic();cpu=time.process_time();read_log=[]
def progress(stage,**kw):
record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name,
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start)
record.update(kw);save(dest/'progress.json',record)
save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'],
training_contract_sha256=sha(root/'training-contract.json')))
roles,missing=eligible_roles(split,f['successful'])
try:
if model_name=='GBRT':
progress('LOAD_TRAIN_ONLY')
x,y=training_arrays(root,f,split,'train',read_log)
save(dest/'training-target-access.json',read_log)
model=xgb_model(split['seed'],c['xgb_threads'])
progress('FIT',nodes=len(y),geometries=len(roles['train']))
model.fit(x,y,verbose=False)
checkpoint=dest/'model.ubj';model.save_model(checkpoint)
(dest/'xgboost-config.json').write_text(model.get_booster().save_config())
extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=model.get_params())
elif model_name=='FNN':
import torch
torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed'])
if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable')
device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)
torch.backends.cuda.matmul.allow_tf32=False
progress('LOAD_FNN_TRAIN_VALIDATION_ONLY')
x,y=training_arrays(root,f,split,'fnn_learning',read_log)
vx,vy=training_arrays(root,f,split,'fnn_validation',read_log)
save(dest/'training-target-access.json',read_log)
x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device)
vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device)
model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001)
best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt'
for epoch in range(1,31):
model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device)
for begin in range(0,len(y),1024):
ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True)
prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2)
if not torch.isfinite(loss):raise ValueError('Nonfinite training loss')
loss.backward();optimizer.step();total+=loss.detach()*len(ids)
model.eval();sse=torch.zeros((),device=device)
with torch.no_grad():
for begin in range(0,len(vy),65536):
sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2)
validation=float((sse/len(vy)).item());train=float((total/len(y)).item())
best,stale,improved,stop=early_stop_update(best,stale,validation)
if improved:
torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()},
'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint)
best_epoch=epoch
row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch,
stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start)
history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row)
if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap')
if stop:break
restored=torch.load(checkpoint,map_location='cpu',weights_only=True)
model.load_state_dict(restored['state_dict'])
extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best,
train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']),
validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False,
input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
peak_reserved_bytes=torch.cuda.max_memory_reserved(device))
else:raise ValueError('Unknown model')
save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,
checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing,
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra))
progress('FIT_COMPLETE_NO_TEST_SCORES');return 0
except Exception as exc:
peaks={}
if model_name=='FNN':
try:
import torch
if torch.cuda.is_initialized():
peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
peak_reserved_bytes=torch.cuda.max_memory_reserved())
except Exception as peak_error:peaks={'peak_read_error':str(peak_error)}
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN'])
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
@@ -0,0 +1,82 @@
"""Single-GPU wall accounting and recorded/device VRAM checks including failure."""
import hashlib
import json
import math
from pathlib import Path
import subprocess
import threading
import time
import psutil
from gre24_job_guard import supervise,stop_group
def gpu_state(uuid=None):
args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits']
if uuid:args+=['-i',uuid]
out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout
rows=[]
for line in out.splitlines():
u,mem,util=[v.strip() for v in line.split(',')]
rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util)))
if not rows:raise RuntimeError('No GPU state returned')
return rows
def recorded_peaks(root):
peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
def walk(value,path,digest):
if isinstance(value,dict):
for k,v in value.items():
if k in peaks:
if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0:
raise ValueError('Invalid CUDA peak record')
if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest}
else:walk(v,path,digest)
elif isinstance(value,list):
for v in value:walk(v,path,digest)
for path in sorted(Path(root).glob('*.json')):
raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest())
return peaks,witnesses
def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap):
start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0}
maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
def scan():
current,seen=recorded_peaks(fit_root)
for k,v in current.items():
if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k]
if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']:
monitor['reasons'].append('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();state=gpu_state(uuid)[0];monitor['samples']+=1
monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes'])
if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']:
monitor['reasons'].append('DEVICE_VRAM_LIMIT')
except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc))
if monitor['reasons']:
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
done.wait(.5)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit,
job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30,
disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2)
finally:
done.set();thread.join(timeout=12)
if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP')
try:scan()
except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc))
if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS')
elapsed=time.monotonic()-start
if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT')
receipt['underlying_worker_status']=receipt['status']
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses,
total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap)
return receipt
@@ -0,0 +1,160 @@
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
"""
import fcntl
import ctypes
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import time
import psutil
from s2_recovery_monitoring import disk_usage,Monitor
def save(path, data):
path = Path(path)
tmp = path.with_suffix(path.suffix + '.tmp')
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
tmp.replace(path)
def tree_usage(pid):
try:
root = psutil.Process(pid)
processes = [root] + root.children(recursive=True)
except psutil.NoSuchProcess:
return 0., 0, 0
cpu = 0.
rss = count = 0
for p in processes:
try:
t = p.cpu_times()
# Linux child times include reaped descendants, not live children.
cpu += t.user + t.system + t.children_user + t.children_system
rss += p.memory_info().rss
count += 1
except psutil.NoSuchProcess:
pass
return cpu, rss, count
def stop_group(pid):
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
def supervise(command, root, name, limits, *, poll=.02, cuda_device='', numerical_threads=1):
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
root = Path(root)
root.mkdir(parents=True, exist_ok=True)
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
with (root/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
started_path = root/(name+'.started.json')
receipt_path = root/(name+'.receipt.json')
if started_path.exists() or receipt_path.exists():
raise FileExistsError('Attempt already exists; no silent retry')
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Every limit must be finite and positive')
ledger_path = root/'ledger.json'
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
raise RuntimeError('Total CPU exhausted before spawn')
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
raise RuntimeError('Disk exhausted before spawn')
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
start = last = time.monotonic()
cpu_peak = rss_peak = max_processes = max_gap = 0
cpu=rss=count=used_disk=0
reason = None
exit_status = usage = None
env = dict(os.environ, OMP_NUM_THREADS=str(numerical_threads), OPENBLAS_NUM_THREADS=str(numerical_threads),
MKL_NUM_THREADS=str(numerical_threads), NUMEXPR_NUM_THREADS=str(numerical_threads),
CUDA_VISIBLE_DEVICES=cuda_device)
with (root/(name+'.log')).open('x') as log:
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
env=env, start_new_session=True)
try:
while True:
now = time.monotonic()
max_gap = max(max_gap, now-last)
last = now
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
cpu_peak = max(cpu_peak,cpu)
rss_peak = max(rss_peak,rss)
max_processes = max(max_processes,count)
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
if now-start >= limits['job_wall_seconds']:
reason = 'wall_limit'
elif rss > limits['tree_rss_bytes']:
reason = 'tree_rss_limit'
elif count > limits['max_processes']:
reason = 'process_count_limit'
elif used_disk >= limits['disk_bytes']:
reason = 'disk_limit'
elif cpu_peak >= limits['job_cpu_seconds']:
reason = 'job_cpu_limit'
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
reason = 'total_cpu_limit'
if reason:
stop_group(proc.pid)
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
if pid:
exit_status, usage = status, child_usage
break
time.sleep(poll)
finally:
# No detached work may outlive a supervised attempt.
stop_group(proc.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(proc.pid, 0)
proc.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.
orphan_peak = 0
# Killed descendants are adopted here rather than losing their
# final CPU usage when the root worker cannot reap them itself.
while True:
try:
_, _, orphan_usage = os.wait4(-proc.pid, 0)
except ChildProcessError:
break
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
elapsed = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
if reason is None:
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
command=command,returncode=proc.returncode,stop_reason=reason,
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
reaped_orphan_cpu_seconds=orphan_cpu,
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
gpu_visible=bool(cuda_device),cuda_device=cuda_device,
numerical_threads=numerical_threads,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
ledger['cpu_seconds_charged'] += charged
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
return receipt
@@ -0,0 +1,42 @@
"""Fixed Gre24 model choices; no test-driven selection or node subsampling."""
import numpy as np
def xgb_model(seed,threads=8):
from xgboost import XGBRegressor
return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1,
objective='reg:squarederror',random_state=seed,n_jobs=threads)
def fnn_model():
import torch
layers=[];width=19
for _ in range(4):
layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100
layers.append(torch.nn.Linear(100,1))
return torch.nn.Sequential(*layers)
def eligible_roles(split,successful):
successful=set(successful)
roles={role:[c for c in split[role] if c in successful]
for role in ['train','test','fnn_learning','fnn_validation']}
if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap')
if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap')
if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']):
raise ValueError('FNN learning/validation do not partition train')
missing={role:[c for c in split[role] if c not in successful] for role in roles}
return roles,missing
def require_training_role(case,split,role):
if role not in ['train','fnn_learning','fnn_validation']:
raise PermissionError('Only frozen training-side roles may read targets')
if case in split['test'] or case not in split[role]:
raise PermissionError('Case is not in the requested training role')
def early_stop_update(best,stale,score,patience=5):
if not np.isfinite(score):raise ValueError('Nonfinite validation MSE')
improved=score<best
return (score if improved else best),(0 if improved else stale+1),improved,(not improved and stale+1>=patience)
@@ -0,0 +1,102 @@
"""Predictions frozen before test-target scoring, all successful tet nodes."""
import argparse
import json
from pathlib import Path
import time
import numpy as np
from gre24_prepare_v2 import sha
from gre24_features_v2 import per_geometry_metrics
from gre24_models import fnn_model
from gre24_fit import contract
from s2_recovery_supervisor import save
def predict(root):
c,prep,splits=contract(root)
models=json.loads((root/'models-freeze.json').read_text())
dest=root/'predictions';dest.mkdir(exist_ok=False)
start=time.process_time();rows=[]
for split in splits:
available={'naive':None}
for name in ['GBRT','FNN']:
r=models['fits'][f"{split['index']}/{name}"]
if r['status']!='PASS':continue
if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze')
if name=='GBRT':
from xgboost import XGBRegressor
m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint'])
else:
import torch
torch.set_num_threads(4);m=fnn_model()
m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval()
available[name]=m
for case in split['test']:
if case not in prep['successful']:
rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={}))
continue
r=prep['successful'][case];source=Path(r['output']);outputs={}
for name in ['features.npy','Vf.npy','raw_ids.npy']:
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed')
x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
# No seconds/target file is opened in this phase.
for name,m in available.items():
if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False)
elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32))
else:
chunks=[]
with torch.no_grad():
for a in range(0,len(x),65536):
chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy())
pred=np.concatenate(chunks)
if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction')
folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True)
path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds'])
outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True)
rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs,
raw_ids_sha256=r['output_sha256']['raw_ids.npy']))
save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start))
save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows,
worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds'))
def score(root):
c,prep,splits=contract(root);frozen=root/'predictions-freeze.json'
if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring')
freeze=json.loads(frozen.read_text());rows=[];start=time.process_time()
for r in freeze['rows']:
if r['status']!='PREDICTED':
rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue
p=prep['successful'][r['case']];source=Path(p['output'])
if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed')
y=np.load(source/'seconds.npy',allow_pickle=False);scores={}
for name,value in r['predictions'].items():
if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed')
pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred)
rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores))
summary={}
for name in ['naive','GBRT','FNN']:
summary[name]={}
for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']:
per=[]
for split in splits:
a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores']
and r['scores'][name][metric] is not None]
per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a),
missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None,
median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None))
# Do not average only whichever model splits happened to finish.
avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']}
summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg,
full_requested_coverage=all(r['missing']==0 for r in per))
save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen)))
save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'],
status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start,
scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True,
limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements',
't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices',
'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit']))
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score'])
a=p.parse_args();(predict if a.phase=='predict' else score)(a.root)
@@ -0,0 +1,87 @@
[
{
"index": 0,
"seed": 91,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 1,
"seed": 92,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 2,
"seed": 93,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 3,
"seed": 94,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 4,
"seed": 95,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
}
]
@@ -0,0 +1,16 @@
{
"code_sha256": {
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"tools/gre24_fit.py": "a3fc9364820ef23f237475cf2594d1089d86b87763aba4afae52a1b32d7d9ab6",
"tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0",
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa"
},
"preparation_freeze_sha256": "508a8997b875e9a978db40c5eed3a6407b3e512bb7916398e5bb688dd935262e",
"splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809",
"dataset_version": "synthetic_only",
"protocol": "synthetic smoke, not scientific data",
"xgb_threads": 1,
"fnn_cpu_threads": 2,
"gpu_reserved_byte_cap": 16106127360
}
@@ -0,0 +1,40 @@
{
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-02",
"0",
"FNN"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 4.8226309999999994,
"wall_seconds": 4.549783353984822,
"peak_tree_rss_bytes": 495280128,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 0.36855653702514246,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"numerical_threads": 2,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null,
"underlying_worker_status": "STOPPED_OR_FAILED",
"device_monitor": {
"reasons": [],
"peak_device_bytes": 9437184,
"samples": 8
},
"recorded_peaks": {
"peak_allocated_bytes": 0,
"peak_reserved_bytes": 0
},
"peak_witnesses": {},
"total_gpu_wall_seconds": 4.554046397970524,
"peak_cap_bytes": 16106127360
}
@@ -0,0 +1,27 @@
{
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-02",
"0",
"GBRT"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.5470069999999998,
"wall_seconds": 1.6621895720018074,
"peak_tree_rss_bytes": 148852736,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 221184,
"max_poll_gap_seconds": 0.12116481299744919,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,73 @@
{
"status": "FAILED",
"receipts": {
"GBRT": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-02",
"0",
"GBRT"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.5470069999999998,
"wall_seconds": 1.6621895720018074,
"peak_tree_rss_bytes": 148852736,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 221184,
"max_poll_gap_seconds": 0.12116481299744919,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"FNN": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-02",
"0",
"FNN"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 4.8226309999999994,
"wall_seconds": 4.549783353984822,
"peak_tree_rss_bytes": 495280128,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 0.36855653702514246,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"numerical_threads": 2,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null,
"underlying_worker_status": "STOPPED_OR_FAILED",
"device_monitor": {
"reasons": [],
"peak_device_bytes": 9437184,
"samples": 8
},
"recorded_peaks": {
"peak_allocated_bytes": 0,
"peak_reserved_bytes": 0
},
"peak_witnesses": {},
"total_gpu_wall_seconds": 4.554046397970524,
"peak_cap_bytes": 16106127360
}
},
"GPU_seconds": 4.601264455995988
}
@@ -0,0 +1,8 @@
{
"type": "RuntimeError",
"message": "Invalid device argument ",
"traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_fit.py\", line 77, in fit\n device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)\n File \"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/lib/python3.10/site-packages/torch/cuda/memory.py\", line 344, in reset_peak_memory_stats\n return torch._C._cuda_resetPeakMemoryStats(device)\nRuntimeError: Invalid device argument \n",
"CPU_s": 2.9871277210000002,
"wall_s": 2.784001157968305,
"automatic_retry": false
}
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 4.8226309999999994,
"attempts": [
{
"name": "worker",
"status": "STOPPED_OR_FAILED",
"cpu_seconds": 4.8226309999999994
}
]
}
@@ -0,0 +1,8 @@
{
"pid": 480269,
"unix": 1794757154.6071362,
"split": 0,
"model": "FNN",
"seed": 91,
"training_contract_sha256": "760d10e1df8863e5d4c34701a0a18678a6a2f9ede56b2cc83e95b6bbfab5848f"
}
@@ -0,0 +1,8 @@
Traceback (most recent call last):
File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 138, in <module>
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 77, in fit
device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)
File "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/lib/python3.10/site-packages/torch/cuda/memory.py", line 344, in reset_peak_memory_stats
return torch._C._cuda_resetPeakMemoryStats(device)
RuntimeError: Invalid device argument
@@ -0,0 +1,27 @@
{
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-02",
"0",
"FNN"
],
"returncode": 1,
"stop_reason": null,
"cpu_seconds": 4.8226309999999994,
"wall_seconds": 4.549783353984822,
"peak_tree_rss_bytes": 495280128,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 16384,
"max_poll_gap_seconds": 0.36855653702514246,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"numerical_threads": 2,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,18 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-02",
"0",
"FNN"
],
"limits": {
"total_cpu_seconds": 120.0,
"job_cpu_seconds": 120.0,
"job_wall_seconds": 57.0,
"tree_rss_bytes": 25769803776,
"disk_bytes": 3221225472,
"max_processes": 1
},
"started_unix": 1794757153.3916488
}
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 1.5470069999999998,
"attempts": [
{
"name": "worker",
"status": "PASS",
"cpu_seconds": 1.5470069999999998
}
]
}
@@ -0,0 +1,9 @@
{
"stage": "FIT_COMPLETE_NO_TEST_SCORES",
"pid": 480263,
"unix": 1794757153.0647538,
"split": 0,
"model": "GBRT",
"CPU_s": 0.495211623,
"wall_s": 0.4952992910402827
}
@@ -0,0 +1,58 @@
{
"status": "FIT_COMPLETE_NO_TEST_SCORES",
"split": 0,
"model": "GBRT",
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/model.ubj",
"checkpoint_sha256": "0630f79ad1b6783fa31ed5b6d94e345a896212187676f65168f4e819889ec74b",
"missing_preparation": {
"train": [],
"test": [],
"fnn_learning": [],
"fnn_validation": []
},
"CPU_s": 0.49491478599999994,
"wall_s": 0.4950045870500617,
"train_nodes": 256,
"train_geometries": 2,
"parameters": {
"objective": "reg:squarederror",
"base_score": null,
"booster": null,
"callbacks": null,
"colsample_bylevel": null,
"colsample_bynode": null,
"colsample_bytree": null,
"device": null,
"early_stopping_rounds": null,
"enable_categorical": false,
"eval_metric": null,
"feature_types": null,
"gamma": null,
"grow_policy": null,
"importance_type": null,
"interaction_constraints": null,
"learning_rate": 0.1,
"max_bin": null,
"max_cat_threshold": null,
"max_cat_to_onehot": null,
"max_delta_step": null,
"max_depth": 8,
"max_leaves": null,
"min_child_weight": null,
"missing": "NaN (library default missing marker)",
"monotone_constraints": null,
"multi_strategy": null,
"n_estimators": 100,
"n_jobs": 1,
"num_parallel_tree": null,
"random_state": 91,
"reg_alpha": null,
"reg_lambda": null,
"sampling_method": null,
"scale_pos_weight": null,
"subsample": null,
"tree_method": null,
"validate_parameters": null,
"verbosity": null
}
}
@@ -0,0 +1,8 @@
{
"pid": 480263,
"unix": 1794757152.5694737,
"split": 0,
"model": "GBRT",
"seed": 91,
"training_contract_sha256": "760d10e1df8863e5d4c34701a0a18678a6a2f9ede56b2cc83e95b6bbfab5848f"
}
@@ -0,0 +1,18 @@
[
{
"case": "learn",
"role": "train",
"nodes": 128,
"source": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/learn",
"feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
"target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc"
},
{
"case": "validation",
"role": "train",
"nodes": 128,
"source": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/validation",
"feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
"target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3"
}
]
@@ -0,0 +1,27 @@
{
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-02",
"0",
"GBRT"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.5470069999999998,
"wall_seconds": 1.6621895720018074,
"peak_tree_rss_bytes": 148852736,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 221184,
"max_poll_gap_seconds": 0.12116481299744919,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,18 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-02",
"0",
"GBRT"
],
"limits": {
"total_cpu_seconds": 120,
"job_cpu_seconds": 120,
"job_wall_seconds": 120,
"tree_rss_bytes": 4294967296,
"disk_bytes": 268435456,
"max_processes": 1
},
"started_unix": 1794757151.6780915
}
@@ -0,0 +1 @@
{"learner":{"generic_param":{"device":"cpu","fail_on_invalid_gpu_id":"0","n_jobs":"1","nthread":"1","random_state":"91","seed":"91","seed_per_iteration":"0","validate_parameters":"1"},"gradient_booster":{"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"100"},"gbtree_train_param":{"process_type":"default","tree_method":"auto","updater":"grow_quantile_histmaker","updater_seq":"grow_quantile_histmaker"},"name":"gbtree","specified_updater":false,"tree_train_param":{"alpha":"0","cache_opt":"1","colsample_bylevel":"1","colsample_bynode":"1","colsample_bytree":"1","eta":"0.100000001","gamma":"0","grow_policy":"depthwise","interaction_constraints":"","lambda":"1","learning_rate":"0.100000001","max_bin":"256","max_cat_threshold":"64","max_cat_to_onehot":"4","max_delta_step":"0","max_depth":"8","max_leaves":"0","min_child_weight":"1","min_split_loss":"0","monotone_constraints":"()","refresh_leaf":"1","reg_alpha":"0","reg_lambda":"1","sampling_method":"uniform","sketch_ratio":"2","sparse_threshold":"0.20000000000000001","subsample":"1"},"updater":[{"hist_train_param":{"debug_synchronize":"0","max_cached_hist_node":"65536"},"name":"grow_quantile_histmaker"}]},"learner_model_param":{"base_score":"3.0167848E-1","boost_from_average":"1","num_class":"0","num_feature":"19","num_target":"1"},"learner_train_param":{"booster":"gbtree","disable_default_eval_metric":"0","multi_strategy":"one_output_per_tree","objective":"reg:squarederror"},"metrics":[{"name":"rmse"}],"objective":{"name":"reg:squarederror","reg_loss_param":{"scale_pos_weight":"1"}}},"version":[2,1,4]}
@@ -0,0 +1,40 @@
{
"successful": {
"learn": {
"output": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/learn",
"nodes": 128,
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
},
"validation": {
"output": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/validation",
"nodes": 128,
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
},
"test": {
"output": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/test",
"nodes": 128,
"t_scale_seconds": 2.0,
"output_sha256": {
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
}
}
},
"failed": []
}
@@ -0,0 +1,138 @@
"""One fixed split/model fit; reads training-side targets only, no test scores."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
import numpy as np
from gre24_prepare_v2 import sha
from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update
from s2_recovery_supervisor import save
def contract(root):
c=json.loads((root/'training-contract.json').read_text())
for p,h in c['code_sha256'].items():
if sha(p)!=h:raise ValueError('Pinned training source changed: '+p)
if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']:
raise ValueError('Preparation freeze changed')
if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed')
return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text())
def training_arrays(root,freeze,split,role,read_log):
selected=[case for case in split[role] if case in freeze['successful']]
n=sum(freeze['successful'][case]['nodes'] for case in selected)
if not n:raise ValueError('No successful training rows')
x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0
for case in selected:
require_training_role(case,split,role)
r=freeze['successful'][case];source=Path(r['output'])
for name in ['features.npy','target.npy']:
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed')
a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False)
count=len(b)
if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch')
x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count
read_log.append(dict(case=case,role=role,nodes=count,source=str(source),
feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy']))
if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input')
return x,y
def fit(root,index,model_name):
c,f,splits=contract(root);split=splits[index]
dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True)
if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite')
start=time.monotonic();cpu=time.process_time();read_log=[]
def progress(stage,**kw):
record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name,
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start)
record.update(kw);save(dest/'progress.json',record)
save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'],
training_contract_sha256=sha(root/'training-contract.json')))
roles,missing=eligible_roles(split,f['successful'])
try:
if model_name=='GBRT':
progress('LOAD_TRAIN_ONLY')
x,y=training_arrays(root,f,split,'train',read_log)
save(dest/'training-target-access.json',read_log)
model=xgb_model(split['seed'],c['xgb_threads'])
progress('FIT',nodes=len(y),geometries=len(roles['train']))
model.fit(x,y,verbose=False)
checkpoint=dest/'model.ubj';model.save_model(checkpoint)
(dest/'xgboost-config.json').write_text(model.get_booster().save_config())
params=model.get_params()
# Preserve XGBoost's default missing=NaN as an explicit textual parameter;
# this is not a nonfinite observation or a change to the fitted estimator.
params={k:('NaN (library default missing marker)' if isinstance(v,float) and np.isnan(v) else v)
for k,v in params.items()}
extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=params)
elif model_name=='FNN':
import torch
torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed'])
if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable')
device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)
torch.backends.cuda.matmul.allow_tf32=False
progress('LOAD_FNN_TRAIN_VALIDATION_ONLY')
x,y=training_arrays(root,f,split,'fnn_learning',read_log)
vx,vy=training_arrays(root,f,split,'fnn_validation',read_log)
save(dest/'training-target-access.json',read_log)
x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device)
vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device)
model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001)
best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt'
for epoch in range(1,31):
model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device)
for begin in range(0,len(y),1024):
ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True)
prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2)
if not torch.isfinite(loss):raise ValueError('Nonfinite training loss')
loss.backward();optimizer.step();total+=loss.detach()*len(ids)
model.eval();sse=torch.zeros((),device=device)
with torch.no_grad():
for begin in range(0,len(vy),65536):
sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2)
validation=float((sse/len(vy)).item());train=float((total/len(y)).item())
best,stale,improved,stop=early_stop_update(best,stale,validation)
if improved:
torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()},
'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint)
best_epoch=epoch
row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch,
stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start)
history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row)
if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap')
if stop:break
restored=torch.load(checkpoint,map_location='cpu',weights_only=True)
model.load_state_dict(restored['state_dict'])
extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best,
train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']),
validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False,
input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
peak_reserved_bytes=torch.cuda.max_memory_reserved(device))
else:raise ValueError('Unknown model')
save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,
checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing,
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra))
progress('FIT_COMPLETE_NO_TEST_SCORES');return 0
except Exception as exc:
peaks={}
if model_name=='FNN':
try:
import torch
if torch.cuda.is_initialized():
peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
peak_reserved_bytes=torch.cuda.max_memory_reserved())
except Exception as peak_error:peaks={'peak_read_error':str(peak_error)}
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN'])
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
@@ -0,0 +1,82 @@
"""Single-GPU wall accounting and recorded/device VRAM checks including failure."""
import hashlib
import json
import math
from pathlib import Path
import subprocess
import threading
import time
import psutil
from gre24_job_guard import supervise,stop_group
def gpu_state(uuid=None):
args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits']
if uuid:args+=['-i',uuid]
out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout
rows=[]
for line in out.splitlines():
u,mem,util=[v.strip() for v in line.split(',')]
rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util)))
if not rows:raise RuntimeError('No GPU state returned')
return rows
def recorded_peaks(root):
peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
def walk(value,path,digest):
if isinstance(value,dict):
for k,v in value.items():
if k in peaks:
if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0:
raise ValueError('Invalid CUDA peak record')
if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest}
else:walk(v,path,digest)
elif isinstance(value,list):
for v in value:walk(v,path,digest)
for path in sorted(Path(root).glob('*.json')):
raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest())
return peaks,witnesses
def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap):
start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0}
maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
def scan():
current,seen=recorded_peaks(fit_root)
for k,v in current.items():
if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k]
if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']:
monitor['reasons'].append('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();state=gpu_state(uuid)[0];monitor['samples']+=1
monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes'])
if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']:
monitor['reasons'].append('DEVICE_VRAM_LIMIT')
except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc))
if monitor['reasons']:
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
done.wait(.5)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit,
job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30,
disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2)
finally:
done.set();thread.join(timeout=12)
if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP')
try:scan()
except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc))
if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS')
elapsed=time.monotonic()-start
if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT')
receipt['underlying_worker_status']=receipt['status']
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses,
total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap)
return receipt
@@ -0,0 +1,160 @@
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
"""
import fcntl
import ctypes
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import time
import psutil
from s2_recovery_monitoring import disk_usage,Monitor
def save(path, data):
path = Path(path)
tmp = path.with_suffix(path.suffix + '.tmp')
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
tmp.replace(path)
def tree_usage(pid):
try:
root = psutil.Process(pid)
processes = [root] + root.children(recursive=True)
except psutil.NoSuchProcess:
return 0., 0, 0
cpu = 0.
rss = count = 0
for p in processes:
try:
t = p.cpu_times()
# Linux child times include reaped descendants, not live children.
cpu += t.user + t.system + t.children_user + t.children_system
rss += p.memory_info().rss
count += 1
except psutil.NoSuchProcess:
pass
return cpu, rss, count
def stop_group(pid):
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
def supervise(command, root, name, limits, *, poll=.02, cuda_device='', numerical_threads=1):
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
root = Path(root)
root.mkdir(parents=True, exist_ok=True)
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
with (root/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
started_path = root/(name+'.started.json')
receipt_path = root/(name+'.receipt.json')
if started_path.exists() or receipt_path.exists():
raise FileExistsError('Attempt already exists; no silent retry')
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Every limit must be finite and positive')
ledger_path = root/'ledger.json'
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
raise RuntimeError('Total CPU exhausted before spawn')
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
raise RuntimeError('Disk exhausted before spawn')
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
start = last = time.monotonic()
cpu_peak = rss_peak = max_processes = max_gap = 0
cpu=rss=count=used_disk=0
reason = None
exit_status = usage = None
env = dict(os.environ, OMP_NUM_THREADS=str(numerical_threads), OPENBLAS_NUM_THREADS=str(numerical_threads),
MKL_NUM_THREADS=str(numerical_threads), NUMEXPR_NUM_THREADS=str(numerical_threads),
CUDA_VISIBLE_DEVICES=cuda_device)
with (root/(name+'.log')).open('x') as log:
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
env=env, start_new_session=True)
try:
while True:
now = time.monotonic()
max_gap = max(max_gap, now-last)
last = now
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
cpu_peak = max(cpu_peak,cpu)
rss_peak = max(rss_peak,rss)
max_processes = max(max_processes,count)
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
if now-start >= limits['job_wall_seconds']:
reason = 'wall_limit'
elif rss > limits['tree_rss_bytes']:
reason = 'tree_rss_limit'
elif count > limits['max_processes']:
reason = 'process_count_limit'
elif used_disk >= limits['disk_bytes']:
reason = 'disk_limit'
elif cpu_peak >= limits['job_cpu_seconds']:
reason = 'job_cpu_limit'
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
reason = 'total_cpu_limit'
if reason:
stop_group(proc.pid)
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
if pid:
exit_status, usage = status, child_usage
break
time.sleep(poll)
finally:
# No detached work may outlive a supervised attempt.
stop_group(proc.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(proc.pid, 0)
proc.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.
orphan_peak = 0
# Killed descendants are adopted here rather than losing their
# final CPU usage when the root worker cannot reap them itself.
while True:
try:
_, _, orphan_usage = os.wait4(-proc.pid, 0)
except ChildProcessError:
break
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
elapsed = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
if reason is None:
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
command=command,returncode=proc.returncode,stop_reason=reason,
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
reaped_orphan_cpu_seconds=orphan_cpu,
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
gpu_visible=bool(cuda_device),cuda_device=cuda_device,
numerical_threads=numerical_threads,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
ledger['cpu_seconds_charged'] += charged
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
return receipt
@@ -0,0 +1,42 @@
"""Fixed Gre24 model choices; no test-driven selection or node subsampling."""
import numpy as np
def xgb_model(seed,threads=8):
from xgboost import XGBRegressor
return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1,
objective='reg:squarederror',random_state=seed,n_jobs=threads)
def fnn_model():
import torch
layers=[];width=19
for _ in range(4):
layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100
layers.append(torch.nn.Linear(100,1))
return torch.nn.Sequential(*layers)
def eligible_roles(split,successful):
successful=set(successful)
roles={role:[c for c in split[role] if c in successful]
for role in ['train','test','fnn_learning','fnn_validation']}
if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap')
if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap')
if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']):
raise ValueError('FNN learning/validation do not partition train')
missing={role:[c for c in split[role] if c not in successful] for role in roles}
return roles,missing
def require_training_role(case,split,role):
if role not in ['train','fnn_learning','fnn_validation']:
raise PermissionError('Only frozen training-side roles may read targets')
if case in split['test'] or case not in split[role]:
raise PermissionError('Case is not in the requested training role')
def early_stop_update(best,stale,score,patience=5):
if not np.isfinite(score):raise ValueError('Nonfinite validation MSE')
improved=score<best
return (score if improved else best),(0 if improved else stale+1),improved,(not improved and stale+1>=patience)
@@ -0,0 +1,102 @@
"""Predictions frozen before test-target scoring, all successful tet nodes."""
import argparse
import json
from pathlib import Path
import time
import numpy as np
from gre24_prepare_v2 import sha
from gre24_features_v2 import per_geometry_metrics
from gre24_models import fnn_model
from gre24_fit import contract
from s2_recovery_supervisor import save
def predict(root):
c,prep,splits=contract(root)
models=json.loads((root/'models-freeze.json').read_text())
dest=root/'predictions';dest.mkdir(exist_ok=False)
start=time.process_time();rows=[]
for split in splits:
available={'naive':None}
for name in ['GBRT','FNN']:
r=models['fits'][f"{split['index']}/{name}"]
if r['status']!='PASS':continue
if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze')
if name=='GBRT':
from xgboost import XGBRegressor
m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint'])
else:
import torch
torch.set_num_threads(4);m=fnn_model()
m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval()
available[name]=m
for case in split['test']:
if case not in prep['successful']:
rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={}))
continue
r=prep['successful'][case];source=Path(r['output']);outputs={}
for name in ['features.npy','Vf.npy','raw_ids.npy']:
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed')
x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
# No seconds/target file is opened in this phase.
for name,m in available.items():
if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False)
elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32))
else:
chunks=[]
with torch.no_grad():
for a in range(0,len(x),65536):
chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy())
pred=np.concatenate(chunks)
if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction')
folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True)
path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds'])
outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True)
rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs,
raw_ids_sha256=r['output_sha256']['raw_ids.npy']))
save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start))
save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows,
worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds'))
def score(root):
c,prep,splits=contract(root);frozen=root/'predictions-freeze.json'
if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring')
freeze=json.loads(frozen.read_text());rows=[];start=time.process_time()
for r in freeze['rows']:
if r['status']!='PREDICTED':
rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue
p=prep['successful'][r['case']];source=Path(p['output'])
if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed')
y=np.load(source/'seconds.npy',allow_pickle=False);scores={}
for name,value in r['predictions'].items():
if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed')
pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred)
rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores))
summary={}
for name in ['naive','GBRT','FNN']:
summary[name]={}
for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']:
per=[]
for split in splits:
a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores']
and r['scores'][name][metric] is not None]
per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a),
missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None,
median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None))
# Do not average only whichever model splits happened to finish.
avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']}
summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg,
full_requested_coverage=all(r['missing']==0 for r in per))
save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen)))
save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'],
status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start,
scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True,
limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements',
't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices',
'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit']))
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score'])
a=p.parse_args();(predict if a.phase=='predict' else score)(a.root)
@@ -0,0 +1,87 @@
[
{
"index": 0,
"seed": 91,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 1,
"seed": 92,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 2,
"seed": 93,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 3,
"seed": 94,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
},
{
"index": 4,
"seed": 95,
"train": [
"learn",
"validation"
],
"test": [
"test"
],
"fnn_learning": [
"learn"
],
"fnn_validation": [
"validation"
]
}
]
@@ -0,0 +1,16 @@
{
"code_sha256": {
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"tools/gre24_fit.py": "29ac7c26d4164593afddb679481702e04cfb3b05e1f83ab4c56edea7162d37a0",
"tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0",
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa"
},
"preparation_freeze_sha256": "41f1ab982ce91eae8c71a5080f53aa8909e6eb1089d376e56b91a40c573aaa13",
"splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809",
"dataset_version": "synthetic_only",
"protocol": "synthetic smoke, not scientific data",
"xgb_threads": 1,
"fnn_cpu_threads": 2,
"gpu_reserved_byte_cap": 16106127360
}
@@ -0,0 +1,49 @@
{
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"0",
"FNN"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 5.798009,
"wall_seconds": 5.534140970034059,
"peak_tree_rss_bytes": 1042665472,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 159744,
"max_poll_gap_seconds": 0.39918523997766897,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"numerical_threads": 2,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null,
"underlying_worker_status": "PASS",
"device_monitor": {
"reasons": [],
"peak_device_bytes": 435159040,
"samples": 10
},
"recorded_peaks": {
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672
},
"peak_witnesses": {
"peak_allocated_bytes": {
"path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json",
"sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af"
},
"peak_reserved_bytes": {
"path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json",
"sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af"
}
},
"total_gpu_wall_seconds": 5.538356595963705,
"peak_cap_bytes": 16106127360
}
@@ -0,0 +1,27 @@
{
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"0",
"GBRT"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.6755669999999998,
"wall_seconds": 1.7543383440352045,
"peak_tree_rss_bytes": 148721664,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 221184,
"max_poll_gap_seconds": 0.22388055600458756,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,145 @@
{
"status": "PASS",
"receipts": {
"GBRT": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"0",
"GBRT"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.6755669999999998,
"wall_seconds": 1.7543383440352045,
"peak_tree_rss_bytes": 148721664,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 221184,
"max_poll_gap_seconds": 0.22388055600458756,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"FNN": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"0",
"FNN"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 5.798009,
"wall_seconds": 5.534140970034059,
"peak_tree_rss_bytes": 1042665472,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 159744,
"max_poll_gap_seconds": 0.39918523997766897,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"numerical_threads": 2,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null,
"underlying_worker_status": "PASS",
"device_monitor": {
"reasons": [],
"peak_device_bytes": 435159040,
"samples": 10
},
"recorded_peaks": {
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672
},
"peak_witnesses": {
"peak_allocated_bytes": {
"path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json",
"sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af"
},
"peak_reserved_bytes": {
"path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json",
"sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af"
}
},
"total_gpu_wall_seconds": 5.538356595963705,
"peak_cap_bytes": 16106127360
},
"predict": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"predict"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 3.362805,
"wall_seconds": 3.0489734660368413,
"peak_tree_rss_bytes": 519958528,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.46637159201782197,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"score": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"score"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 0.6084579999999999,
"wall_seconds": 0.7152220709831454,
"peak_tree_rss_bytes": 111345664,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.11809522699331865,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
},
"GPU_seconds": 5.584548475977499,
"worker_CPU_seconds": 11.444839,
"checks": [
"real GBRT and FNN entrypoints",
"test target absent during fits",
"test seconds unavailable during prediction",
"all predictions frozen before scoring",
"missing splits not silently averaged",
"failure VRAM peaks retained"
],
"scientific_data_read": false,
"coordinator_CPU_seconds": 2.8703529999999997
}
@@ -0,0 +1,152 @@
[
{
"epoch": 1,
"training_mse": 0.07795825600624084,
"validation_mse": 0.060705915093421936,
"best_epoch": 1,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.7462023249827325
},
{
"epoch": 2,
"training_mse": 0.06311292946338654,
"validation_mse": 0.04847054183483124,
"best_epoch": 2,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.751589720021002
},
{
"epoch": 3,
"training_mse": 0.05059317126870155,
"validation_mse": 0.03820352628827095,
"best_epoch": 3,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.7556245460291393
},
{
"epoch": 4,
"training_mse": 0.04009852185845375,
"validation_mse": 0.02991129830479622,
"best_epoch": 4,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.7596782970358618
},
{
"epoch": 5,
"training_mse": 0.031606074422597885,
"validation_mse": 0.02256850153207779,
"best_epoch": 5,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.7651306050247513
},
{
"epoch": 6,
"training_mse": 0.024003881961107254,
"validation_mse": 0.0158725343644619,
"best_epoch": 6,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.769136717019137
},
{
"epoch": 7,
"training_mse": 0.017005659639835358,
"validation_mse": 0.010371999815106392,
"best_epoch": 7,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.773159590025898
},
{
"epoch": 8,
"training_mse": 0.011204666458070278,
"validation_mse": 0.006224535871297121,
"best_epoch": 8,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.7771851680008695
},
{
"epoch": 9,
"training_mse": 0.00675668939948082,
"validation_mse": 0.003289248328655958,
"best_epoch": 9,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.7813589150318876
},
{
"epoch": 10,
"training_mse": 0.0034951232373714447,
"validation_mse": 0.0018917981069535017,
"best_epoch": 10,
"stale_epochs": 0,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.7855790790054016
},
{
"epoch": 11,
"training_mse": 0.001763510867021978,
"validation_mse": 0.0024419045075774193,
"best_epoch": 10,
"stale_epochs": 1,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.788880237028934
},
{
"epoch": 12,
"training_mse": 0.0019949353300035,
"validation_mse": 0.004658875055611134,
"best_epoch": 10,
"stale_epochs": 2,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.792193347995635
},
{
"epoch": 13,
"training_mse": 0.003945866599678993,
"validation_mse": 0.007049876265227795,
"best_epoch": 10,
"stale_epochs": 3,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.7955055559868924
},
{
"epoch": 14,
"training_mse": 0.006167569197714329,
"validation_mse": 0.008173784241080284,
"best_epoch": 10,
"stale_epochs": 4,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.798879029985983
},
{
"epoch": 15,
"training_mse": 0.007227514870464802,
"validation_mse": 0.007775146048516035,
"best_epoch": 10,
"stale_epochs": 5,
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672,
"wall_s": 3.8023696839809418
}
]
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 5.798009,
"attempts": [
{
"name": "worker",
"status": "PASS",
"cpu_seconds": 5.798009
}
]
}
@@ -0,0 +1,9 @@
{
"stage": "FIT_COMPLETE_NO_TEST_SCORES",
"pid": 480379,
"unix": 1794757202.323634,
"split": 0,
"model": "FNN",
"CPU_s": 4.038258777,
"wall_s": 3.8051335139898583
}
@@ -0,0 +1,27 @@
{
"status": "FIT_COMPLETE_NO_TEST_SCORES",
"split": 0,
"model": "FNN",
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/model.pt",
"checkpoint_sha256": "737d54bf01b17dbd87756fcf43bb9b3333597d05a28c94d8cd4bc309e682c0b2",
"missing_preparation": {
"train": [],
"test": [],
"fnn_learning": [],
"fnn_validation": []
},
"CPU_s": 4.038131669,
"wall_s": 3.8050071470206603,
"epochs": 15,
"best_epoch": 10,
"best_validation_mse": 0.0018917981069535017,
"train_nodes": 128,
"validation_nodes": 128,
"train_geometries": 1,
"validation_geometries": 1,
"restored_best": true,
"amp": false,
"input_standardization": "none; use eq12 columns",
"peak_allocated_bytes": 18742272,
"peak_reserved_bytes": 23068672
}
@@ -0,0 +1,8 @@
{
"pid": 480379,
"unix": 1794757198.5185177,
"split": 0,
"model": "FNN",
"seed": 91,
"training_contract_sha256": "c86ab1d6ef22262200789c1b0d7aadd112b68862a64275a1c657a9d3f9f39dfd"
}
@@ -0,0 +1,18 @@
[
{
"case": "learn",
"role": "fnn_learning",
"nodes": 128,
"source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/learn",
"feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
"target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc"
},
{
"case": "validation",
"role": "fnn_validation",
"nodes": 128,
"source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/validation",
"feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
"target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3"
}
]
@@ -0,0 +1,27 @@
{
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"0",
"FNN"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 5.798009,
"wall_seconds": 5.534140970034059,
"peak_tree_rss_bytes": 1042665472,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 159744,
"max_poll_gap_seconds": 0.39918523997766897,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"numerical_threads": 2,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,18 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"0",
"FNN"
],
"limits": {
"total_cpu_seconds": 120.0,
"job_cpu_seconds": 120.0,
"job_wall_seconds": 57.0,
"tree_rss_bytes": 25769803776,
"disk_bytes": 3221225472,
"max_processes": 1
},
"started_unix": 1794757197.6559174
}
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 1.6755669999999998,
"attempts": [
{
"name": "worker",
"status": "PASS",
"cpu_seconds": 1.6755669999999998
}
]
}
@@ -0,0 +1,9 @@
{
"stage": "FIT_COMPLETE_NO_TEST_SCORES",
"pid": 480373,
"unix": 1794757197.3915205,
"split": 0,
"model": "GBRT",
"CPU_s": 0.6673245510000001,
"wall_s": 0.6674595809890889
}
@@ -0,0 +1,58 @@
{
"status": "FIT_COMPLETE_NO_TEST_SCORES",
"split": 0,
"model": "GBRT",
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/model.ubj",
"checkpoint_sha256": "0630f79ad1b6783fa31ed5b6d94e345a896212187676f65168f4e819889ec74b",
"missing_preparation": {
"train": [],
"test": [],
"fnn_learning": [],
"fnn_validation": []
},
"CPU_s": 0.6670500859999999,
"wall_s": 0.6671876410255209,
"train_nodes": 256,
"train_geometries": 2,
"parameters": {
"objective": "reg:squarederror",
"base_score": null,
"booster": null,
"callbacks": null,
"colsample_bylevel": null,
"colsample_bynode": null,
"colsample_bytree": null,
"device": null,
"early_stopping_rounds": null,
"enable_categorical": false,
"eval_metric": null,
"feature_types": null,
"gamma": null,
"grow_policy": null,
"importance_type": null,
"interaction_constraints": null,
"learning_rate": 0.1,
"max_bin": null,
"max_cat_threshold": null,
"max_cat_to_onehot": null,
"max_delta_step": null,
"max_depth": 8,
"max_leaves": null,
"min_child_weight": null,
"missing": "NaN (library default missing marker)",
"monotone_constraints": null,
"multi_strategy": null,
"n_estimators": 100,
"n_jobs": 1,
"num_parallel_tree": null,
"random_state": 91,
"reg_alpha": null,
"reg_lambda": null,
"sampling_method": null,
"scale_pos_weight": null,
"subsample": null,
"tree_method": null,
"validate_parameters": null,
"verbosity": null
}
}
@@ -0,0 +1,8 @@
{
"pid": 480373,
"unix": 1794757196.7240794,
"split": 0,
"model": "GBRT",
"seed": 91,
"training_contract_sha256": "c86ab1d6ef22262200789c1b0d7aadd112b68862a64275a1c657a9d3f9f39dfd"
}
@@ -0,0 +1,18 @@
[
{
"case": "learn",
"role": "train",
"nodes": 128,
"source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/learn",
"feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
"target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc"
},
{
"case": "validation",
"role": "train",
"nodes": 128,
"source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/validation",
"feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
"target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3"
}
]
@@ -0,0 +1,27 @@
{
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"0",
"GBRT"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1.6755669999999998,
"wall_seconds": 1.7543383440352045,
"peak_tree_rss_bytes": 148721664,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 221184,
"max_poll_gap_seconds": 0.22388055600458756,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,18 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_fit.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"0",
"GBRT"
],
"limits": {
"total_cpu_seconds": 120,
"job_cpu_seconds": 120,
"job_wall_seconds": 120,
"tree_rss_bytes": 4294967296,
"disk_bytes": 268435456,
"max_processes": 1
},
"started_unix": 1794757195.8518412
}
@@ -0,0 +1 @@
{"learner":{"generic_param":{"device":"cpu","fail_on_invalid_gpu_id":"0","n_jobs":"1","nthread":"1","random_state":"91","seed":"91","seed_per_iteration":"0","validate_parameters":"1"},"gradient_booster":{"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"100"},"gbtree_train_param":{"process_type":"default","tree_method":"auto","updater":"grow_quantile_histmaker","updater_seq":"grow_quantile_histmaker"},"name":"gbtree","specified_updater":false,"tree_train_param":{"alpha":"0","cache_opt":"1","colsample_bylevel":"1","colsample_bynode":"1","colsample_bytree":"1","eta":"0.100000001","gamma":"0","grow_policy":"depthwise","interaction_constraints":"","lambda":"1","learning_rate":"0.100000001","max_bin":"256","max_cat_threshold":"64","max_cat_to_onehot":"4","max_delta_step":"0","max_depth":"8","max_leaves":"0","min_child_weight":"1","min_split_loss":"0","monotone_constraints":"()","refresh_leaf":"1","reg_alpha":"0","reg_lambda":"1","sampling_method":"uniform","sketch_ratio":"2","sparse_threshold":"0.20000000000000001","subsample":"1"},"updater":[{"hist_train_param":{"debug_synchronize":"0","max_cached_hist_node":"65536"},"name":"grow_quantile_histmaker"}]},"learner_model_param":{"base_score":"3.0167848E-1","boost_from_average":"1","num_class":"0","num_feature":"19","num_target":"1"},"learner_train_param":{"booster":"gbtree","disable_default_eval_metric":"0","multi_strategy":"one_output_per_tree","objective":"reg:squarederror"},"metrics":[{"name":"rmse"}],"objective":{"name":"reg:squarederror","reg_loss_param":{"scale_pos_weight":"1"}}},"version":[2,1,4]}
@@ -0,0 +1,38 @@
{
"fits": {
"0/GBRT": {
"status": "PASS",
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/model.ubj",
"checkpoint_sha256": "0630f79ad1b6783fa31ed5b6d94e345a896212187676f65168f4e819889ec74b"
},
"0/FNN": {
"status": "PASS",
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/model.pt",
"checkpoint_sha256": "737d54bf01b17dbd87756fcf43bb9b3333597d05a28c94d8cd4bc309e682c0b2"
},
"1/GBRT": {
"status": "NOT_ATTEMPTED_SYNTHETIC"
},
"1/FNN": {
"status": "NOT_ATTEMPTED_SYNTHETIC"
},
"2/GBRT": {
"status": "NOT_ATTEMPTED_SYNTHETIC"
},
"2/FNN": {
"status": "NOT_ATTEMPTED_SYNTHETIC"
},
"3/GBRT": {
"status": "NOT_ATTEMPTED_SYNTHETIC"
},
"3/FNN": {
"status": "NOT_ATTEMPTED_SYNTHETIC"
},
"4/GBRT": {
"status": "NOT_ATTEMPTED_SYNTHETIC"
},
"4/FNN": {
"status": "NOT_ATTEMPTED_SYNTHETIC"
}
}
}
@@ -0,0 +1,3 @@
{
"peak_reserved_bytes": 17179869184
}
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 3.362805,
"attempts": [
{
"name": "worker",
"status": "PASS",
"cpu_seconds": 3.362805
}
]
}
@@ -0,0 +1,26 @@
{
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"predict"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 3.362805,
"wall_seconds": 3.0489734660368413,
"peak_tree_rss_bytes": 519958528,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.46637159201782197,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"cuda_device": "",
"numerical_threads": 1,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,17 @@
{
"command": [
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
"tools/gre24_score.py",
"artifacts/gre24-reproduction-2026-11-15/integration-03",
"predict"
],
"limits": {
"total_cpu_seconds": 120,
"job_cpu_seconds": 120,
"job_wall_seconds": 120,
"tree_rss_bytes": 4294967296,
"disk_bytes": 268435456,
"max_processes": 1
},
"started_unix": 1794757203.1936562
}
Loaded 100 of 124 files, more files were not shown because too many files have changed in this diff. Show more