Freeze Gre24 model chain and retain synthetic acceptance evidence
This commit is contained in:
1 parent
8409910b1e
commit
eb9ed0c762
124 files changed
+5993
No files matched your search
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"status": "COMPLETE",
|
||||
"accounting_complete": true,
|
||||
"records": [
|
||||
{
|
||||
"case": "learn",
|
||||
"worker": {
|
||||
"status": "PASS"
|
||||
},
|
||||
"result_path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn/result.json",
|
||||
"result_sha256": "9110e0f3a43d41602c1b17281544163455d2ae18b4f3509fca72aae063253751"
|
||||
},
|
||||
{
|
||||
"case": "validation",
|
||||
"worker": {
|
||||
"status": "PASS"
|
||||
},
|
||||
"result_path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation/result.json",
|
||||
"result_sha256": "2e7bf2831b9d0136b848c76aa3ed7e974f0dcbd26e97369b288b18bda9d57031"
|
||||
},
|
||||
{
|
||||
"case": "test",
|
||||
"worker": {
|
||||
"status": "PASS"
|
||||
},
|
||||
"result_path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test/result.json",
|
||||
"result_sha256": "7c8a234f50a9ab9424b78226cfd4d3709b140ff63cef2138aa19e2eb20e22c86"
|
||||
}
|
||||
],
|
||||
"worker_cpu_seconds": 0.0,
|
||||
"initial_cpu_reserve_seconds": 0.0
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"parts": [
|
||||
"learn",
|
||||
"validation",
|
||||
"test"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"case": "learn",
|
||||
"geometry": {
|
||||
"tet_nodes": 128,
|
||||
"outside_tet_nodes": 0
|
||||
},
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
|
||||
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"case": "learn",
|
||||
"geometry": {
|
||||
"tet_nodes": 128,
|
||||
"outside_tet_nodes": 0
|
||||
},
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
|
||||
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"pid": 480565,
|
||||
"unix": 1794757290.256113,
|
||||
"plan_sha256": "6e410499203d3e13862b271ad29b31f62cab20b2fc5530aa79b30156124c83d3",
|
||||
"stage": "WAIT_PREPARATION"
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"fits": {
|
||||
"0/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"1/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"2/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"3/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"4/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"0/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"1/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"2/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"3/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"4/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
}
|
||||
},
|
||||
"charged_worker_cpu_seconds": 0.0,
|
||||
"charged_gpu_seconds": 0.0,
|
||||
"uuid": null,
|
||||
"training_contract_sha256": "c37fbd2bcb5fbd4b12a9bab5cb4aa0b06f9ca5fab638fd46e09e03a7bcae32f9",
|
||||
"all_fits_terminal": true,
|
||||
"test_scores_read": false
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"code_sha256": {
|
||||
"tools/run_gre24_models.py": "2a5393a5009a2eb1bdd470370a9183b80aa7029d6b69a545831706a0139eb774",
|
||||
"tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688",
|
||||
"tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0",
|
||||
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
|
||||
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa"
|
||||
},
|
||||
"total_worker_cpu_seconds": 1000,
|
||||
"postfit_cpu_reserve_seconds": 998,
|
||||
"total_gpu_seconds": 60,
|
||||
"initial_gpu_seconds": 0,
|
||||
"dataset_version": "synthetic_budget_only",
|
||||
"protocol": "synthetic",
|
||||
"xgb_threads": 1,
|
||||
"fnn_cpu_threads": 1,
|
||||
"xgb_per_fit_cpu_seconds": 10,
|
||||
"fnn_per_fit_cpu_seconds": 10,
|
||||
"gpu_reserved_byte_cap": 16106127360
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 1.566765,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "worker",
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 1.566765
|
||||
}
|
||||
]
|
||||
}
|
||||
Whitespace-only changes.
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
|
||||
"predict"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.566765,
|
||||
"wall_seconds": 1.4599632910103537,
|
||||
"peak_tree_rss_bytes": 110755840,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.363760320993606,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 4,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
|
||||
"predict"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 1000.0,
|
||||
"job_cpu_seconds": 1000.0,
|
||||
"job_wall_seconds": 7200,
|
||||
"tree_rss_bytes": 25769803776,
|
||||
"disk_bytes": 3221225472,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1794757290.2597783
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 1.299077,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "worker",
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 1.299077
|
||||
}
|
||||
]
|
||||
}
|
||||
Whitespace-only changes.
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
|
||||
"score"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.299077,
|
||||
"wall_seconds": 1.1043277920107357,
|
||||
"peak_tree_rss_bytes": 111403008,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.21797074796631932,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 4,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
|
||||
"score"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 998.433235,
|
||||
"job_cpu_seconds": 998.433235,
|
||||
"job_wall_seconds": 7200,
|
||||
"tree_rss_bytes": 25769803776,
|
||||
"disk_bytes": 3221225472,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1794757291.7210498
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"test_case_visits": 5,
|
||||
"worker_CPU_s": 0.006219989999999953
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"models_freeze_sha256": "d2c1c2d58053e3e42902604bbfdcd0fe6d07ec1d8893f548207abc01f44ec563",
|
||||
"rows": [
|
||||
{
|
||||
"split": 0,
|
||||
"case": "test",
|
||||
"status": "PREDICTED",
|
||||
"predictions": {
|
||||
"naive": {
|
||||
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-0/naive/test.npy",
|
||||
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
|
||||
"nodes": 128,
|
||||
"no_clipping": true
|
||||
}
|
||||
},
|
||||
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"case": "test",
|
||||
"status": "PREDICTED",
|
||||
"predictions": {
|
||||
"naive": {
|
||||
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-1/naive/test.npy",
|
||||
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
|
||||
"nodes": 128,
|
||||
"no_clipping": true
|
||||
}
|
||||
},
|
||||
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"case": "test",
|
||||
"status": "PREDICTED",
|
||||
"predictions": {
|
||||
"naive": {
|
||||
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-2/naive/test.npy",
|
||||
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
|
||||
"nodes": 128,
|
||||
"no_clipping": true
|
||||
}
|
||||
},
|
||||
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"case": "test",
|
||||
"status": "PREDICTED",
|
||||
"predictions": {
|
||||
"naive": {
|
||||
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-3/naive/test.npy",
|
||||
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
|
||||
"nodes": 128,
|
||||
"no_clipping": true
|
||||
}
|
||||
},
|
||||
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"case": "test",
|
||||
"status": "PREDICTED",
|
||||
"predictions": {
|
||||
"naive": {
|
||||
"path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-4/naive/test.npy",
|
||||
"sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29",
|
||||
"nodes": 128,
|
||||
"no_clipping": true
|
||||
}
|
||||
},
|
||||
"raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d"
|
||||
}
|
||||
],
|
||||
"worker_CPU_s": 0.0065045639999998794,
|
||||
"test_targets_read": false,
|
||||
"device": "CPU",
|
||||
"prediction_units": "seconds"
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"successful": {
|
||||
"learn": {
|
||||
"output": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn",
|
||||
"nodes": 128,
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
|
||||
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
},
|
||||
"result_sha256": "9110e0f3a43d41602c1b17281544163455d2ae18b4f3509fca72aae063253751",
|
||||
"outside_tet_nodes": 0
|
||||
},
|
||||
"validation": {
|
||||
"output": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation",
|
||||
"nodes": 128,
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
|
||||
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
},
|
||||
"result_sha256": "2e7bf2831b9d0136b848c76aa3ed7e974f0dcbd26e97369b288b18bda9d57031",
|
||||
"outside_tet_nodes": 0
|
||||
},
|
||||
"test": {
|
||||
"output": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test",
|
||||
"nodes": 128,
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
},
|
||||
"result_sha256": "7c8a234f50a9ab9424b78226cfd4d3709b140ff63cef2138aa19e2eb20e22c86",
|
||||
"outside_tet_nodes": 0
|
||||
}
|
||||
},
|
||||
"failed": [],
|
||||
"requested": 3,
|
||||
"preparation_complete_sha256": "84d3c0674e4f128f7ac2bb88b09d6376976211efabde3e8746f239b68c5b835a",
|
||||
"replacement": false,
|
||||
"cumulative_initial_worker_cpu_seconds": 0.0
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"status": "SCORED",
|
||||
"fits": {
|
||||
"0/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"1/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"2/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"3/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"4/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"0/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"1/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"2/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"3/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"4/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
}
|
||||
},
|
||||
"postfit": [
|
||||
{
|
||||
"phase": "predict",
|
||||
"receipt": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
|
||||
"predict"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.566765,
|
||||
"wall_seconds": 1.4599632910103537,
|
||||
"peak_tree_rss_bytes": 110755840,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.363760320993606,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 4,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"phase": "score",
|
||||
"receipt": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
|
||||
"score"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.299077,
|
||||
"wall_seconds": 1.1043277920107357,
|
||||
"peak_tree_rss_bytes": 111403008,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.21797074796631932,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 4,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"charged_worker_cpu_seconds": 2.865842,
|
||||
"charged_gpu_seconds": 0.0,
|
||||
"total_worker_cpu_cap": 1000,
|
||||
"total_gpu_cap": 60,
|
||||
"dataset_version": "synthetic_budget_only",
|
||||
"paper_numbers_diagnostic_only": true
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
{
|
||||
"dataset_version": "synthetic_budget_only",
|
||||
"protocol": "synthetic",
|
||||
"status": "SCORED_WITH_EXPLICIT_COVERAGE",
|
||||
"summary": {
|
||||
"naive": {
|
||||
"rrmse_rms": {
|
||||
"per_split": [
|
||||
{
|
||||
"split": 0,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.1607788938732118,
|
||||
"median": 1.1607788938732118,
|
||||
"max": 1.1607788938732118
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.1607788938732118,
|
||||
"median": 1.1607788938732118,
|
||||
"max": 1.1607788938732118
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.1607788938732118,
|
||||
"median": 1.1607788938732118,
|
||||
"max": 1.1607788938732118
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.1607788938732118,
|
||||
"median": 1.1607788938732118,
|
||||
"max": 1.1607788938732118
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.1607788938732118,
|
||||
"median": 1.1607788938732118,
|
||||
"max": 1.1607788938732118
|
||||
}
|
||||
],
|
||||
"mean_of_five_split_summaries": {
|
||||
"min": 1.1607788938732118,
|
||||
"median": 1.1607788938732118,
|
||||
"max": 1.1607788938732118
|
||||
},
|
||||
"full_requested_coverage": true
|
||||
},
|
||||
"rrmse_maximum": {
|
||||
"per_split": [
|
||||
{
|
||||
"split": 0,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 0.9117259819436264,
|
||||
"median": 0.9117259819436264,
|
||||
"max": 0.9117259819436264
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 0.9117259819436264,
|
||||
"median": 0.9117259819436264,
|
||||
"max": 0.9117259819436264
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 0.9117259819436264,
|
||||
"median": 0.9117259819436264,
|
||||
"max": 0.9117259819436264
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 0.9117259819436264,
|
||||
"median": 0.9117259819436264,
|
||||
"max": 0.9117259819436264
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 0.9117259819436264,
|
||||
"median": 0.9117259819436264,
|
||||
"max": 0.9117259819436264
|
||||
}
|
||||
],
|
||||
"mean_of_five_split_summaries": {
|
||||
"min": 0.9117259819436263,
|
||||
"median": 0.9117259819436263,
|
||||
"max": 0.9117259819436263
|
||||
},
|
||||
"full_requested_coverage": true
|
||||
},
|
||||
"rrmse_mean": {
|
||||
"per_split": [
|
||||
{
|
||||
"split": 0,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.173094770814564,
|
||||
"median": 1.173094770814564,
|
||||
"max": 1.173094770814564
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.173094770814564,
|
||||
"median": 1.173094770814564,
|
||||
"max": 1.173094770814564
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.173094770814564,
|
||||
"median": 1.173094770814564,
|
||||
"max": 1.173094770814564
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.173094770814564,
|
||||
"median": 1.173094770814564,
|
||||
"max": 1.173094770814564
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 1,
|
||||
"missing": 0,
|
||||
"min": 1.173094770814564,
|
||||
"median": 1.173094770814564,
|
||||
"max": 1.173094770814564
|
||||
}
|
||||
],
|
||||
"mean_of_five_split_summaries": {
|
||||
"min": 1.173094770814564,
|
||||
"median": 1.173094770814564,
|
||||
"max": 1.173094770814564
|
||||
},
|
||||
"full_requested_coverage": true
|
||||
}
|
||||
},
|
||||
"GBRT": {
|
||||
"rrmse_rms": {
|
||||
"per_split": [
|
||||
{
|
||||
"split": 0,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
}
|
||||
],
|
||||
"mean_of_five_split_summaries": {
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
"full_requested_coverage": false
|
||||
},
|
||||
"rrmse_maximum": {
|
||||
"per_split": [
|
||||
{
|
||||
"split": 0,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
}
|
||||
],
|
||||
"mean_of_five_split_summaries": {
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
"full_requested_coverage": false
|
||||
},
|
||||
"rrmse_mean": {
|
||||
"per_split": [
|
||||
{
|
||||
"split": 0,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
}
|
||||
],
|
||||
"mean_of_five_split_summaries": {
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
"full_requested_coverage": false
|
||||
}
|
||||
},
|
||||
"FNN": {
|
||||
"rrmse_rms": {
|
||||
"per_split": [
|
||||
{
|
||||
"split": 0,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
}
|
||||
],
|
||||
"mean_of_five_split_summaries": {
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
"full_requested_coverage": false
|
||||
},
|
||||
"rrmse_maximum": {
|
||||
"per_split": [
|
||||
{
|
||||
"split": 0,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
}
|
||||
],
|
||||
"mean_of_five_split_summaries": {
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
"full_requested_coverage": false
|
||||
},
|
||||
"rrmse_mean": {
|
||||
"per_split": [
|
||||
{
|
||||
"split": 0,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"requested_test_geometries": 1,
|
||||
"scored": 0,
|
||||
"missing": 1,
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
}
|
||||
],
|
||||
"mean_of_five_split_summaries": {
|
||||
"min": null,
|
||||
"median": null,
|
||||
"max": null
|
||||
},
|
||||
"full_requested_coverage": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"worker_CPU_s": 0.004958694000000152,
|
||||
"scores_sha256": "0cb25cf6f6224e7cf2a5707bf99d85acada792d9fbf80166cfd4cf7ab3e0784e",
|
||||
"paper_numbers_diagnostic_only": true,
|
||||
"limitations": [
|
||||
"v5 data instead of paper v4",
|
||||
"R1 discretisation choices are explicit supplements",
|
||||
"t_set substitutes unavailable applied Q",
|
||||
"FNN fill-only and early stopping are declared choices",
|
||||
"Random geometry splits may leak near-duplicate geometry families",
|
||||
"Failed preparations not replaced; coverage is explicit"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
{
|
||||
"rows": [
|
||||
{
|
||||
"split": 0,
|
||||
"case": "test",
|
||||
"status": "SCORED",
|
||||
"scores": {
|
||||
"naive": {
|
||||
"nodes": 128,
|
||||
"rmse": 0.7029024782641866,
|
||||
"denominators": {
|
||||
"rms": 0.6055438137049401,
|
||||
"maximum": 0.770958042421619,
|
||||
"mean": 0.5991864389405733
|
||||
},
|
||||
"undefined_metrics": [],
|
||||
"rrmse_rms": 1.1607788938732118,
|
||||
"rrmse_maximum": 0.9117259819436264,
|
||||
"rrmse_mean": 1.173094770814564
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"case": "test",
|
||||
"status": "SCORED",
|
||||
"scores": {
|
||||
"naive": {
|
||||
"nodes": 128,
|
||||
"rmse": 0.7029024782641866,
|
||||
"denominators": {
|
||||
"rms": 0.6055438137049401,
|
||||
"maximum": 0.770958042421619,
|
||||
"mean": 0.5991864389405733
|
||||
},
|
||||
"undefined_metrics": [],
|
||||
"rrmse_rms": 1.1607788938732118,
|
||||
"rrmse_maximum": 0.9117259819436264,
|
||||
"rrmse_mean": 1.173094770814564
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"case": "test",
|
||||
"status": "SCORED",
|
||||
"scores": {
|
||||
"naive": {
|
||||
"nodes": 128,
|
||||
"rmse": 0.7029024782641866,
|
||||
"denominators": {
|
||||
"rms": 0.6055438137049401,
|
||||
"maximum": 0.770958042421619,
|
||||
"mean": 0.5991864389405733
|
||||
},
|
||||
"undefined_metrics": [],
|
||||
"rrmse_rms": 1.1607788938732118,
|
||||
"rrmse_maximum": 0.9117259819436264,
|
||||
"rrmse_mean": 1.173094770814564
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"split": 3,
|
||||
"case": "test",
|
||||
"status": "SCORED",
|
||||
"scores": {
|
||||
"naive": {
|
||||
"nodes": 128,
|
||||
"rmse": 0.7029024782641866,
|
||||
"denominators": {
|
||||
"rms": 0.6055438137049401,
|
||||
"maximum": 0.770958042421619,
|
||||
"mean": 0.5991864389405733
|
||||
},
|
||||
"undefined_metrics": [],
|
||||
"rrmse_rms": 1.1607788938732118,
|
||||
"rrmse_maximum": 0.9117259819436264,
|
||||
"rrmse_mean": 1.173094770814564
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"case": "test",
|
||||
"status": "SCORED",
|
||||
"scores": {
|
||||
"naive": {
|
||||
"nodes": 128,
|
||||
"rmse": 0.7029024782641866,
|
||||
"denominators": {
|
||||
"rms": 0.6055438137049401,
|
||||
"maximum": 0.770958042421619,
|
||||
"mean": 0.5991864389405733
|
||||
},
|
||||
"undefined_metrics": [],
|
||||
"rrmse_rms": 1.1607788938732118,
|
||||
"rrmse_maximum": 0.9117259819436264,
|
||||
"rrmse_mean": 1.173094770814564
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"predictions_freeze_sha256": "5cc930becfef50f13d4c724fd884f8de4cf95e47afd09d0250c3bae7a2f1b868"
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
[
|
||||
{
|
||||
"index": 0,
|
||||
"seed": 91,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"seed": 92,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"seed": 93,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 3,
|
||||
"seed": 94,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 4,
|
||||
"seed": 95,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,121 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"checks": [
|
||||
"CPU reserve prevents all fits",
|
||||
"no GPU starts",
|
||||
"terminal model freeze exists before prediction",
|
||||
"naive score survives missing models",
|
||||
"actual postfit CPU charged"
|
||||
],
|
||||
"result": {
|
||||
"status": "SCORED",
|
||||
"fits": {
|
||||
"0/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"1/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"2/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"3/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"4/GBRT": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"0/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"1/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"2/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"3/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
},
|
||||
"4/FNN": {
|
||||
"status": "NOT_ATTEMPTED",
|
||||
"reason": "SHARED_CPU_CAP_RESERVE"
|
||||
}
|
||||
},
|
||||
"postfit": [
|
||||
{
|
||||
"phase": "predict",
|
||||
"receipt": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
|
||||
"predict"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.566765,
|
||||
"wall_seconds": 1.4599632910103537,
|
||||
"peak_tree_rss_bytes": 110755840,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.363760320993606,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 4,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"phase": "score",
|
||||
"receipt": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/budget-chain-test",
|
||||
"score"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.299077,
|
||||
"wall_seconds": 1.1043277920107357,
|
||||
"peak_tree_rss_bytes": 111403008,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.21797074796631932,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 4,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"charged_worker_cpu_seconds": 2.865842,
|
||||
"charged_gpu_seconds": 0.0,
|
||||
"total_worker_cpu_cap": 1000,
|
||||
"total_gpu_cap": 60,
|
||||
"dataset_version": "synthetic_budget_only",
|
||||
"paper_numbers_diagnostic_only": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"case": "test",
|
||||
"geometry": {
|
||||
"tet_nodes": 128,
|
||||
"outside_tet_nodes": 0
|
||||
},
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"case": "test",
|
||||
"geometry": {
|
||||
"tet_nodes": 128,
|
||||
"outside_tet_nodes": 0
|
||||
},
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"code_sha256": {
|
||||
"tools/run_gre24_models.py": "2a5393a5009a2eb1bdd470370a9183b80aa7029d6b69a545831706a0139eb774",
|
||||
"tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688",
|
||||
"tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0",
|
||||
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
|
||||
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa"
|
||||
},
|
||||
"total_worker_cpu_seconds": 1000,
|
||||
"postfit_cpu_reserve_seconds": 998,
|
||||
"total_gpu_seconds": 60,
|
||||
"initial_gpu_seconds": 0,
|
||||
"dataset_version": "synthetic_budget_only",
|
||||
"protocol": "synthetic",
|
||||
"xgb_threads": 1,
|
||||
"fnn_cpu_threads": 1,
|
||||
"xgb_per_fit_cpu_seconds": 10,
|
||||
"fnn_per_fit_cpu_seconds": 10,
|
||||
"gpu_reserved_byte_cap": 16106127360,
|
||||
"preparation_freeze_sha256": "b2095d362c98db182025439f05891a766c72fda41bc8a392337a0a99926ad7f0",
|
||||
"splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809"
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"case": "validation",
|
||||
"geometry": {
|
||||
"tet_nodes": 128,
|
||||
"outside_tet_nodes": 0
|
||||
},
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
|
||||
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"case": "validation",
|
||||
"geometry": {
|
||||
"tet_nodes": 128,
|
||||
"outside_tet_nodes": 0
|
||||
},
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
|
||||
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-01",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"returncode": 1,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.5006439999999999,
|
||||
"wall_seconds": 1.512771409994457,
|
||||
"peak_tree_rss_bytes": 148819968,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 225280,
|
||||
"max_poll_gap_seconds": 0.11697613802971318,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"status": "FAILED",
|
||||
"receipts": {
|
||||
"GBRT": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-01",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"returncode": 1,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.5006439999999999,
|
||||
"wall_seconds": 1.512771409994457,
|
||||
"peak_tree_rss_bytes": 148819968,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 225280,
|
||||
"max_poll_gap_seconds": 0.11697613802971318,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
},
|
||||
"GPU_seconds": 0
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"type": "ValueError",
|
||||
"message": "Out of range float values are not JSON compliant: nan",
|
||||
"traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_fit.py\", line 113, in fit\n save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,\n File \"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py\", line 23, in save\n tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\\n')\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/__init__.py\", line 238, in dumps\n **kw).encode(obj)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 201, in encode\n chunks = list(chunks)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 431, in _iterencode\n yield from _iterencode_dict(o, _current_indent_level)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 405, in _iterencode_dict\n yield from chunks\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 397, in _iterencode_dict\n yield _floatstr(value)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 239, in floatstr\n raise ValueError(\nValueError: Out of range float values are not JSON compliant: nan\n",
|
||||
"CPU_s": 0.490331745,
|
||||
"wall_s": 0.4909153020125814,
|
||||
"automatic_retry": false
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 1.5006439999999999,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "worker",
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"cpu_seconds": 1.5006439999999999
|
||||
}
|
||||
]
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"stage": "FIT",
|
||||
"pid": 480171,
|
||||
"unix": 1794757056.8670084,
|
||||
"split": 0,
|
||||
"model": "GBRT",
|
||||
"CPU_s": 0.03734539199999998,
|
||||
"wall_s": 0.03740089898929,
|
||||
"nodes": 256,
|
||||
"geometries": 2
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"pid": 480171,
|
||||
"unix": 1794757056.82963,
|
||||
"split": 0,
|
||||
"model": "GBRT",
|
||||
"seed": 91,
|
||||
"training_contract_sha256": "4180d820c98034ab333cf402e03de38f5979af34d3b6078b1c098ea40727e658"
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
[
|
||||
{
|
||||
"case": "learn",
|
||||
"role": "train",
|
||||
"nodes": 128,
|
||||
"source": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/learn",
|
||||
"feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
|
||||
"target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc"
|
||||
},
|
||||
{
|
||||
"case": "validation",
|
||||
"role": "train",
|
||||
"nodes": 128,
|
||||
"source": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/validation",
|
||||
"feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
|
||||
"target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,20 @@
|
||||
Traceback (most recent call last):
|
||||
File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 133, in <module>
|
||||
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
|
||||
File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 113, in fit
|
||||
save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,
|
||||
File "/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py", line 23, in save
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/__init__.py", line 238, in dumps
|
||||
**kw).encode(obj)
|
||||
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 201, in encode
|
||||
chunks = list(chunks)
|
||||
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 431, in _iterencode
|
||||
yield from _iterencode_dict(o, _current_indent_level)
|
||||
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 405, in _iterencode_dict
|
||||
yield from chunks
|
||||
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 397, in _iterencode_dict
|
||||
yield _floatstr(value)
|
||||
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 239, in floatstr
|
||||
raise ValueError(
|
||||
ValueError: Out of range float values are not JSON compliant: nan
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-01",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"returncode": 1,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.5006439999999999,
|
||||
"wall_seconds": 1.512771409994457,
|
||||
"peak_tree_rss_bytes": 148819968,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 225280,
|
||||
"max_poll_gap_seconds": 0.11697613802971318,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-01",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 120,
|
||||
"job_cpu_seconds": 120,
|
||||
"job_wall_seconds": 120,
|
||||
"tree_rss_bytes": 4294967296,
|
||||
"disk_bytes": 268435456,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1794757055.9604247
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"learner":{"generic_param":{"device":"cpu","fail_on_invalid_gpu_id":"0","n_jobs":"1","nthread":"1","random_state":"91","seed":"91","seed_per_iteration":"0","validate_parameters":"1"},"gradient_booster":{"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"100"},"gbtree_train_param":{"process_type":"default","tree_method":"auto","updater":"grow_quantile_histmaker","updater_seq":"grow_quantile_histmaker"},"name":"gbtree","specified_updater":false,"tree_train_param":{"alpha":"0","cache_opt":"1","colsample_bylevel":"1","colsample_bynode":"1","colsample_bytree":"1","eta":"0.100000001","gamma":"0","grow_policy":"depthwise","interaction_constraints":"","lambda":"1","learning_rate":"0.100000001","max_bin":"256","max_cat_threshold":"64","max_cat_to_onehot":"4","max_delta_step":"0","max_depth":"8","max_leaves":"0","min_child_weight":"1","min_split_loss":"0","monotone_constraints":"()","refresh_leaf":"1","reg_alpha":"0","reg_lambda":"1","sampling_method":"uniform","sketch_ratio":"2","sparse_threshold":"0.20000000000000001","subsample":"1"},"updater":[{"hist_train_param":{"debug_synchronize":"0","max_cached_hist_node":"65536"},"name":"grow_quantile_histmaker"}]},"learner_model_param":{"base_score":"3.0167848E-1","boost_from_average":"1","num_class":"0","num_feature":"19","num_target":"1"},"learner_train_param":{"booster":"gbtree","disable_default_eval_metric":"0","multi_strategy":"one_output_per_tree","objective":"reg:squarederror"},"metrics":[{"name":"rmse"}],"objective":{"name":"reg:squarederror","reg_loss_param":{"scale_pos_weight":"1"}}},"version":[2,1,4]}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"successful": {
|
||||
"learn": {
|
||||
"output": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/learn",
|
||||
"nodes": 128,
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
|
||||
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
},
|
||||
"validation": {
|
||||
"output": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/validation",
|
||||
"nodes": 128,
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
|
||||
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
},
|
||||
"test": {
|
||||
"output": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/test",
|
||||
"nodes": 128,
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
}
|
||||
},
|
||||
"failed": []
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
"""One fixed split/model fit; reads training-side targets only, no test scores."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
from gre24_prepare_v2 import sha
|
||||
from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def contract(root):
|
||||
c=json.loads((root/'training-contract.json').read_text())
|
||||
for p,h in c['code_sha256'].items():
|
||||
if sha(p)!=h:raise ValueError('Pinned training source changed: '+p)
|
||||
if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']:
|
||||
raise ValueError('Preparation freeze changed')
|
||||
if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed')
|
||||
return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text())
|
||||
|
||||
|
||||
def training_arrays(root,freeze,split,role,read_log):
|
||||
selected=[case for case in split[role] if case in freeze['successful']]
|
||||
n=sum(freeze['successful'][case]['nodes'] for case in selected)
|
||||
if not n:raise ValueError('No successful training rows')
|
||||
x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0
|
||||
for case in selected:
|
||||
require_training_role(case,split,role)
|
||||
r=freeze['successful'][case];source=Path(r['output'])
|
||||
for name in ['features.npy','target.npy']:
|
||||
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed')
|
||||
a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
|
||||
b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False)
|
||||
count=len(b)
|
||||
if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch')
|
||||
x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count
|
||||
read_log.append(dict(case=case,role=role,nodes=count,source=str(source),
|
||||
feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy']))
|
||||
if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input')
|
||||
return x,y
|
||||
|
||||
|
||||
def fit(root,index,model_name):
|
||||
c,f,splits=contract(root);split=splits[index]
|
||||
dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True)
|
||||
if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite')
|
||||
start=time.monotonic();cpu=time.process_time();read_log=[]
|
||||
def progress(stage,**kw):
|
||||
record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name,
|
||||
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start)
|
||||
record.update(kw);save(dest/'progress.json',record)
|
||||
save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'],
|
||||
training_contract_sha256=sha(root/'training-contract.json')))
|
||||
roles,missing=eligible_roles(split,f['successful'])
|
||||
try:
|
||||
if model_name=='GBRT':
|
||||
progress('LOAD_TRAIN_ONLY')
|
||||
x,y=training_arrays(root,f,split,'train',read_log)
|
||||
save(dest/'training-target-access.json',read_log)
|
||||
model=xgb_model(split['seed'],c['xgb_threads'])
|
||||
progress('FIT',nodes=len(y),geometries=len(roles['train']))
|
||||
model.fit(x,y,verbose=False)
|
||||
checkpoint=dest/'model.ubj';model.save_model(checkpoint)
|
||||
(dest/'xgboost-config.json').write_text(model.get_booster().save_config())
|
||||
extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=model.get_params())
|
||||
elif model_name=='FNN':
|
||||
import torch
|
||||
torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed'])
|
||||
if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable')
|
||||
device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)
|
||||
torch.backends.cuda.matmul.allow_tf32=False
|
||||
progress('LOAD_FNN_TRAIN_VALIDATION_ONLY')
|
||||
x,y=training_arrays(root,f,split,'fnn_learning',read_log)
|
||||
vx,vy=training_arrays(root,f,split,'fnn_validation',read_log)
|
||||
save(dest/'training-target-access.json',read_log)
|
||||
x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device)
|
||||
vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device)
|
||||
model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001)
|
||||
best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt'
|
||||
for epoch in range(1,31):
|
||||
model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device)
|
||||
for begin in range(0,len(y),1024):
|
||||
ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True)
|
||||
prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2)
|
||||
if not torch.isfinite(loss):raise ValueError('Nonfinite training loss')
|
||||
loss.backward();optimizer.step();total+=loss.detach()*len(ids)
|
||||
model.eval();sse=torch.zeros((),device=device)
|
||||
with torch.no_grad():
|
||||
for begin in range(0,len(vy),65536):
|
||||
sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2)
|
||||
validation=float((sse/len(vy)).item());train=float((total/len(y)).item())
|
||||
best,stale,improved,stop=early_stop_update(best,stale,validation)
|
||||
if improved:
|
||||
torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()},
|
||||
'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint)
|
||||
best_epoch=epoch
|
||||
row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch,
|
||||
stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start)
|
||||
history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row)
|
||||
if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap')
|
||||
if stop:break
|
||||
restored=torch.load(checkpoint,map_location='cpu',weights_only=True)
|
||||
model.load_state_dict(restored['state_dict'])
|
||||
extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best,
|
||||
train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']),
|
||||
validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False,
|
||||
input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved(device))
|
||||
else:raise ValueError('Unknown model')
|
||||
save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,
|
||||
checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing,
|
||||
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra))
|
||||
progress('FIT_COMPLETE_NO_TEST_SCORES');return 0
|
||||
except Exception as exc:
|
||||
peaks={}
|
||||
if model_name=='FNN':
|
||||
try:
|
||||
import torch
|
||||
if torch.cuda.is_initialized():
|
||||
peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
||||
except Exception as peak_error:peaks={'peak_read_error':str(peak_error)}
|
||||
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
|
||||
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN'])
|
||||
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
"""Single-GPU wall accounting and recorded/device VRAM checks including failure."""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from gre24_job_guard import supervise,stop_group
|
||||
|
||||
|
||||
def gpu_state(uuid=None):
|
||||
args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits']
|
||||
if uuid:args+=['-i',uuid]
|
||||
out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout
|
||||
rows=[]
|
||||
for line in out.splitlines():
|
||||
u,mem,util=[v.strip() for v in line.split(',')]
|
||||
rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util)))
|
||||
if not rows:raise RuntimeError('No GPU state returned')
|
||||
return rows
|
||||
|
||||
|
||||
def recorded_peaks(root):
|
||||
peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
||||
def walk(value,path,digest):
|
||||
if isinstance(value,dict):
|
||||
for k,v in value.items():
|
||||
if k in peaks:
|
||||
if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0:
|
||||
raise ValueError('Invalid CUDA peak record')
|
||||
if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest}
|
||||
else:walk(v,path,digest)
|
||||
elif isinstance(value,list):
|
||||
for v in value:walk(v,path,digest)
|
||||
for path in sorted(Path(root).glob('*.json')):
|
||||
raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest())
|
||||
return peaks,witnesses
|
||||
|
||||
|
||||
def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap):
|
||||
start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0}
|
||||
maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
||||
def scan():
|
||||
current,seen=recorded_peaks(fit_root)
|
||||
for k,v in current.items():
|
||||
if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k]
|
||||
if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']:
|
||||
monitor['reasons'].append('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();state=gpu_state(uuid)[0];monitor['samples']+=1
|
||||
monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes'])
|
||||
if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']:
|
||||
monitor['reasons'].append('DEVICE_VRAM_LIMIT')
|
||||
except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc))
|
||||
if monitor['reasons']:
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
done.wait(.5)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit,
|
||||
job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30,
|
||||
disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2)
|
||||
finally:
|
||||
done.set();thread.join(timeout=12)
|
||||
if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP')
|
||||
try:scan()
|
||||
except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc))
|
||||
if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS')
|
||||
elapsed=time.monotonic()-start
|
||||
if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT')
|
||||
receipt['underlying_worker_status']=receipt['status']
|
||||
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
|
||||
receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses,
|
||||
total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap)
|
||||
return receipt
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
|
||||
|
||||
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
||||
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
|
||||
"""
|
||||
import fcntl
|
||||
import ctypes
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from s2_recovery_monitoring import disk_usage,Monitor
|
||||
|
||||
|
||||
def save(path, data):
|
||||
path = Path(path)
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def tree_usage(pid):
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
processes = [root] + root.children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return 0., 0, 0
|
||||
cpu = 0.
|
||||
rss = count = 0
|
||||
for p in processes:
|
||||
try:
|
||||
t = p.cpu_times()
|
||||
# Linux child times include reaped descendants, not live children.
|
||||
cpu += t.user + t.system + t.children_user + t.children_system
|
||||
rss += p.memory_info().rss
|
||||
count += 1
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
return cpu, rss, count
|
||||
|
||||
|
||||
def stop_group(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def supervise(command, root, name, limits, *, poll=.02, cuda_device='', numerical_threads=1):
|
||||
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
||||
root = Path(root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
|
||||
with (root/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
started_path = root/(name+'.started.json')
|
||||
receipt_path = root/(name+'.receipt.json')
|
||||
if started_path.exists() or receipt_path.exists():
|
||||
raise FileExistsError('Attempt already exists; no silent retry')
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Every limit must be finite and positive')
|
||||
ledger_path = root/'ledger.json'
|
||||
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
||||
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
||||
raise RuntimeError('Total CPU exhausted before spawn')
|
||||
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
|
||||
raise RuntimeError('Disk exhausted before spawn')
|
||||
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
||||
start = last = time.monotonic()
|
||||
cpu_peak = rss_peak = max_processes = max_gap = 0
|
||||
cpu=rss=count=used_disk=0
|
||||
reason = None
|
||||
exit_status = usage = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS=str(numerical_threads), OPENBLAS_NUM_THREADS=str(numerical_threads),
|
||||
MKL_NUM_THREADS=str(numerical_threads), NUMEXPR_NUM_THREADS=str(numerical_threads),
|
||||
CUDA_VISIBLE_DEVICES=cuda_device)
|
||||
with (root/(name+'.log')).open('x') as log:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
||||
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env, start_new_session=True)
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
max_gap = max(max_gap, now-last)
|
||||
last = now
|
||||
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
|
||||
cpu_peak = max(cpu_peak,cpu)
|
||||
rss_peak = max(rss_peak,rss)
|
||||
max_processes = max(max_processes,count)
|
||||
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
|
||||
if now-start >= limits['job_wall_seconds']:
|
||||
reason = 'wall_limit'
|
||||
elif rss > limits['tree_rss_bytes']:
|
||||
reason = 'tree_rss_limit'
|
||||
elif count > limits['max_processes']:
|
||||
reason = 'process_count_limit'
|
||||
elif used_disk >= limits['disk_bytes']:
|
||||
reason = 'disk_limit'
|
||||
elif cpu_peak >= limits['job_cpu_seconds']:
|
||||
reason = 'job_cpu_limit'
|
||||
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
||||
reason = 'total_cpu_limit'
|
||||
if reason:
|
||||
stop_group(proc.pid)
|
||||
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
exit_status, usage = status, child_usage
|
||||
break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
# No detached work may outlive a supervised attempt.
|
||||
stop_group(proc.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.
|
||||
orphan_peak = 0
|
||||
# Killed descendants are adopted here rather than losing their
|
||||
# final CPU usage when the root worker cannot reap them itself.
|
||||
while True:
|
||||
try:
|
||||
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
||||
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
||||
elapsed = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
||||
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
|
||||
if reason is None:
|
||||
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
||||
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
||||
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
||||
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
||||
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
||||
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
||||
command=command,returncode=proc.returncode,stop_reason=reason,
|
||||
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
||||
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
||||
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
||||
reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
||||
gpu_visible=bool(cuda_device),cuda_device=cuda_device,
|
||||
numerical_threads=numerical_threads,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
|
||||
ledger['cpu_seconds_charged'] += charged
|
||||
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
||||
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
|
||||
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
|
||||
return receipt
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Fixed Gre24 model choices; no test-driven selection or node subsampling."""
|
||||
import numpy as np
|
||||
|
||||
|
||||
def xgb_model(seed,threads=8):
|
||||
from xgboost import XGBRegressor
|
||||
return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1,
|
||||
objective='reg:squarederror',random_state=seed,n_jobs=threads)
|
||||
|
||||
|
||||
def fnn_model():
|
||||
import torch
|
||||
layers=[];width=19
|
||||
for _ in range(4):
|
||||
layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100
|
||||
layers.append(torch.nn.Linear(100,1))
|
||||
return torch.nn.Sequential(*layers)
|
||||
|
||||
|
||||
def eligible_roles(split,successful):
|
||||
successful=set(successful)
|
||||
roles={role:[c for c in split[role] if c in successful]
|
||||
for role in ['train','test','fnn_learning','fnn_validation']}
|
||||
if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap')
|
||||
if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap')
|
||||
if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']):
|
||||
raise ValueError('FNN learning/validation do not partition train')
|
||||
missing={role:[c for c in split[role] if c not in successful] for role in roles}
|
||||
return roles,missing
|
||||
|
||||
|
||||
def require_training_role(case,split,role):
|
||||
if role not in ['train','fnn_learning','fnn_validation']:
|
||||
raise PermissionError('Only frozen training-side roles may read targets')
|
||||
if case in split['test'] or case not in split[role]:
|
||||
raise PermissionError('Case is not in the requested training role')
|
||||
|
||||
|
||||
def early_stop_update(best,stale,score,patience=5):
|
||||
if not np.isfinite(score):raise ValueError('Nonfinite validation MSE')
|
||||
improved=score<best
|
||||
return (score if improved else best),(0 if improved else stale+1),improved,(not improved and stale+1>=patience)
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
"""Predictions frozen before test-target scoring, all successful tet nodes."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import numpy as np
|
||||
from gre24_prepare_v2 import sha
|
||||
from gre24_features_v2 import per_geometry_metrics
|
||||
from gre24_models import fnn_model
|
||||
from gre24_fit import contract
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def predict(root):
|
||||
c,prep,splits=contract(root)
|
||||
models=json.loads((root/'models-freeze.json').read_text())
|
||||
dest=root/'predictions';dest.mkdir(exist_ok=False)
|
||||
start=time.process_time();rows=[]
|
||||
for split in splits:
|
||||
available={'naive':None}
|
||||
for name in ['GBRT','FNN']:
|
||||
r=models['fits'][f"{split['index']}/{name}"]
|
||||
if r['status']!='PASS':continue
|
||||
if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze')
|
||||
if name=='GBRT':
|
||||
from xgboost import XGBRegressor
|
||||
m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint'])
|
||||
else:
|
||||
import torch
|
||||
torch.set_num_threads(4);m=fnn_model()
|
||||
m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval()
|
||||
available[name]=m
|
||||
for case in split['test']:
|
||||
if case not in prep['successful']:
|
||||
rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={}))
|
||||
continue
|
||||
r=prep['successful'][case];source=Path(r['output']);outputs={}
|
||||
for name in ['features.npy','Vf.npy','raw_ids.npy']:
|
||||
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed')
|
||||
x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
|
||||
# No seconds/target file is opened in this phase.
|
||||
for name,m in available.items():
|
||||
if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False)
|
||||
elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32))
|
||||
else:
|
||||
chunks=[]
|
||||
with torch.no_grad():
|
||||
for a in range(0,len(x),65536):
|
||||
chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy())
|
||||
pred=np.concatenate(chunks)
|
||||
if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction')
|
||||
folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True)
|
||||
path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds'])
|
||||
outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True)
|
||||
rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs,
|
||||
raw_ids_sha256=r['output_sha256']['raw_ids.npy']))
|
||||
save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start))
|
||||
save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows,
|
||||
worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds'))
|
||||
|
||||
|
||||
def score(root):
|
||||
c,prep,splits=contract(root);frozen=root/'predictions-freeze.json'
|
||||
if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring')
|
||||
freeze=json.loads(frozen.read_text());rows=[];start=time.process_time()
|
||||
for r in freeze['rows']:
|
||||
if r['status']!='PREDICTED':
|
||||
rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue
|
||||
p=prep['successful'][r['case']];source=Path(p['output'])
|
||||
if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed')
|
||||
y=np.load(source/'seconds.npy',allow_pickle=False);scores={}
|
||||
for name,value in r['predictions'].items():
|
||||
if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed')
|
||||
pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred)
|
||||
rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores))
|
||||
summary={}
|
||||
for name in ['naive','GBRT','FNN']:
|
||||
summary[name]={}
|
||||
for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']:
|
||||
per=[]
|
||||
for split in splits:
|
||||
a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores']
|
||||
and r['scores'][name][metric] is not None]
|
||||
per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a),
|
||||
missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None,
|
||||
median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None))
|
||||
# Do not average only whichever model splits happened to finish.
|
||||
avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']}
|
||||
summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg,
|
||||
full_requested_coverage=all(r['missing']==0 for r in per))
|
||||
save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen)))
|
||||
save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'],
|
||||
status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start,
|
||||
scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True,
|
||||
limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements',
|
||||
't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices',
|
||||
'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit']))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score'])
|
||||
a=p.parse_args();(predict if a.phase=='predict' else score)(a.root)
|
||||
@@ -0,0 +1,87 @@
|
||||
[
|
||||
{
|
||||
"index": 0,
|
||||
"seed": 91,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"seed": 92,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"seed": 93,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 3,
|
||||
"seed": 94,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 4,
|
||||
"seed": 95,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"code_sha256": {
|
||||
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"tools/gre24_fit.py": "a3fc9364820ef23f237475cf2594d1089d86b87763aba4afae52a1b32d7d9ab6",
|
||||
"tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0",
|
||||
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
|
||||
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa"
|
||||
},
|
||||
"preparation_freeze_sha256": "508a8997b875e9a978db40c5eed3a6407b3e512bb7916398e5bb688dd935262e",
|
||||
"splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809",
|
||||
"dataset_version": "synthetic_only",
|
||||
"protocol": "synthetic smoke, not scientific data",
|
||||
"xgb_threads": 1,
|
||||
"fnn_cpu_threads": 2,
|
||||
"gpu_reserved_byte_cap": 16106127360
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-02",
|
||||
"0",
|
||||
"FNN"
|
||||
],
|
||||
"returncode": 1,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 4.8226309999999994,
|
||||
"wall_seconds": 4.549783353984822,
|
||||
"peak_tree_rss_bytes": 495280128,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 16384,
|
||||
"max_poll_gap_seconds": 0.36855653702514246,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"numerical_threads": 2,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"underlying_worker_status": "STOPPED_OR_FAILED",
|
||||
"device_monitor": {
|
||||
"reasons": [],
|
||||
"peak_device_bytes": 9437184,
|
||||
"samples": 8
|
||||
},
|
||||
"recorded_peaks": {
|
||||
"peak_allocated_bytes": 0,
|
||||
"peak_reserved_bytes": 0
|
||||
},
|
||||
"peak_witnesses": {},
|
||||
"total_gpu_wall_seconds": 4.554046397970524,
|
||||
"peak_cap_bytes": 16106127360
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-02",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.5470069999999998,
|
||||
"wall_seconds": 1.6621895720018074,
|
||||
"peak_tree_rss_bytes": 148852736,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 221184,
|
||||
"max_poll_gap_seconds": 0.12116481299744919,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"status": "FAILED",
|
||||
"receipts": {
|
||||
"GBRT": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-02",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.5470069999999998,
|
||||
"wall_seconds": 1.6621895720018074,
|
||||
"peak_tree_rss_bytes": 148852736,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 221184,
|
||||
"max_poll_gap_seconds": 0.12116481299744919,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
},
|
||||
"FNN": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-02",
|
||||
"0",
|
||||
"FNN"
|
||||
],
|
||||
"returncode": 1,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 4.8226309999999994,
|
||||
"wall_seconds": 4.549783353984822,
|
||||
"peak_tree_rss_bytes": 495280128,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 16384,
|
||||
"max_poll_gap_seconds": 0.36855653702514246,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"numerical_threads": 2,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"underlying_worker_status": "STOPPED_OR_FAILED",
|
||||
"device_monitor": {
|
||||
"reasons": [],
|
||||
"peak_device_bytes": 9437184,
|
||||
"samples": 8
|
||||
},
|
||||
"recorded_peaks": {
|
||||
"peak_allocated_bytes": 0,
|
||||
"peak_reserved_bytes": 0
|
||||
},
|
||||
"peak_witnesses": {},
|
||||
"total_gpu_wall_seconds": 4.554046397970524,
|
||||
"peak_cap_bytes": 16106127360
|
||||
}
|
||||
},
|
||||
"GPU_seconds": 4.601264455995988
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"type": "RuntimeError",
|
||||
"message": "Invalid device argument ",
|
||||
"traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_fit.py\", line 77, in fit\n device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)\n File \"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/lib/python3.10/site-packages/torch/cuda/memory.py\", line 344, in reset_peak_memory_stats\n return torch._C._cuda_resetPeakMemoryStats(device)\nRuntimeError: Invalid device argument \n",
|
||||
"CPU_s": 2.9871277210000002,
|
||||
"wall_s": 2.784001157968305,
|
||||
"automatic_retry": false
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 4.8226309999999994,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "worker",
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"cpu_seconds": 4.8226309999999994
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"pid": 480269,
|
||||
"unix": 1794757154.6071362,
|
||||
"split": 0,
|
||||
"model": "FNN",
|
||||
"seed": 91,
|
||||
"training_contract_sha256": "760d10e1df8863e5d4c34701a0a18678a6a2f9ede56b2cc83e95b6bbfab5848f"
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
Traceback (most recent call last):
|
||||
File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 138, in <module>
|
||||
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
|
||||
File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 77, in fit
|
||||
device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)
|
||||
File "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/lib/python3.10/site-packages/torch/cuda/memory.py", line 344, in reset_peak_memory_stats
|
||||
return torch._C._cuda_resetPeakMemoryStats(device)
|
||||
RuntimeError: Invalid device argument
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-02",
|
||||
"0",
|
||||
"FNN"
|
||||
],
|
||||
"returncode": 1,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 4.8226309999999994,
|
||||
"wall_seconds": 4.549783353984822,
|
||||
"peak_tree_rss_bytes": 495280128,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 16384,
|
||||
"max_poll_gap_seconds": 0.36855653702514246,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"numerical_threads": 2,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-02",
|
||||
"0",
|
||||
"FNN"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 120.0,
|
||||
"job_cpu_seconds": 120.0,
|
||||
"job_wall_seconds": 57.0,
|
||||
"tree_rss_bytes": 25769803776,
|
||||
"disk_bytes": 3221225472,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1794757153.3916488
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 1.5470069999999998,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "worker",
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 1.5470069999999998
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"stage": "FIT_COMPLETE_NO_TEST_SCORES",
|
||||
"pid": 480263,
|
||||
"unix": 1794757153.0647538,
|
||||
"split": 0,
|
||||
"model": "GBRT",
|
||||
"CPU_s": 0.495211623,
|
||||
"wall_s": 0.4952992910402827
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"status": "FIT_COMPLETE_NO_TEST_SCORES",
|
||||
"split": 0,
|
||||
"model": "GBRT",
|
||||
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/model.ubj",
|
||||
"checkpoint_sha256": "0630f79ad1b6783fa31ed5b6d94e345a896212187676f65168f4e819889ec74b",
|
||||
"missing_preparation": {
|
||||
"train": [],
|
||||
"test": [],
|
||||
"fnn_learning": [],
|
||||
"fnn_validation": []
|
||||
},
|
||||
"CPU_s": 0.49491478599999994,
|
||||
"wall_s": 0.4950045870500617,
|
||||
"train_nodes": 256,
|
||||
"train_geometries": 2,
|
||||
"parameters": {
|
||||
"objective": "reg:squarederror",
|
||||
"base_score": null,
|
||||
"booster": null,
|
||||
"callbacks": null,
|
||||
"colsample_bylevel": null,
|
||||
"colsample_bynode": null,
|
||||
"colsample_bytree": null,
|
||||
"device": null,
|
||||
"early_stopping_rounds": null,
|
||||
"enable_categorical": false,
|
||||
"eval_metric": null,
|
||||
"feature_types": null,
|
||||
"gamma": null,
|
||||
"grow_policy": null,
|
||||
"importance_type": null,
|
||||
"interaction_constraints": null,
|
||||
"learning_rate": 0.1,
|
||||
"max_bin": null,
|
||||
"max_cat_threshold": null,
|
||||
"max_cat_to_onehot": null,
|
||||
"max_delta_step": null,
|
||||
"max_depth": 8,
|
||||
"max_leaves": null,
|
||||
"min_child_weight": null,
|
||||
"missing": "NaN (library default missing marker)",
|
||||
"monotone_constraints": null,
|
||||
"multi_strategy": null,
|
||||
"n_estimators": 100,
|
||||
"n_jobs": 1,
|
||||
"num_parallel_tree": null,
|
||||
"random_state": 91,
|
||||
"reg_alpha": null,
|
||||
"reg_lambda": null,
|
||||
"sampling_method": null,
|
||||
"scale_pos_weight": null,
|
||||
"subsample": null,
|
||||
"tree_method": null,
|
||||
"validate_parameters": null,
|
||||
"verbosity": null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"pid": 480263,
|
||||
"unix": 1794757152.5694737,
|
||||
"split": 0,
|
||||
"model": "GBRT",
|
||||
"seed": 91,
|
||||
"training_contract_sha256": "760d10e1df8863e5d4c34701a0a18678a6a2f9ede56b2cc83e95b6bbfab5848f"
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
[
|
||||
{
|
||||
"case": "learn",
|
||||
"role": "train",
|
||||
"nodes": 128,
|
||||
"source": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/learn",
|
||||
"feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
|
||||
"target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc"
|
||||
},
|
||||
{
|
||||
"case": "validation",
|
||||
"role": "train",
|
||||
"nodes": 128,
|
||||
"source": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/validation",
|
||||
"feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
|
||||
"target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3"
|
||||
}
|
||||
]
|
||||
Whitespace-only changes.
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-02",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.5470069999999998,
|
||||
"wall_seconds": 1.6621895720018074,
|
||||
"peak_tree_rss_bytes": 148852736,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 221184,
|
||||
"max_poll_gap_seconds": 0.12116481299744919,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-02",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 120,
|
||||
"job_cpu_seconds": 120,
|
||||
"job_wall_seconds": 120,
|
||||
"tree_rss_bytes": 4294967296,
|
||||
"disk_bytes": 268435456,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1794757151.6780915
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"learner":{"generic_param":{"device":"cpu","fail_on_invalid_gpu_id":"0","n_jobs":"1","nthread":"1","random_state":"91","seed":"91","seed_per_iteration":"0","validate_parameters":"1"},"gradient_booster":{"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"100"},"gbtree_train_param":{"process_type":"default","tree_method":"auto","updater":"grow_quantile_histmaker","updater_seq":"grow_quantile_histmaker"},"name":"gbtree","specified_updater":false,"tree_train_param":{"alpha":"0","cache_opt":"1","colsample_bylevel":"1","colsample_bynode":"1","colsample_bytree":"1","eta":"0.100000001","gamma":"0","grow_policy":"depthwise","interaction_constraints":"","lambda":"1","learning_rate":"0.100000001","max_bin":"256","max_cat_threshold":"64","max_cat_to_onehot":"4","max_delta_step":"0","max_depth":"8","max_leaves":"0","min_child_weight":"1","min_split_loss":"0","monotone_constraints":"()","refresh_leaf":"1","reg_alpha":"0","reg_lambda":"1","sampling_method":"uniform","sketch_ratio":"2","sparse_threshold":"0.20000000000000001","subsample":"1"},"updater":[{"hist_train_param":{"debug_synchronize":"0","max_cached_hist_node":"65536"},"name":"grow_quantile_histmaker"}]},"learner_model_param":{"base_score":"3.0167848E-1","boost_from_average":"1","num_class":"0","num_feature":"19","num_target":"1"},"learner_train_param":{"booster":"gbtree","disable_default_eval_metric":"0","multi_strategy":"one_output_per_tree","objective":"reg:squarederror"},"metrics":[{"name":"rmse"}],"objective":{"name":"reg:squarederror","reg_loss_param":{"scale_pos_weight":"1"}}},"version":[2,1,4]}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"successful": {
|
||||
"learn": {
|
||||
"output": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/learn",
|
||||
"nodes": 128,
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076",
|
||||
"target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
},
|
||||
"validation": {
|
||||
"output": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/validation",
|
||||
"nodes": 128,
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40",
|
||||
"target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
},
|
||||
"test": {
|
||||
"output": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/test",
|
||||
"nodes": 128,
|
||||
"t_scale_seconds": 2.0,
|
||||
"output_sha256": {
|
||||
"seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353",
|
||||
"raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d",
|
||||
"features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0",
|
||||
"Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df"
|
||||
}
|
||||
}
|
||||
},
|
||||
"failed": []
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
"""One fixed split/model fit; reads training-side targets only, no test scores."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
from gre24_prepare_v2 import sha
|
||||
from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def contract(root):
|
||||
c=json.loads((root/'training-contract.json').read_text())
|
||||
for p,h in c['code_sha256'].items():
|
||||
if sha(p)!=h:raise ValueError('Pinned training source changed: '+p)
|
||||
if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']:
|
||||
raise ValueError('Preparation freeze changed')
|
||||
if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed')
|
||||
return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text())
|
||||
|
||||
|
||||
def training_arrays(root,freeze,split,role,read_log):
|
||||
selected=[case for case in split[role] if case in freeze['successful']]
|
||||
n=sum(freeze['successful'][case]['nodes'] for case in selected)
|
||||
if not n:raise ValueError('No successful training rows')
|
||||
x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0
|
||||
for case in selected:
|
||||
require_training_role(case,split,role)
|
||||
r=freeze['successful'][case];source=Path(r['output'])
|
||||
for name in ['features.npy','target.npy']:
|
||||
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed')
|
||||
a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
|
||||
b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False)
|
||||
count=len(b)
|
||||
if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch')
|
||||
x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count
|
||||
read_log.append(dict(case=case,role=role,nodes=count,source=str(source),
|
||||
feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy']))
|
||||
if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input')
|
||||
return x,y
|
||||
|
||||
|
||||
def fit(root,index,model_name):
|
||||
c,f,splits=contract(root);split=splits[index]
|
||||
dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True)
|
||||
if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite')
|
||||
start=time.monotonic();cpu=time.process_time();read_log=[]
|
||||
def progress(stage,**kw):
|
||||
record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name,
|
||||
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start)
|
||||
record.update(kw);save(dest/'progress.json',record)
|
||||
save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'],
|
||||
training_contract_sha256=sha(root/'training-contract.json')))
|
||||
roles,missing=eligible_roles(split,f['successful'])
|
||||
try:
|
||||
if model_name=='GBRT':
|
||||
progress('LOAD_TRAIN_ONLY')
|
||||
x,y=training_arrays(root,f,split,'train',read_log)
|
||||
save(dest/'training-target-access.json',read_log)
|
||||
model=xgb_model(split['seed'],c['xgb_threads'])
|
||||
progress('FIT',nodes=len(y),geometries=len(roles['train']))
|
||||
model.fit(x,y,verbose=False)
|
||||
checkpoint=dest/'model.ubj';model.save_model(checkpoint)
|
||||
(dest/'xgboost-config.json').write_text(model.get_booster().save_config())
|
||||
params=model.get_params()
|
||||
# Preserve XGBoost's default missing=NaN as an explicit textual parameter;
|
||||
# this is not a nonfinite observation or a change to the fitted estimator.
|
||||
params={k:('NaN (library default missing marker)' if isinstance(v,float) and np.isnan(v) else v)
|
||||
for k,v in params.items()}
|
||||
extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=params)
|
||||
elif model_name=='FNN':
|
||||
import torch
|
||||
torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed'])
|
||||
if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable')
|
||||
device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)
|
||||
torch.backends.cuda.matmul.allow_tf32=False
|
||||
progress('LOAD_FNN_TRAIN_VALIDATION_ONLY')
|
||||
x,y=training_arrays(root,f,split,'fnn_learning',read_log)
|
||||
vx,vy=training_arrays(root,f,split,'fnn_validation',read_log)
|
||||
save(dest/'training-target-access.json',read_log)
|
||||
x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device)
|
||||
vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device)
|
||||
model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001)
|
||||
best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt'
|
||||
for epoch in range(1,31):
|
||||
model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device)
|
||||
for begin in range(0,len(y),1024):
|
||||
ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True)
|
||||
prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2)
|
||||
if not torch.isfinite(loss):raise ValueError('Nonfinite training loss')
|
||||
loss.backward();optimizer.step();total+=loss.detach()*len(ids)
|
||||
model.eval();sse=torch.zeros((),device=device)
|
||||
with torch.no_grad():
|
||||
for begin in range(0,len(vy),65536):
|
||||
sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2)
|
||||
validation=float((sse/len(vy)).item());train=float((total/len(y)).item())
|
||||
best,stale,improved,stop=early_stop_update(best,stale,validation)
|
||||
if improved:
|
||||
torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()},
|
||||
'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint)
|
||||
best_epoch=epoch
|
||||
row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch,
|
||||
stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start)
|
||||
history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row)
|
||||
if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap')
|
||||
if stop:break
|
||||
restored=torch.load(checkpoint,map_location='cpu',weights_only=True)
|
||||
model.load_state_dict(restored['state_dict'])
|
||||
extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best,
|
||||
train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']),
|
||||
validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False,
|
||||
input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved(device))
|
||||
else:raise ValueError('Unknown model')
|
||||
save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,
|
||||
checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing,
|
||||
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra))
|
||||
progress('FIT_COMPLETE_NO_TEST_SCORES');return 0
|
||||
except Exception as exc:
|
||||
peaks={}
|
||||
if model_name=='FNN':
|
||||
try:
|
||||
import torch
|
||||
if torch.cuda.is_initialized():
|
||||
peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
||||
except Exception as peak_error:peaks={'peak_read_error':str(peak_error)}
|
||||
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
|
||||
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN'])
|
||||
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
"""Single-GPU wall accounting and recorded/device VRAM checks including failure."""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from gre24_job_guard import supervise,stop_group
|
||||
|
||||
|
||||
def gpu_state(uuid=None):
|
||||
args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits']
|
||||
if uuid:args+=['-i',uuid]
|
||||
out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout
|
||||
rows=[]
|
||||
for line in out.splitlines():
|
||||
u,mem,util=[v.strip() for v in line.split(',')]
|
||||
rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util)))
|
||||
if not rows:raise RuntimeError('No GPU state returned')
|
||||
return rows
|
||||
|
||||
|
||||
def recorded_peaks(root):
|
||||
peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
||||
def walk(value,path,digest):
|
||||
if isinstance(value,dict):
|
||||
for k,v in value.items():
|
||||
if k in peaks:
|
||||
if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0:
|
||||
raise ValueError('Invalid CUDA peak record')
|
||||
if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest}
|
||||
else:walk(v,path,digest)
|
||||
elif isinstance(value,list):
|
||||
for v in value:walk(v,path,digest)
|
||||
for path in sorted(Path(root).glob('*.json')):
|
||||
raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest())
|
||||
return peaks,witnesses
|
||||
|
||||
|
||||
def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap):
|
||||
start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0}
|
||||
maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
||||
def scan():
|
||||
current,seen=recorded_peaks(fit_root)
|
||||
for k,v in current.items():
|
||||
if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k]
|
||||
if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']:
|
||||
monitor['reasons'].append('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();state=gpu_state(uuid)[0];monitor['samples']+=1
|
||||
monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes'])
|
||||
if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']:
|
||||
monitor['reasons'].append('DEVICE_VRAM_LIMIT')
|
||||
except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc))
|
||||
if monitor['reasons']:
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
done.wait(.5)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit,
|
||||
job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30,
|
||||
disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2)
|
||||
finally:
|
||||
done.set();thread.join(timeout=12)
|
||||
if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP')
|
||||
try:scan()
|
||||
except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc))
|
||||
if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS')
|
||||
elapsed=time.monotonic()-start
|
||||
if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT')
|
||||
receipt['underlying_worker_status']=receipt['status']
|
||||
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
|
||||
receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses,
|
||||
total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap)
|
||||
return receipt
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
|
||||
|
||||
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
||||
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
|
||||
"""
|
||||
import fcntl
|
||||
import ctypes
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from s2_recovery_monitoring import disk_usage,Monitor
|
||||
|
||||
|
||||
def save(path, data):
|
||||
path = Path(path)
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def tree_usage(pid):
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
processes = [root] + root.children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return 0., 0, 0
|
||||
cpu = 0.
|
||||
rss = count = 0
|
||||
for p in processes:
|
||||
try:
|
||||
t = p.cpu_times()
|
||||
# Linux child times include reaped descendants, not live children.
|
||||
cpu += t.user + t.system + t.children_user + t.children_system
|
||||
rss += p.memory_info().rss
|
||||
count += 1
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
return cpu, rss, count
|
||||
|
||||
|
||||
def stop_group(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def supervise(command, root, name, limits, *, poll=.02, cuda_device='', numerical_threads=1):
|
||||
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
||||
root = Path(root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
|
||||
with (root/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
started_path = root/(name+'.started.json')
|
||||
receipt_path = root/(name+'.receipt.json')
|
||||
if started_path.exists() or receipt_path.exists():
|
||||
raise FileExistsError('Attempt already exists; no silent retry')
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Every limit must be finite and positive')
|
||||
ledger_path = root/'ledger.json'
|
||||
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
||||
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
||||
raise RuntimeError('Total CPU exhausted before spawn')
|
||||
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
|
||||
raise RuntimeError('Disk exhausted before spawn')
|
||||
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
||||
start = last = time.monotonic()
|
||||
cpu_peak = rss_peak = max_processes = max_gap = 0
|
||||
cpu=rss=count=used_disk=0
|
||||
reason = None
|
||||
exit_status = usage = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS=str(numerical_threads), OPENBLAS_NUM_THREADS=str(numerical_threads),
|
||||
MKL_NUM_THREADS=str(numerical_threads), NUMEXPR_NUM_THREADS=str(numerical_threads),
|
||||
CUDA_VISIBLE_DEVICES=cuda_device)
|
||||
with (root/(name+'.log')).open('x') as log:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
||||
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env, start_new_session=True)
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
max_gap = max(max_gap, now-last)
|
||||
last = now
|
||||
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
|
||||
cpu_peak = max(cpu_peak,cpu)
|
||||
rss_peak = max(rss_peak,rss)
|
||||
max_processes = max(max_processes,count)
|
||||
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
|
||||
if now-start >= limits['job_wall_seconds']:
|
||||
reason = 'wall_limit'
|
||||
elif rss > limits['tree_rss_bytes']:
|
||||
reason = 'tree_rss_limit'
|
||||
elif count > limits['max_processes']:
|
||||
reason = 'process_count_limit'
|
||||
elif used_disk >= limits['disk_bytes']:
|
||||
reason = 'disk_limit'
|
||||
elif cpu_peak >= limits['job_cpu_seconds']:
|
||||
reason = 'job_cpu_limit'
|
||||
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
||||
reason = 'total_cpu_limit'
|
||||
if reason:
|
||||
stop_group(proc.pid)
|
||||
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
exit_status, usage = status, child_usage
|
||||
break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
# No detached work may outlive a supervised attempt.
|
||||
stop_group(proc.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.
|
||||
orphan_peak = 0
|
||||
# Killed descendants are adopted here rather than losing their
|
||||
# final CPU usage when the root worker cannot reap them itself.
|
||||
while True:
|
||||
try:
|
||||
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
||||
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
||||
elapsed = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
||||
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
|
||||
if reason is None:
|
||||
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
||||
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
||||
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
||||
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
||||
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
||||
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
||||
command=command,returncode=proc.returncode,stop_reason=reason,
|
||||
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
||||
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
||||
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
||||
reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
||||
gpu_visible=bool(cuda_device),cuda_device=cuda_device,
|
||||
numerical_threads=numerical_threads,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
|
||||
ledger['cpu_seconds_charged'] += charged
|
||||
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
||||
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
|
||||
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
|
||||
return receipt
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Fixed Gre24 model choices; no test-driven selection or node subsampling."""
|
||||
import numpy as np
|
||||
|
||||
|
||||
def xgb_model(seed,threads=8):
|
||||
from xgboost import XGBRegressor
|
||||
return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1,
|
||||
objective='reg:squarederror',random_state=seed,n_jobs=threads)
|
||||
|
||||
|
||||
def fnn_model():
|
||||
import torch
|
||||
layers=[];width=19
|
||||
for _ in range(4):
|
||||
layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100
|
||||
layers.append(torch.nn.Linear(100,1))
|
||||
return torch.nn.Sequential(*layers)
|
||||
|
||||
|
||||
def eligible_roles(split,successful):
|
||||
successful=set(successful)
|
||||
roles={role:[c for c in split[role] if c in successful]
|
||||
for role in ['train','test','fnn_learning','fnn_validation']}
|
||||
if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap')
|
||||
if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap')
|
||||
if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']):
|
||||
raise ValueError('FNN learning/validation do not partition train')
|
||||
missing={role:[c for c in split[role] if c not in successful] for role in roles}
|
||||
return roles,missing
|
||||
|
||||
|
||||
def require_training_role(case,split,role):
|
||||
if role not in ['train','fnn_learning','fnn_validation']:
|
||||
raise PermissionError('Only frozen training-side roles may read targets')
|
||||
if case in split['test'] or case not in split[role]:
|
||||
raise PermissionError('Case is not in the requested training role')
|
||||
|
||||
|
||||
def early_stop_update(best,stale,score,patience=5):
|
||||
if not np.isfinite(score):raise ValueError('Nonfinite validation MSE')
|
||||
improved=score<best
|
||||
return (score if improved else best),(0 if improved else stale+1),improved,(not improved and stale+1>=patience)
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
"""Predictions frozen before test-target scoring, all successful tet nodes."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import numpy as np
|
||||
from gre24_prepare_v2 import sha
|
||||
from gre24_features_v2 import per_geometry_metrics
|
||||
from gre24_models import fnn_model
|
||||
from gre24_fit import contract
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def predict(root):
|
||||
c,prep,splits=contract(root)
|
||||
models=json.loads((root/'models-freeze.json').read_text())
|
||||
dest=root/'predictions';dest.mkdir(exist_ok=False)
|
||||
start=time.process_time();rows=[]
|
||||
for split in splits:
|
||||
available={'naive':None}
|
||||
for name in ['GBRT','FNN']:
|
||||
r=models['fits'][f"{split['index']}/{name}"]
|
||||
if r['status']!='PASS':continue
|
||||
if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze')
|
||||
if name=='GBRT':
|
||||
from xgboost import XGBRegressor
|
||||
m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint'])
|
||||
else:
|
||||
import torch
|
||||
torch.set_num_threads(4);m=fnn_model()
|
||||
m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval()
|
||||
available[name]=m
|
||||
for case in split['test']:
|
||||
if case not in prep['successful']:
|
||||
rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={}))
|
||||
continue
|
||||
r=prep['successful'][case];source=Path(r['output']);outputs={}
|
||||
for name in ['features.npy','Vf.npy','raw_ids.npy']:
|
||||
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed')
|
||||
x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
|
||||
# No seconds/target file is opened in this phase.
|
||||
for name,m in available.items():
|
||||
if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False)
|
||||
elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32))
|
||||
else:
|
||||
chunks=[]
|
||||
with torch.no_grad():
|
||||
for a in range(0,len(x),65536):
|
||||
chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy())
|
||||
pred=np.concatenate(chunks)
|
||||
if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction')
|
||||
folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True)
|
||||
path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds'])
|
||||
outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True)
|
||||
rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs,
|
||||
raw_ids_sha256=r['output_sha256']['raw_ids.npy']))
|
||||
save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start))
|
||||
save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows,
|
||||
worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds'))
|
||||
|
||||
|
||||
def score(root):
|
||||
c,prep,splits=contract(root);frozen=root/'predictions-freeze.json'
|
||||
if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring')
|
||||
freeze=json.loads(frozen.read_text());rows=[];start=time.process_time()
|
||||
for r in freeze['rows']:
|
||||
if r['status']!='PREDICTED':
|
||||
rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue
|
||||
p=prep['successful'][r['case']];source=Path(p['output'])
|
||||
if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed')
|
||||
y=np.load(source/'seconds.npy',allow_pickle=False);scores={}
|
||||
for name,value in r['predictions'].items():
|
||||
if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed')
|
||||
pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred)
|
||||
rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores))
|
||||
summary={}
|
||||
for name in ['naive','GBRT','FNN']:
|
||||
summary[name]={}
|
||||
for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']:
|
||||
per=[]
|
||||
for split in splits:
|
||||
a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores']
|
||||
and r['scores'][name][metric] is not None]
|
||||
per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a),
|
||||
missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None,
|
||||
median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None))
|
||||
# Do not average only whichever model splits happened to finish.
|
||||
avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']}
|
||||
summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg,
|
||||
full_requested_coverage=all(r['missing']==0 for r in per))
|
||||
save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen)))
|
||||
save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'],
|
||||
status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start,
|
||||
scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True,
|
||||
limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements',
|
||||
't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices',
|
||||
'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit']))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score'])
|
||||
a=p.parse_args();(predict if a.phase=='predict' else score)(a.root)
|
||||
@@ -0,0 +1,87 @@
|
||||
[
|
||||
{
|
||||
"index": 0,
|
||||
"seed": 91,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"seed": 92,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"seed": 93,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 3,
|
||||
"seed": 94,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"index": 4,
|
||||
"seed": 95,
|
||||
"train": [
|
||||
"learn",
|
||||
"validation"
|
||||
],
|
||||
"test": [
|
||||
"test"
|
||||
],
|
||||
"fnn_learning": [
|
||||
"learn"
|
||||
],
|
||||
"fnn_validation": [
|
||||
"validation"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"code_sha256": {
|
||||
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"tools/gre24_fit.py": "29ac7c26d4164593afddb679481702e04cfb3b05e1f83ab4c56edea7162d37a0",
|
||||
"tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0",
|
||||
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
|
||||
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa"
|
||||
},
|
||||
"preparation_freeze_sha256": "41f1ab982ce91eae8c71a5080f53aa8909e6eb1089d376e56b91a40c573aaa13",
|
||||
"splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809",
|
||||
"dataset_version": "synthetic_only",
|
||||
"protocol": "synthetic smoke, not scientific data",
|
||||
"xgb_threads": 1,
|
||||
"fnn_cpu_threads": 2,
|
||||
"gpu_reserved_byte_cap": 16106127360
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"0",
|
||||
"FNN"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 5.798009,
|
||||
"wall_seconds": 5.534140970034059,
|
||||
"peak_tree_rss_bytes": 1042665472,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 159744,
|
||||
"max_poll_gap_seconds": 0.39918523997766897,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"numerical_threads": 2,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"underlying_worker_status": "PASS",
|
||||
"device_monitor": {
|
||||
"reasons": [],
|
||||
"peak_device_bytes": 435159040,
|
||||
"samples": 10
|
||||
},
|
||||
"recorded_peaks": {
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672
|
||||
},
|
||||
"peak_witnesses": {
|
||||
"peak_allocated_bytes": {
|
||||
"path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json",
|
||||
"sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af"
|
||||
},
|
||||
"peak_reserved_bytes": {
|
||||
"path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json",
|
||||
"sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af"
|
||||
}
|
||||
},
|
||||
"total_gpu_wall_seconds": 5.538356595963705,
|
||||
"peak_cap_bytes": 16106127360
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.6755669999999998,
|
||||
"wall_seconds": 1.7543383440352045,
|
||||
"peak_tree_rss_bytes": 148721664,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 221184,
|
||||
"max_poll_gap_seconds": 0.22388055600458756,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"receipts": {
|
||||
"GBRT": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.6755669999999998,
|
||||
"wall_seconds": 1.7543383440352045,
|
||||
"peak_tree_rss_bytes": 148721664,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 221184,
|
||||
"max_poll_gap_seconds": 0.22388055600458756,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
},
|
||||
"FNN": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"0",
|
||||
"FNN"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 5.798009,
|
||||
"wall_seconds": 5.534140970034059,
|
||||
"peak_tree_rss_bytes": 1042665472,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 159744,
|
||||
"max_poll_gap_seconds": 0.39918523997766897,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"numerical_threads": 2,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"underlying_worker_status": "PASS",
|
||||
"device_monitor": {
|
||||
"reasons": [],
|
||||
"peak_device_bytes": 435159040,
|
||||
"samples": 10
|
||||
},
|
||||
"recorded_peaks": {
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672
|
||||
},
|
||||
"peak_witnesses": {
|
||||
"peak_allocated_bytes": {
|
||||
"path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json",
|
||||
"sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af"
|
||||
},
|
||||
"peak_reserved_bytes": {
|
||||
"path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json",
|
||||
"sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af"
|
||||
}
|
||||
},
|
||||
"total_gpu_wall_seconds": 5.538356595963705,
|
||||
"peak_cap_bytes": 16106127360
|
||||
},
|
||||
"predict": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"predict"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 3.362805,
|
||||
"wall_seconds": 3.0489734660368413,
|
||||
"peak_tree_rss_bytes": 519958528,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.46637159201782197,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
},
|
||||
"score": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"score"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 0.6084579999999999,
|
||||
"wall_seconds": 0.7152220709831454,
|
||||
"peak_tree_rss_bytes": 111345664,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.11809522699331865,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
},
|
||||
"GPU_seconds": 5.584548475977499,
|
||||
"worker_CPU_seconds": 11.444839,
|
||||
"checks": [
|
||||
"real GBRT and FNN entrypoints",
|
||||
"test target absent during fits",
|
||||
"test seconds unavailable during prediction",
|
||||
"all predictions frozen before scoring",
|
||||
"missing splits not silently averaged",
|
||||
"failure VRAM peaks retained"
|
||||
],
|
||||
"scientific_data_read": false,
|
||||
"coordinator_CPU_seconds": 2.8703529999999997
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
[
|
||||
{
|
||||
"epoch": 1,
|
||||
"training_mse": 0.07795825600624084,
|
||||
"validation_mse": 0.060705915093421936,
|
||||
"best_epoch": 1,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.7462023249827325
|
||||
},
|
||||
{
|
||||
"epoch": 2,
|
||||
"training_mse": 0.06311292946338654,
|
||||
"validation_mse": 0.04847054183483124,
|
||||
"best_epoch": 2,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.751589720021002
|
||||
},
|
||||
{
|
||||
"epoch": 3,
|
||||
"training_mse": 0.05059317126870155,
|
||||
"validation_mse": 0.03820352628827095,
|
||||
"best_epoch": 3,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.7556245460291393
|
||||
},
|
||||
{
|
||||
"epoch": 4,
|
||||
"training_mse": 0.04009852185845375,
|
||||
"validation_mse": 0.02991129830479622,
|
||||
"best_epoch": 4,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.7596782970358618
|
||||
},
|
||||
{
|
||||
"epoch": 5,
|
||||
"training_mse": 0.031606074422597885,
|
||||
"validation_mse": 0.02256850153207779,
|
||||
"best_epoch": 5,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.7651306050247513
|
||||
},
|
||||
{
|
||||
"epoch": 6,
|
||||
"training_mse": 0.024003881961107254,
|
||||
"validation_mse": 0.0158725343644619,
|
||||
"best_epoch": 6,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.769136717019137
|
||||
},
|
||||
{
|
||||
"epoch": 7,
|
||||
"training_mse": 0.017005659639835358,
|
||||
"validation_mse": 0.010371999815106392,
|
||||
"best_epoch": 7,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.773159590025898
|
||||
},
|
||||
{
|
||||
"epoch": 8,
|
||||
"training_mse": 0.011204666458070278,
|
||||
"validation_mse": 0.006224535871297121,
|
||||
"best_epoch": 8,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.7771851680008695
|
||||
},
|
||||
{
|
||||
"epoch": 9,
|
||||
"training_mse": 0.00675668939948082,
|
||||
"validation_mse": 0.003289248328655958,
|
||||
"best_epoch": 9,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.7813589150318876
|
||||
},
|
||||
{
|
||||
"epoch": 10,
|
||||
"training_mse": 0.0034951232373714447,
|
||||
"validation_mse": 0.0018917981069535017,
|
||||
"best_epoch": 10,
|
||||
"stale_epochs": 0,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.7855790790054016
|
||||
},
|
||||
{
|
||||
"epoch": 11,
|
||||
"training_mse": 0.001763510867021978,
|
||||
"validation_mse": 0.0024419045075774193,
|
||||
"best_epoch": 10,
|
||||
"stale_epochs": 1,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.788880237028934
|
||||
},
|
||||
{
|
||||
"epoch": 12,
|
||||
"training_mse": 0.0019949353300035,
|
||||
"validation_mse": 0.004658875055611134,
|
||||
"best_epoch": 10,
|
||||
"stale_epochs": 2,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.792193347995635
|
||||
},
|
||||
{
|
||||
"epoch": 13,
|
||||
"training_mse": 0.003945866599678993,
|
||||
"validation_mse": 0.007049876265227795,
|
||||
"best_epoch": 10,
|
||||
"stale_epochs": 3,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.7955055559868924
|
||||
},
|
||||
{
|
||||
"epoch": 14,
|
||||
"training_mse": 0.006167569197714329,
|
||||
"validation_mse": 0.008173784241080284,
|
||||
"best_epoch": 10,
|
||||
"stale_epochs": 4,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.798879029985983
|
||||
},
|
||||
{
|
||||
"epoch": 15,
|
||||
"training_mse": 0.007227514870464802,
|
||||
"validation_mse": 0.007775146048516035,
|
||||
"best_epoch": 10,
|
||||
"stale_epochs": 5,
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672,
|
||||
"wall_s": 3.8023696839809418
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 5.798009,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "worker",
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 5.798009
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"stage": "FIT_COMPLETE_NO_TEST_SCORES",
|
||||
"pid": 480379,
|
||||
"unix": 1794757202.323634,
|
||||
"split": 0,
|
||||
"model": "FNN",
|
||||
"CPU_s": 4.038258777,
|
||||
"wall_s": 3.8051335139898583
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "FIT_COMPLETE_NO_TEST_SCORES",
|
||||
"split": 0,
|
||||
"model": "FNN",
|
||||
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/model.pt",
|
||||
"checkpoint_sha256": "737d54bf01b17dbd87756fcf43bb9b3333597d05a28c94d8cd4bc309e682c0b2",
|
||||
"missing_preparation": {
|
||||
"train": [],
|
||||
"test": [],
|
||||
"fnn_learning": [],
|
||||
"fnn_validation": []
|
||||
},
|
||||
"CPU_s": 4.038131669,
|
||||
"wall_s": 3.8050071470206603,
|
||||
"epochs": 15,
|
||||
"best_epoch": 10,
|
||||
"best_validation_mse": 0.0018917981069535017,
|
||||
"train_nodes": 128,
|
||||
"validation_nodes": 128,
|
||||
"train_geometries": 1,
|
||||
"validation_geometries": 1,
|
||||
"restored_best": true,
|
||||
"amp": false,
|
||||
"input_standardization": "none; use eq12 columns",
|
||||
"peak_allocated_bytes": 18742272,
|
||||
"peak_reserved_bytes": 23068672
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"pid": 480379,
|
||||
"unix": 1794757198.5185177,
|
||||
"split": 0,
|
||||
"model": "FNN",
|
||||
"seed": 91,
|
||||
"training_contract_sha256": "c86ab1d6ef22262200789c1b0d7aadd112b68862a64275a1c657a9d3f9f39dfd"
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
[
|
||||
{
|
||||
"case": "learn",
|
||||
"role": "fnn_learning",
|
||||
"nodes": 128,
|
||||
"source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/learn",
|
||||
"feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
|
||||
"target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc"
|
||||
},
|
||||
{
|
||||
"case": "validation",
|
||||
"role": "fnn_validation",
|
||||
"nodes": 128,
|
||||
"source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/validation",
|
||||
"feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
|
||||
"target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3"
|
||||
}
|
||||
]
|
||||
Whitespace-only changes.
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"0",
|
||||
"FNN"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 5.798009,
|
||||
"wall_seconds": 5.534140970034059,
|
||||
"peak_tree_rss_bytes": 1042665472,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 159744,
|
||||
"max_poll_gap_seconds": 0.39918523997766897,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"numerical_threads": 2,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"0",
|
||||
"FNN"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 120.0,
|
||||
"job_cpu_seconds": 120.0,
|
||||
"job_wall_seconds": 57.0,
|
||||
"tree_rss_bytes": 25769803776,
|
||||
"disk_bytes": 3221225472,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1794757197.6559174
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 1.6755669999999998,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "worker",
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 1.6755669999999998
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"stage": "FIT_COMPLETE_NO_TEST_SCORES",
|
||||
"pid": 480373,
|
||||
"unix": 1794757197.3915205,
|
||||
"split": 0,
|
||||
"model": "GBRT",
|
||||
"CPU_s": 0.6673245510000001,
|
||||
"wall_s": 0.6674595809890889
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"status": "FIT_COMPLETE_NO_TEST_SCORES",
|
||||
"split": 0,
|
||||
"model": "GBRT",
|
||||
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/model.ubj",
|
||||
"checkpoint_sha256": "0630f79ad1b6783fa31ed5b6d94e345a896212187676f65168f4e819889ec74b",
|
||||
"missing_preparation": {
|
||||
"train": [],
|
||||
"test": [],
|
||||
"fnn_learning": [],
|
||||
"fnn_validation": []
|
||||
},
|
||||
"CPU_s": 0.6670500859999999,
|
||||
"wall_s": 0.6671876410255209,
|
||||
"train_nodes": 256,
|
||||
"train_geometries": 2,
|
||||
"parameters": {
|
||||
"objective": "reg:squarederror",
|
||||
"base_score": null,
|
||||
"booster": null,
|
||||
"callbacks": null,
|
||||
"colsample_bylevel": null,
|
||||
"colsample_bynode": null,
|
||||
"colsample_bytree": null,
|
||||
"device": null,
|
||||
"early_stopping_rounds": null,
|
||||
"enable_categorical": false,
|
||||
"eval_metric": null,
|
||||
"feature_types": null,
|
||||
"gamma": null,
|
||||
"grow_policy": null,
|
||||
"importance_type": null,
|
||||
"interaction_constraints": null,
|
||||
"learning_rate": 0.1,
|
||||
"max_bin": null,
|
||||
"max_cat_threshold": null,
|
||||
"max_cat_to_onehot": null,
|
||||
"max_delta_step": null,
|
||||
"max_depth": 8,
|
||||
"max_leaves": null,
|
||||
"min_child_weight": null,
|
||||
"missing": "NaN (library default missing marker)",
|
||||
"monotone_constraints": null,
|
||||
"multi_strategy": null,
|
||||
"n_estimators": 100,
|
||||
"n_jobs": 1,
|
||||
"num_parallel_tree": null,
|
||||
"random_state": 91,
|
||||
"reg_alpha": null,
|
||||
"reg_lambda": null,
|
||||
"sampling_method": null,
|
||||
"scale_pos_weight": null,
|
||||
"subsample": null,
|
||||
"tree_method": null,
|
||||
"validate_parameters": null,
|
||||
"verbosity": null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"pid": 480373,
|
||||
"unix": 1794757196.7240794,
|
||||
"split": 0,
|
||||
"model": "GBRT",
|
||||
"seed": 91,
|
||||
"training_contract_sha256": "c86ab1d6ef22262200789c1b0d7aadd112b68862a64275a1c657a9d3f9f39dfd"
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
[
|
||||
{
|
||||
"case": "learn",
|
||||
"role": "train",
|
||||
"nodes": 128,
|
||||
"source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/learn",
|
||||
"feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a",
|
||||
"target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc"
|
||||
},
|
||||
{
|
||||
"case": "validation",
|
||||
"role": "train",
|
||||
"nodes": 128,
|
||||
"source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/validation",
|
||||
"feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499",
|
||||
"target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3"
|
||||
}
|
||||
]
|
||||
Whitespace-only changes.
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1.6755669999999998,
|
||||
"wall_seconds": 1.7543383440352045,
|
||||
"peak_tree_rss_bytes": 148721664,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 221184,
|
||||
"max_poll_gap_seconds": 0.22388055600458756,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_fit.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"0",
|
||||
"GBRT"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 120,
|
||||
"job_cpu_seconds": 120,
|
||||
"job_wall_seconds": 120,
|
||||
"tree_rss_bytes": 4294967296,
|
||||
"disk_bytes": 268435456,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1794757195.8518412
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"learner":{"generic_param":{"device":"cpu","fail_on_invalid_gpu_id":"0","n_jobs":"1","nthread":"1","random_state":"91","seed":"91","seed_per_iteration":"0","validate_parameters":"1"},"gradient_booster":{"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"100"},"gbtree_train_param":{"process_type":"default","tree_method":"auto","updater":"grow_quantile_histmaker","updater_seq":"grow_quantile_histmaker"},"name":"gbtree","specified_updater":false,"tree_train_param":{"alpha":"0","cache_opt":"1","colsample_bylevel":"1","colsample_bynode":"1","colsample_bytree":"1","eta":"0.100000001","gamma":"0","grow_policy":"depthwise","interaction_constraints":"","lambda":"1","learning_rate":"0.100000001","max_bin":"256","max_cat_threshold":"64","max_cat_to_onehot":"4","max_delta_step":"0","max_depth":"8","max_leaves":"0","min_child_weight":"1","min_split_loss":"0","monotone_constraints":"()","refresh_leaf":"1","reg_alpha":"0","reg_lambda":"1","sampling_method":"uniform","sketch_ratio":"2","sparse_threshold":"0.20000000000000001","subsample":"1"},"updater":[{"hist_train_param":{"debug_synchronize":"0","max_cached_hist_node":"65536"},"name":"grow_quantile_histmaker"}]},"learner_model_param":{"base_score":"3.0167848E-1","boost_from_average":"1","num_class":"0","num_feature":"19","num_target":"1"},"learner_train_param":{"booster":"gbtree","disable_default_eval_metric":"0","multi_strategy":"one_output_per_tree","objective":"reg:squarederror"},"metrics":[{"name":"rmse"}],"objective":{"name":"reg:squarederror","reg_loss_param":{"scale_pos_weight":"1"}}},"version":[2,1,4]}
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"fits": {
|
||||
"0/GBRT": {
|
||||
"status": "PASS",
|
||||
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/model.ubj",
|
||||
"checkpoint_sha256": "0630f79ad1b6783fa31ed5b6d94e345a896212187676f65168f4e819889ec74b"
|
||||
},
|
||||
"0/FNN": {
|
||||
"status": "PASS",
|
||||
"checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/model.pt",
|
||||
"checkpoint_sha256": "737d54bf01b17dbd87756fcf43bb9b3333597d05a28c94d8cd4bc309e682c0b2"
|
||||
},
|
||||
"1/GBRT": {
|
||||
"status": "NOT_ATTEMPTED_SYNTHETIC"
|
||||
},
|
||||
"1/FNN": {
|
||||
"status": "NOT_ATTEMPTED_SYNTHETIC"
|
||||
},
|
||||
"2/GBRT": {
|
||||
"status": "NOT_ATTEMPTED_SYNTHETIC"
|
||||
},
|
||||
"2/FNN": {
|
||||
"status": "NOT_ATTEMPTED_SYNTHETIC"
|
||||
},
|
||||
"3/GBRT": {
|
||||
"status": "NOT_ATTEMPTED_SYNTHETIC"
|
||||
},
|
||||
"3/FNN": {
|
||||
"status": "NOT_ATTEMPTED_SYNTHETIC"
|
||||
},
|
||||
"4/GBRT": {
|
||||
"status": "NOT_ATTEMPTED_SYNTHETIC"
|
||||
},
|
||||
"4/FNN": {
|
||||
"status": "NOT_ATTEMPTED_SYNTHETIC"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"peak_reserved_bytes": 17179869184
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 3.362805,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "worker",
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 3.362805
|
||||
}
|
||||
]
|
||||
}
|
||||
Whitespace-only changes.
+26
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"predict"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 3.362805,
|
||||
"wall_seconds": 3.0489734660368413,
|
||||
"peak_tree_rss_bytes": 519958528,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.46637159201782197,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"cuda_device": "",
|
||||
"numerical_threads": 1,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python",
|
||||
"tools/gre24_score.py",
|
||||
"artifacts/gre24-reproduction-2026-11-15/integration-03",
|
||||
"predict"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 120,
|
||||
"job_cpu_seconds": 120,
|
||||
"job_wall_seconds": 120,
|
||||
"tree_rss_bytes": 4294967296,
|
||||
"disk_bytes": 268435456,
|
||||
"max_processes": 1
|
||||
},
|
||||
"started_unix": 1794757203.1936562
|
||||
}
|
||||
Loaded 100 of 124 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user