From eb9ed0c762be74863449c8a6146cc414418b1aa6 Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Sun, 15 Nov 2026 23:54:50 +0800 Subject: [PATCH] Freeze Gre24 model chain and retain synthetic acceptance evidence --- .../budget-chain-test/complete.json | 32 ++ .../budget-chain-test/contract.json | 7 + .../budget-chain-test/learn-prep-result.json | 16 + .../budget-chain-test/learn/result.json | 16 + .../model-chain-started.json | 6 + .../budget-chain-test/models-freeze.json | 50 ++ .../budget-chain-test/plan.json | 21 + .../postfit-guards/predict/ledger.json | 10 + .../postfit-guards/predict/worker.log | 0 .../predict/worker.receipt.json | 26 + .../predict/worker.started.json | 17 + .../postfit-guards/score/ledger.json | 10 + .../postfit-guards/score/worker.log | 0 .../postfit-guards/score/worker.receipt.json | 26 + .../postfit-guards/score/worker.started.json | 17 + .../prediction-progress.json | 4 + .../budget-chain-test/predictions-freeze.json | 79 +++ .../budget-chain-test/preparation-freeze.json | 50 ++ .../reproduction-complete.json | 111 ++++ .../budget-chain-test/score-summary.json | 519 ++++++++++++++++++ .../budget-chain-test/scores.json | 105 ++++ .../budget-chain-test/splits.json | 87 +++ .../budget-chain-test/test-acceptance.json | 121 ++++ .../budget-chain-test/test-prep-result.json | 15 + .../budget-chain-test/test/result.json | 15 + .../budget-chain-test/training-contract.json | 23 + .../validation-prep-result.json | 16 + .../budget-chain-test/validation/result.json | 16 + .../integration-01/GBRT-guard.json | 27 + .../integration-01/acceptance.json | 33 ++ .../fits/split-0/GBRT/failure.json | 8 + .../fits/split-0/GBRT/ledger.json | 10 + .../fits/split-0/GBRT/progress.json | 11 + .../fits/split-0/GBRT/started.json | 8 + .../split-0/GBRT/training-target-access.json | 18 + .../fits/split-0/GBRT/worker.log | 20 + .../fits/split-0/GBRT/worker.receipt.json | 27 + .../fits/split-0/GBRT/worker.started.json | 18 + .../fits/split-0/GBRT/xgboost-config.json | 1 + .../integration-01/preparation-freeze.json | 40 ++ .../source-snapshot/gre24_fit.py | 133 +++++ .../source-snapshot/gre24_gpu_guard.py | 82 +++ .../source-snapshot/gre24_job_guard.py | 160 ++++++ .../source-snapshot/gre24_models.py | 42 ++ .../source-snapshot/gre24_score.py | 102 ++++ .../integration-01/splits.json | 87 +++ .../integration-01/training-contract.json | 16 + .../integration-02/FNN-guard.json | 40 ++ .../integration-02/GBRT-guard.json | 27 + .../integration-02/acceptance.json | 73 +++ .../fits/split-0/FNN/failure.json | 8 + .../fits/split-0/FNN/ledger.json | 10 + .../fits/split-0/FNN/started.json | 8 + .../fits/split-0/FNN/worker.log | 8 + .../fits/split-0/FNN/worker.receipt.json | 27 + .../fits/split-0/FNN/worker.started.json | 18 + .../fits/split-0/GBRT/ledger.json | 10 + .../fits/split-0/GBRT/progress.json | 9 + .../fits/split-0/GBRT/result.json | 58 ++ .../fits/split-0/GBRT/started.json | 8 + .../split-0/GBRT/training-target-access.json | 18 + .../fits/split-0/GBRT/worker.log | 0 .../fits/split-0/GBRT/worker.receipt.json | 27 + .../fits/split-0/GBRT/worker.started.json | 18 + .../fits/split-0/GBRT/xgboost-config.json | 1 + .../integration-02/preparation-freeze.json | 40 ++ .../source-snapshot/gre24_fit.py | 138 +++++ .../source-snapshot/gre24_gpu_guard.py | 82 +++ .../source-snapshot/gre24_job_guard.py | 160 ++++++ .../source-snapshot/gre24_models.py | 42 ++ .../source-snapshot/gre24_score.py | 102 ++++ .../integration-02/splits.json | 87 +++ .../integration-02/training-contract.json | 16 + .../integration-03/FNN-guard.json | 49 ++ .../integration-03/GBRT-guard.json | 27 + .../integration-03/acceptance.json | 145 +++++ .../fits/split-0/FNN/history.json | 152 +++++ .../fits/split-0/FNN/ledger.json | 10 + .../fits/split-0/FNN/progress.json | 9 + .../fits/split-0/FNN/result.json | 27 + .../fits/split-0/FNN/started.json | 8 + .../split-0/FNN/training-target-access.json | 18 + .../fits/split-0/FNN/worker.log | 0 .../fits/split-0/FNN/worker.receipt.json | 27 + .../fits/split-0/FNN/worker.started.json | 18 + .../fits/split-0/GBRT/ledger.json | 10 + .../fits/split-0/GBRT/progress.json | 9 + .../fits/split-0/GBRT/result.json | 58 ++ .../fits/split-0/GBRT/started.json | 8 + .../split-0/GBRT/training-target-access.json | 18 + .../fits/split-0/GBRT/worker.log | 0 .../fits/split-0/GBRT/worker.receipt.json | 27 + .../fits/split-0/GBRT/worker.started.json | 18 + .../fits/split-0/GBRT/xgboost-config.json | 1 + .../integration-03/models-freeze.json | 38 ++ .../integration-03/peak-check/failure.json | 3 + .../integration-03/predict-guard/ledger.json | 10 + .../integration-03/predict-guard/worker.log | 0 .../predict-guard/worker.receipt.json | 26 + .../predict-guard/worker.started.json | 17 + .../integration-03/prediction-progress.json | 4 + .../integration-03/predictions-freeze.json | 91 +++ .../integration-03/preparation-freeze.json | 40 ++ .../integration-03/score-guard/ledger.json | 10 + .../integration-03/score-guard/worker.log | 0 .../score-guard/worker.receipt.json | 26 + .../score-guard/worker.started.json | 17 + .../integration-03/score-summary.json | 519 ++++++++++++++++++ .../integration-03/scores.json | 131 +++++ .../integration-03/splits.json | 87 +++ .../integration-03/training-contract.json | 16 + .../model-synthetic-receipt.json | 8 + .../v5-protocol/model-plan.json | 63 +++ .../v5-protocol/model-prelaunch-audit.json | 76 +++ docs/gre24-model-chain-2026-11-15.md | 23 + tools/gre24_fit.py | 139 +++++ tools/gre24_gpu_guard.py | 82 +++ tools/gre24_job_guard.py | 160 ++++++ tools/gre24_models.py | 42 ++ tools/gre24_score.py | 104 ++++ tools/run_gre24_models.py | 142 +++++ tools/test_gre24_chain_budget.py | 44 ++ tools/test_gre24_fit_integration.py | 78 +++ tools/test_gre24_models.py | 65 +++ 124 files changed, 5993 insertions(+) create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/complete.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/contract.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn-prep-result.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn/result.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/model-chain-started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/models-freeze.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/plan.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/ledger.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.log create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/ledger.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.log create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/prediction-progress.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions-freeze.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/preparation-freeze.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/reproduction-complete.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/score-summary.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/scores.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/splits.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-acceptance.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-prep-result.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test/result.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/training-contract.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation-prep-result.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation/result.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/GBRT-guard.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/acceptance.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/failure.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/ledger.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/progress.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/training-target-access.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.log create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/xgboost-config.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/preparation-freeze.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_fit.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_gpu_guard.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_job_guard.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_models.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_score.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/splits.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-01/training-contract.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/FNN-guard.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/GBRT-guard.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/acceptance.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/failure.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/ledger.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.log create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/ledger.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/progress.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/result.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/training-target-access.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.log create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/xgboost-config.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/preparation-freeze.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_fit.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_gpu_guard.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_job_guard.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_models.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_score.py create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/splits.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-02/training-contract.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/FNN-guard.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/GBRT-guard.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/acceptance.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/history.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/ledger.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/progress.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/training-target-access.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.log create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/ledger.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/progress.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/result.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/training-target-access.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.log create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/xgboost-config.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/models-freeze.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/peak-check/failure.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/ledger.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.log create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/prediction-progress.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/predictions-freeze.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/preparation-freeze.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/ledger.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.log create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.started.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/score-summary.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/scores.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/splits.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/integration-03/training-contract.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/model-synthetic-receipt.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json create mode 100644 artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-prelaunch-audit.json create mode 100644 docs/gre24-model-chain-2026-11-15.md create mode 100644 tools/gre24_fit.py create mode 100644 tools/gre24_gpu_guard.py create mode 100644 tools/gre24_job_guard.py create mode 100644 tools/gre24_models.py create mode 100644 tools/gre24_score.py create mode 100644 tools/run_gre24_models.py create mode 100644 tools/test_gre24_chain_budget.py create mode 100644 tools/test_gre24_fit_integration.py create mode 100644 tools/test_gre24_models.py diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/complete.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/complete.json new file mode 100644 index 000000000..94b199beb --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/complete.json @@ -0,0 +1,32 @@ +{ + "status": "COMPLETE", + "accounting_complete": true, + "records": [ + { + "case": "learn", + "worker": { + "status": "PASS" + }, + "result_path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn/result.json", + "result_sha256": "9110e0f3a43d41602c1b17281544163455d2ae18b4f3509fca72aae063253751" + }, + { + "case": "validation", + "worker": { + "status": "PASS" + }, + "result_path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation/result.json", + "result_sha256": "2e7bf2831b9d0136b848c76aa3ed7e974f0dcbd26e97369b288b18bda9d57031" + }, + { + "case": "test", + "worker": { + "status": "PASS" + }, + "result_path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test/result.json", + "result_sha256": "7c8a234f50a9ab9424b78226cfd4d3709b140ff63cef2138aa19e2eb20e22c86" + } + ], + "worker_cpu_seconds": 0.0, + "initial_cpu_reserve_seconds": 0.0 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/contract.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/contract.json new file mode 100644 index 000000000..5dfe3e643 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/contract.json @@ -0,0 +1,7 @@ +{ + "parts": [ + "learn", + "validation", + "test" + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn-prep-result.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn-prep-result.json new file mode 100644 index 000000000..68858b087 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn-prep-result.json @@ -0,0 +1,16 @@ +{ + "status": "PASS", + "case": "learn", + "geometry": { + "tet_nodes": 128, + "outside_tet_nodes": 0 + }, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076", + "target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn/result.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn/result.json new file mode 100644 index 000000000..68858b087 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn/result.json @@ -0,0 +1,16 @@ +{ + "status": "PASS", + "case": "learn", + "geometry": { + "tet_nodes": 128, + "outside_tet_nodes": 0 + }, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076", + "target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/model-chain-started.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/model-chain-started.json new file mode 100644 index 000000000..b0b839a6e --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/model-chain-started.json @@ -0,0 +1,6 @@ +{ + "pid": 480565, + "unix": 1794757290.256113, + "plan_sha256": "6e410499203d3e13862b271ad29b31f62cab20b2fc5530aa79b30156124c83d3", + "stage": "WAIT_PREPARATION" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/models-freeze.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/models-freeze.json new file mode 100644 index 000000000..7393bceaf --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/models-freeze.json @@ -0,0 +1,50 @@ +{ + "fits": { + "0/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "1/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "2/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "3/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "4/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "0/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "1/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "2/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "3/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "4/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + } + }, + "charged_worker_cpu_seconds": 0.0, + "charged_gpu_seconds": 0.0, + "uuid": null, + "training_contract_sha256": "c37fbd2bcb5fbd4b12a9bab5cb4aa0b06f9ca5fab638fd46e09e03a7bcae32f9", + "all_fits_terminal": true, + "test_scores_read": false +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/plan.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/plan.json new file mode 100644 index 000000000..cac72077a --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/plan.json @@ -0,0 +1,21 @@ +{ + "code_sha256": { + "tools/run_gre24_models.py": "2a5393a5009a2eb1bdd470370a9183b80aa7029d6b69a545831706a0139eb774", + "tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688", + "tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0", + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4", + "tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa" + }, + "total_worker_cpu_seconds": 1000, + "postfit_cpu_reserve_seconds": 998, + "total_gpu_seconds": 60, + "initial_gpu_seconds": 0, + "dataset_version": "synthetic_budget_only", + "protocol": "synthetic", + "xgb_threads": 1, + "fnn_cpu_threads": 1, + "xgb_per_fit_cpu_seconds": 10, + "fnn_per_fit_cpu_seconds": 10, + "gpu_reserved_byte_cap": 16106127360 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/ledger.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/ledger.json new file mode 100644 index 000000000..c905d4e15 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 1.566765, + "attempts": [ + { + "name": "worker", + "status": "PASS", + "cpu_seconds": 1.566765 + } + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.log b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.log new file mode 100644 index 000000000..e69de29bb diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.receipt.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.receipt.json new file mode 100644 index 000000000..c290a59e8 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.receipt.json @@ -0,0 +1,26 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test", + "predict" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.566765, + "wall_seconds": 1.4599632910103537, + "peak_tree_rss_bytes": 110755840, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.363760320993606, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 4, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.started.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.started.json new file mode 100644 index 000000000..501971bba --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/predict/worker.started.json @@ -0,0 +1,17 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test", + "predict" + ], + "limits": { + "total_cpu_seconds": 1000.0, + "job_cpu_seconds": 1000.0, + "job_wall_seconds": 7200, + "tree_rss_bytes": 25769803776, + "disk_bytes": 3221225472, + "max_processes": 1 + }, + "started_unix": 1794757290.2597783 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/ledger.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/ledger.json new file mode 100644 index 000000000..34980b667 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 1.299077, + "attempts": [ + { + "name": "worker", + "status": "PASS", + "cpu_seconds": 1.299077 + } + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.log b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.log new file mode 100644 index 000000000..e69de29bb diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.receipt.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.receipt.json new file mode 100644 index 000000000..cd94aa33d --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.receipt.json @@ -0,0 +1,26 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test", + "score" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.299077, + "wall_seconds": 1.1043277920107357, + "peak_tree_rss_bytes": 111403008, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.21797074796631932, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 4, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.started.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.started.json new file mode 100644 index 000000000..bd5d428f5 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/postfit-guards/score/worker.started.json @@ -0,0 +1,17 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test", + "score" + ], + "limits": { + "total_cpu_seconds": 998.433235, + "job_cpu_seconds": 998.433235, + "job_wall_seconds": 7200, + "tree_rss_bytes": 25769803776, + "disk_bytes": 3221225472, + "max_processes": 1 + }, + "started_unix": 1794757291.7210498 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/prediction-progress.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/prediction-progress.json new file mode 100644 index 000000000..34d685a1d --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/prediction-progress.json @@ -0,0 +1,4 @@ +{ + "test_case_visits": 5, + "worker_CPU_s": 0.006219989999999953 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions-freeze.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions-freeze.json new file mode 100644 index 000000000..877cd6bda --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions-freeze.json @@ -0,0 +1,79 @@ +{ + "models_freeze_sha256": "d2c1c2d58053e3e42902604bbfdcd0fe6d07ec1d8893f548207abc01f44ec563", + "rows": [ + { + "split": 0, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-0/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + }, + { + "split": 1, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-1/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + }, + { + "split": 2, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-2/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + }, + { + "split": 3, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-3/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + }, + { + "split": 4, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/predictions/split-4/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + } + ], + "worker_CPU_s": 0.0065045639999998794, + "test_targets_read": false, + "device": "CPU", + "prediction_units": "seconds" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/preparation-freeze.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/preparation-freeze.json new file mode 100644 index 000000000..0b2bd9e3c --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/preparation-freeze.json @@ -0,0 +1,50 @@ +{ + "successful": { + "learn": { + "output": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/learn", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076", + "target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + }, + "result_sha256": "9110e0f3a43d41602c1b17281544163455d2ae18b4f3509fca72aae063253751", + "outside_tet_nodes": 0 + }, + "validation": { + "output": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40", + "target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + }, + "result_sha256": "2e7bf2831b9d0136b848c76aa3ed7e974f0dcbd26e97369b288b18bda9d57031", + "outside_tet_nodes": 0 + }, + "test": { + "output": "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + }, + "result_sha256": "7c8a234f50a9ab9424b78226cfd4d3709b140ff63cef2138aa19e2eb20e22c86", + "outside_tet_nodes": 0 + } + }, + "failed": [], + "requested": 3, + "preparation_complete_sha256": "84d3c0674e4f128f7ac2bb88b09d6376976211efabde3e8746f239b68c5b835a", + "replacement": false, + "cumulative_initial_worker_cpu_seconds": 0.0 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/reproduction-complete.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/reproduction-complete.json new file mode 100644 index 000000000..cbf0fe6af --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/reproduction-complete.json @@ -0,0 +1,111 @@ +{ + "status": "SCORED", + "fits": { + "0/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "1/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "2/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "3/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "4/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "0/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "1/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "2/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "3/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "4/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + } + }, + "postfit": [ + { + "phase": "predict", + "receipt": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test", + "predict" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.566765, + "wall_seconds": 1.4599632910103537, + "peak_tree_rss_bytes": 110755840, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.363760320993606, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 4, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + } + }, + { + "phase": "score", + "receipt": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test", + "score" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.299077, + "wall_seconds": 1.1043277920107357, + "peak_tree_rss_bytes": 111403008, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.21797074796631932, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 4, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + } + } + ], + "charged_worker_cpu_seconds": 2.865842, + "charged_gpu_seconds": 0.0, + "total_worker_cpu_cap": 1000, + "total_gpu_cap": 60, + "dataset_version": "synthetic_budget_only", + "paper_numbers_diagnostic_only": true +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/score-summary.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/score-summary.json new file mode 100644 index 000000000..f7519e6a9 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/score-summary.json @@ -0,0 +1,519 @@ +{ + "dataset_version": "synthetic_budget_only", + "protocol": "synthetic", + "status": "SCORED_WITH_EXPLICIT_COVERAGE", + "summary": { + "naive": { + "rrmse_rms": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + } + ], + "mean_of_five_split_summaries": { + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + "full_requested_coverage": true + }, + "rrmse_maximum": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + } + ], + "mean_of_five_split_summaries": { + "min": 0.9117259819436263, + "median": 0.9117259819436263, + "max": 0.9117259819436263 + }, + "full_requested_coverage": true + }, + "rrmse_mean": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + } + ], + "mean_of_five_split_summaries": { + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + "full_requested_coverage": true + } + }, + "GBRT": { + "rrmse_rms": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + }, + "rrmse_maximum": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + }, + "rrmse_mean": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + } + }, + "FNN": { + "rrmse_rms": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + }, + "rrmse_maximum": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + }, + "rrmse_mean": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + } + } + }, + "worker_CPU_s": 0.004958694000000152, + "scores_sha256": "0cb25cf6f6224e7cf2a5707bf99d85acada792d9fbf80166cfd4cf7ab3e0784e", + "paper_numbers_diagnostic_only": true, + "limitations": [ + "v5 data instead of paper v4", + "R1 discretisation choices are explicit supplements", + "t_set substitutes unavailable applied Q", + "FNN fill-only and early stopping are declared choices", + "Random geometry splits may leak near-duplicate geometry families", + "Failed preparations not replaced; coverage is explicit" + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/scores.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/scores.json new file mode 100644 index 000000000..e9b2394e5 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/scores.json @@ -0,0 +1,105 @@ +{ + "rows": [ + { + "split": 0, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + } + } + }, + { + "split": 1, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + } + } + }, + { + "split": 2, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + } + } + }, + { + "split": 3, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + } + } + }, + { + "split": 4, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + } + } + } + ], + "predictions_freeze_sha256": "5cc930becfef50f13d4c724fd884f8de4cf95e47afd09d0250c3bae7a2f1b868" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/splits.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/splits.json new file mode 100644 index 000000000..1a7c33f89 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/splits.json @@ -0,0 +1,87 @@ +[ + { + "index": 0, + "seed": 91, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 1, + "seed": 92, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 2, + "seed": 93, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 3, + "seed": 94, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 4, + "seed": 95, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + } +] diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-acceptance.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-acceptance.json new file mode 100644 index 000000000..27c77eaf6 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-acceptance.json @@ -0,0 +1,121 @@ +{ + "status": "PASS", + "checks": [ + "CPU reserve prevents all fits", + "no GPU starts", + "terminal model freeze exists before prediction", + "naive score survives missing models", + "actual postfit CPU charged" + ], + "result": { + "status": "SCORED", + "fits": { + "0/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "1/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "2/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "3/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "4/GBRT": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "0/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "1/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "2/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "3/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + }, + "4/FNN": { + "status": "NOT_ATTEMPTED", + "reason": "SHARED_CPU_CAP_RESERVE" + } + }, + "postfit": [ + { + "phase": "predict", + "receipt": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test", + "predict" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.566765, + "wall_seconds": 1.4599632910103537, + "peak_tree_rss_bytes": 110755840, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.363760320993606, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 4, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + } + }, + { + "phase": "score", + "receipt": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test", + "score" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.299077, + "wall_seconds": 1.1043277920107357, + "peak_tree_rss_bytes": 111403008, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.21797074796631932, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 4, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + } + } + ], + "charged_worker_cpu_seconds": 2.865842, + "charged_gpu_seconds": 0.0, + "total_worker_cpu_cap": 1000, + "total_gpu_cap": 60, + "dataset_version": "synthetic_budget_only", + "paper_numbers_diagnostic_only": true + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-prep-result.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-prep-result.json new file mode 100644 index 000000000..845a5d48a --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-prep-result.json @@ -0,0 +1,15 @@ +{ + "status": "PASS", + "case": "test", + "geometry": { + "tet_nodes": 128, + "outside_tet_nodes": 0 + }, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test/result.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test/result.json new file mode 100644 index 000000000..845a5d48a --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test/result.json @@ -0,0 +1,15 @@ +{ + "status": "PASS", + "case": "test", + "geometry": { + "tet_nodes": 128, + "outside_tet_nodes": 0 + }, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/training-contract.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/training-contract.json new file mode 100644 index 000000000..4f5f55434 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/training-contract.json @@ -0,0 +1,23 @@ +{ + "code_sha256": { + "tools/run_gre24_models.py": "2a5393a5009a2eb1bdd470370a9183b80aa7029d6b69a545831706a0139eb774", + "tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688", + "tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0", + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4", + "tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa" + }, + "total_worker_cpu_seconds": 1000, + "postfit_cpu_reserve_seconds": 998, + "total_gpu_seconds": 60, + "initial_gpu_seconds": 0, + "dataset_version": "synthetic_budget_only", + "protocol": "synthetic", + "xgb_threads": 1, + "fnn_cpu_threads": 1, + "xgb_per_fit_cpu_seconds": 10, + "fnn_per_fit_cpu_seconds": 10, + "gpu_reserved_byte_cap": 16106127360, + "preparation_freeze_sha256": "b2095d362c98db182025439f05891a766c72fda41bc8a392337a0a99926ad7f0", + "splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation-prep-result.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation-prep-result.json new file mode 100644 index 000000000..2622392b5 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation-prep-result.json @@ -0,0 +1,16 @@ +{ + "status": "PASS", + "case": "validation", + "geometry": { + "tet_nodes": 128, + "outside_tet_nodes": 0 + }, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40", + "target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation/result.json b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation/result.json new file mode 100644 index 000000000..2622392b5 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/budget-chain-test/validation/result.json @@ -0,0 +1,16 @@ +{ + "status": "PASS", + "case": "validation", + "geometry": { + "tet_nodes": 128, + "outside_tet_nodes": 0 + }, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40", + "target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/GBRT-guard.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/GBRT-guard.json new file mode 100644 index 000000000..09c8c43c9 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/GBRT-guard.json @@ -0,0 +1,27 @@ +{ + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-01", + "0", + "GBRT" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 1.5006439999999999, + "wall_seconds": 1.512771409994457, + "peak_tree_rss_bytes": 148819968, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 225280, + "max_poll_gap_seconds": 0.11697613802971318, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/acceptance.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/acceptance.json new file mode 100644 index 000000000..927755810 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/acceptance.json @@ -0,0 +1,33 @@ +{ + "status": "FAILED", + "receipts": { + "GBRT": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-01", + "0", + "GBRT" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 1.5006439999999999, + "wall_seconds": 1.512771409994457, + "peak_tree_rss_bytes": 148819968, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 225280, + "max_poll_gap_seconds": 0.11697613802971318, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + } + }, + "GPU_seconds": 0 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/failure.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/failure.json new file mode 100644 index 000000000..43cc26261 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/failure.json @@ -0,0 +1,8 @@ +{ + "type": "ValueError", + "message": "Out of range float values are not JSON compliant: nan", + "traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_fit.py\", line 113, in fit\n save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,\n File \"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py\", line 23, in save\n tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\\n')\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/__init__.py\", line 238, in dumps\n **kw).encode(obj)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 201, in encode\n chunks = list(chunks)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 431, in _iterencode\n yield from _iterencode_dict(o, _current_indent_level)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 405, in _iterencode_dict\n yield from chunks\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 397, in _iterencode_dict\n yield _floatstr(value)\n File \"/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py\", line 239, in floatstr\n raise ValueError(\nValueError: Out of range float values are not JSON compliant: nan\n", + "CPU_s": 0.490331745, + "wall_s": 0.4909153020125814, + "automatic_retry": false +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/ledger.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/ledger.json new file mode 100644 index 000000000..36aaa5899 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 1.5006439999999999, + "attempts": [ + { + "name": "worker", + "status": "STOPPED_OR_FAILED", + "cpu_seconds": 1.5006439999999999 + } + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/progress.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/progress.json new file mode 100644 index 000000000..9382f8140 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/progress.json @@ -0,0 +1,11 @@ +{ + "stage": "FIT", + "pid": 480171, + "unix": 1794757056.8670084, + "split": 0, + "model": "GBRT", + "CPU_s": 0.03734539199999998, + "wall_s": 0.03740089898929, + "nodes": 256, + "geometries": 2 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/started.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/started.json new file mode 100644 index 000000000..2dad1c3c5 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/started.json @@ -0,0 +1,8 @@ +{ + "pid": 480171, + "unix": 1794757056.82963, + "split": 0, + "model": "GBRT", + "seed": 91, + "training_contract_sha256": "4180d820c98034ab333cf402e03de38f5979af34d3b6078b1c098ea40727e658" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/training-target-access.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/training-target-access.json new file mode 100644 index 000000000..570fd75ec --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/training-target-access.json @@ -0,0 +1,18 @@ +[ + { + "case": "learn", + "role": "train", + "nodes": 128, + "source": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/learn", + "feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc" + }, + { + "case": "validation", + "role": "train", + "nodes": 128, + "source": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/validation", + "feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3" + } +] diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.log b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.log new file mode 100644 index 000000000..d71164ad3 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.log @@ -0,0 +1,20 @@ +Traceback (most recent call last): + File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 133, in + a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model)) + File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 113, in fit + save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name, + File "/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py", line 23, in save + tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') + File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/__init__.py", line 238, in dumps + **kw).encode(obj) + File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 201, in encode + chunks = list(chunks) + File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 431, in _iterencode + yield from _iterencode_dict(o, _current_indent_level) + File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 405, in _iterencode_dict + yield from chunks + File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 397, in _iterencode_dict + yield _floatstr(value) + File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/json/encoder.py", line 239, in floatstr + raise ValueError( +ValueError: Out of range float values are not JSON compliant: nan diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.receipt.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.receipt.json new file mode 100644 index 000000000..09c8c43c9 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.receipt.json @@ -0,0 +1,27 @@ +{ + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-01", + "0", + "GBRT" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 1.5006439999999999, + "wall_seconds": 1.512771409994457, + "peak_tree_rss_bytes": 148819968, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 225280, + "max_poll_gap_seconds": 0.11697613802971318, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.started.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.started.json new file mode 100644 index 000000000..9b2e5417f --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/worker.started.json @@ -0,0 +1,18 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-01", + "0", + "GBRT" + ], + "limits": { + "total_cpu_seconds": 120, + "job_cpu_seconds": 120, + "job_wall_seconds": 120, + "tree_rss_bytes": 4294967296, + "disk_bytes": 268435456, + "max_processes": 1 + }, + "started_unix": 1794757055.9604247 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/xgboost-config.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/xgboost-config.json new file mode 100644 index 000000000..4bd921477 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/fits/split-0/GBRT/xgboost-config.json @@ -0,0 +1 @@ +{"learner":{"generic_param":{"device":"cpu","fail_on_invalid_gpu_id":"0","n_jobs":"1","nthread":"1","random_state":"91","seed":"91","seed_per_iteration":"0","validate_parameters":"1"},"gradient_booster":{"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"100"},"gbtree_train_param":{"process_type":"default","tree_method":"auto","updater":"grow_quantile_histmaker","updater_seq":"grow_quantile_histmaker"},"name":"gbtree","specified_updater":false,"tree_train_param":{"alpha":"0","cache_opt":"1","colsample_bylevel":"1","colsample_bynode":"1","colsample_bytree":"1","eta":"0.100000001","gamma":"0","grow_policy":"depthwise","interaction_constraints":"","lambda":"1","learning_rate":"0.100000001","max_bin":"256","max_cat_threshold":"64","max_cat_to_onehot":"4","max_delta_step":"0","max_depth":"8","max_leaves":"0","min_child_weight":"1","min_split_loss":"0","monotone_constraints":"()","refresh_leaf":"1","reg_alpha":"0","reg_lambda":"1","sampling_method":"uniform","sketch_ratio":"2","sparse_threshold":"0.20000000000000001","subsample":"1"},"updater":[{"hist_train_param":{"debug_synchronize":"0","max_cached_hist_node":"65536"},"name":"grow_quantile_histmaker"}]},"learner_model_param":{"base_score":"3.0167848E-1","boost_from_average":"1","num_class":"0","num_feature":"19","num_target":"1"},"learner_train_param":{"booster":"gbtree","disable_default_eval_metric":"0","multi_strategy":"one_output_per_tree","objective":"reg:squarederror"},"metrics":[{"name":"rmse"}],"objective":{"name":"reg:squarederror","reg_loss_param":{"scale_pos_weight":"1"}}},"version":[2,1,4]} \ No newline at end of file diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/preparation-freeze.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/preparation-freeze.json new file mode 100644 index 000000000..c0af11dbe --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/preparation-freeze.json @@ -0,0 +1,40 @@ +{ + "successful": { + "learn": { + "output": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/learn", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076", + "target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } + }, + "validation": { + "output": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/validation", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40", + "target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } + }, + "test": { + "output": "artifacts/gre24-reproduction-2026-11-15/integration-01/parts/test", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } + } + }, + "failed": [] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_fit.py b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_fit.py new file mode 100644 index 000000000..f2eb262f8 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_fit.py @@ -0,0 +1,133 @@ +"""One fixed split/model fit; reads training-side targets only, no test scores.""" +import argparse +import json +import os +from pathlib import Path +import time +import traceback +import numpy as np +from gre24_prepare_v2 import sha +from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update +from s2_recovery_supervisor import save + + +def contract(root): + c=json.loads((root/'training-contract.json').read_text()) + for p,h in c['code_sha256'].items(): + if sha(p)!=h:raise ValueError('Pinned training source changed: '+p) + if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']: + raise ValueError('Preparation freeze changed') + if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed') + return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text()) + + +def training_arrays(root,freeze,split,role,read_log): + selected=[case for case in split[role] if case in freeze['successful']] + n=sum(freeze['successful'][case]['nodes'] for case in selected) + if not n:raise ValueError('No successful training rows') + x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0 + for case in selected: + require_training_role(case,split,role) + r=freeze['successful'][case];source=Path(r['output']) + for name in ['features.npy','target.npy']: + if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed') + a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False) + b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False) + count=len(b) + if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch') + x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count + read_log.append(dict(case=case,role=role,nodes=count,source=str(source), + feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy'])) + if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input') + return x,y + + +def fit(root,index,model_name): + c,f,splits=contract(root);split=splits[index] + dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True) + if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite') + start=time.monotonic();cpu=time.process_time();read_log=[] + def progress(stage,**kw): + record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name, + CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start) + record.update(kw);save(dest/'progress.json',record) + save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'], + training_contract_sha256=sha(root/'training-contract.json'))) + roles,missing=eligible_roles(split,f['successful']) + try: + if model_name=='GBRT': + progress('LOAD_TRAIN_ONLY') + x,y=training_arrays(root,f,split,'train',read_log) + save(dest/'training-target-access.json',read_log) + model=xgb_model(split['seed'],c['xgb_threads']) + progress('FIT',nodes=len(y),geometries=len(roles['train'])) + model.fit(x,y,verbose=False) + checkpoint=dest/'model.ubj';model.save_model(checkpoint) + (dest/'xgboost-config.json').write_text(model.get_booster().save_config()) + extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=model.get_params()) + elif model_name=='FNN': + import torch + torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed']) + if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable') + device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device) + torch.backends.cuda.matmul.allow_tf32=False + progress('LOAD_FNN_TRAIN_VALIDATION_ONLY') + x,y=training_arrays(root,f,split,'fnn_learning',read_log) + vx,vy=training_arrays(root,f,split,'fnn_validation',read_log) + save(dest/'training-target-access.json',read_log) + x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device) + vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device) + model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001) + best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt' + for epoch in range(1,31): + model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device) + for begin in range(0,len(y),1024): + ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True) + prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2) + if not torch.isfinite(loss):raise ValueError('Nonfinite training loss') + loss.backward();optimizer.step();total+=loss.detach()*len(ids) + model.eval();sse=torch.zeros((),device=device) + with torch.no_grad(): + for begin in range(0,len(vy),65536): + sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2) + validation=float((sse/len(vy)).item());train=float((total/len(y)).item()) + best,stale,improved,stop=early_stop_update(best,stale,validation) + if improved: + torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()}, + 'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint) + best_epoch=epoch + row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch, + stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device), + peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start) + history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row) + if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap') + if stop:break + restored=torch.load(checkpoint,map_location='cpu',weights_only=True) + model.load_state_dict(restored['state_dict']) + extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best, + train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']), + validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False, + input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device), + peak_reserved_bytes=torch.cuda.max_memory_reserved(device)) + else:raise ValueError('Unknown model') + save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name, + checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing, + CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra)) + progress('FIT_COMPLETE_NO_TEST_SCORES');return 0 + except Exception as exc: + peaks={} + if model_name=='FNN': + try: + import torch + if torch.cuda.is_initialized(): + peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(), + peak_reserved_bytes=torch.cuda.max_memory_reserved()) + except Exception as peak_error:peaks={'peak_read_error':str(peak_error)} + save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(), + CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks)) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN']) + a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model)) diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_gpu_guard.py b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_gpu_guard.py new file mode 100644 index 000000000..dd87718e8 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_gpu_guard.py @@ -0,0 +1,82 @@ +"""Single-GPU wall accounting and recorded/device VRAM checks including failure.""" +import hashlib +import json +import math +from pathlib import Path +import subprocess +import threading +import time +import psutil +from gre24_job_guard import supervise,stop_group + + +def gpu_state(uuid=None): + args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits'] + if uuid:args+=['-i',uuid] + out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout + rows=[] + for line in out.splitlines(): + u,mem,util=[v.strip() for v in line.split(',')] + rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util))) + if not rows:raise RuntimeError('No GPU state returned') + return rows + + +def recorded_peaks(root): + peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={} + def walk(value,path,digest): + if isinstance(value,dict): + for k,v in value.items(): + if k in peaks: + if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0: + raise ValueError('Invalid CUDA peak record') + if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest} + else:walk(v,path,digest) + elif isinstance(value,list): + for v in value:walk(v,path,digest) + for path in sorted(Path(root).glob('*.json')): + raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest()) + return peaks,witnesses + + +def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap): + start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0} + maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={} + def scan(): + current,seen=recorded_peaks(fit_root) + for k,v in current.items(): + if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k] + if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']: + monitor['reasons'].append('RECORDED_VRAM_LIMIT') + def watch(): + while not done.is_set(): + try: + scan();state=gpu_state(uuid)[0];monitor['samples']+=1 + monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes']) + if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']: + monitor['reasons'].append('DEVICE_VRAM_LIMIT') + except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc)) + if monitor['reasons']: + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + done.wait(.5) + thread=threading.Thread(target=watch,daemon=True);thread.start() + try: + receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit, + job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30, + disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2) + finally: + done.set();thread.join(timeout=12) + if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP') + try:scan() + except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc)) + if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS') + elapsed=time.monotonic()-start + if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT') + receipt['underlying_worker_status']=receipt['status'] + if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0]) + receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses, + total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap) + return receipt diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_job_guard.py b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_job_guard.py new file mode 100644 index 000000000..a5e451cd0 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_job_guard.py @@ -0,0 +1,160 @@ +"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts. + +Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot +is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged. +""" +import fcntl +import ctypes +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import time + +import psutil +from s2_recovery_monitoring import disk_usage,Monitor + + +def save(path, data): + path = Path(path) + tmp = path.with_suffix(path.suffix + '.tmp') + tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') + tmp.replace(path) + + +def tree_usage(pid): + try: + root = psutil.Process(pid) + processes = [root] + root.children(recursive=True) + except psutil.NoSuchProcess: + return 0., 0, 0 + cpu = 0. + rss = count = 0 + for p in processes: + try: + t = p.cpu_times() + # Linux child times include reaped descendants, not live children. + cpu += t.user + t.system + t.children_user + t.children_system + rss += p.memory_info().rss + count += 1 + except psutil.NoSuchProcess: + pass + return cpu, rss, count + + +def stop_group(pid): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def supervise(command, root, name, limits, *, poll=.02, cuda_device='', numerical_threads=1): + """One locked attempt; no automatic retry/replacement; account failed CPU.""" + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + monitor=Monitor(root/(name+'.monitor-errors.jsonl')) + with (root/'execution.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + started_path = root/(name+'.started.json') + receipt_path = root/(name+'.receipt.json') + if started_path.exists() or receipt_path.exists(): + raise FileExistsError('Attempt already exists; no silent retry') + if any(not math.isfinite(v) or v <= 0 for v in limits.values()): + raise ValueError('Every limit must be finite and positive') + ledger_path = root/'ledger.json' + ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]} + if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']: + raise RuntimeError('Total CPU exhausted before spawn') + if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']: + raise RuntimeError('Disk exhausted before spawn') + save(started_path, {'command':command,'limits':limits,'started_unix':time.time()}) + start = last = time.monotonic() + cpu_peak = rss_peak = max_processes = max_gap = 0 + cpu=rss=count=used_disk=0 + reason = None + exit_status = usage = None + env = dict(os.environ, OMP_NUM_THREADS=str(numerical_threads), OPENBLAS_NUM_THREADS=str(numerical_threads), + MKL_NUM_THREADS=str(numerical_threads), NUMEXPR_NUM_THREADS=str(numerical_threads), + CUDA_VISIBLE_DEVICES=cuda_device) + with (root/(name+'.log')).open('x') as log: + libc = ctypes.CDLL(None, use_errno=True) + previous_subreaper = ctypes.c_int() + if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping') + proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, + env=env, start_new_session=True) + try: + while True: + now = time.monotonic() + max_gap = max(max_gap, now-last) + last = now + (cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count)) + cpu_peak = max(cpu_peak,cpu) + rss_peak = max(rss_peak,rss) + max_processes = max(max_processes,count) + used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk) + if now-start >= limits['job_wall_seconds']: + reason = 'wall_limit' + elif rss > limits['tree_rss_bytes']: + reason = 'tree_rss_limit' + elif count > limits['max_processes']: + reason = 'process_count_limit' + elif used_disk >= limits['disk_bytes']: + reason = 'disk_limit' + elif cpu_peak >= limits['job_cpu_seconds']: + reason = 'job_cpu_limit' + elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']: + reason = 'total_cpu_limit' + if reason: + stop_group(proc.pid) + pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG) + if pid: + exit_status, usage = status, child_usage + break + time.sleep(poll) + finally: + # No detached work may outlive a supervised attempt. + stop_group(proc.pid) + if exit_status is None: + _, exit_status, usage = os.wait4(proc.pid, 0) + proc.returncode = os.waitstatus_to_exitcode(exit_status) + orphan_cpu = 0. + orphan_peak = 0 + # Killed descendants are adopted here rather than losing their + # final CPU usage when the root worker cannot reap them itself. + while True: + try: + _, _, orphan_usage = os.wait4(-proc.pid, 0) + except ChildProcessError: + break + orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime + orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024) + if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state') + elapsed = time.monotonic()-start + charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) + rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak) + used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll) + if reason is None: + if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit' + elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit' + elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit' + elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit' + elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit' + receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED', + command=command,returncode=proc.returncode,stop_reason=reason, + cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak, + peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk, + max_poll_gap_seconds=max_gap,poll_target_seconds=poll, + reaped_orphan_cpu_seconds=orphan_cpu, + supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure', + gpu_visible=bool(cuda_device),cuda_device=cuda_device, + numerical_threads=numerical_threads,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error) + ledger['cpu_seconds_charged'] += charged + ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged}) + monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll) + monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll) + return receipt diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_models.py b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_models.py new file mode 100644 index 000000000..723c9a487 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_models.py @@ -0,0 +1,42 @@ +"""Fixed Gre24 model choices; no test-driven selection or node subsampling.""" +import numpy as np + + +def xgb_model(seed,threads=8): + from xgboost import XGBRegressor + return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1, + objective='reg:squarederror',random_state=seed,n_jobs=threads) + + +def fnn_model(): + import torch + layers=[];width=19 + for _ in range(4): + layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100 + layers.append(torch.nn.Linear(100,1)) + return torch.nn.Sequential(*layers) + + +def eligible_roles(split,successful): + successful=set(successful) + roles={role:[c for c in split[role] if c in successful] + for role in ['train','test','fnn_learning','fnn_validation']} + if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap') + if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap') + if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']): + raise ValueError('FNN learning/validation do not partition train') + missing={role:[c for c in split[role] if c not in successful] for role in roles} + return roles,missing + + +def require_training_role(case,split,role): + if role not in ['train','fnn_learning','fnn_validation']: + raise PermissionError('Only frozen training-side roles may read targets') + if case in split['test'] or case not in split[role]: + raise PermissionError('Case is not in the requested training role') + + +def early_stop_update(best,stale,score,patience=5): + if not np.isfinite(score):raise ValueError('Nonfinite validation MSE') + improved=score=patience) diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_score.py b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_score.py new file mode 100644 index 000000000..d2f30310e --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/source-snapshot/gre24_score.py @@ -0,0 +1,102 @@ +"""Predictions frozen before test-target scoring, all successful tet nodes.""" +import argparse +import json +from pathlib import Path +import time +import numpy as np +from gre24_prepare_v2 import sha +from gre24_features_v2 import per_geometry_metrics +from gre24_models import fnn_model +from gre24_fit import contract +from s2_recovery_supervisor import save + + +def predict(root): + c,prep,splits=contract(root) + models=json.loads((root/'models-freeze.json').read_text()) + dest=root/'predictions';dest.mkdir(exist_ok=False) + start=time.process_time();rows=[] + for split in splits: + available={'naive':None} + for name in ['GBRT','FNN']: + r=models['fits'][f"{split['index']}/{name}"] + if r['status']!='PASS':continue + if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze') + if name=='GBRT': + from xgboost import XGBRegressor + m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint']) + else: + import torch + torch.set_num_threads(4);m=fnn_model() + m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval() + available[name]=m + for case in split['test']: + if case not in prep['successful']: + rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={})) + continue + r=prep['successful'][case];source=Path(r['output']);outputs={} + for name in ['features.npy','Vf.npy','raw_ids.npy']: + if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed') + x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False) + # No seconds/target file is opened in this phase. + for name,m in available.items(): + if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False) + elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32)) + else: + chunks=[] + with torch.no_grad(): + for a in range(0,len(x),65536): + chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy()) + pred=np.concatenate(chunks) + if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction') + folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True) + path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds']) + outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True) + rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs, + raw_ids_sha256=r['output_sha256']['raw_ids.npy'])) + save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start)) + save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows, + worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds')) + + +def score(root): + c,prep,splits=contract(root);frozen=root/'predictions-freeze.json' + if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring') + freeze=json.loads(frozen.read_text());rows=[];start=time.process_time() + for r in freeze['rows']: + if r['status']!='PREDICTED': + rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue + p=prep['successful'][r['case']];source=Path(p['output']) + if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed') + y=np.load(source/'seconds.npy',allow_pickle=False);scores={} + for name,value in r['predictions'].items(): + if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed') + pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred) + rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores)) + summary={} + for name in ['naive','GBRT','FNN']: + summary[name]={} + for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']: + per=[] + for split in splits: + a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores'] + and r['scores'][name][metric] is not None] + per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a), + missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None, + median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None)) + # Do not average only whichever model splits happened to finish. + avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']} + summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg, + full_requested_coverage=all(r['missing']==0 for r in per)) + save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen))) + save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'], + status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start, + scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True, + limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements', + 't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices', + 'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit'])) + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']) + a=p.parse_args();(predict if a.phase=='predict' else score)(a.root) diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/splits.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/splits.json new file mode 100644 index 000000000..1a7c33f89 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/splits.json @@ -0,0 +1,87 @@ +[ + { + "index": 0, + "seed": 91, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 1, + "seed": 92, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 2, + "seed": 93, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 3, + "seed": 94, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 4, + "seed": 95, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + } +] diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-01/training-contract.json b/artifacts/gre24-reproduction-2026-11-15/integration-01/training-contract.json new file mode 100644 index 000000000..d41368960 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-01/training-contract.json @@ -0,0 +1,16 @@ +{ + "code_sha256": { + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/gre24_fit.py": "a3fc9364820ef23f237475cf2594d1089d86b87763aba4afae52a1b32d7d9ab6", + "tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0", + "tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4", + "tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa" + }, + "preparation_freeze_sha256": "508a8997b875e9a978db40c5eed3a6407b3e512bb7916398e5bb688dd935262e", + "splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809", + "dataset_version": "synthetic_only", + "protocol": "synthetic smoke, not scientific data", + "xgb_threads": 1, + "fnn_cpu_threads": 2, + "gpu_reserved_byte_cap": 16106127360 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/FNN-guard.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/FNN-guard.json new file mode 100644 index 000000000..8970d1b7a --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/FNN-guard.json @@ -0,0 +1,40 @@ +{ + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-02", + "0", + "FNN" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 4.8226309999999994, + "wall_seconds": 4.549783353984822, + "peak_tree_rss_bytes": 495280128, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 0.36855653702514246, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "numerical_threads": 2, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null, + "underlying_worker_status": "STOPPED_OR_FAILED", + "device_monitor": { + "reasons": [], + "peak_device_bytes": 9437184, + "samples": 8 + }, + "recorded_peaks": { + "peak_allocated_bytes": 0, + "peak_reserved_bytes": 0 + }, + "peak_witnesses": {}, + "total_gpu_wall_seconds": 4.554046397970524, + "peak_cap_bytes": 16106127360 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/GBRT-guard.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/GBRT-guard.json new file mode 100644 index 000000000..a5d978fef --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/GBRT-guard.json @@ -0,0 +1,27 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-02", + "0", + "GBRT" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.5470069999999998, + "wall_seconds": 1.6621895720018074, + "peak_tree_rss_bytes": 148852736, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 221184, + "max_poll_gap_seconds": 0.12116481299744919, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/acceptance.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/acceptance.json new file mode 100644 index 000000000..d680e4dae --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/acceptance.json @@ -0,0 +1,73 @@ +{ + "status": "FAILED", + "receipts": { + "GBRT": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-02", + "0", + "GBRT" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.5470069999999998, + "wall_seconds": 1.6621895720018074, + "peak_tree_rss_bytes": 148852736, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 221184, + "max_poll_gap_seconds": 0.12116481299744919, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "FNN": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-02", + "0", + "FNN" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 4.8226309999999994, + "wall_seconds": 4.549783353984822, + "peak_tree_rss_bytes": 495280128, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 0.36855653702514246, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "numerical_threads": 2, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null, + "underlying_worker_status": "STOPPED_OR_FAILED", + "device_monitor": { + "reasons": [], + "peak_device_bytes": 9437184, + "samples": 8 + }, + "recorded_peaks": { + "peak_allocated_bytes": 0, + "peak_reserved_bytes": 0 + }, + "peak_witnesses": {}, + "total_gpu_wall_seconds": 4.554046397970524, + "peak_cap_bytes": 16106127360 + } + }, + "GPU_seconds": 4.601264455995988 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/failure.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/failure.json new file mode 100644 index 000000000..37e2afa86 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/failure.json @@ -0,0 +1,8 @@ +{ + "type": "RuntimeError", + "message": "Invalid device argument ", + "traceback": "Traceback (most recent call last):\n File \"/home/frank14f/optane/MeltMind/tools/gre24_fit.py\", line 77, in fit\n device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)\n File \"/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/lib/python3.10/site-packages/torch/cuda/memory.py\", line 344, in reset_peak_memory_stats\n return torch._C._cuda_resetPeakMemoryStats(device)\nRuntimeError: Invalid device argument \n", + "CPU_s": 2.9871277210000002, + "wall_s": 2.784001157968305, + "automatic_retry": false +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/ledger.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/ledger.json new file mode 100644 index 000000000..37ddac5c2 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 4.8226309999999994, + "attempts": [ + { + "name": "worker", + "status": "STOPPED_OR_FAILED", + "cpu_seconds": 4.8226309999999994 + } + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/started.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/started.json new file mode 100644 index 000000000..1d5b1f351 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/started.json @@ -0,0 +1,8 @@ +{ + "pid": 480269, + "unix": 1794757154.6071362, + "split": 0, + "model": "FNN", + "seed": 91, + "training_contract_sha256": "760d10e1df8863e5d4c34701a0a18678a6a2f9ede56b2cc83e95b6bbfab5848f" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.log b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.log new file mode 100644 index 000000000..0579072d3 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.log @@ -0,0 +1,8 @@ +Traceback (most recent call last): + File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 138, in + a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model)) + File "/home/frank14f/optane/MeltMind/tools/gre24_fit.py", line 77, in fit + device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device) + File "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/lib/python3.10/site-packages/torch/cuda/memory.py", line 344, in reset_peak_memory_stats + return torch._C._cuda_resetPeakMemoryStats(device) +RuntimeError: Invalid device argument diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.receipt.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.receipt.json new file mode 100644 index 000000000..6b7d1c1a1 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.receipt.json @@ -0,0 +1,27 @@ +{ + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-02", + "0", + "FNN" + ], + "returncode": 1, + "stop_reason": null, + "cpu_seconds": 4.8226309999999994, + "wall_seconds": 4.549783353984822, + "peak_tree_rss_bytes": 495280128, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 16384, + "max_poll_gap_seconds": 0.36855653702514246, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "numerical_threads": 2, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.started.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.started.json new file mode 100644 index 000000000..abaec4b4a --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/FNN/worker.started.json @@ -0,0 +1,18 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-02", + "0", + "FNN" + ], + "limits": { + "total_cpu_seconds": 120.0, + "job_cpu_seconds": 120.0, + "job_wall_seconds": 57.0, + "tree_rss_bytes": 25769803776, + "disk_bytes": 3221225472, + "max_processes": 1 + }, + "started_unix": 1794757153.3916488 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/ledger.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/ledger.json new file mode 100644 index 000000000..c943f56dd --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 1.5470069999999998, + "attempts": [ + { + "name": "worker", + "status": "PASS", + "cpu_seconds": 1.5470069999999998 + } + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/progress.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/progress.json new file mode 100644 index 000000000..aa6e37808 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/progress.json @@ -0,0 +1,9 @@ +{ + "stage": "FIT_COMPLETE_NO_TEST_SCORES", + "pid": 480263, + "unix": 1794757153.0647538, + "split": 0, + "model": "GBRT", + "CPU_s": 0.495211623, + "wall_s": 0.4952992910402827 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/result.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/result.json new file mode 100644 index 000000000..70de4ac53 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/result.json @@ -0,0 +1,58 @@ +{ + "status": "FIT_COMPLETE_NO_TEST_SCORES", + "split": 0, + "model": "GBRT", + "checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/model.ubj", + "checkpoint_sha256": "0630f79ad1b6783fa31ed5b6d94e345a896212187676f65168f4e819889ec74b", + "missing_preparation": { + "train": [], + "test": [], + "fnn_learning": [], + "fnn_validation": [] + }, + "CPU_s": 0.49491478599999994, + "wall_s": 0.4950045870500617, + "train_nodes": 256, + "train_geometries": 2, + "parameters": { + "objective": "reg:squarederror", + "base_score": null, + "booster": null, + "callbacks": null, + "colsample_bylevel": null, + "colsample_bynode": null, + "colsample_bytree": null, + "device": null, + "early_stopping_rounds": null, + "enable_categorical": false, + "eval_metric": null, + "feature_types": null, + "gamma": null, + "grow_policy": null, + "importance_type": null, + "interaction_constraints": null, + "learning_rate": 0.1, + "max_bin": null, + "max_cat_threshold": null, + "max_cat_to_onehot": null, + "max_delta_step": null, + "max_depth": 8, + "max_leaves": null, + "min_child_weight": null, + "missing": "NaN (library default missing marker)", + "monotone_constraints": null, + "multi_strategy": null, + "n_estimators": 100, + "n_jobs": 1, + "num_parallel_tree": null, + "random_state": 91, + "reg_alpha": null, + "reg_lambda": null, + "sampling_method": null, + "scale_pos_weight": null, + "subsample": null, + "tree_method": null, + "validate_parameters": null, + "verbosity": null + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/started.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/started.json new file mode 100644 index 000000000..729751515 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/started.json @@ -0,0 +1,8 @@ +{ + "pid": 480263, + "unix": 1794757152.5694737, + "split": 0, + "model": "GBRT", + "seed": 91, + "training_contract_sha256": "760d10e1df8863e5d4c34701a0a18678a6a2f9ede56b2cc83e95b6bbfab5848f" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/training-target-access.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/training-target-access.json new file mode 100644 index 000000000..b20f14707 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/training-target-access.json @@ -0,0 +1,18 @@ +[ + { + "case": "learn", + "role": "train", + "nodes": 128, + "source": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/learn", + "feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc" + }, + { + "case": "validation", + "role": "train", + "nodes": 128, + "source": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/validation", + "feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3" + } +] diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.log b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.log new file mode 100644 index 000000000..e69de29bb diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.receipt.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.receipt.json new file mode 100644 index 000000000..a5d978fef --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.receipt.json @@ -0,0 +1,27 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-02", + "0", + "GBRT" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.5470069999999998, + "wall_seconds": 1.6621895720018074, + "peak_tree_rss_bytes": 148852736, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 221184, + "max_poll_gap_seconds": 0.12116481299744919, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.started.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.started.json new file mode 100644 index 000000000..c773bb6db --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/worker.started.json @@ -0,0 +1,18 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-02", + "0", + "GBRT" + ], + "limits": { + "total_cpu_seconds": 120, + "job_cpu_seconds": 120, + "job_wall_seconds": 120, + "tree_rss_bytes": 4294967296, + "disk_bytes": 268435456, + "max_processes": 1 + }, + "started_unix": 1794757151.6780915 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/xgboost-config.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/xgboost-config.json new file mode 100644 index 000000000..4bd921477 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/fits/split-0/GBRT/xgboost-config.json @@ -0,0 +1 @@ +{"learner":{"generic_param":{"device":"cpu","fail_on_invalid_gpu_id":"0","n_jobs":"1","nthread":"1","random_state":"91","seed":"91","seed_per_iteration":"0","validate_parameters":"1"},"gradient_booster":{"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"100"},"gbtree_train_param":{"process_type":"default","tree_method":"auto","updater":"grow_quantile_histmaker","updater_seq":"grow_quantile_histmaker"},"name":"gbtree","specified_updater":false,"tree_train_param":{"alpha":"0","cache_opt":"1","colsample_bylevel":"1","colsample_bynode":"1","colsample_bytree":"1","eta":"0.100000001","gamma":"0","grow_policy":"depthwise","interaction_constraints":"","lambda":"1","learning_rate":"0.100000001","max_bin":"256","max_cat_threshold":"64","max_cat_to_onehot":"4","max_delta_step":"0","max_depth":"8","max_leaves":"0","min_child_weight":"1","min_split_loss":"0","monotone_constraints":"()","refresh_leaf":"1","reg_alpha":"0","reg_lambda":"1","sampling_method":"uniform","sketch_ratio":"2","sparse_threshold":"0.20000000000000001","subsample":"1"},"updater":[{"hist_train_param":{"debug_synchronize":"0","max_cached_hist_node":"65536"},"name":"grow_quantile_histmaker"}]},"learner_model_param":{"base_score":"3.0167848E-1","boost_from_average":"1","num_class":"0","num_feature":"19","num_target":"1"},"learner_train_param":{"booster":"gbtree","disable_default_eval_metric":"0","multi_strategy":"one_output_per_tree","objective":"reg:squarederror"},"metrics":[{"name":"rmse"}],"objective":{"name":"reg:squarederror","reg_loss_param":{"scale_pos_weight":"1"}}},"version":[2,1,4]} \ No newline at end of file diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/preparation-freeze.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/preparation-freeze.json new file mode 100644 index 000000000..c75583754 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/preparation-freeze.json @@ -0,0 +1,40 @@ +{ + "successful": { + "learn": { + "output": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/learn", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076", + "target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } + }, + "validation": { + "output": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/validation", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40", + "target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } + }, + "test": { + "output": "artifacts/gre24-reproduction-2026-11-15/integration-02/parts/test", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } + } + }, + "failed": [] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_fit.py b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_fit.py new file mode 100644 index 000000000..eda1bcde9 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_fit.py @@ -0,0 +1,138 @@ +"""One fixed split/model fit; reads training-side targets only, no test scores.""" +import argparse +import json +import os +from pathlib import Path +import time +import traceback +import numpy as np +from gre24_prepare_v2 import sha +from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update +from s2_recovery_supervisor import save + + +def contract(root): + c=json.loads((root/'training-contract.json').read_text()) + for p,h in c['code_sha256'].items(): + if sha(p)!=h:raise ValueError('Pinned training source changed: '+p) + if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']: + raise ValueError('Preparation freeze changed') + if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed') + return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text()) + + +def training_arrays(root,freeze,split,role,read_log): + selected=[case for case in split[role] if case in freeze['successful']] + n=sum(freeze['successful'][case]['nodes'] for case in selected) + if not n:raise ValueError('No successful training rows') + x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0 + for case in selected: + require_training_role(case,split,role) + r=freeze['successful'][case];source=Path(r['output']) + for name in ['features.npy','target.npy']: + if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed') + a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False) + b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False) + count=len(b) + if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch') + x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count + read_log.append(dict(case=case,role=role,nodes=count,source=str(source), + feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy'])) + if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input') + return x,y + + +def fit(root,index,model_name): + c,f,splits=contract(root);split=splits[index] + dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True) + if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite') + start=time.monotonic();cpu=time.process_time();read_log=[] + def progress(stage,**kw): + record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name, + CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start) + record.update(kw);save(dest/'progress.json',record) + save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'], + training_contract_sha256=sha(root/'training-contract.json'))) + roles,missing=eligible_roles(split,f['successful']) + try: + if model_name=='GBRT': + progress('LOAD_TRAIN_ONLY') + x,y=training_arrays(root,f,split,'train',read_log) + save(dest/'training-target-access.json',read_log) + model=xgb_model(split['seed'],c['xgb_threads']) + progress('FIT',nodes=len(y),geometries=len(roles['train'])) + model.fit(x,y,verbose=False) + checkpoint=dest/'model.ubj';model.save_model(checkpoint) + (dest/'xgboost-config.json').write_text(model.get_booster().save_config()) + params=model.get_params() + # Preserve XGBoost's default missing=NaN as an explicit textual parameter; + # this is not a nonfinite observation or a change to the fitted estimator. + params={k:('NaN (library default missing marker)' if isinstance(v,float) and np.isnan(v) else v) + for k,v in params.items()} + extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=params) + elif model_name=='FNN': + import torch + torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed']) + if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable') + device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device) + torch.backends.cuda.matmul.allow_tf32=False + progress('LOAD_FNN_TRAIN_VALIDATION_ONLY') + x,y=training_arrays(root,f,split,'fnn_learning',read_log) + vx,vy=training_arrays(root,f,split,'fnn_validation',read_log) + save(dest/'training-target-access.json',read_log) + x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device) + vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device) + model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001) + best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt' + for epoch in range(1,31): + model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device) + for begin in range(0,len(y),1024): + ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True) + prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2) + if not torch.isfinite(loss):raise ValueError('Nonfinite training loss') + loss.backward();optimizer.step();total+=loss.detach()*len(ids) + model.eval();sse=torch.zeros((),device=device) + with torch.no_grad(): + for begin in range(0,len(vy),65536): + sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2) + validation=float((sse/len(vy)).item());train=float((total/len(y)).item()) + best,stale,improved,stop=early_stop_update(best,stale,validation) + if improved: + torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()}, + 'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint) + best_epoch=epoch + row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch, + stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device), + peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start) + history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row) + if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap') + if stop:break + restored=torch.load(checkpoint,map_location='cpu',weights_only=True) + model.load_state_dict(restored['state_dict']) + extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best, + train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']), + validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False, + input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device), + peak_reserved_bytes=torch.cuda.max_memory_reserved(device)) + else:raise ValueError('Unknown model') + save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name, + checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing, + CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra)) + progress('FIT_COMPLETE_NO_TEST_SCORES');return 0 + except Exception as exc: + peaks={} + if model_name=='FNN': + try: + import torch + if torch.cuda.is_initialized(): + peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(), + peak_reserved_bytes=torch.cuda.max_memory_reserved()) + except Exception as peak_error:peaks={'peak_read_error':str(peak_error)} + save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(), + CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks)) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN']) + a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model)) diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_gpu_guard.py b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_gpu_guard.py new file mode 100644 index 000000000..dd87718e8 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_gpu_guard.py @@ -0,0 +1,82 @@ +"""Single-GPU wall accounting and recorded/device VRAM checks including failure.""" +import hashlib +import json +import math +from pathlib import Path +import subprocess +import threading +import time +import psutil +from gre24_job_guard import supervise,stop_group + + +def gpu_state(uuid=None): + args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits'] + if uuid:args+=['-i',uuid] + out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout + rows=[] + for line in out.splitlines(): + u,mem,util=[v.strip() for v in line.split(',')] + rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util))) + if not rows:raise RuntimeError('No GPU state returned') + return rows + + +def recorded_peaks(root): + peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={} + def walk(value,path,digest): + if isinstance(value,dict): + for k,v in value.items(): + if k in peaks: + if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0: + raise ValueError('Invalid CUDA peak record') + if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest} + else:walk(v,path,digest) + elif isinstance(value,list): + for v in value:walk(v,path,digest) + for path in sorted(Path(root).glob('*.json')): + raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest()) + return peaks,witnesses + + +def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap): + start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0} + maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={} + def scan(): + current,seen=recorded_peaks(fit_root) + for k,v in current.items(): + if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k] + if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']: + monitor['reasons'].append('RECORDED_VRAM_LIMIT') + def watch(): + while not done.is_set(): + try: + scan();state=gpu_state(uuid)[0];monitor['samples']+=1 + monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes']) + if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']: + monitor['reasons'].append('DEVICE_VRAM_LIMIT') + except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc)) + if monitor['reasons']: + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + done.wait(.5) + thread=threading.Thread(target=watch,daemon=True);thread.start() + try: + receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit, + job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30, + disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2) + finally: + done.set();thread.join(timeout=12) + if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP') + try:scan() + except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc)) + if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS') + elapsed=time.monotonic()-start + if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT') + receipt['underlying_worker_status']=receipt['status'] + if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0]) + receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses, + total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap) + return receipt diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_job_guard.py b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_job_guard.py new file mode 100644 index 000000000..a5e451cd0 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_job_guard.py @@ -0,0 +1,160 @@ +"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts. + +Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot +is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged. +""" +import fcntl +import ctypes +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import time + +import psutil +from s2_recovery_monitoring import disk_usage,Monitor + + +def save(path, data): + path = Path(path) + tmp = path.with_suffix(path.suffix + '.tmp') + tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') + tmp.replace(path) + + +def tree_usage(pid): + try: + root = psutil.Process(pid) + processes = [root] + root.children(recursive=True) + except psutil.NoSuchProcess: + return 0., 0, 0 + cpu = 0. + rss = count = 0 + for p in processes: + try: + t = p.cpu_times() + # Linux child times include reaped descendants, not live children. + cpu += t.user + t.system + t.children_user + t.children_system + rss += p.memory_info().rss + count += 1 + except psutil.NoSuchProcess: + pass + return cpu, rss, count + + +def stop_group(pid): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def supervise(command, root, name, limits, *, poll=.02, cuda_device='', numerical_threads=1): + """One locked attempt; no automatic retry/replacement; account failed CPU.""" + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + monitor=Monitor(root/(name+'.monitor-errors.jsonl')) + with (root/'execution.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + started_path = root/(name+'.started.json') + receipt_path = root/(name+'.receipt.json') + if started_path.exists() or receipt_path.exists(): + raise FileExistsError('Attempt already exists; no silent retry') + if any(not math.isfinite(v) or v <= 0 for v in limits.values()): + raise ValueError('Every limit must be finite and positive') + ledger_path = root/'ledger.json' + ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]} + if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']: + raise RuntimeError('Total CPU exhausted before spawn') + if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']: + raise RuntimeError('Disk exhausted before spawn') + save(started_path, {'command':command,'limits':limits,'started_unix':time.time()}) + start = last = time.monotonic() + cpu_peak = rss_peak = max_processes = max_gap = 0 + cpu=rss=count=used_disk=0 + reason = None + exit_status = usage = None + env = dict(os.environ, OMP_NUM_THREADS=str(numerical_threads), OPENBLAS_NUM_THREADS=str(numerical_threads), + MKL_NUM_THREADS=str(numerical_threads), NUMEXPR_NUM_THREADS=str(numerical_threads), + CUDA_VISIBLE_DEVICES=cuda_device) + with (root/(name+'.log')).open('x') as log: + libc = ctypes.CDLL(None, use_errno=True) + previous_subreaper = ctypes.c_int() + if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping') + proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, + env=env, start_new_session=True) + try: + while True: + now = time.monotonic() + max_gap = max(max_gap, now-last) + last = now + (cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count)) + cpu_peak = max(cpu_peak,cpu) + rss_peak = max(rss_peak,rss) + max_processes = max(max_processes,count) + used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk) + if now-start >= limits['job_wall_seconds']: + reason = 'wall_limit' + elif rss > limits['tree_rss_bytes']: + reason = 'tree_rss_limit' + elif count > limits['max_processes']: + reason = 'process_count_limit' + elif used_disk >= limits['disk_bytes']: + reason = 'disk_limit' + elif cpu_peak >= limits['job_cpu_seconds']: + reason = 'job_cpu_limit' + elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']: + reason = 'total_cpu_limit' + if reason: + stop_group(proc.pid) + pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG) + if pid: + exit_status, usage = status, child_usage + break + time.sleep(poll) + finally: + # No detached work may outlive a supervised attempt. + stop_group(proc.pid) + if exit_status is None: + _, exit_status, usage = os.wait4(proc.pid, 0) + proc.returncode = os.waitstatus_to_exitcode(exit_status) + orphan_cpu = 0. + orphan_peak = 0 + # Killed descendants are adopted here rather than losing their + # final CPU usage when the root worker cannot reap them itself. + while True: + try: + _, _, orphan_usage = os.wait4(-proc.pid, 0) + except ChildProcessError: + break + orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime + orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024) + if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state') + elapsed = time.monotonic()-start + charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) + rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak) + used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll) + if reason is None: + if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit' + elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit' + elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit' + elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit' + elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit' + receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED', + command=command,returncode=proc.returncode,stop_reason=reason, + cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak, + peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk, + max_poll_gap_seconds=max_gap,poll_target_seconds=poll, + reaped_orphan_cpu_seconds=orphan_cpu, + supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure', + gpu_visible=bool(cuda_device),cuda_device=cuda_device, + numerical_threads=numerical_threads,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error) + ledger['cpu_seconds_charged'] += charged + ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged}) + monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll) + monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll) + return receipt diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_models.py b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_models.py new file mode 100644 index 000000000..723c9a487 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_models.py @@ -0,0 +1,42 @@ +"""Fixed Gre24 model choices; no test-driven selection or node subsampling.""" +import numpy as np + + +def xgb_model(seed,threads=8): + from xgboost import XGBRegressor + return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1, + objective='reg:squarederror',random_state=seed,n_jobs=threads) + + +def fnn_model(): + import torch + layers=[];width=19 + for _ in range(4): + layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100 + layers.append(torch.nn.Linear(100,1)) + return torch.nn.Sequential(*layers) + + +def eligible_roles(split,successful): + successful=set(successful) + roles={role:[c for c in split[role] if c in successful] + for role in ['train','test','fnn_learning','fnn_validation']} + if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap') + if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap') + if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']): + raise ValueError('FNN learning/validation do not partition train') + missing={role:[c for c in split[role] if c not in successful] for role in roles} + return roles,missing + + +def require_training_role(case,split,role): + if role not in ['train','fnn_learning','fnn_validation']: + raise PermissionError('Only frozen training-side roles may read targets') + if case in split['test'] or case not in split[role]: + raise PermissionError('Case is not in the requested training role') + + +def early_stop_update(best,stale,score,patience=5): + if not np.isfinite(score):raise ValueError('Nonfinite validation MSE') + improved=score=patience) diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_score.py b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_score.py new file mode 100644 index 000000000..d2f30310e --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/source-snapshot/gre24_score.py @@ -0,0 +1,102 @@ +"""Predictions frozen before test-target scoring, all successful tet nodes.""" +import argparse +import json +from pathlib import Path +import time +import numpy as np +from gre24_prepare_v2 import sha +from gre24_features_v2 import per_geometry_metrics +from gre24_models import fnn_model +from gre24_fit import contract +from s2_recovery_supervisor import save + + +def predict(root): + c,prep,splits=contract(root) + models=json.loads((root/'models-freeze.json').read_text()) + dest=root/'predictions';dest.mkdir(exist_ok=False) + start=time.process_time();rows=[] + for split in splits: + available={'naive':None} + for name in ['GBRT','FNN']: + r=models['fits'][f"{split['index']}/{name}"] + if r['status']!='PASS':continue + if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze') + if name=='GBRT': + from xgboost import XGBRegressor + m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint']) + else: + import torch + torch.set_num_threads(4);m=fnn_model() + m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval() + available[name]=m + for case in split['test']: + if case not in prep['successful']: + rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={})) + continue + r=prep['successful'][case];source=Path(r['output']);outputs={} + for name in ['features.npy','Vf.npy','raw_ids.npy']: + if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed') + x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False) + # No seconds/target file is opened in this phase. + for name,m in available.items(): + if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False) + elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32)) + else: + chunks=[] + with torch.no_grad(): + for a in range(0,len(x),65536): + chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy()) + pred=np.concatenate(chunks) + if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction') + folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True) + path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds']) + outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True) + rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs, + raw_ids_sha256=r['output_sha256']['raw_ids.npy'])) + save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start)) + save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows, + worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds')) + + +def score(root): + c,prep,splits=contract(root);frozen=root/'predictions-freeze.json' + if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring') + freeze=json.loads(frozen.read_text());rows=[];start=time.process_time() + for r in freeze['rows']: + if r['status']!='PREDICTED': + rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue + p=prep['successful'][r['case']];source=Path(p['output']) + if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed') + y=np.load(source/'seconds.npy',allow_pickle=False);scores={} + for name,value in r['predictions'].items(): + if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed') + pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred) + rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores)) + summary={} + for name in ['naive','GBRT','FNN']: + summary[name]={} + for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']: + per=[] + for split in splits: + a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores'] + and r['scores'][name][metric] is not None] + per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a), + missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None, + median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None)) + # Do not average only whichever model splits happened to finish. + avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']} + summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg, + full_requested_coverage=all(r['missing']==0 for r in per)) + save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen))) + save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'], + status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start, + scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True, + limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements', + 't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices', + 'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit'])) + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']) + a=p.parse_args();(predict if a.phase=='predict' else score)(a.root) diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/splits.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/splits.json new file mode 100644 index 000000000..1a7c33f89 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/splits.json @@ -0,0 +1,87 @@ +[ + { + "index": 0, + "seed": 91, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 1, + "seed": 92, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 2, + "seed": 93, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 3, + "seed": 94, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 4, + "seed": 95, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + } +] diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-02/training-contract.json b/artifacts/gre24-reproduction-2026-11-15/integration-02/training-contract.json new file mode 100644 index 000000000..81fed29f0 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-02/training-contract.json @@ -0,0 +1,16 @@ +{ + "code_sha256": { + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/gre24_fit.py": "29ac7c26d4164593afddb679481702e04cfb3b05e1f83ab4c56edea7162d37a0", + "tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0", + "tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4", + "tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa" + }, + "preparation_freeze_sha256": "41f1ab982ce91eae8c71a5080f53aa8909e6eb1089d376e56b91a40c573aaa13", + "splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809", + "dataset_version": "synthetic_only", + "protocol": "synthetic smoke, not scientific data", + "xgb_threads": 1, + "fnn_cpu_threads": 2, + "gpu_reserved_byte_cap": 16106127360 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/FNN-guard.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/FNN-guard.json new file mode 100644 index 000000000..a4cea65b6 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/FNN-guard.json @@ -0,0 +1,49 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "0", + "FNN" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 5.798009, + "wall_seconds": 5.534140970034059, + "peak_tree_rss_bytes": 1042665472, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 159744, + "max_poll_gap_seconds": 0.39918523997766897, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "numerical_threads": 2, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null, + "underlying_worker_status": "PASS", + "device_monitor": { + "reasons": [], + "peak_device_bytes": 435159040, + "samples": 10 + }, + "recorded_peaks": { + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672 + }, + "peak_witnesses": { + "peak_allocated_bytes": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json", + "sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af" + }, + "peak_reserved_bytes": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json", + "sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af" + } + }, + "total_gpu_wall_seconds": 5.538356595963705, + "peak_cap_bytes": 16106127360 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/GBRT-guard.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/GBRT-guard.json new file mode 100644 index 000000000..dcedf839b --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/GBRT-guard.json @@ -0,0 +1,27 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "0", + "GBRT" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.6755669999999998, + "wall_seconds": 1.7543383440352045, + "peak_tree_rss_bytes": 148721664, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 221184, + "max_poll_gap_seconds": 0.22388055600458756, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/acceptance.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/acceptance.json new file mode 100644 index 000000000..9bb5041dd --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/acceptance.json @@ -0,0 +1,145 @@ +{ + "status": "PASS", + "receipts": { + "GBRT": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "0", + "GBRT" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.6755669999999998, + "wall_seconds": 1.7543383440352045, + "peak_tree_rss_bytes": 148721664, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 221184, + "max_poll_gap_seconds": 0.22388055600458756, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "FNN": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "0", + "FNN" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 5.798009, + "wall_seconds": 5.534140970034059, + "peak_tree_rss_bytes": 1042665472, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 159744, + "max_poll_gap_seconds": 0.39918523997766897, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "numerical_threads": 2, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null, + "underlying_worker_status": "PASS", + "device_monitor": { + "reasons": [], + "peak_device_bytes": 435159040, + "samples": 10 + }, + "recorded_peaks": { + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672 + }, + "peak_witnesses": { + "peak_allocated_bytes": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json", + "sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af" + }, + "peak_reserved_bytes": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json", + "sha256": "bf5b93c67f1b9b13e9051cbd6df373d3c6097d56cdeadbb193e51a49405417af" + } + }, + "total_gpu_wall_seconds": 5.538356595963705, + "peak_cap_bytes": 16106127360 + }, + "predict": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "predict" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 3.362805, + "wall_seconds": 3.0489734660368413, + "peak_tree_rss_bytes": 519958528, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.46637159201782197, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "score": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "score" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 0.6084579999999999, + "wall_seconds": 0.7152220709831454, + "peak_tree_rss_bytes": 111345664, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.11809522699331865, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + } + }, + "GPU_seconds": 5.584548475977499, + "worker_CPU_seconds": 11.444839, + "checks": [ + "real GBRT and FNN entrypoints", + "test target absent during fits", + "test seconds unavailable during prediction", + "all predictions frozen before scoring", + "missing splits not silently averaged", + "failure VRAM peaks retained" + ], + "scientific_data_read": false, + "coordinator_CPU_seconds": 2.8703529999999997 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/history.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/history.json new file mode 100644 index 000000000..845a29ee8 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/history.json @@ -0,0 +1,152 @@ +[ + { + "epoch": 1, + "training_mse": 0.07795825600624084, + "validation_mse": 0.060705915093421936, + "best_epoch": 1, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.7462023249827325 + }, + { + "epoch": 2, + "training_mse": 0.06311292946338654, + "validation_mse": 0.04847054183483124, + "best_epoch": 2, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.751589720021002 + }, + { + "epoch": 3, + "training_mse": 0.05059317126870155, + "validation_mse": 0.03820352628827095, + "best_epoch": 3, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.7556245460291393 + }, + { + "epoch": 4, + "training_mse": 0.04009852185845375, + "validation_mse": 0.02991129830479622, + "best_epoch": 4, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.7596782970358618 + }, + { + "epoch": 5, + "training_mse": 0.031606074422597885, + "validation_mse": 0.02256850153207779, + "best_epoch": 5, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.7651306050247513 + }, + { + "epoch": 6, + "training_mse": 0.024003881961107254, + "validation_mse": 0.0158725343644619, + "best_epoch": 6, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.769136717019137 + }, + { + "epoch": 7, + "training_mse": 0.017005659639835358, + "validation_mse": 0.010371999815106392, + "best_epoch": 7, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.773159590025898 + }, + { + "epoch": 8, + "training_mse": 0.011204666458070278, + "validation_mse": 0.006224535871297121, + "best_epoch": 8, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.7771851680008695 + }, + { + "epoch": 9, + "training_mse": 0.00675668939948082, + "validation_mse": 0.003289248328655958, + "best_epoch": 9, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.7813589150318876 + }, + { + "epoch": 10, + "training_mse": 0.0034951232373714447, + "validation_mse": 0.0018917981069535017, + "best_epoch": 10, + "stale_epochs": 0, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.7855790790054016 + }, + { + "epoch": 11, + "training_mse": 0.001763510867021978, + "validation_mse": 0.0024419045075774193, + "best_epoch": 10, + "stale_epochs": 1, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.788880237028934 + }, + { + "epoch": 12, + "training_mse": 0.0019949353300035, + "validation_mse": 0.004658875055611134, + "best_epoch": 10, + "stale_epochs": 2, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.792193347995635 + }, + { + "epoch": 13, + "training_mse": 0.003945866599678993, + "validation_mse": 0.007049876265227795, + "best_epoch": 10, + "stale_epochs": 3, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.7955055559868924 + }, + { + "epoch": 14, + "training_mse": 0.006167569197714329, + "validation_mse": 0.008173784241080284, + "best_epoch": 10, + "stale_epochs": 4, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.798879029985983 + }, + { + "epoch": 15, + "training_mse": 0.007227514870464802, + "validation_mse": 0.007775146048516035, + "best_epoch": 10, + "stale_epochs": 5, + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672, + "wall_s": 3.8023696839809418 + } +] diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/ledger.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/ledger.json new file mode 100644 index 000000000..104423280 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 5.798009, + "attempts": [ + { + "name": "worker", + "status": "PASS", + "cpu_seconds": 5.798009 + } + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/progress.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/progress.json new file mode 100644 index 000000000..18204af3d --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/progress.json @@ -0,0 +1,9 @@ +{ + "stage": "FIT_COMPLETE_NO_TEST_SCORES", + "pid": 480379, + "unix": 1794757202.323634, + "split": 0, + "model": "FNN", + "CPU_s": 4.038258777, + "wall_s": 3.8051335139898583 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json new file mode 100644 index 000000000..2537fc610 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/result.json @@ -0,0 +1,27 @@ +{ + "status": "FIT_COMPLETE_NO_TEST_SCORES", + "split": 0, + "model": "FNN", + "checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/model.pt", + "checkpoint_sha256": "737d54bf01b17dbd87756fcf43bb9b3333597d05a28c94d8cd4bc309e682c0b2", + "missing_preparation": { + "train": [], + "test": [], + "fnn_learning": [], + "fnn_validation": [] + }, + "CPU_s": 4.038131669, + "wall_s": 3.8050071470206603, + "epochs": 15, + "best_epoch": 10, + "best_validation_mse": 0.0018917981069535017, + "train_nodes": 128, + "validation_nodes": 128, + "train_geometries": 1, + "validation_geometries": 1, + "restored_best": true, + "amp": false, + "input_standardization": "none; use eq12 columns", + "peak_allocated_bytes": 18742272, + "peak_reserved_bytes": 23068672 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/started.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/started.json new file mode 100644 index 000000000..df9e37d9b --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/started.json @@ -0,0 +1,8 @@ +{ + "pid": 480379, + "unix": 1794757198.5185177, + "split": 0, + "model": "FNN", + "seed": 91, + "training_contract_sha256": "c86ab1d6ef22262200789c1b0d7aadd112b68862a64275a1c657a9d3f9f39dfd" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/training-target-access.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/training-target-access.json new file mode 100644 index 000000000..bab2c0144 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/training-target-access.json @@ -0,0 +1,18 @@ +[ + { + "case": "learn", + "role": "fnn_learning", + "nodes": 128, + "source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/learn", + "feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc" + }, + { + "case": "validation", + "role": "fnn_validation", + "nodes": 128, + "source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/validation", + "feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3" + } +] diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.log b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.log new file mode 100644 index 000000000..e69de29bb diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.receipt.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.receipt.json new file mode 100644 index 000000000..0e5625a68 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.receipt.json @@ -0,0 +1,27 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "0", + "FNN" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 5.798009, + "wall_seconds": 5.534140970034059, + "peak_tree_rss_bytes": 1042665472, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 159744, + "max_poll_gap_seconds": 0.39918523997766897, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "cuda_device": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "numerical_threads": 2, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.started.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.started.json new file mode 100644 index 000000000..12e9d8304 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/worker.started.json @@ -0,0 +1,18 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "0", + "FNN" + ], + "limits": { + "total_cpu_seconds": 120.0, + "job_cpu_seconds": 120.0, + "job_wall_seconds": 57.0, + "tree_rss_bytes": 25769803776, + "disk_bytes": 3221225472, + "max_processes": 1 + }, + "started_unix": 1794757197.6559174 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/ledger.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/ledger.json new file mode 100644 index 000000000..a00209cb5 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 1.6755669999999998, + "attempts": [ + { + "name": "worker", + "status": "PASS", + "cpu_seconds": 1.6755669999999998 + } + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/progress.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/progress.json new file mode 100644 index 000000000..656a01d86 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/progress.json @@ -0,0 +1,9 @@ +{ + "stage": "FIT_COMPLETE_NO_TEST_SCORES", + "pid": 480373, + "unix": 1794757197.3915205, + "split": 0, + "model": "GBRT", + "CPU_s": 0.6673245510000001, + "wall_s": 0.6674595809890889 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/result.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/result.json new file mode 100644 index 000000000..e8f9ad64a --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/result.json @@ -0,0 +1,58 @@ +{ + "status": "FIT_COMPLETE_NO_TEST_SCORES", + "split": 0, + "model": "GBRT", + "checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/model.ubj", + "checkpoint_sha256": "0630f79ad1b6783fa31ed5b6d94e345a896212187676f65168f4e819889ec74b", + "missing_preparation": { + "train": [], + "test": [], + "fnn_learning": [], + "fnn_validation": [] + }, + "CPU_s": 0.6670500859999999, + "wall_s": 0.6671876410255209, + "train_nodes": 256, + "train_geometries": 2, + "parameters": { + "objective": "reg:squarederror", + "base_score": null, + "booster": null, + "callbacks": null, + "colsample_bylevel": null, + "colsample_bynode": null, + "colsample_bytree": null, + "device": null, + "early_stopping_rounds": null, + "enable_categorical": false, + "eval_metric": null, + "feature_types": null, + "gamma": null, + "grow_policy": null, + "importance_type": null, + "interaction_constraints": null, + "learning_rate": 0.1, + "max_bin": null, + "max_cat_threshold": null, + "max_cat_to_onehot": null, + "max_delta_step": null, + "max_depth": 8, + "max_leaves": null, + "min_child_weight": null, + "missing": "NaN (library default missing marker)", + "monotone_constraints": null, + "multi_strategy": null, + "n_estimators": 100, + "n_jobs": 1, + "num_parallel_tree": null, + "random_state": 91, + "reg_alpha": null, + "reg_lambda": null, + "sampling_method": null, + "scale_pos_weight": null, + "subsample": null, + "tree_method": null, + "validate_parameters": null, + "verbosity": null + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/started.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/started.json new file mode 100644 index 000000000..add2def25 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/started.json @@ -0,0 +1,8 @@ +{ + "pid": 480373, + "unix": 1794757196.7240794, + "split": 0, + "model": "GBRT", + "seed": 91, + "training_contract_sha256": "c86ab1d6ef22262200789c1b0d7aadd112b68862a64275a1c657a9d3f9f39dfd" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/training-target-access.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/training-target-access.json new file mode 100644 index 000000000..fd09fdf5b --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/training-target-access.json @@ -0,0 +1,18 @@ +[ + { + "case": "learn", + "role": "train", + "nodes": 128, + "source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/learn", + "feature_sha256": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "target_sha256": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc" + }, + { + "case": "validation", + "role": "train", + "nodes": 128, + "source": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/validation", + "feature_sha256": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "target_sha256": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3" + } +] diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.log b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.log new file mode 100644 index 000000000..e69de29bb diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.receipt.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.receipt.json new file mode 100644 index 000000000..dcedf839b --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.receipt.json @@ -0,0 +1,27 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "0", + "GBRT" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1.6755669999999998, + "wall_seconds": 1.7543383440352045, + "peak_tree_rss_bytes": 148721664, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 221184, + "max_poll_gap_seconds": 0.22388055600458756, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.started.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.started.json new file mode 100644 index 000000000..9a9608f4c --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/worker.started.json @@ -0,0 +1,18 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_fit.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "0", + "GBRT" + ], + "limits": { + "total_cpu_seconds": 120, + "job_cpu_seconds": 120, + "job_wall_seconds": 120, + "tree_rss_bytes": 4294967296, + "disk_bytes": 268435456, + "max_processes": 1 + }, + "started_unix": 1794757195.8518412 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/xgboost-config.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/xgboost-config.json new file mode 100644 index 000000000..4bd921477 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/xgboost-config.json @@ -0,0 +1 @@ +{"learner":{"generic_param":{"device":"cpu","fail_on_invalid_gpu_id":"0","n_jobs":"1","nthread":"1","random_state":"91","seed":"91","seed_per_iteration":"0","validate_parameters":"1"},"gradient_booster":{"gbtree_model_param":{"num_parallel_tree":"1","num_trees":"100"},"gbtree_train_param":{"process_type":"default","tree_method":"auto","updater":"grow_quantile_histmaker","updater_seq":"grow_quantile_histmaker"},"name":"gbtree","specified_updater":false,"tree_train_param":{"alpha":"0","cache_opt":"1","colsample_bylevel":"1","colsample_bynode":"1","colsample_bytree":"1","eta":"0.100000001","gamma":"0","grow_policy":"depthwise","interaction_constraints":"","lambda":"1","learning_rate":"0.100000001","max_bin":"256","max_cat_threshold":"64","max_cat_to_onehot":"4","max_delta_step":"0","max_depth":"8","max_leaves":"0","min_child_weight":"1","min_split_loss":"0","monotone_constraints":"()","refresh_leaf":"1","reg_alpha":"0","reg_lambda":"1","sampling_method":"uniform","sketch_ratio":"2","sparse_threshold":"0.20000000000000001","subsample":"1"},"updater":[{"hist_train_param":{"debug_synchronize":"0","max_cached_hist_node":"65536"},"name":"grow_quantile_histmaker"}]},"learner_model_param":{"base_score":"3.0167848E-1","boost_from_average":"1","num_class":"0","num_feature":"19","num_target":"1"},"learner_train_param":{"booster":"gbtree","disable_default_eval_metric":"0","multi_strategy":"one_output_per_tree","objective":"reg:squarederror"},"metrics":[{"name":"rmse"}],"objective":{"name":"reg:squarederror","reg_loss_param":{"scale_pos_weight":"1"}}},"version":[2,1,4]} \ No newline at end of file diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/models-freeze.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/models-freeze.json new file mode 100644 index 000000000..c77e7191f --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/models-freeze.json @@ -0,0 +1,38 @@ +{ + "fits": { + "0/GBRT": { + "status": "PASS", + "checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/GBRT/model.ubj", + "checkpoint_sha256": "0630f79ad1b6783fa31ed5b6d94e345a896212187676f65168f4e819889ec74b" + }, + "0/FNN": { + "status": "PASS", + "checkpoint": "artifacts/gre24-reproduction-2026-11-15/integration-03/fits/split-0/FNN/model.pt", + "checkpoint_sha256": "737d54bf01b17dbd87756fcf43bb9b3333597d05a28c94d8cd4bc309e682c0b2" + }, + "1/GBRT": { + "status": "NOT_ATTEMPTED_SYNTHETIC" + }, + "1/FNN": { + "status": "NOT_ATTEMPTED_SYNTHETIC" + }, + "2/GBRT": { + "status": "NOT_ATTEMPTED_SYNTHETIC" + }, + "2/FNN": { + "status": "NOT_ATTEMPTED_SYNTHETIC" + }, + "3/GBRT": { + "status": "NOT_ATTEMPTED_SYNTHETIC" + }, + "3/FNN": { + "status": "NOT_ATTEMPTED_SYNTHETIC" + }, + "4/GBRT": { + "status": "NOT_ATTEMPTED_SYNTHETIC" + }, + "4/FNN": { + "status": "NOT_ATTEMPTED_SYNTHETIC" + } + } +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/peak-check/failure.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/peak-check/failure.json new file mode 100644 index 000000000..4e1fa33ba --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/peak-check/failure.json @@ -0,0 +1,3 @@ +{ + "peak_reserved_bytes": 17179869184 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/ledger.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/ledger.json new file mode 100644 index 000000000..ec0308313 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 3.362805, + "attempts": [ + { + "name": "worker", + "status": "PASS", + "cpu_seconds": 3.362805 + } + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.log b/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.log new file mode 100644 index 000000000..e69de29bb diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.receipt.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.receipt.json new file mode 100644 index 000000000..1db7626e3 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.receipt.json @@ -0,0 +1,26 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "predict" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 3.362805, + "wall_seconds": 3.0489734660368413, + "peak_tree_rss_bytes": 519958528, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.46637159201782197, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.started.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.started.json new file mode 100644 index 000000000..e1c4c59ab --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/predict-guard/worker.started.json @@ -0,0 +1,17 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "predict" + ], + "limits": { + "total_cpu_seconds": 120, + "job_cpu_seconds": 120, + "job_wall_seconds": 120, + "tree_rss_bytes": 4294967296, + "disk_bytes": 268435456, + "max_processes": 1 + }, + "started_unix": 1794757203.1936562 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/prediction-progress.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/prediction-progress.json new file mode 100644 index 000000000..7396a9fc9 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/prediction-progress.json @@ -0,0 +1,4 @@ +{ + "test_case_visits": 5, + "worker_CPU_s": 1.9683088469999999 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/predictions-freeze.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/predictions-freeze.json new file mode 100644 index 000000000..ca8966c71 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/predictions-freeze.json @@ -0,0 +1,91 @@ +{ + "models_freeze_sha256": "fa75e754cdf7961602066e88e6d9b189a4470fddc6bb0c3d6098b1772287af14", + "rows": [ + { + "split": 0, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/predictions/split-0/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + }, + "GBRT": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/predictions/split-0/GBRT/test.npy", + "sha256": "7e1475f28547094ef396c5663ffb4ddc413cde4ec4ec07d397bd5e03bd79ee2a", + "nodes": 128, + "no_clipping": true + }, + "FNN": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/predictions/split-0/FNN/test.npy", + "sha256": "d8f8d7d2837a35943125ec13afef55d76c3e9bc7254ccf50deccf00eb6cbb6ce", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + }, + { + "split": 1, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/predictions/split-1/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + }, + { + "split": 2, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/predictions/split-2/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + }, + { + "split": 3, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/predictions/split-3/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + }, + { + "split": 4, + "case": "test", + "status": "PREDICTED", + "predictions": { + "naive": { + "path": "artifacts/gre24-reproduction-2026-11-15/integration-03/predictions/split-4/naive/test.npy", + "sha256": "06900d6a34bdeeaa5080d560d75726737007da4e037e024ecdf7596482676d29", + "nodes": 128, + "no_clipping": true + } + }, + "raw_ids_sha256": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d" + } + ], + "worker_CPU_s": 1.9684696389999998, + "test_targets_read": false, + "device": "CPU", + "prediction_units": "seconds" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/preparation-freeze.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/preparation-freeze.json new file mode 100644 index 000000000..6898fb214 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/preparation-freeze.json @@ -0,0 +1,40 @@ +{ + "successful": { + "learn": { + "output": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/learn", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "b4de52373d35475720b485d2410603bcdcfbd6ff0f8abc0505b21677347dc076", + "target.npy": "96318947361faaaa7b5a8597eb7cbdcf0e396f5b78997285abd6cdf9b52214fc", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "d7a48bc6fc22f90016ce2272df2b0defc7ab7a112e6ca5d9dc9250f67b92185a", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } + }, + "validation": { + "output": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/validation", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "152b3d4259c125efcddfcfcda38911ce770a954aefd9feb0bf37c32f57081f40", + "target.npy": "2de93fc78efb894a67264d5f0a2ce5e69b310312eb41fcbe2f63a500eddf3ef3", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7c6cc1ff2fcccafb35c07193565d87936311e89febec7d0a080abbfd9142e499", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } + }, + "test": { + "output": "artifacts/gre24-reproduction-2026-11-15/integration-03/parts/test", + "nodes": 128, + "t_scale_seconds": 2.0, + "output_sha256": { + "seconds.npy": "69876518da187c72404e321b3ac9ad31bb8a84033157683502278d12f4554353", + "raw_ids.npy": "384b73789fda94f946b5b2e1d02a26d749024460bccef47e00a20042b00aee1d", + "features.npy": "7d94ef4c54e285bcbeb6a9b9953fe94910e8a5819fff7cca49198941b873f7f0", + "Vf.npy": "4e4f11c0ff5145e55665c9ae288048e387f2cb0deaec3384695b9365e6e3e6df" + } + } + }, + "failed": [] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/ledger.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/ledger.json new file mode 100644 index 000000000..87eec2352 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 0.6084579999999999, + "attempts": [ + { + "name": "worker", + "status": "PASS", + "cpu_seconds": 0.6084579999999999 + } + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.log b/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.log new file mode 100644 index 000000000..e69de29bb diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.receipt.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.receipt.json new file mode 100644 index 000000000..ad434fb14 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.receipt.json @@ -0,0 +1,26 @@ +{ + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "score" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 0.6084579999999999, + "wall_seconds": 0.7152220709831454, + "peak_tree_rss_bytes": 111345664, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.11809522699331865, + "poll_target_seconds": 0.1, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": false, + "cuda_device": "", + "numerical_threads": 1, + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.started.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.started.json new file mode 100644 index 000000000..1a9c37895 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/score-guard/worker.started.json @@ -0,0 +1,17 @@ +{ + "command": [ + "/home/frank14f/optane/MeltMind/artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/gre24_score.py", + "artifacts/gre24-reproduction-2026-11-15/integration-03", + "score" + ], + "limits": { + "total_cpu_seconds": 120, + "job_cpu_seconds": 120, + "job_wall_seconds": 120, + "tree_rss_bytes": 4294967296, + "disk_bytes": 268435456, + "max_processes": 1 + }, + "started_unix": 1794757206.2439442 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/score-summary.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/score-summary.json new file mode 100644 index 000000000..74d28dce3 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/score-summary.json @@ -0,0 +1,519 @@ +{ + "dataset_version": "synthetic_only", + "protocol": "synthetic smoke, not scientific data", + "status": "SCORED_WITH_EXPLICIT_COVERAGE", + "summary": { + "naive": { + "rrmse_rms": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + } + ], + "mean_of_five_split_summaries": { + "min": 1.1607788938732118, + "median": 1.1607788938732118, + "max": 1.1607788938732118 + }, + "full_requested_coverage": true + }, + "rrmse_maximum": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.9117259819436264, + "median": 0.9117259819436264, + "max": 0.9117259819436264 + } + ], + "mean_of_five_split_summaries": { + "min": 0.9117259819436263, + "median": 0.9117259819436263, + "max": 0.9117259819436263 + }, + "full_requested_coverage": true + }, + "rrmse_mean": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + } + ], + "mean_of_five_split_summaries": { + "min": 1.173094770814564, + "median": 1.173094770814564, + "max": 1.173094770814564 + }, + "full_requested_coverage": true + } + }, + "GBRT": { + "rrmse_rms": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.02095071847807076, + "median": 0.02095071847807076, + "max": 0.02095071847807076 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + }, + "rrmse_maximum": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.016455601043112972, + "median": 0.016455601043112972, + "max": 0.016455601043112972 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + }, + "rrmse_mean": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.021173005833544523, + "median": 0.021173005833544523, + "max": 0.021173005833544523 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + } + }, + "FNN": { + "rrmse_rms": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.13810242497024947, + "median": 0.13810242497024947, + "max": 0.13810242497024947 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + }, + "rrmse_maximum": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.10847162166660623, + "median": 0.10847162166660623, + "max": 0.10847162166660623 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + }, + "rrmse_mean": { + "per_split": [ + { + "split": 0, + "requested_test_geometries": 1, + "scored": 1, + "missing": 0, + "min": 0.13956769323125362, + "median": 0.13956769323125362, + "max": 0.13956769323125362 + }, + { + "split": 1, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 2, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 3, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + }, + { + "split": 4, + "requested_test_geometries": 1, + "scored": 0, + "missing": 1, + "min": null, + "median": null, + "max": null + } + ], + "mean_of_five_split_summaries": { + "min": null, + "median": null, + "max": null + }, + "full_requested_coverage": false + } + } + }, + "worker_CPU_s": 0.0035956970000000643, + "scores_sha256": "13cef650a67f1c8e1ce9198b41325f3f6a2be35377da46f121eccdc7fdc4a197", + "paper_numbers_diagnostic_only": true, + "limitations": [ + "v5 data instead of paper v4", + "R1 discretisation choices are explicit supplements", + "t_set substitutes unavailable applied Q", + "FNN fill-only and early stopping are declared choices", + "Random geometry splits may leak near-duplicate geometry families", + "Failed preparations not replaced; coverage is explicit" + ] +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/scores.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/scores.json new file mode 100644 index 000000000..20e10a9dd --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/scores.json @@ -0,0 +1,131 @@ +{ + "rows": [ + { + "split": 0, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + }, + "GBRT": { + "nodes": 128, + "rmse": 0.012686577967069528, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 0.02095071847807076, + "rrmse_maximum": 0.016455601043112972, + "rrmse_mean": 0.021173005833544523 + }, + "FNN": { + "nodes": 128, + "rmse": 0.08362706909838521, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 0.13810242497024947, + "rrmse_maximum": 0.10847162166660623, + "rrmse_mean": 0.13956769323125362 + } + } + }, + { + "split": 1, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + } + } + }, + { + "split": 2, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + } + } + }, + { + "split": 3, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + } + } + }, + { + "split": 4, + "case": "test", + "status": "SCORED", + "scores": { + "naive": { + "nodes": 128, + "rmse": 0.7029024782641866, + "denominators": { + "rms": 0.6055438137049401, + "maximum": 0.770958042421619, + "mean": 0.5991864389405733 + }, + "undefined_metrics": [], + "rrmse_rms": 1.1607788938732118, + "rrmse_maximum": 0.9117259819436264, + "rrmse_mean": 1.173094770814564 + } + } + } + ], + "predictions_freeze_sha256": "7432f743f45200b35afab36379cd944f843e51895bcf40b97bf57b4bebd775e3" +} diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/splits.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/splits.json new file mode 100644 index 000000000..1a7c33f89 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/splits.json @@ -0,0 +1,87 @@ +[ + { + "index": 0, + "seed": 91, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 1, + "seed": 92, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 2, + "seed": 93, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 3, + "seed": 94, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + }, + { + "index": 4, + "seed": 95, + "train": [ + "learn", + "validation" + ], + "test": [ + "test" + ], + "fnn_learning": [ + "learn" + ], + "fnn_validation": [ + "validation" + ] + } +] diff --git a/artifacts/gre24-reproduction-2026-11-15/integration-03/training-contract.json b/artifacts/gre24-reproduction-2026-11-15/integration-03/training-contract.json new file mode 100644 index 000000000..fe3af3050 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/integration-03/training-contract.json @@ -0,0 +1,16 @@ +{ + "code_sha256": { + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688", + "tools/gre24_score.py": "f0d2f190e5c3be5ae25205156b5944b365ca5250c3871ce094a3f94f84a0f1f0", + "tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4", + "tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa" + }, + "preparation_freeze_sha256": "ccabbbfac0452272dc0dd6c468a8b15bbe302d98043ba0c154f59e83edeb636f", + "splits_sha256": "4f2e9459bc231dff203e8d839427afbd5480cbc6f2efb58a485cd415ecdde809", + "dataset_version": "synthetic_only", + "protocol": "synthetic smoke, not scientific data", + "xgb_threads": 1, + "fnn_cpu_threads": 2, + "gpu_reserved_byte_cap": 16106127360 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/model-synthetic-receipt.json b/artifacts/gre24-reproduction-2026-11-15/model-synthetic-receipt.json new file mode 100644 index 000000000..99146056c --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/model-synthetic-receipt.json @@ -0,0 +1,8 @@ +{ + "tests": 5, + "passed": true, + "failures": 0, + "errors": 0, + "CPU_s": 6.148973, + "GPU_used": false +} \ No newline at end of file diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json b/artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json new file mode 100644 index 000000000..f1842920d --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json @@ -0,0 +1,63 @@ +{ + "version": "Gre24-v5-model-chain-v1", + "dataset_version": "v5", + "protocol": "5 independent random 312test/312train geometry splits; v5 data deviation", + "authorizing_rulings": [ + "1794755127.457709", + "1794755474.778649", + "1794755594.435869", + "1794756165.827889", + "1794756271.768399" + ], + "code_sha256": { + "tools/run_gre24_models.py": "103ecb6dc8a40481f72da02a02e0a41069a9a747ad0b7af4418c6ed06f0dcbc9", + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688", + "tools/gre24_score.py": "dbb11b57b7b7b2178ac11fd7243c934adb899d666a4c717dfb6a51ee1c65c705", + "tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4", + "tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa", + "tools/gre24_prepare_v2.py": "7637e55009005f6a8c77114159d7ac0d3c024c7add3e320afc486927537b78bc", + "tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf", + "tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919" + }, + "total_worker_cpu_seconds": 144000, + "total_gpu_seconds": 7200, + "initial_gpu_seconds": 10.185812931973487, + "postfit_cpu_reserve_seconds": 900, + "xgb_per_fit_cpu_seconds": 7200, + "fnn_per_fit_cpu_seconds": 14400, + "xgb_threads": 8, + "fnn_cpu_threads": 2, + "gpu_reserved_byte_cap": 16106127360, + "ordering": "All five GBRT, then five FNN, freeze every terminal/checkpoint, CPU prediction freeze, CPU scoring", + "FNN": { + "hidden_layers": 4, + "width": 100, + "activation": "ReLU", + "optimizer": "Adam", + "learning_rate": 0.001, + "batch_size": 1024, + "maximum_epochs": 30, + "patience": 5, + "restore_best": true, + "AMP": false, + "input_standardization": "none beyond Eq12", + "validation": "32 fixed training geometries per split; no replacement after prep failure" + }, + "GBRT": { + "library": "xgboost2.1.4", + "trees": 100, + "max_depth": 8, + "learning_rate": 0.1, + "objective": "reg:squarederror", + "other_parameters": "library defaults, saved per fit" + }, + "limits_notes": "No retry or replacement; CPU preparation+oldcost+test reserve+all fits+postfit share40h; waits/setup/failures in FNN stage count toward shared2GPUh", + "acceptance_receipts": { + "artifacts/gre24-reproduction-2026-11-15/integration-03/acceptance.json": "008c40416c992e27b7111a7a5580ef151be29849fa5f275f70e7cc22ca196397", + "artifacts/gre24-reproduction-2026-11-15/budget-chain-test/test-acceptance.json": "e2d11ca8901285999e7e71b2f94f6f14a586252cc5797881a5eede72fdd716a3", + "artifacts/gre24-reproduction-2026-11-15/model-synthetic-receipt.json": "90958f1e44239c9c776a7c297ba2b848193e208ab74881ba63c62bb04328bb15" + }, + "timestamp": 1794757401.999023 +} diff --git a/artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-prelaunch-audit.json b/artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-prelaunch-audit.json new file mode 100644 index 000000000..9e57dae94 --- /dev/null +++ b/artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-prelaunch-audit.json @@ -0,0 +1,76 @@ +{ + "unix": 1794758038.4473786, + "preparation_pid": 479550, + "preparation_cmd": [ + "artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python", + "tools/run_gre24_protocol_preparation.py", + "artifacts/gre24-reproduction-2026-11-15/v5-protocol" + ], + "workers": [ + { + "pid": 480925, + "cpu_seconds": 26.98 + }, + { + "pid": 480894, + "cpu_seconds": 44.97 + }, + { + "pid": 480804, + "cpu_seconds": 94.08 + }, + { + "pid": 480788, + "cpu_seconds": 186.39000000000001 + }, + { + "pid": 480737, + "cpu_seconds": 419.2 + }, + { + "pid": 480710, + "cpu_seconds": 422.23 + }, + { + "pid": 480430, + "cpu_seconds": 791.86 + }, + { + "pid": 479849, + "cpu_seconds": 1311.6599999999999 + } + ], + "progress": { + "completed": 61, + "attempted": 69, + "total": 624, + "active": { + "7": 11, + "6": 36, + "5": 52, + "4": 53, + "1": 63, + "0": 65, + "2": 67, + "3": 68 + }, + "settled_worker_cpu_seconds": 7962.913921999999, + "reserved_active_worker_cpu_seconds": 14416, + "stop_reason": null, + "first_part_checked": true, + "elapsed_seconds": 1627.0575121869915 + }, + "terminal_status_counts": { + "PASS": 61 + }, + "live_worker_cpu_seconds": 3297.33, + "remaining_cpu_seconds_sampled": 132739.756078, + "preparation_pins_match": true, + "model_pins_match": true, + "contract_sha256": "9f8f87cb36108d51a7bdf90f006b709fd2483a9c4d9cc6658ee8d722f634a248", + "model_plan_sha256": "4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4", + "splits_sha256": "76bb712f5c40c3c5bc3ec6ffc670e8259456802d36facd5f4282049704c239da", + "acceptance_pins_match": true, + "existing_model_chain_files": [], + "training_started": false +} diff --git a/docs/gre24-model-chain-2026-11-15.md b/docs/gre24-model-chain-2026-11-15.md new file mode 100644 index 000000000..010d9b30f --- /dev/null +++ b/docs/gre24-model-chain-2026-11-15.md @@ -0,0 +1,23 @@ +# Gre24 固定模型接续与验收 + +今晚目标是复现 Greif 原方法并明确与原论文的差异,服务于新件填充代理的强基线核查;论文数值仅作诊断,不以调参追平为验收。沿 Claude 1794755127、1794755474、1794755594、1794756165 和 1794756271 的已定范围,v5 624 件、五次独立 312/312 随机几何划分继续,Grok 唯一 v4 中转独立进行。原 S2/S3 结果与失败不变。 + +准备实现已在 8409910 固定,并获 Claude 1794756599 实读核查;数学细节沿 Mold 55eb0d48 原稿。v5、全 tet 节点域、壁面代理、t_set 代替不可得 Q、仅 fill-time 输出等偏离完整保留,不能标作严格同数据同输出复现。不存在新 R4 实验或参数搜索。 + +## 固定实现及进入条件 + +模型计划 `artifacts/gre24-reproduction-2026-11-15/v5-protocol/model-plan.json` 的 SHA256 为 `4e3ddf31ab829955def1c450babb37b2c6b1291b1ac97ce1819ee4d079783bd4`,内含各源码 SHA 与三份验收回执 SHA。模型等待链使用独占锁和不可覆盖的 started 记录。仅原准备队列 624 全终态、无 global stop、费用完整时冻结成功/失败名单并进入训练;失败件不替换、不重抽划分。等待状态不代表训练开始。 + +先五个 GBRT,后五个 FNN;每个 GBRT 固定 100 树、深度 8、学习率 0.1,其余 XGBoost 2.1.4 默认值逐次落盘。FNN 为四层各 100 ReLU、Adam 0.001、batch 1024、FP32、最多 30 epoch;固定训练侧 32 件验证、patience 5、恢复最佳。无额外输入标准化。全部模型终态及 checkpoint 冻结后才生成测试预测,预测冻结后才评分。缺失 split 的五次平均保持 NA。 + +总预算仍为 144000 worker CPU 秒与 7200 GPU 秒。准备、历史单件、合成预留、拟合与后处理共用 CPU 预算,后处理保留 900 秒;GBRT 单次最多 7200 CPU 秒,FNN 最多 14400 CPU 秒,实际还受剩余总量限制。GPU 历史合成费用 `10.185812931973487` 秒计一次;后续 GPU 等待、设置和失败计入同一总帽。单 GPU 显存上限 15 GiB,进程树内存 24 GiB,失败记录的显存峰值同样检查。无自动重试。 + +## 已完成验收与限制 + +模型单元检查 5 项通过。integration-03 实际运行合成 GBRT/FNN、checkpoint、CPU 预测及评分全部 PASS:训练时测试 target 故意不存在,预测时测试 seconds 不可读;缺失 split 不静默平均,失败 JSON 的显存峰值被识别。此为合成验收,不是真实数据结果。 + +integration-01 暴露 XGBoost 默认 NaN 参数的 JSON 记录问题;integration-02 暴露 CUDA 设备初始化前重置显存峰值的问题。已修复,原失败与当时源码保留。三次合成 GPU 用量分别为 0、4.601264455995988、5.584548475977499 秒。预算链合成验收验证真实入口在拟合预算不足时不启动任何模型/GPU,仍按顺序冻结并对 naive 评分。 + +验收后仅增补 raw_nodes/域外节点统计报告字段,数值路径未改。准备源码未改。源码与完整终态记录可供独立审阅;这里没有声称团队已逐行审查新模型实现,也没有将拟议计算写作完成。 + +23:53:58 接续前实测:原 session 18323 / PID 479550 存活,61 PASS、8 活跃、无失败和全局停止;准备/模型/验收 pins 全匹配,无既有模型等待链、无真实训练。已结算 7962.913922 CPU 秒,加采样活跃 3297.33 秒,剩余约 132739.756078 秒;采样值并非终态账本。原始见 `model-prelaunch-audit.json`。唯一等待链启动后的 PID/session/状态另由实际启动回执记录。 diff --git a/tools/gre24_fit.py b/tools/gre24_fit.py new file mode 100644 index 000000000..ca2f55392 --- /dev/null +++ b/tools/gre24_fit.py @@ -0,0 +1,139 @@ +"""One fixed split/model fit; reads training-side targets only, no test scores.""" +import argparse +import json +import os +from pathlib import Path +import time +import traceback +import numpy as np +from gre24_prepare_v2 import sha +from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update +from s2_recovery_supervisor import save + + +def contract(root): + c=json.loads((root/'training-contract.json').read_text()) + for p,h in c['code_sha256'].items(): + if sha(p)!=h:raise ValueError('Pinned training source changed: '+p) + if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']: + raise ValueError('Preparation freeze changed') + if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed') + return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text()) + + +def training_arrays(root,freeze,split,role,read_log): + selected=[case for case in split[role] if case in freeze['successful']] + n=sum(freeze['successful'][case]['nodes'] for case in selected) + if not n:raise ValueError('No successful training rows') + x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0 + for case in selected: + require_training_role(case,split,role) + r=freeze['successful'][case];source=Path(r['output']) + for name in ['features.npy','target.npy']: + if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed') + a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False) + b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False) + count=len(b) + if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch') + x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count + read_log.append(dict(case=case,role=role,nodes=count,source=str(source), + feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy'])) + if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input') + return x,y + + +def fit(root,index,model_name): + c,f,splits=contract(root);split=splits[index] + dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True) + if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite') + start=time.monotonic();cpu=time.process_time();read_log=[] + def progress(stage,**kw): + record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name, + CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start) + record.update(kw);save(dest/'progress.json',record) + save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'], + training_contract_sha256=sha(root/'training-contract.json'))) + roles,missing=eligible_roles(split,f['successful']) + try: + if model_name=='GBRT': + progress('LOAD_TRAIN_ONLY') + x,y=training_arrays(root,f,split,'train',read_log) + save(dest/'training-target-access.json',read_log) + model=xgb_model(split['seed'],c['xgb_threads']) + progress('FIT',nodes=len(y),geometries=len(roles['train'])) + model.fit(x,y,verbose=False) + checkpoint=dest/'model.ubj';model.save_model(checkpoint) + (dest/'xgboost-config.json').write_text(model.get_booster().save_config()) + params=model.get_params() + # Preserve XGBoost's default missing=NaN as an explicit textual parameter; + # this is not a nonfinite observation or a change to the fitted estimator. + params={k:('NaN (library default missing marker)' if isinstance(v,float) and np.isnan(v) else v) + for k,v in params.items()} + extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=params) + elif model_name=='FNN': + import torch + torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed']) + if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable') + torch.cuda.set_device(0) # Initialise the selected device before resetting its allocator counters. + device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device) + torch.backends.cuda.matmul.allow_tf32=False + progress('LOAD_FNN_TRAIN_VALIDATION_ONLY') + x,y=training_arrays(root,f,split,'fnn_learning',read_log) + vx,vy=training_arrays(root,f,split,'fnn_validation',read_log) + save(dest/'training-target-access.json',read_log) + x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device) + vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device) + model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001) + best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt' + for epoch in range(1,31): + model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device) + for begin in range(0,len(y),1024): + ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True) + prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2) + if not torch.isfinite(loss):raise ValueError('Nonfinite training loss') + loss.backward();optimizer.step();total+=loss.detach()*len(ids) + model.eval();sse=torch.zeros((),device=device) + with torch.no_grad(): + for begin in range(0,len(vy),65536): + sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2) + validation=float((sse/len(vy)).item());train=float((total/len(y)).item()) + best,stale,improved,stop=early_stop_update(best,stale,validation) + if improved: + torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()}, + 'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint) + best_epoch=epoch + row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch, + stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device), + peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start) + history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row) + if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap') + if stop:break + restored=torch.load(checkpoint,map_location='cpu',weights_only=True) + model.load_state_dict(restored['state_dict']) + extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best, + train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']), + validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False, + input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device), + peak_reserved_bytes=torch.cuda.max_memory_reserved(device)) + else:raise ValueError('Unknown model') + save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name, + checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing, + CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra)) + progress('FIT_COMPLETE_NO_TEST_SCORES');return 0 + except Exception as exc: + peaks={} + if model_name=='FNN': + try: + import torch + if torch.cuda.is_initialized(): + peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(), + peak_reserved_bytes=torch.cuda.max_memory_reserved()) + except Exception as peak_error:peaks={'peak_read_error':str(peak_error)} + save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(), + CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks)) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN']) + a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model)) diff --git a/tools/gre24_gpu_guard.py b/tools/gre24_gpu_guard.py new file mode 100644 index 000000000..dd87718e8 --- /dev/null +++ b/tools/gre24_gpu_guard.py @@ -0,0 +1,82 @@ +"""Single-GPU wall accounting and recorded/device VRAM checks including failure.""" +import hashlib +import json +import math +from pathlib import Path +import subprocess +import threading +import time +import psutil +from gre24_job_guard import supervise,stop_group + + +def gpu_state(uuid=None): + args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits'] + if uuid:args+=['-i',uuid] + out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout + rows=[] + for line in out.splitlines(): + u,mem,util=[v.strip() for v in line.split(',')] + rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util))) + if not rows:raise RuntimeError('No GPU state returned') + return rows + + +def recorded_peaks(root): + peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={} + def walk(value,path,digest): + if isinstance(value,dict): + for k,v in value.items(): + if k in peaks: + if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0: + raise ValueError('Invalid CUDA peak record') + if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest} + else:walk(v,path,digest) + elif isinstance(value,list): + for v in value:walk(v,path,digest) + for path in sorted(Path(root).glob('*.json')): + raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest()) + return peaks,witnesses + + +def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap): + start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0} + maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={} + def scan(): + current,seen=recorded_peaks(fit_root) + for k,v in current.items(): + if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k] + if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']: + monitor['reasons'].append('RECORDED_VRAM_LIMIT') + def watch(): + while not done.is_set(): + try: + scan();state=gpu_state(uuid)[0];monitor['samples']+=1 + monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes']) + if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']: + monitor['reasons'].append('DEVICE_VRAM_LIMIT') + except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc)) + if monitor['reasons']: + for p in psutil.Process().children(recursive=True): + try: + if p.cmdline()==command:stop_group(p.pid) + except psutil.NoSuchProcess:pass + done.wait(.5) + thread=threading.Thread(target=watch,daemon=True);thread.start() + try: + receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit, + job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30, + disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2) + finally: + done.set();thread.join(timeout=12) + if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP') + try:scan() + except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc)) + if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS') + elapsed=time.monotonic()-start + if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT') + receipt['underlying_worker_status']=receipt['status'] + if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0]) + receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses, + total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap) + return receipt diff --git a/tools/gre24_job_guard.py b/tools/gre24_job_guard.py new file mode 100644 index 000000000..a5e451cd0 --- /dev/null +++ b/tools/gre24_job_guard.py @@ -0,0 +1,160 @@ +"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts. + +Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot +is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged. +""" +import fcntl +import ctypes +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import time + +import psutil +from s2_recovery_monitoring import disk_usage,Monitor + + +def save(path, data): + path = Path(path) + tmp = path.with_suffix(path.suffix + '.tmp') + tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') + tmp.replace(path) + + +def tree_usage(pid): + try: + root = psutil.Process(pid) + processes = [root] + root.children(recursive=True) + except psutil.NoSuchProcess: + return 0., 0, 0 + cpu = 0. + rss = count = 0 + for p in processes: + try: + t = p.cpu_times() + # Linux child times include reaped descendants, not live children. + cpu += t.user + t.system + t.children_user + t.children_system + rss += p.memory_info().rss + count += 1 + except psutil.NoSuchProcess: + pass + return cpu, rss, count + + +def stop_group(pid): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def supervise(command, root, name, limits, *, poll=.02, cuda_device='', numerical_threads=1): + """One locked attempt; no automatic retry/replacement; account failed CPU.""" + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + monitor=Monitor(root/(name+'.monitor-errors.jsonl')) + with (root/'execution.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + started_path = root/(name+'.started.json') + receipt_path = root/(name+'.receipt.json') + if started_path.exists() or receipt_path.exists(): + raise FileExistsError('Attempt already exists; no silent retry') + if any(not math.isfinite(v) or v <= 0 for v in limits.values()): + raise ValueError('Every limit must be finite and positive') + ledger_path = root/'ledger.json' + ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]} + if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']: + raise RuntimeError('Total CPU exhausted before spawn') + if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']: + raise RuntimeError('Disk exhausted before spawn') + save(started_path, {'command':command,'limits':limits,'started_unix':time.time()}) + start = last = time.monotonic() + cpu_peak = rss_peak = max_processes = max_gap = 0 + cpu=rss=count=used_disk=0 + reason = None + exit_status = usage = None + env = dict(os.environ, OMP_NUM_THREADS=str(numerical_threads), OPENBLAS_NUM_THREADS=str(numerical_threads), + MKL_NUM_THREADS=str(numerical_threads), NUMEXPR_NUM_THREADS=str(numerical_threads), + CUDA_VISIBLE_DEVICES=cuda_device) + with (root/(name+'.log')).open('x') as log: + libc = ctypes.CDLL(None, use_errno=True) + previous_subreaper = ctypes.c_int() + if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping') + proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, + env=env, start_new_session=True) + try: + while True: + now = time.monotonic() + max_gap = max(max_gap, now-last) + last = now + (cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count)) + cpu_peak = max(cpu_peak,cpu) + rss_peak = max(rss_peak,rss) + max_processes = max(max_processes,count) + used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk) + if now-start >= limits['job_wall_seconds']: + reason = 'wall_limit' + elif rss > limits['tree_rss_bytes']: + reason = 'tree_rss_limit' + elif count > limits['max_processes']: + reason = 'process_count_limit' + elif used_disk >= limits['disk_bytes']: + reason = 'disk_limit' + elif cpu_peak >= limits['job_cpu_seconds']: + reason = 'job_cpu_limit' + elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']: + reason = 'total_cpu_limit' + if reason: + stop_group(proc.pid) + pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG) + if pid: + exit_status, usage = status, child_usage + break + time.sleep(poll) + finally: + # No detached work may outlive a supervised attempt. + stop_group(proc.pid) + if exit_status is None: + _, exit_status, usage = os.wait4(proc.pid, 0) + proc.returncode = os.waitstatus_to_exitcode(exit_status) + orphan_cpu = 0. + orphan_peak = 0 + # Killed descendants are adopted here rather than losing their + # final CPU usage when the root worker cannot reap them itself. + while True: + try: + _, _, orphan_usage = os.wait4(-proc.pid, 0) + except ChildProcessError: + break + orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime + orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024) + if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state') + elapsed = time.monotonic()-start + charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) + rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak) + used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll) + if reason is None: + if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit' + elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit' + elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit' + elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit' + elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit' + receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED', + command=command,returncode=proc.returncode,stop_reason=reason, + cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak, + peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk, + max_poll_gap_seconds=max_gap,poll_target_seconds=poll, + reaped_orphan_cpu_seconds=orphan_cpu, + supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure', + gpu_visible=bool(cuda_device),cuda_device=cuda_device, + numerical_threads=numerical_threads,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error) + ledger['cpu_seconds_charged'] += charged + ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged}) + monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll) + monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll) + return receipt diff --git a/tools/gre24_models.py b/tools/gre24_models.py new file mode 100644 index 000000000..723c9a487 --- /dev/null +++ b/tools/gre24_models.py @@ -0,0 +1,42 @@ +"""Fixed Gre24 model choices; no test-driven selection or node subsampling.""" +import numpy as np + + +def xgb_model(seed,threads=8): + from xgboost import XGBRegressor + return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1, + objective='reg:squarederror',random_state=seed,n_jobs=threads) + + +def fnn_model(): + import torch + layers=[];width=19 + for _ in range(4): + layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100 + layers.append(torch.nn.Linear(100,1)) + return torch.nn.Sequential(*layers) + + +def eligible_roles(split,successful): + successful=set(successful) + roles={role:[c for c in split[role] if c in successful] + for role in ['train','test','fnn_learning','fnn_validation']} + if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap') + if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap') + if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']): + raise ValueError('FNN learning/validation do not partition train') + missing={role:[c for c in split[role] if c not in successful] for role in roles} + return roles,missing + + +def require_training_role(case,split,role): + if role not in ['train','fnn_learning','fnn_validation']: + raise PermissionError('Only frozen training-side roles may read targets') + if case in split['test'] or case not in split[role]: + raise PermissionError('Case is not in the requested training role') + + +def early_stop_update(best,stale,score,patience=5): + if not np.isfinite(score):raise ValueError('Nonfinite validation MSE') + improved=score=patience) diff --git a/tools/gre24_score.py b/tools/gre24_score.py new file mode 100644 index 000000000..20f8a1193 --- /dev/null +++ b/tools/gre24_score.py @@ -0,0 +1,104 @@ +"""Predictions frozen before test-target scoring, all successful tet nodes.""" +import argparse +import json +from pathlib import Path +import time +import numpy as np +from gre24_prepare_v2 import sha +from gre24_features_v2 import per_geometry_metrics +from gre24_models import fnn_model +from gre24_fit import contract +from s2_recovery_supervisor import save + + +def predict(root): + c,prep,splits=contract(root) + models=json.loads((root/'models-freeze.json').read_text()) + dest=root/'predictions';dest.mkdir(exist_ok=False) + start=time.process_time();rows=[] + for split in splits: + available={'naive':None} + for name in ['GBRT','FNN']: + r=models['fits'][f"{split['index']}/{name}"] + if r['status']!='PASS':continue + if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze') + if name=='GBRT': + from xgboost import XGBRegressor + m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint']) + else: + import torch + torch.set_num_threads(4);m=fnn_model() + m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval() + available[name]=m + for case in split['test']: + if case not in prep['successful']: + rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={})) + continue + r=prep['successful'][case];source=Path(r['output']);outputs={} + for name in ['features.npy','Vf.npy','raw_ids.npy']: + if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed') + x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False) + # No seconds/target file is opened in this phase. + for name,m in available.items(): + if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False) + elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32)) + else: + chunks=[] + with torch.no_grad(): + for a in range(0,len(x),65536): + chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy()) + pred=np.concatenate(chunks) + if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction') + folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True) + path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds']) + outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True) + rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs, + raw_ids_sha256=r['output_sha256']['raw_ids.npy'])) + save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start)) + save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows, + worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds')) + + +def score(root): + c,prep,splits=contract(root);frozen=root/'predictions-freeze.json' + if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring') + freeze=json.loads(frozen.read_text());rows=[];start=time.process_time() + for r in freeze['rows']: + if r['status']!='PREDICTED': + rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue + p=prep['successful'][r['case']];source=Path(p['output']) + if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed') + y=np.load(source/'seconds.npy',allow_pickle=False);scores={} + for name,value in r['predictions'].items(): + if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed') + pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred) + rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores)) + summary={} + for name in ['naive','GBRT','FNN']: + summary[name]={} + for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']: + per=[] + for split in splits: + a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores'] + and r['scores'][name][metric] is not None] + per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a), + missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None, + median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None)) + # Do not average only whichever model splits happened to finish. + avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']} + summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg, + full_requested_coverage=all(r['missing']==0 for r in per)) + save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen))) + save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'], + status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start, + node_domain='All retained tetrahedral nodes including walls, not S2 queries; auxiliary nodes separately reported', + outside_tet_node_counts={case:r.get('outside_tet_nodes',0) for case,r in prep['successful'].items() if r.get('outside_tet_nodes',0)}, + scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True, + limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements', + 't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices', + 'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit'])) + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']) + a=p.parse_args();(predict if a.phase=='predict' else score)(a.root) diff --git a/tools/run_gre24_models.py b/tools/run_gre24_models.py new file mode 100644 index 000000000..14b3b7024 --- /dev/null +++ b/tools/run_gre24_models.py @@ -0,0 +1,142 @@ +"""One sequential fixed-model chain after terminal preparation, within shared caps.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import sys +import time +import psutil +from gre24_prepare_v2 import sha +from gre24_job_guard import supervise,save +from gre24_gpu_guard import gpu_state,gpu_fit + + +def freeze_preparation(root): + complete=json.loads((root/'complete.json').read_text());c=json.loads((root/'contract.json').read_text()) + if complete['status']!='COMPLETE' or not complete['accounting_complete'] or len(complete['records'])!=len(c['parts']): + raise ValueError('Preparation globally stopped or missing terminal accounting') + good={};fail=[] + for r in complete['records']: + if r['worker']['status']!='PASS': + fail.append(dict(case=r['case'],guard_status=r['worker']['status'],reason=r['worker']['stop_reason']));continue + p=Path(r['result_path']) + if sha(p)!=r['result_sha256']:raise ValueError('Preparation result changed') + result=json.loads(p.read_text()) + good[r['case']]=dict(output=str(p.parent),nodes=result['geometry']['tet_nodes'], + raw_nodes=result['geometry'].get('raw_nodes',result['geometry']['tet_nodes']), + t_scale_seconds=result['t_scale_seconds'],output_sha256=result['output_sha256'],result_sha256=sha(p), + outside_tet_nodes=result['geometry']['outside_tet_nodes']) + frozen=dict(successful=good,failed=fail,requested=len(c['parts']), + preparation_complete_sha256=sha(root/'complete.json'),replacement=False, + cumulative_initial_worker_cpu_seconds=complete['worker_cpu_seconds']+complete['initial_cpu_reserve_seconds']) + save(root/'preparation-freeze.json',frozen);return frozen + + +def run(root,plan_path): + root=Path(root);plan_path=Path(plan_path) + with (root/'model-chain.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + return run_locked(root,plan_path) + + +def run_locked(root,plan_path): + plan=json.loads(plan_path.read_text()) + for p,h in plan['code_sha256'].items(): + if sha(p)!=h:raise ValueError('Model plan source changed: '+p) + started=root/'model-chain-started.json' + if started.exists():raise FileExistsError('Never restart model chain') + save(started,dict(pid=os.getpid(),unix=time.time(),plan_sha256=sha(plan_path),stage='WAIT_PREPARATION')) + missing_since=None + while not (root/'complete.json').exists(): + parent=json.loads((root/'queue-started.json').read_text()) + try: + process=psutil.Process(parent['pid']) + alive=process.create_time()<=parent['unix'] and process.status()!=psutil.STATUS_ZOMBIE and any('run_gre24_protocol_preparation.py' in a for a in process.cmdline()) + except psutil.NoSuchProcess:alive=False + if not alive: + missing_since=missing_since or time.monotonic() + if time.monotonic()-missing_since>30:raise RuntimeError('Preparation exited without complete receipt; no restart') + else:missing_since=None + save(root/'model-chain-progress.json',dict(stage='WAIT_PREPARATION',unix=time.time())) + time.sleep(10) + frozen=freeze_preparation(root) + split_path=root/'splits.json';splits=json.loads(split_path.read_text()) + c=dict(plan,preparation_freeze_sha256=sha(root/'preparation-freeze.json'),splits_sha256=sha(split_path)) + save(root/'training-contract.json',c) + spent=frozen['cumulative_initial_worker_cpu_seconds'];fits={};gpu_started=None;gpu_uuid=None;gpu_lock=None + gpu_previous=plan.get('initial_gpu_seconds',0.) + os.sched_setaffinity(0,range(8)) + for name in ['GBRT','FNN']: + for split in splits: + index=split['index'];key=f'{index}/{name}';dest=root/'fits'/f'split-{index}'/name + remain=plan['total_worker_cpu_seconds']-spent-plan['postfit_cpu_reserve_seconds'] + if remain<=5: + fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_CPU_CAP_RESERVE');continue + command=[sys.executable,'tools/gre24_fit.py',str(root),str(index),name] + if name=='GBRT': + cap=min(plan['xgb_per_fit_cpu_seconds'],remain) + receipt=supervise(command,dest,'worker',dict(total_cpu_seconds=cap,job_cpu_seconds=cap, + job_wall_seconds=7200,tree_rss_bytes=24*2**30,disk_bytes=3*2**30,max_processes=1), + poll=.1,numerical_threads=plan['xgb_threads']) + else: + if gpu_started is None:gpu_started=time.monotonic() + remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started) + if remaining_gpu<=20: + fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_GPU_CAP');continue + # Fixed one device once selected, three idle polls; waiting counts toward 2 GPU-hours. + idle=0 + while idle<3 and time.monotonic()-gpu_started+gpu_previous