From f8a6ed3f81e0ed43609441dc588aab05a4294260 Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Sun, 15 Nov 2026 02:37:29 +0800 Subject: [PATCH] Implement bounded S2 preparation and sealed pilot execution --- .../gpu-env-diagnosis/ledger.json | 10 + .../gpu-env-diagnosis/synthetic.log | 0 .../gpu-env-diagnosis/synthetic.receipt.json | 22 + .../gpu-env-diagnosis/synthetic.started.json | 16 + .../gpu-guard-synthetic-tests-final.txt | 21 + .../gpu-guard-synthetic-tests-locked.txt | 12 + .../gpu-guard-synthetic-tests-v2.txt | 21 + .../gpu-guard-synthetic-tests.txt | 8 + .../implementation-receipt.json | 40 ++ .../initial-synthetic-failure.json | 30 + .../contract.json | 648 ++++++++++++++++++ .../public/implementation-acceptance.json | 39 ++ .../public/learning-curve-x-axis.csv | 16 + .../public/ten-part-plan.csv | 11 + docs/s2-pilot-implementation-2026-11-15.md | 47 ++ ...en-part-measurement-contract-2026-11-15.md | 40 ++ tools/prepare_s2_pilot_inputs.py | 88 +++ tools/prepare_s2_ten_part_measurement.py | 72 ++ tools/run_s2_pilot_preparation.py | 124 ++++ tools/run_s2_pilot_training.py | 126 ++++ tools/run_s2_ten_part_measurement.py | 49 ++ tools/s2_gpu_process_supervisor.py | 165 +++++ tools/s2_pilot_data.py | 52 ++ tools/s2_pilot_gpu_guard.py | 106 +++ tools/s2_pilot_prepare.py | 56 ++ tools/s2_pilot_train.py | 48 ++ tools/s2_preprocess.py | 214 ++++++ tools/s2_preprocess_pilot.py | 232 +++++++ tools/summarize_s2_ten_part_measurement.py | 74 ++ tools/test_s2_pilot_data.py | 37 + tools/test_s2_pilot_entry.py | 22 + tools/test_s2_pilot_gpu_guard.py | 49 ++ tools/test_s2_pilot_preparation.py | 29 + tools/test_s2_pilot_train.py | 47 ++ tools/test_s2_preprocess.py | 55 ++ 35 files changed, 2626 insertions(+) create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/ledger.json create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.log create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.receipt.json create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.started.json create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-final.txt create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-locked.txt create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-v2.txt create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests.txt create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/implementation-receipt.json create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/initial-synthetic-failure.json create mode 100644 artifacts/s2-ten-part-measurement-2026-11-15/contract.json create mode 100644 artifacts/s2-ten-part-measurement-2026-11-15/public/implementation-acceptance.json create mode 100644 artifacts/s2-ten-part-measurement-2026-11-15/public/learning-curve-x-axis.csv create mode 100644 artifacts/s2-ten-part-measurement-2026-11-15/public/ten-part-plan.csv create mode 100644 docs/s2-pilot-implementation-2026-11-15.md create mode 100644 docs/s2-ten-part-measurement-contract-2026-11-15.md create mode 100644 tools/prepare_s2_pilot_inputs.py create mode 100644 tools/prepare_s2_ten_part_measurement.py create mode 100644 tools/run_s2_pilot_preparation.py create mode 100644 tools/run_s2_pilot_training.py create mode 100644 tools/run_s2_ten_part_measurement.py create mode 100644 tools/s2_gpu_process_supervisor.py create mode 100644 tools/s2_pilot_data.py create mode 100644 tools/s2_pilot_gpu_guard.py create mode 100644 tools/s2_pilot_prepare.py create mode 100644 tools/s2_pilot_train.py create mode 100644 tools/s2_preprocess.py create mode 100644 tools/s2_preprocess_pilot.py create mode 100644 tools/summarize_s2_ten_part_measurement.py create mode 100644 tools/test_s2_pilot_data.py create mode 100644 tools/test_s2_pilot_entry.py create mode 100644 tools/test_s2_pilot_gpu_guard.py create mode 100644 tools/test_s2_pilot_preparation.py create mode 100644 tools/test_s2_pilot_train.py create mode 100644 tools/test_s2_preprocess.py diff --git a/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/ledger.json b/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/ledger.json new file mode 100644 index 000000000..ef48c6b12 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/ledger.json @@ -0,0 +1,10 @@ +{ + "cpu_seconds_charged": 0.021509, + "attempts": [ + { + "name": "synthetic", + "status": "STOPPED_OR_FAILED", + "cpu_seconds": 0.021509 + } + ] +} diff --git a/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.log b/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.log new file mode 100644 index 000000000..e69de29bb diff --git a/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.receipt.json b/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.receipt.json new file mode 100644 index 000000000..c5b2b6135 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.receipt.json @@ -0,0 +1,22 @@ +{ + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "-c", + "import os; assert os.environ[\"CUDA_VISIBLE_DEVICES\"]==\"synthetic-device\"" + ], + "returncode": 0, + "stop_reason": "rss_limit_after_exit", + "cpu_seconds": 0.021509, + "wall_seconds": 0.022533220006152987, + "peak_tree_rss_bytes": 382091264, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.020913485961500555, + "poll_target_seconds": 0.02, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "gpu_uuid": "synthetic-device", + "retry": false +} diff --git a/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.started.json b/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.started.json new file mode 100644 index 000000000..6de7645c5 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/gpu-env-diagnosis/synthetic.started.json @@ -0,0 +1,16 @@ +{ + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "-c", + "import os; assert os.environ[\"CUDA_VISIBLE_DEVICES\"]==\"synthetic-device\"" + ], + "limits": { + "total_cpu_seconds": 5, + "job_cpu_seconds": 2, + "job_wall_seconds": 2, + "tree_rss_bytes": 134217728, + "disk_bytes": 1048576, + "max_processes": 1 + }, + "started_unix": 1794681093.7772284 +} diff --git a/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-final.txt b/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-final.txt new file mode 100644 index 000000000..9c3696e2d --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-final.txt @@ -0,0 +1,21 @@ +test_exact_device_memory_units (__main__.GPUUsageTests) ... ok +test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use (__main__.GPUUsageTests) ... ok +test_missing_or_ambiguous_counters_fail_closed (__main__.GPUUsageTests) ... ok +test_vram_monitor_error_and_whole_host_rss_stop_without_second_run (__main__.GPUUsageTests) ... {"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "DEVICE_VRAM_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 17179869184, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 2.6650377549231052e-06, "reason": "DEVICE_VRAM_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.00118093698984012, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0018433370278216898, "test_labels_read": false, "test_predictions": false, "retries": 0} +/home/frank14f/optane/MeltMind/tools/test_s2_pilot_gpu_guard.py:42: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmp8t5zwxfd/pilot.lock' mode='a' encoding='UTF-8'> + guard.run(root) +ResourceWarning: Enable tracemalloc to get the object allocation traceback +{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "GPU_MONITOR_FAILED", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 1.6629928722977638e-06, "reason": "GPU_MONITOR_FAILED", "samples": 0, "error": "synthetic monitor failure"}, "total_run_wall_seconds": 0.0004254329833202064, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0009375379886478186, "test_labels_read": false, "test_predictions": false, "retries": 0} +/home/frank14f/optane/MeltMind/tools/test_s2_pilot_gpu_guard.py:42: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmpackzyjel/pilot.lock' mode='a' encoding='UTF-8'> + guard.run(root) +ResourceWarning: Enable tracemalloc to get the object allocation traceback +{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "WHOLE_RUN_HOST_RSS_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 18253611008, "max_poll_gap_seconds": 1.4830147847533226e-06, "reason": "WHOLE_RUN_HOST_RSS_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.0007409480167552829, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0012677110498771071, "test_labels_read": false, "test_predictions": false, "retries": 0} +/home/frank14f/optane/MeltMind/tools/test_s2_pilot_gpu_guard.py:42: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmpeb4lk0qx/pilot.lock' mode='a' encoding='UTF-8'> + guard.run(root) +ResourceWarning: Enable tracemalloc to get the object allocation traceback +ok + +---------------------------------------------------------------------- +Ran 4 tests in 0.039s + +OK diff --git a/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-locked.txt b/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-locked.txt new file mode 100644 index 000000000..c36370ee8 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-locked.txt @@ -0,0 +1,12 @@ +test_exact_device_memory_units (__main__.GPUUsageTests) ... ok +test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use (__main__.GPUUsageTests) ... ok +test_missing_or_ambiguous_counters_fail_closed (__main__.GPUUsageTests) ... ok +test_vram_monitor_error_and_whole_host_rss_stop_without_second_run (__main__.GPUUsageTests) ... {"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "DEVICE_VRAM_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 17179869184, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 2.435001078993082e-06, "reason": "DEVICE_VRAM_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.0010300629655830562, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0018631540006026626, "test_labels_read": false, "test_predictions": false, "retries": 0} +{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "GPU_MONITOR_FAILED", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 1.4629913493990898e-06, "reason": "GPU_MONITOR_FAILED", "samples": 0, "error": "synthetic monitor failure"}, "total_run_wall_seconds": 0.00040111603448167443, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.000856155005749315, "test_labels_read": false, "test_predictions": false, "retries": 0} +{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "STOPPED_OR_FAILED", "stop_reason": "WHOLE_RUN_HOST_RSS_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 18253611008, "max_poll_gap_seconds": 1.4929682947695255e-06, "reason": "WHOLE_RUN_HOST_RSS_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.0006760350079275668, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0011414529872126877, "test_labels_read": false, "test_predictions": false, "retries": 0} +ok + +---------------------------------------------------------------------- +Ran 4 tests in 0.036s + +OK diff --git a/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-v2.txt b/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-v2.txt new file mode 100644 index 000000000..914ce8167 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests-v2.txt @@ -0,0 +1,21 @@ +test_exact_device_memory_units (__main__.GPUUsageTests) ... ok +test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use (__main__.GPUUsageTests) ... ok +test_missing_or_ambiguous_counters_fail_closed (__main__.GPUUsageTests) ... ok +test_vram_monitor_error_and_whole_host_rss_stop_without_second_run (__main__.GPUUsageTests) ... {"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "first", "arm": "N", "width": 128}, "status": "STOPPED_OR_FAILED", "stop_reason": "DEVICE_VRAM_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 17179869184, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 1.9430299289524555e-06, "reason": "DEVICE_VRAM_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.0008959799888543785, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.001290674030315131, "test_labels_read": false, "test_predictions": false, "retries": 0} +/home/frank14f/optane/MeltMind/tools/test_s2_pilot_gpu_guard.py:42: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmp52rtox8_/pilot.lock' mode='a' encoding='UTF-8'> + guard.run(root) +ResourceWarning: Enable tracemalloc to get the object allocation traceback +{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "first", "arm": "N", "width": 128}, "status": "STOPPED_OR_FAILED", "stop_reason": "GPU_MONITOR_FAILED", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 0, "max_poll_gap_seconds": 1.5130499377846718e-06, "reason": "GPU_MONITOR_FAILED", "samples": 0, "error": "synthetic monitor failure"}, "total_run_wall_seconds": 0.0003517029690556228, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0006604759837500751, "test_labels_read": false, "test_predictions": false, "retries": 0} +/home/frank14f/optane/MeltMind/tools/test_s2_pilot_gpu_guard.py:42: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmp4o7msp4q/pilot.lock' mode='a' encoding='UTF-8'> + guard.run(root) +ResourceWarning: Enable tracemalloc to get the object allocation traceback +{"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "first", "arm": "N", "width": 128}, "status": "STOPPED_OR_FAILED", "stop_reason": "WHOLE_RUN_HOST_RSS_LIMIT", "gpu_visible": true, "gpu_uuid": "synthetic-device", "device_monitor": {"peak_device_memory_bytes": 5242880, "peak_whole_run_tree_rss_bytes": 18253611008, "max_poll_gap_seconds": 1.1619995348155499e-06, "reason": "WHOLE_RUN_HOST_RSS_LIMIT", "samples": 1}, "total_run_wall_seconds": 0.0005933789652772248, "result_sha256": null}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 0.0009194929734803736, "test_labels_read": false, "test_predictions": false, "retries": 0} +/home/frank14f/optane/MeltMind/tools/test_s2_pilot_gpu_guard.py:42: ResourceWarning: unclosed file <_io.TextIOWrapper name='/tmp/tmpqnjbmk42/pilot.lock' mode='a' encoding='UTF-8'> + guard.run(root) +ResourceWarning: Enable tracemalloc to get the object allocation traceback +ok + +---------------------------------------------------------------------- +Ran 4 tests in 0.053s + +OK diff --git a/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests.txt b/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests.txt new file mode 100644 index 000000000..0b4c5f4ec --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/gpu-guard-synthetic-tests.txt @@ -0,0 +1,8 @@ +test_exact_device_memory_units (__main__.GPUUsageTests) ... ok +test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use (__main__.GPUUsageTests) ... ok +test_missing_or_ambiguous_counters_fail_closed (__main__.GPUUsageTests) ... ok + +---------------------------------------------------------------------- +Ran 3 tests in 0.026s + +OK diff --git a/artifacts/s2-pilot-implementation-2026-11-15/implementation-receipt.json b/artifacts/s2-pilot-implementation-2026-11-15/implementation-receipt.json new file mode 100644 index 000000000..59da3bc65 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/implementation-receipt.json @@ -0,0 +1,40 @@ +{ + "status": "SYNTHETIC_CHECKS_PASS_REAL_PILOT_NOT_RUN", + "checks": { + "label_seal": 4, + "preparation_cache_and_projection": 2, + "training_primitives_and_four_model_backward": 4, + "GPU_guard_synthetic": 4, + "training_entry_role_seal": 2 + }, + "total_tests": 16, + "passed": 16, + "receipts_source": "Observed tool unittest outputs; GPU guard final raw log preserved", + "observed_seconds": { + "label_seal": 0.036, + "preparation": 0.008, + "training": 1.569, + "gpu_guard": 0.036 + }, + "initial_failure": "initial-synthetic-failure.json retained; 128MiB test budget below inherited torch fork HWM; diagnostic worker exited0 and visibility was correct; test budget1GiB, production caps unchanged", + "real_project_labels_read": false, + "real_training": false, + "GPU_execution": false, + "synthetic_CPU_optimizer_updates": true, + "source_sha256": { + "tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97", + "tools/test_s2_pilot_data.py": "cd9a5f69a5207d0ea62e8c150df737992ab640a08d82b182d504c9559e729562", + "tools/s2_preprocess_pilot.py": "46cec13e13c0b94e5a51444b5d5ee78579dec9a3aa42ed24294e0a0deec55833", + "tools/s2_pilot_prepare.py": "0987a5fe49816557d2cd8ccaeaf01de5effcb5d080221f2c93eb834719d413d2", + "tools/run_s2_pilot_preparation.py": "08f07593a5f5f434ff1bcfdf952b9079500e2378ea3afb2e4d7e152db1a58def", + "tools/prepare_s2_pilot_inputs.py": "8a04afe82c0add692c7cd5182b3fd70919ed9979f417af7223fba64753ca3144", + "tools/test_s2_pilot_preparation.py": "71667a38b352ad64e588653b7d3545e71a473fb151de775e515e174b5358403c", + "tools/s2_pilot_train.py": "b0563201d269fca1240ebf8e68d197ed16f6209c3fe4b9e6d4ee33da45977c59", + "tools/test_s2_pilot_train.py": "12b192fd6774583ec606125c6c2d7be33af0bbf060cc18c7012b9b7939a65505", + "tools/run_s2_pilot_training.py": "65286cce3b0511ea9bc010d11e3ab47a4be8637803888fa624e79683b4b2ab36", + "tools/test_s2_pilot_entry.py": "d02fba59429268e77dbb8fc4a37f24a0218266d851ca0c4442a6f7c05c42177e", + "tools/s2_gpu_process_supervisor.py": "9d26e0531a444191d1bc5f813a582d1b91fe93fe24a08fd010fedf295063de67", + "tools/s2_pilot_gpu_guard.py": "fe4f3414a85d0205c92067e88bdd8da20d44f42a27d1f5e552a731f0dae9977c", + "tools/test_s2_pilot_gpu_guard.py": "e335d24ba4745dc409b4d37aa8705c0e2f2b7e665c299469b154cfbfb3e84644" + } +} diff --git a/artifacts/s2-pilot-implementation-2026-11-15/initial-synthetic-failure.json b/artifacts/s2-pilot-implementation-2026-11-15/initial-synthetic-failure.json new file mode 100644 index 000000000..1281d1945 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/initial-synthetic-failure.json @@ -0,0 +1,30 @@ +{ + "test": "explicit_visibility_reaches_worker_and_receipt_without_gpu_use", + "failure": "STOPPED_OR_FAILED != PASS", + "observed_tests": 3, + "passed": 2, + "elapsed_seconds": 0.026, + "diagnosis_receipt": { + "status": "STOPPED_OR_FAILED", + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "-c", + "import os; assert os.environ[\"CUDA_VISIBLE_DEVICES\"]==\"synthetic-device\"" + ], + "returncode": 0, + "stop_reason": "rss_limit_after_exit", + "cpu_seconds": 0.021509, + "wall_seconds": 0.022533220006152987, + "peak_tree_rss_bytes": 382091264, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 4096, + "max_poll_gap_seconds": 0.020913485961500555, + "poll_target_seconds": 0.02, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "gpu_uuid": "synthetic-device", + "retry": false + }, + "scientific_or_GPU_execution": false +} diff --git a/artifacts/s2-ten-part-measurement-2026-11-15/contract.json b/artifacts/s2-ten-part-measurement-2026-11-15/contract.json new file mode 100644 index 000000000..a07b2d821 --- /dev/null +++ b/artifacts/s2-ten-part-measurement-2026-11-15/contract.json @@ -0,0 +1,648 @@ +{ + "version": "S2-ten-part-preparation-v1", + "authorizing_rulings": [ + "1794679459.056999" + ], + "fixed_split_commit": "fb827178eca01ff71f9854c3eb3e985cf5d808fa", + "cases": [ + { + "case": "00001130_c2db313c2ce749d784c610d2_step_002", + "source_key": "c2db313c2ce749d784c610d2", + "numeric_prefix": 1130, + "tet_nodes": 8276, + "tet_cells": 44778, + "tertile": 1, + "selection": "random", + "t_set": 0.6, + "t_set_provenance": { + "member": "00001130_c2db313c2ce749d784c610d2_step_002.log", + "setting": { + "line": 250, + "text": " Fill time = 0.60 s", + "context_before": [ + { + "line": 246, + "text": "" + }, + { + "line": 247, + "text": " Filling Control:" + }, + { + "line": 248, + "text": " ---------------" + }, + { + "line": 249, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 247, + "text": " Filling Control:" + }, + "seconds": 0.6 + }, + "process_headings": [ + { + "line": 225, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 280, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00001130_c2db313c2ce749d784c610d2_step_002/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00001130_c2db313c2ce749d784c610d2_step_002/geometry.npz": "83d3fd6e45b3f4060a51d9b0a8ae6c2f9570bb3dc2837fcbf9810d03dddf96d0", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9" + } + }, + { + "case": "00001324_5bf4a74af99f405392406e40_step_000", + "source_key": "5bf4a74af99f405392406e40", + "numeric_prefix": 1324, + "tet_nodes": 10273, + "tet_cells": 52188, + "tertile": 1, + "selection": "random", + "t_set": 0.08, + "t_set_provenance": { + "member": "00001324_5bf4a74af99f405392406e40_step_000.log", + "setting": { + "line": 234, + "text": " Fill time = 0.08 s", + "context_before": [ + { + "line": 230, + "text": "" + }, + { + "line": 231, + "text": " Filling Control:" + }, + { + "line": 232, + "text": " ---------------" + }, + { + "line": 233, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 231, + "text": " Filling Control:" + }, + "seconds": 0.08 + }, + "process_headings": [ + { + "line": 209, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 264, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00001324_5bf4a74af99f405392406e40_step_000/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00001324_5bf4a74af99f405392406e40_step_000/geometry.npz": "9dd06e69fe03e0df6425d288c734be4e44f565149674ba6608e7950b49b2f2e6", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9" + } + }, + { + "case": "00000842_2c1efcc32d094e9fb9e1a88e_step_002", + "source_key": "2c1efcc32d094e9fb9e1a88e", + "numeric_prefix": 842, + "tet_nodes": 11321, + "tet_cells": 60183, + "tertile": 1, + "selection": "random", + "t_set": 0.8, + "t_set_provenance": { + "member": "00000842_2c1efcc32d094e9fb9e1a88e_step_002.log", + "setting": { + "line": 242, + "text": " Fill time = 0.80 s", + "context_before": [ + { + "line": 238, + "text": "" + }, + { + "line": 239, + "text": " Filling Control:" + }, + { + "line": 240, + "text": " ---------------" + }, + { + "line": 241, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 239, + "text": " Filling Control:" + }, + "seconds": 0.8 + }, + "process_headings": [ + { + "line": 217, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 272, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000842_2c1efcc32d094e9fb9e1a88e_step_002/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000842_2c1efcc32d094e9fb9e1a88e_step_002/geometry.npz": "9847232d81bcafda4f9186c4e989d5bde5ce12cc7e47bb18d27e1dff69e9a2b6", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9" + } + }, + { + "case": "00000367_23c957e8e7b2428282a13ae2_step_000", + "source_key": "23c957e8e7b2428282a13ae2", + "numeric_prefix": 367, + "tet_nodes": 24732, + "tet_cells": 134182, + "tertile": 2, + "selection": "random", + "t_set": 2.4, + "t_set_provenance": { + "member": "00000367_23c957e8e7b2428282a13ae2_step_000.log", + "setting": { + "line": 242, + "text": " Fill time = 2.40 s", + "context_before": [ + { + "line": 238, + "text": "" + }, + { + "line": 239, + "text": " Filling Control:" + }, + { + "line": 240, + "text": " ---------------" + }, + { + "line": 241, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 239, + "text": " Filling Control:" + }, + "seconds": 2.4 + }, + "process_headings": [ + { + "line": 217, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 272, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000367_23c957e8e7b2428282a13ae2_step_000/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000367_23c957e8e7b2428282a13ae2_step_000/geometry.npz": "9905f78ff7fcdaa8afda652f161ec19afc8abf162684c1136f84af074338eebd", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9" + } + }, + { + "case": "00000746_6cb3ddbb49974d7db0e97135_step_001", + "source_key": "6cb3ddbb49974d7db0e97135", + "numeric_prefix": 746, + "tet_nodes": 29264, + "tet_cells": 160435, + "tertile": 2, + "selection": "random", + "t_set": 1.1, + "t_set_provenance": { + "member": "00000746_6cb3ddbb49974d7db0e97135_step_001.log", + "setting": { + "line": 242, + "text": " Fill time = 1.10 s", + "context_before": [ + { + "line": 238, + "text": "" + }, + { + "line": 239, + "text": " Filling Control:" + }, + { + "line": 240, + "text": " ---------------" + }, + { + "line": 241, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 239, + "text": " Filling Control:" + }, + "seconds": 1.1 + }, + "process_headings": [ + { + "line": 217, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 272, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000746_6cb3ddbb49974d7db0e97135_step_001/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000746_6cb3ddbb49974d7db0e97135_step_001/geometry.npz": "372c6c720c9a0a73fadf420039eb4f22b4269f4b6cfdec76390334264d2361cf", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9" + } + }, + { + "case": "00000777_a25fb37392a44447b9684f16_step_002", + "source_key": "a25fb37392a44447b9684f16", + "numeric_prefix": 777, + "tet_nodes": 31810, + "tet_cells": 165433, + "tertile": 2, + "selection": "random", + "t_set": 0.02, + "t_set_provenance": { + "member": "00000777_a25fb37392a44447b9684f16_step_002.log", + "setting": { + "line": 240, + "text": " Fill time = 0.02 s", + "context_before": [ + { + "line": 236, + "text": "" + }, + { + "line": 237, + "text": " Filling Control:" + }, + { + "line": 238, + "text": " ---------------" + }, + { + "line": 239, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 237, + "text": " Filling Control:" + }, + "seconds": 0.02 + }, + "process_headings": [ + { + "line": 215, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 270, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000777_a25fb37392a44447b9684f16_step_002/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000777_a25fb37392a44447b9684f16_step_002/geometry.npz": "a46462c17951f656c336c1ee98fc05c2007b1448cb138fe057e1332b8e26f8e4", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9" + } + }, + { + "case": "00001552_b0a9c4d836294846891615fe_step_000", + "source_key": "b0a9c4d836294846891615fe", + "numeric_prefix": 1552, + "tet_nodes": 34818, + "tet_cells": 190189, + "tertile": 3, + "selection": "random", + "t_set": 0.05, + "t_set_provenance": { + "member": "00001552_b0a9c4d836294846891615fe_step_000.log", + "setting": { + "line": 234, + "text": " Fill time = 0.05 s", + "context_before": [ + { + "line": 230, + "text": "" + }, + { + "line": 231, + "text": " Filling Control:" + }, + { + "line": 232, + "text": " ---------------" + }, + { + "line": 233, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 231, + "text": " Filling Control:" + }, + "seconds": 0.05 + }, + "process_headings": [ + { + "line": 209, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 264, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00001552_b0a9c4d836294846891615fe_step_000/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00001552_b0a9c4d836294846891615fe_step_000/geometry.npz": "38edc0c43412332f82c61e10ad8aad1260ecff0c9fa11660aa22e0b4799358ca", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9" + } + }, + { + "case": "00001379_5b1d6e2599ea47d498a04256_step_000", + "source_key": "5b1d6e2599ea47d498a04256", + "numeric_prefix": 1379, + "tet_nodes": 38253, + "tet_cells": 211370, + "tertile": 3, + "selection": "random", + "t_set": 0.3, + "t_set_provenance": { + "member": "00001379_5b1d6e2599ea47d498a04256_step_000.log", + "setting": { + "line": 242, + "text": " Fill time = 0.30 s", + "context_before": [ + { + "line": 238, + "text": "" + }, + { + "line": 239, + "text": " Filling Control:" + }, + { + "line": 240, + "text": " ---------------" + }, + { + "line": 241, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 239, + "text": " Filling Control:" + }, + "seconds": 0.3 + }, + "process_headings": [ + { + "line": 217, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 272, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00001379_5b1d6e2599ea47d498a04256_step_000/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00001379_5b1d6e2599ea47d498a04256_step_000/geometry.npz": "f869b4fc5cdc09f144c2967857a5b587f653c67f33ae6994b915f42d700edb56", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9" + } + }, + { + "case": "00000610_04e799936ae446b9a3d8d2b8_step_001", + "source_key": "04e799936ae446b9a3d8d2b8", + "numeric_prefix": 610, + "tet_nodes": 55883, + "tet_cells": 301287, + "tertile": 3, + "selection": "random", + "t_set": 0.1, + "t_set_provenance": { + "member": "00000610_04e799936ae446b9a3d8d2b8_step_001.log", + "setting": { + "line": 240, + "text": " Fill time = 0.10 s", + "context_before": [ + { + "line": 236, + "text": "" + }, + { + "line": 237, + "text": " Filling Control:" + }, + { + "line": 238, + "text": " ---------------" + }, + { + "line": 239, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 237, + "text": " Filling Control:" + }, + "seconds": 0.1 + }, + "process_headings": [ + { + "line": 215, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 270, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000610_04e799936ae446b9a3d8d2b8_step_001/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000610_04e799936ae446b9a3d8d2b8_step_001/geometry.npz": "6349b2cd50c253e8a23f5e066b2f14f896d4702817da2f9896f7704f20bd0b2b", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9" + } + }, + { + "case": "00000666_6f67a59c16b949a4b66dcfe1_step_000", + "source_key": "6f67a59c16b949a4b66dcfe1", + "numeric_prefix": 666, + "tet_nodes": 187345, + "tet_cells": 1045371, + "tertile": 3, + "selection": "fixed_largest", + "t_set": 0.4, + "t_set_provenance": { + "member": "00000666_6f67a59c16b949a4b66dcfe1_step_000.log", + "setting": { + "line": 250, + "text": " Fill time = 0.40 s", + "context_before": [ + { + "line": 246, + "text": "" + }, + { + "line": 247, + "text": " Filling Control:" + }, + { + "line": 248, + "text": " ---------------" + }, + { + "line": 249, + "text": " Filling control type = Automatic" + } + ], + "section": { + "line": 247, + "text": " Filling Control:" + }, + "seconds": 0.4 + }, + "process_headings": [ + { + "line": 225, + "text": "Process Settings:" + } + ], + "filling_phase": { + "line": 280, + "text": "Filling Phase: Status: V = Velocity control" + }, + "setting_before_filling_phase": true, + "no_time_series_values_read": true + }, + "g1_geometry": "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000666_6f67a59c16b949a4b66dcfe1_step_000/geometry.npz", + "input_sha256": { + "artifacts/s2-g1-candidates-2026-11-14/custodian/fingerprints/00000666_6f67a59c16b949a4b66dcfe1_step_000/geometry.npz": "98f2bdfb62e853f78f40aeea4bb0b6610ff0d575095d1902922384b20b0e189b", + "artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json": "bdbea5bdf2f7bda2d39d5d9a63e9114f708ac978c14779fb32efd92f47282bf9", + "artifacts/intake-2026-11-13/samples/00000666_6f67a59c16b949a4b66dcfe1_step_000.npz": "ee626a32e6e796a9efc7c01953ebf2cfff94dd3556c8a6cc25a0e24e6ec9584c", + "artifacts/routes-2026-11-13/00000666_6f67a59c16b949a4b66dcfe1_step_000.boundary_distance_raw.npy": "d779abaab78c928851f57489409eced35c188e0afa3fea5046110d593710fb28", + "artifacts/o2-rep-v1.1a-2026-11-14/00000666_6f67a59c16b949a4b66dcfe1_step_000_original/nodes.npz": "3fcb0cec8ea3b1cd4691de674b26e1be097e8eba7110357f98052606898c07b8", + "artifacts/o2-rep-v1.1a-2026-11-14/00000666_6f67a59c16b949a4b66dcfe1_step_000_original/result.json": "edabaddfec9b09e27a9a327afbcb2b5a7e6868f9c57a9e4a620e378451d2ed76" + }, + "legacy_geometry": "artifacts/intake-2026-11-13/samples/00000666_6f67a59c16b949a4b66dcfe1_step_000.npz", + "existing_distance": "artifacts/routes-2026-11-13/00000666_6f67a59c16b949a4b66dcfe1_step_000.boundary_distance_raw.npy", + "existing_o2_nodes": "artifacts/o2-rep-v1.1a-2026-11-14/00000666_6f67a59c16b949a4b66dcfe1_step_000_original/nodes.npz", + "existing_o2_result": "artifacts/o2-rep-v1.1a-2026-11-14/00000666_6f67a59c16b949a4b66dcfe1_step_000_original/result.json" + } + ], + "code_sha256": { + "tools/s2_preprocess.py": "0bb59c6b5be166b379c924ca68aacfcaca6737386b71be61a43dc5433ca53419", + "tools/run_s2_ten_part_measurement.py": "4108352a44368bd5581b21df029a84db5f00afad04de3ff73716ad4fa214296c", + "tools/prepare_s2_ten_part_measurement.py": "a6c8ec22ebd541a2e68f72f4c3049c893adc9f55e4a706f3e9c2d4681f1a8a6f", + "tools/test_s2_preprocess.py": "6f70d4392d3800b8146fbdd92b3c06b08638e45167fefb7261bf7ecf30f3faf4", + "tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02", + "tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "tools/g1_candidate_fingerprints.py": "2d027bd95f8da2cbfc0381d051f6a37949a5abe6b27e6b37d747d43c5b6612a0", + "tools/g1_registration_cost.py": "c3d722a8cfd067dfd726f8a8729e72c37dbc63e2c24ba1b0e3d767d47f7756ba", + "tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70", + "tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7", + "tools/run_e6_real_kernel.py": "45a3ade5d1cbc52828b83dcb4a1b884e83063c7eea3f68851e464f4d1e2ce68a", + "tools/o2_rep.py": "8c784ea04f6c14fd47ab79e1f2160c19ed48344bf24e6ab4255ac23ab3eb2003", + "tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4", + "tools/run_volume_quadrature_trial.py": "eff0c71b4cb795b1b9992a1111b2f28c23b5f8f5e34b483f79512644008ac725", + "tools/projector_remesh_probe.py": "90411c11dde855d3a17d651c859a0a5c8d3241c47375a0487eea1c9631abc55a", + "tools/route_feasibility.py": "00e6b67d9079fc9c5d371064d5bfeb1a9950ce8b677bbe2c89c9976f45fb23d2", + "tools/physical_support_probe.py": "4513745d0f2c00e5b5b545ef87382efb2cf8057b1166068b3c85b7c1854a91f7", + "tools/analyze_support_capacity_conditions.py": "6bfc3ab91c48ff1c28a4b6d56092c7c8bd1f73e7f263c54bf857858cf84fd2c9" + }, + "plan_sha256": "89fe54bad7a17a9712b31ee7750f29581a2d1de43d4e6203daf807d783cec5fc", + "archive": "data.zip", + "archive_stat": { + "status": "official_size_and_md5_pass", + "source": "/home/frank14f/optane/MeltMind/data.zip", + "source_url": "https://zenodo.org/records/18598121", + "bytes": 32757812726, + "md5": "75ec15002234c2cdcdf4c751bdefab85", + "sha256": "d151f018294832fd80fbd23e865642cb16c860da150f245033d010c388fa0241", + "mtime_ns": 1794548962944204621, + "inode": 12582980, + "members": 1872, + "triplets": 624, + "source_tokens": 249, + "max_group": 34, + "total_uncompressed_bytes": 43966308383, + "finished_utc": "2026-11-13T08:04:57.180660+00:00", + "wall_seconds": 97.27923265600111, + "peak_rss_kib": 32056, + "crc_scope": "ZIP stored CRC inventory only; selected members checked on read; no full decompression", + "grouping_scope": "Filename source tokens, not verified independent CAD families" + }, + "limits": { + "stage_wall_seconds": 10800, + "stage_total_cpu_seconds": 10800, + "stage_disk_bytes": 5368709120, + "per_part_wall_seconds": 1800, + "per_part_rss_bytes": 1342177280, + "parallel_workers": 1, + "numerical_threads": 1, + "retries": 0, + "replacements": 0 + }, + "input_policy": "Geometry-prefix VTU only; explicit geometry NPY whitelist for legacy 666 container; no fill_time array decoding. Existing process setting provenance only. No fitted statistics or labels.", + "scales": "full-node lumped lower weighted median d: t_part=4*median; R=10*median; full-part lumped center and Rg; no fallback", + "draw": "reuse any frozen S1 primary draw; otherwise PCG64(SeedSequence([2026111400,prefix])), 2000 PPS replacement slots, d>0, ascending tet/raw IDs", + "eligibility": "S2-v1-no-gate-q: d>0, O2 valid, N_eff>=64, complete incident-edge block with at least one finite valid endpoint; finite g and edge summary", + "cache_policy": "Reuse immutable compatible geometry/distance/O2; report new and reused stages separately. Nine missing distance/O2 fields are computed once. Largest 666 full O2 reused at drawn nodes.", + "failure_policy": "Each preselected part attempted at most once; retain failure and partial outputs; continue other parts within total deadline; never impute unreachable R2 graph or zero scale", + "output": "base10 all nodes; R2-G24 all nodes20; common query IDs/counts, g2 and E4 with coverage/rejections; no base standardizer fit, labels, optimizer, training or GPU", + "pilot": "Subsequent stage only, after ten-part evidence; 4 runs total<=2 GPU-h, each<=30min, test fold sealed, no retry" +} diff --git a/artifacts/s2-ten-part-measurement-2026-11-15/public/implementation-acceptance.json b/artifacts/s2-ten-part-measurement-2026-11-15/public/implementation-acceptance.json new file mode 100644 index 000000000..03664805f --- /dev/null +++ b/artifacts/s2-ten-part-measurement-2026-11-15/public/implementation-acceptance.json @@ -0,0 +1,39 @@ +{ + "status": "PASS", + "new_adapter_synthetic_checks": 6, + "passed": 6, + "observed_unittest_seconds": 0.035, + "names": [ + "geometry-only whitelist", + "fixed draws and original IDs", + "nonpositive scale rejection", + "S2 no gate eligibility", + "all incident edge endpoints", + "stage wall and shared CPU budget" + ], + "unchanged_supervisor_previous_checks": 6, + "previous_receipt": "artifacts/s2-e1-2026-11-14/preflight/synthetic-test-receipt.json", + "previous_receipt_sha256": "279e4e199e014af369edf9501077c2977d90f9054efcb85fe677f1c6944da6c2", + "supervisor_source_matches_prior": true, + "source_sha256": { + "tools/s2_preprocess.py": "0bb59c6b5be166b379c924ca68aacfcaca6737386b71be61a43dc5433ca53419", + "tools/run_s2_ten_part_measurement.py": "4108352a44368bd5581b21df029a84db5f00afad04de3ff73716ad4fa214296c", + "tools/prepare_s2_ten_part_measurement.py": "a6c8ec22ebd541a2e68f72f4c3049c893adc9f55e4a706f3e9c2d4681f1a8a6f", + "tools/test_s2_preprocess.py": "6f70d4392d3800b8146fbdd92b3c06b08638e45167fefb7261bf7ecf30f3faf4", + "tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02", + "tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "tools/g1_candidate_fingerprints.py": "2d027bd95f8da2cbfc0381d051f6a37949a5abe6b27e6b37d747d43c5b6612a0", + "tools/g1_registration_cost.py": "c3d722a8cfd067dfd726f8a8729e72c37dbc63e2c24ba1b0e3d767d47f7756ba", + "tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70", + "tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7", + "tools/run_e6_real_kernel.py": "45a3ade5d1cbc52828b83dcb4a1b884e83063c7eea3f68851e464f4d1e2ce68a", + "tools/o2_rep.py": "8c784ea04f6c14fd47ab79e1f2160c19ed48344bf24e6ab4255ac23ab3eb2003", + "tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4", + "tools/run_volume_quadrature_trial.py": "eff0c71b4cb795b1b9992a1111b2f28c23b5f8f5e34b483f79512644008ac725", + "tools/projector_remesh_probe.py": "90411c11dde855d3a17d651c859a0a5c8d3241c47375a0487eea1c9631abc55a", + "tools/route_feasibility.py": "00e6b67d9079fc9c5d371064d5bfeb1a9950ce8b677bbe2c89c9976f45fb23d2", + "tools/physical_support_probe.py": "4513745d0f2c00e5b5b545ef87382efb2cf8057b1166068b3c85b7c1854a91f7", + "tools/analyze_support_capacity_conditions.py": "6bfc3ab91c48ff1c28a4b6d56092c7c8bd1f73e7f263c54bf857858cf84fd2c9" + }, + "labels_fit_gpu": false +} diff --git a/artifacts/s2-ten-part-measurement-2026-11-15/public/learning-curve-x-axis.csv b/artifacts/s2-ten-part-measurement-2026-11-15/public/learning-curve-x-axis.csv new file mode 100644 index 000000000..fd8a61df5 --- /dev/null +++ b/artifacts/s2-ten-part-measurement-2026-11-15/public/learning-curve-x-axis.csv @@ -0,0 +1,16 @@ +fold,component_fraction_percent,actual_components,actual_parts,actual_nodes,x_axis_label +0,25,35,145,4319375,component fraction +0,50,69,254,7524426,component fraction +0,100,137,387,11863762,component fraction +1,25,34,85,2401741,component fraction +1,50,67,180,4549249,component fraction +1,100,134,409,11386113,component fraction +2,25,34,90,2998926,component fraction +2,50,67,203,6659657,component fraction +2,100,133,411,12229477,component fraction +3,25,33,128,4348385,component fraction +3,50,66,236,7631328,component fraction +3,100,132,355,11031937,component fraction +4,25,34,58,1748016,component fraction +4,50,67,144,4743744,component fraction +4,100,133,386,11535306,component fraction diff --git a/artifacts/s2-ten-part-measurement-2026-11-15/public/ten-part-plan.csv b/artifacts/s2-ten-part-measurement-2026-11-15/public/ten-part-plan.csv new file mode 100644 index 000000000..605a98a6b --- /dev/null +++ b/artifacts/s2-ten-part-measurement-2026-11-15/public/ten-part-plan.csv @@ -0,0 +1,11 @@ +case,source_key,numeric_prefix,tet_nodes,tet_cells,tertile,selection +00001130_c2db313c2ce749d784c610d2_step_002,c2db313c2ce749d784c610d2,1130,8276,44778,1,random +00001324_5bf4a74af99f405392406e40_step_000,5bf4a74af99f405392406e40,1324,10273,52188,1,random +00000842_2c1efcc32d094e9fb9e1a88e_step_002,2c1efcc32d094e9fb9e1a88e,842,11321,60183,1,random +00000367_23c957e8e7b2428282a13ae2_step_000,23c957e8e7b2428282a13ae2,367,24732,134182,2,random +00000746_6cb3ddbb49974d7db0e97135_step_001,6cb3ddbb49974d7db0e97135,746,29264,160435,2,random +00000777_a25fb37392a44447b9684f16_step_002,a25fb37392a44447b9684f16,777,31810,165433,2,random +00001552_b0a9c4d836294846891615fe_step_000,b0a9c4d836294846891615fe,1552,34818,190189,3,random +00001379_5b1d6e2599ea47d498a04256_step_000,5b1d6e2599ea47d498a04256,1379,38253,211370,3,random +00000610_04e799936ae446b9a3d8d2b8_step_001,04e799936ae446b9a3d8d2b8,610,55883,301287,3,random +00000666_6f67a59c16b949a4b66dcfe1_step_000,6f67a59c16b949a4b66dcfe1,666,187345,1045371,3,fixed_largest diff --git a/docs/s2-pilot-implementation-2026-11-15.md b/docs/s2-pilot-implementation-2026-11-15.md new file mode 100644 index 000000000..3f74c5925 --- /dev/null +++ b/docs/s2-pilot-implementation-2026-11-15.md @@ -0,0 +1,47 @@ +# S2 pilot 接续实现与合成验收 + +本包是实现及合成检查,**没有启动真实训练、标签读取或 GPU 计算**。十件无标签测量另有正在运行的固定队列;它的源码和输入契约不因这里新增实现而变化。已有 N/E/R2 模型代码沿 Claude 接受的 `f3532962`,组件/拆分沿已接受 `fb827178`。 + +## 已获裁量及输入准备 + +Claude [1794680129.303719](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794680129303719) 固定 pilot 外折 0、seed 0;387 学习 +112 内验证 =499 件。125 测试件封存。本端选择只准备必要的 499 件,不执行可选的测试几何准备。 + +独立准备预算 60 worker CPU-h、≤16 workers、≤100 GiB;每件整条链 ≤30 min /1.25 GiB,0 重试/替换。十件 3h/5GiB 独立账本不挪用。十件完成后按成功的九件新算样本 pooled CPU/node(排除复用的 666)外推全部 499 件,并留 25% 操作余量;磁盘按实测最大 bytes/node 外推并加25%。若超限先报告。另报告中位/p95/max情景,不把线性外推当保证。 + +首 50 件全部终止前不启动第 51 件;检查点按“复用完整准备 / 新准备”两类分别估计每节点 CPU,计入失败 CPU,再加剩余节点成本;缺观测类别用初始率。磁盘按已处理节点率外推。超 60 CPU-h/100 GiB 则停止报告。运行中还监控全局 CPU/磁盘;原始超限与部分产物保留。 + +失败学习/验证件按 Claude 裁量只在 pilot 中明确删除,不补件;保留原组件身份。若十件已有失败,接续记录继承失败而不重算同件。此成功者子集政策不是主矩阵完整性条件的豁免。 + +新 `s2_preprocess_pilot.py` 是 v1 的单独适配,不改十件运行中的 `s2_preprocess.py`:新增旧 S1 主 draw/O2 原姿态缓存复用,必须精确匹配查询 ID、次数、原始 ID、节点数与半径;重新应用 S2 资格,不重新积分旧 O2,不使用 S1 gate-q 筛选位图。 + +## 标签封存 + +`role-manifest.json` 先固定学习、验证和封存测试 ID。`s2_pilot_data.py` 在打开任何 archive 前拒绝测试、未知或角色冲突 ID;只解码获准件 PointData 下精确名为 Fill_time 的数组,按原始 VTU ID 对应 tet 节点。其他仿真字段不解码,不用 LOG 求解输出。只对共同合格查询验证非负有限标签;目标为 T/t_set。无合格查询或有效查询标签异常则失败,不按标签值另择查询。 + +已完成十件输入按不可变结果/文件哈希直接复用;准备成功才提取非测试标签。训练加载器再次按身份拒绝测试,核 base/query/label 哈希,并验证共同 ID 与次数一致。学习输入标准化只允许学习件映射;验证/测试传入会在读数组前拒绝。 + +## 训练入口 + +固定顺序 N128、E128、N256、E256,全部 seed 0、外折0、100% component fraction。所有 tet 节点进入相同 encoder;N/E 共用固定资格、两列谱隙,E 额外四列边读数,原模型不改。 + +每 run 都在资源监督计时内,以实际成功学习件全节点拟合 base 标准化并冻结;跨 run 要求相同 fingerprint。这里重复的是相同输入统计验证,不是几何/谱场重算。学习/验证各按实际成功件及保留组件报告,不能冒称原完整池。 + +AdamW 1e-3、weight_decay1e-5、cosine 200 epochs、batch一件、patience20。每 epoch 用独立 PCG64(seed0) 顺序遍历学习件;每件次数归一 /组件内成功件数 /成功组件数,再乘学习件总数,作为均匀打乱件步骤的全局目标无偏权重。不把每件权重重新归一为1。损失和标准化 FP32,AMP float16;非有限 loss/gradient/parameter 失败,不静默切精度、缩配或重试。 + +验证沿原逐件秒单位相对 L2,再组件内等件、组件等权;用原 float64 标签计算,不把真值先量化为 float32。保存最低验证误差 checkpoint;20 epoch无改善停止,最多200。四个有效 run齐备才按两臂验证误差均值选一次全局宽,并列小宽。失败/不完整时不从剩余 run 偷选宽。 + +## 监督 + +单 V100 UUID 明确写入运行契约。总 pilot ≤2 h 墙钟,单 run ≤30 min,全部加载、输入统计、训练、验证和保存均在 run guard 内。实际单 run 预留少量监督收尾时间;不是额外试验预算。 + +`s2_gpu_process_supervisor.py` 从既有 CPU 监督器仅增加显式 CUDA visibility 参数与回执字段,保留 wait4 CPU/HWM、进程组终止、subreaper、排它/不可重跑规则;原 E1 监督器不改。设备监控独立读取 nvidia-smi,整卡显存 >15 GiB、整棵 guard+worker 树 RSS >16 GiB、监控失败均终止本 run;同时保存 PyTorch allocated/reserved峰值并复核。先检测指定卡空闲;其他进程不被终止。轮询可能瞬时超调,保留实际峰值与间隔,超调不能报PASS。 + +任何一个真实 run失败则停止后续试验并报告;0自动重试、0缩配。所有测试折标签/预测保持封存。准备失败件剔除与训练数值失败是两类不同事件。 + +## 合成证据 + +16/16 新 pilot 实现检查通过:标签封存4、缓存身份/CPU外推2、训练权重/四种实际模型合成反向/容量选择4、GPU监督模拟4、训练入口隔离2。四种模型只在9个合成节点上做 CPU 参数更新,没有真实数据或 GPU。此前十件适配6项与原监督6项记录另附,未重跑旧科学计算。 + +首次 GPU环境传递测试 2/3 通过:128 MiB测试上限低于 PyTorch 父进程 fork/exec 瞬时HWM。单独诊断确认子进程 exit0、CUDA环境正确,失败原因为 `rss_limit_after_exit`,峰值382,091,264 bytes。保留失败及诊断,测试预算改1GiB后通过;正式16GiB主存/15GiB显存上限未改。模拟显存超限、设备监控故障及整棵树超内存均确认停止、无下一run。测试中的资源锁关闭警告也已修正,最终日志无该警告。 + +这是合成验收,不能证明真实499件准备完毕、V100内存适配、训练收敛或预测收益。下一步先完成十件实测及预算核对,再准备非测试池,之后以固定实现提交 Claude 主审和实际有界 pilot;不新增用户逐批审批。 diff --git a/docs/s2-ten-part-measurement-contract-2026-11-15.md b/docs/s2-ten-part-measurement-contract-2026-11-15.md new file mode 100644 index 000000000..efc105d0d --- /dev/null +++ b/docs/s2-ten-part-measurement-contract-2026-11-15.md @@ -0,0 +1,40 @@ +# S2 十件完整准备测量:执行契约与适配 + +接续 Claude 已接受的组件/拆分 `fb827178`,以及 [1794679459.056999](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794679459056999) 的十件→pilot 顺序。当前队列只做十件无标签准备;pilot 的四次训练、总 ≤2 GPU-h、单次 ≤30 min、单 V100、15 GiB VRAM/16 GiB host、0 重试及封存测试折沿既定规则,尚未运行。 + +研究目标是新件填充代理的泛化与样本效率。固定来源组件提供防泄漏约束,不是 CAD 母型真值;十件成本测量仅检验准备流水线可执行性及资源,并不证明预测收益。沿既有 Gre24 图距近似和 Transolver N/E 对照,不重开已完成 G1/盲审、不否定仍开放的 O1 类型/连通问题。 + +## 名单与预算 + +复用原 `ten_part_plan.csv`,顺序 1130、1324、842、367、746、777、1552、1379、610、666;三分位各三件加固定最大件,不重选、不替换。唯一顺序队列,一 worker、一数学线程;单件完整进程树 ≤30 min / 1.25 GiB,整个十件阶段 ≤3 h 墙钟、≤3 CPU-h(worker + supervisor)、≤5 GiB 新磁盘。超限保留失败、无重试,剩余件仍仅在总预算内执行。既有输入不复制计为新科学计算,但新输出计磁盘。 + +原 `s2_e1_supervisor.py` 未改,SHA 与先前六项监督验收一致。新协调器把每件墙钟缩到阶段剩余时间,并扣除 supervisor CPU 后传递共享 CPU 上限;已有 attempt 或批启动标志时拒绝再次启动。RSS/磁盘采用轮询,保留实际间隔和超调,不冒称内核硬配额。 + +## 输入与复用 + +冻结契约 `artifacts/s2-ten-part-measurement-2026-11-15/contract.json` 在第一 worker 前生成,记录源码、原名单、逐件输入 SHA256、给定 t_set 及其已核查 LOG 设置出处。 + +- 全部十件复用 G1 tet-only points/boundary/center/Rg,不再计算指纹或配准。九件缺 tetra 拓扑时从原 VTU 几何前缀提取,遇 PointData/CellData 则停止报错;只解码 Points/connectivity/offsets/types。读取 YML 的门坐标。 +- 666 在旧 intake/routes/O2 中有兼容结果。严格白名单仅打开旧 NPZ 的 points_raw、tetra、node_volume_raw、boundary_triangles、edges、gate_location_raw 六个成员,其他成员不解码。距离及 O2 原姿态全节点缓存校验后复用;O2 只按新固定 draw 取子集,不重复积分。 +- 其余九件补缺失精确边界距离,按既有 SurfaceDistance 保守三角面搜索及 30 点 brute 检查;不使用近邻截断或厚度替代。 +- t_set 只来自先前核验的 Process Settings/Filling Control 给定值;不读取 Fill_time 场、真实终止时间、压力或流率时间序列。原始文件完整保留。 + +这是复用兼容缓存的实际生产路径成本,不是十件全冷启动成本;结果逐阶段注明新算/复用,不能用 666 复用耗时估计全新最大件成本。 + +## 科学与数据契约 + +尺度遵循既定全节点体积集总下加权中位数 d,t_part=4 median、R=10 median;非正值失败,无 fallback。全件 center/Rg/volume 与节点质量一致校验。 + +抽样:已有 24 件的 S1 主 draw 若存在就原样复用;其他件 seed=[2026111400,prefix]、PCG64、正距离节点体积 PPS 有放回 2,000 槽,ID 按原 tet 子网/原 VTU 升序,所有臂复用同一次抽样和次数。666 的旧等价测试 draw 不冒充主 draw。 + +S2 新资格单独版本化为 `S2-v1-no-gate-q`:d>0、O2-rep v1.1a valid、N_eff≥64、所有真实入射边端均评估、至少一个有效有限端、摘要和 g 有限。保持原 S1 边块规则;删除模型不用的 gate-q 筛选,保留原门坐标。只在固定抽中节点上积分,源仍为全件四点求积,不缩源。 + +输出每件全节点 base10、固定 draw ID/次数、g2、E4、资格/覆盖/拒绝原因,以及 R2-G24 全节点 20 列。N/E 无限制 tet-edge 门图距与 R2 排除 wall-wall 边的 Di/Df 分开;R2 不可达整件基线路径失败,不插补。若 R2 失败,保留已完成 N/E 部分并记 PARTIAL_R2_FAILED,不把部分完成写成完整成功。零中位数或早期失败也原样保留。所有原始无效及拒绝记录保留。 + +本次不拟合 base standardizer;后续必须按冻结外折/学习池全节点拟合,同一统计给 N/E,不用验证或测试节点。无标签读取、优化器、训练或 GPU。 + +## 实现验收与报告约束 + +新增 6/6 合成检查通过:白名单不会打开标签成员、抽样和原 ID/次数不变、非正中位数失败、S2 不继承 gate 筛选、完整真实入射边汇总、阶段剩余墙钟和 supervisor CPU 扣账。监督器先前 6/6 记录及源码哈希同时附带;没有重复运行旧全套几何或盲审检查。 + +按 Claude 最新要求,后续每张学习曲线表/图的 25/50% 点必须标 **component fraction**,同时给每折实际件数和节点数。固定 15 行计数见 `public/learning-curve-x-axis.csv`;25% 的 58–145 件差异保留,不重抽样,也不把它解释成等件数预算。100% 主对比不改。 diff --git a/tools/prepare_s2_pilot_inputs.py b/tools/prepare_s2_pilot_inputs.py new file mode 100644 index 000000000..5ebe638c9 --- /dev/null +++ b/tools/prepare_s2_pilot_inputs.py @@ -0,0 +1,88 @@ +"""Freeze fold-0 non-test preparation after completed ten-part measurement.""" +import json +from pathlib import Path +import shutil +from freeze_g1_components import sha + +ROOT=Path('artifacts/s2-pilot-inputs-2026-11-15') +TEN=Path('artifacts/s2-ten-part-measurement-2026-11-15') + + +def main(): + measured=json.loads((TEN/'public/measurement-result.json').read_text()) + ten_contract=json.loads((TEN/'contract.json').read_text()) + assert measured['attempted']==10 and not measured['not_run'],'Finish fixed ten-part evidence first' + assert not ROOT.exists(),'Do not overwrite prior preparation' + parts=json.loads(Path('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json').read_text()) + role_lists={r:sorted(p['case'] for p in parts if p['assignments'][0]['role']==r) for r in ['learning','validation','test']} + assert [len(role_lists[k]) for k in ['learning','validation','test']]==[387,112,125] + manifest=dict(fold=0,seed=0,pool=100,learning_parts=role_lists['learning'],validation_parts=role_lists['validation'],sealed_test_parts=role_lists['test'], + split_sha256=sha('artifacts/s2-g1-components-2026-11-15/public/splits.json'),ruling='1794680129.303719') + by_case={p['case']:p for p in parts} + plan=json.loads(Path('artifacts/s2-g1-candidates-2026-11-14/custodian/plan.json').read_text()) + g1=json.loads(Path('artifacts/s2-g1-registration-2026-11-14/custodian/plan.json').read_text())['geometry_cache'] + settings_path=Path('artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json') + settings={r['member'].removesuffix('.log'):r for r in json.loads(settings_path.read_text())} + geometry_counts={r['case']:r for r in __import__('csv').DictReader(open('artifacts/s2-e1-2026-11-14/preflight/geometry_counts.csv'))} + passed={r['case']:r for r in measured['rows'] if r['status']=='PASS' and r.get('preparation_status')=='COMPLETE'} + prior_failed={r['case'] for r in measured['rows'] if r['case'] not in passed} + candidates=sorted(role_lists['learning']+role_lists['validation']);cases=[] + for case in candidates: + p=by_case[case];meta=geometry_counts[case];setting=settings[case] + # Census field spelling is checked below, rather than guessed at execution. + tets=int(meta['tet_cells']) + spec=dict(case=case,numeric_prefix=int(case.split('_')[0]),source_key=p['source_key'],component_id=p['component_id'], + tet_nodes=p['nodes'],tet_cells=tets,t_set=setting['setting']['seconds'],t_set_provenance=setting, + g1_geometry=g1[case]['path'],input_sha256={g1[case]['path']:g1[case]['sha256'],str(settings_path):sha(settings_path)}) + if case in prior_failed: + spec['prior_failed_preparation']=str(TEN/'parts'/case) + elif case in passed: + source=TEN/'parts'/case;spec.update(reuse_preparation=str(source),reuse_result_sha256=sha(source/'result.json')) + else: + count_root=Path('artifacts/s1-panel-precheck-2026-11-14')/case + if (count_root/'result.json').exists(): + result=json.loads((count_root/'result.json').read_text());assert result['status']=='COUNT_COMPLETE' + spec.update(legacy_geometry=str(count_root/'geometry.npz'),existing_distance=str(count_root/'boundary_distance_raw.npy')) + for name in ['geometry.npz','boundary_distance_raw.npy']:spec['input_sha256'][str(count_root/name)]=result['output_sha256'][name] + primary=Path('artifacts/s1-sampled-draw-2026-11-14')/case/'draw.npz' + if primary.exists(): + dr=json.loads((primary.parent/'receipt.json').read_text());assert dr['status']=='DRAW_COMPLETE' + spec['existing_primary_draw']=str(primary);spec['input_sha256'][str(primary)]=dr['output_sha256']['draw.npz'] + o2=Path('artifacts/s1-sampled-o2-2026-11-14/panel')/(case+'_original') + if (o2/'result.json').exists(): + old=json.loads((o2/'result.json').read_text());assert old['status']=='COMPLETE' + spec.update(existing_sampled_o2_nodes=str(o2/'nodes.npz'),existing_sampled_o2_result=str(o2/'result.json')) + spec['input_sha256'][str(o2/'nodes.npz')]=old['output_sha256']['nodes.npz'];spec['input_sha256'][str(o2/'result.json')]=sha(o2/'result.json') + cases.append(spec) + assert len(cases)==499 and not ({s['case'] for s in cases}&set(role_lists['test'])) + new=[r for r in measured['rows'] if r['case'] in passed and not r['case'].startswith('00000666')] + assert new,'Need measured newly prepared cases' + rate=sum(r['cpu_seconds'] for r in new)/sum(r['nodes'] for r in new) + bpn=max(r['output_disk_bytes']/r['nodes'] for r in measured['rows'] if r['case'] in passed) + # Conservative planning charges the full measured preparation rate even to + # reusable parts, and reserves 25% for label extraction / corpus differences. + projected_cpu=rate*sum(s['tet_nodes'] for s in cases)*1.25/3600 + projected_gib=bpn*sum(s['tet_nodes'] for s in cases)*1.25/2**30 + ROOT.mkdir();(ROOT/'public').mkdir() + (ROOT/'role-manifest.json').write_text(json.dumps(manifest,indent=2)+'\n') + added=['tools/s2_preprocess_pilot.py','tools/s2_pilot_prepare.py','tools/s2_pilot_data.py','tools/run_s2_pilot_preparation.py','tools/prepare_s2_pilot_inputs.py','tools/test_s2_pilot_data.py','tools/test_s2_pilot_preparation.py'] + paths=list(ten_contract['code_sha256'])+added + c=dict(ten_contract,cases=cases,version='S2-pilot-non-test-preparation-v1',code_sha256={p:sha(p) for p in paths}, + authorizing_rulings=['1794679459.056999','1794680129.303719'],role_manifest_sha256=sha(ROOT/'role-manifest.json'), + ten_measurement_sha256=sha(TEN/'public/measurement-result.json'),initial_cpu_seconds_per_node=rate, + initial_projection_cpu_hours=projected_cpu,initial_projection_gib=projected_gib, + projection_method='Pooled CPU/node from successful nine new measurement cases excluding cached 666, times all 499 roster nodes, plus 25% reserve; max measured output bytes/node plus 25%', + limits=dict(worker_cpu_hours=60,workers=16,disk_bytes=100*2**30,per_part_wall_seconds=1800,per_part_tree_rss_bytes=1342177280,retries=0,replacements=0), + input_policy='Only fixed fold0 learning/validation labels may be decoded; sealed tests rejected before IO. Fill_time only. No fitting or GPU.', + failure_policy='Retain/drop failed pilot parts with reason; no replacements or retries; original component identity preserved; not a main-matrix completeness waiver', + checkpoint='Exactly first 50 terminal jobs before launching job 50; separate reused/new observed per-node CPU rates, extrapolate disk; stop/report if over total cap') + (ROOT/'contract.json').write_text(json.dumps(c,indent=2)+'\n') + report=dict(status='READY_WITHIN_PLANNING_CAPS' if projected_cpu<=60 and projected_gib<=100 else 'PROJECTION_OVER_CAP_REPORT_BEFORE_LAUNCH', + parts=499,learning=387,validation=112,sealed_test=125,fold=0,seed=0,projected_worker_cpu_hours=projected_cpu,projected_gib=projected_gib, + cpu_seconds_per_node=rate,full_preparations_reused=sum(bool(s.get('reuse_preparation')) for s in cases), + S1_sampled_O2_reused=sum(bool(s.get('existing_sampled_o2_nodes')) for s in cases),contract_sha256=sha(ROOT/'contract.json'), + role_manifest_sha256=sha(ROOT/'role-manifest.json'),labels_read=False,workers_started=False) + (ROOT/'public/prelaunch-projection.json').write_text(json.dumps(report,indent=2)+'\n');print(json.dumps(report)) + + +if __name__=='__main__':main() diff --git a/tools/prepare_s2_ten_part_measurement.py b/tools/prepare_s2_ten_part_measurement.py new file mode 100644 index 000000000..e216cda9e --- /dev/null +++ b/tools/prepare_s2_ten_part_measurement.py @@ -0,0 +1,72 @@ +"""Freeze the existing ten-part list and data/code whitelist before any worker.""" +import csv +import json +from pathlib import Path +import shutil +from freeze_g1_components import sha + +ROOT=Path('artifacts/s2-ten-part-measurement-2026-11-15') + + +def main(): + ROOT.mkdir(exist_ok=False);(ROOT/'public').mkdir() + plan_path=Path('artifacts/s2-e1-2026-11-14/preflight/ten_part_plan.csv') + rows=list(csv.DictReader(plan_path.open())) + assert [int(r['numeric_prefix']) for r in rows]==[1130,1324,842,367,746,777,1552,1379,610,666] + for r in rows: + for k in ['numeric_prefix','tet_nodes','tet_cells','tertile']:r[k]=int(r[k]) + log_path=Path('artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json') + settings={r['member'].removesuffix('.log'):r for r in json.loads(log_path.read_text())} + g1=json.loads(Path('artifacts/s2-g1-registration-2026-11-14/custodian/plan.json').read_text())['geometry_cache'] + original_o2=Path('artifacts/o2-rep-v1.1a-2026-11-14') + old_contract=json.loads((original_o2/'execution_contract.json').read_text()) + for r in rows: + case=r['case'];setting=settings[case] + assert setting['setting_before_filling_phase'] and setting['setting']['seconds']>0 + r.update(t_set=setting['setting']['seconds'],t_set_provenance=setting, + g1_geometry=g1[case]['path'],input_sha256={g1[case]['path']:g1[case]['sha256'],str(log_path):sha(log_path)}) + primary=Path('artifacts/s1-sampled-draw-2026-11-14')/case/'draw.npz' + if primary.exists():r['existing_primary_draw']=str(primary);r['input_sha256'][str(primary)]=sha(primary) + if r['numeric_prefix']==666: + old=next(s for s in old_contract['cases'] if s['case']==case) + result_path=original_o2/(case+'_original')/'result.json';result=json.loads(result_path.read_text()) + r.update(legacy_geometry=old['sample'],existing_distance=old['distance'],existing_o2_nodes=str(result_path.parent/'nodes.npz'),existing_o2_result=str(result_path)) + for path in [r['legacy_geometry'],r['existing_distance']]:r['input_sha256'][path]=old_contract['input_sha256'][path] + r['input_sha256'][r['existing_o2_nodes']]=result['full_output_sha256']['nodes.npz'] + r['input_sha256'][str(result_path)]=sha(result_path) + for p,h in r['input_sha256'].items():assert sha(p)==h,p + paths=['tools/s2_preprocess.py','tools/run_s2_ten_part_measurement.py','tools/prepare_s2_ten_part_measurement.py','tools/test_s2_preprocess.py', + 'tools/s2_e1_supervisor.py','tools/freeze_g1_components.py','tools/g1_candidate_fingerprints.py','tools/g1_registration_cost.py', + 'tools/audit_samples.py','tools/boundary_distance.py','tools/run_e6_real_kernel.py','tools/o2_rep.py','tools/s2_models/features.py', + 'tools/run_volume_quadrature_trial.py','tools/projector_remesh_probe.py','tools/route_feasibility.py','tools/physical_support_probe.py','tools/analyze_support_capacity_conditions.py'] + contract=dict(version='S2-ten-part-preparation-v1',authorizing_rulings=['1794679459.056999'],fixed_split_commit='fb827178eca01ff71f9854c3eb3e985cf5d808fa', + cases=rows,code_sha256={p:sha(p) for p in paths},plan_sha256=sha(plan_path), + archive='data.zip',archive_stat=json.loads(Path('artifacts/intake-2026-11-13/archive_verification.json').read_text()), + limits=dict(stage_wall_seconds=10800,stage_total_cpu_seconds=10800,stage_disk_bytes=5*2**30,per_part_wall_seconds=1800,per_part_rss_bytes=1342177280,parallel_workers=1,numerical_threads=1,retries=0,replacements=0), + input_policy='Geometry-prefix VTU only; explicit geometry NPY whitelist for legacy 666 container; no fill_time array decoding. Existing process setting provenance only. No fitted statistics or labels.', + scales='full-node lumped lower weighted median d: t_part=4*median; R=10*median; full-part lumped center and Rg; no fallback', + draw='reuse any frozen S1 primary draw; otherwise PCG64(SeedSequence([2026111400,prefix])), 2000 PPS replacement slots, d>0, ascending tet/raw IDs', + eligibility='S2-v1-no-gate-q: d>0, O2 valid, N_eff>=64, complete incident-edge block with at least one finite valid endpoint; finite g and edge summary', + cache_policy='Reuse immutable compatible geometry/distance/O2; report new and reused stages separately. Nine missing distance/O2 fields are computed once. Largest 666 full O2 reused at drawn nodes.', + failure_policy='Each preselected part attempted at most once; retain failure and partial outputs; continue other parts within total deadline; never impute unreachable R2 graph or zero scale', + output='base10 all nodes; R2-G24 all nodes20; common query IDs/counts, g2 and E4 with coverage/rejections; no base standardizer fit, labels, optimizer, training or GPU', + pilot='Subsequent stage only, after ten-part evidence; 4 runs total<=2 GPU-h, each<=30min, test fold sealed, no retry') + (ROOT/'contract.json').write_text(json.dumps(contract,indent=2)+'\n') + shutil.copy2(plan_path,ROOT/'public/ten-part-plan.csv') + test=dict(status='PASS',new_adapter_synthetic_checks=6,passed=6,observed_unittest_seconds=.035, + names=['geometry-only whitelist','fixed draws and original IDs','nonpositive scale rejection','S2 no gate eligibility','all incident edge endpoints','stage wall and shared CPU budget'], + unchanged_supervisor_previous_checks=6,previous_receipt='artifacts/s2-e1-2026-11-14/preflight/synthetic-test-receipt.json', + previous_receipt_sha256=sha('artifacts/s2-e1-2026-11-14/preflight/synthetic-test-receipt.json'), + supervisor_source_matches_prior=sha('tools/s2_e1_supervisor.py')=='6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02', + source_sha256=contract['code_sha256'],labels_fit_gpu=False) + assert test['supervisor_source_matches_prior'] + (ROOT/'public/implementation-acceptance.json').write_text(json.dumps(test,indent=2)+'\n') + splits=json.loads(Path('artifacts/s2-g1-components-2026-11-15/public/splits.json').read_text()) + with (ROOT/'public/learning-curve-x-axis.csv').open('x') as f: + w=csv.writer(f);w.writerow(['fold','component_fraction_percent','actual_components','actual_parts','actual_nodes','x_axis_label']) + for fold in splits['folds']: + for pool,count in fold['counts']['pools'].items():w.writerow([fold['fold'],pool,count['components'],count['parts'],count['nodes'],'component fraction']) + print(json.dumps(dict(status='CONTRACT_FROZEN_NO_WORKER_YET',contract_sha256=sha(ROOT/'contract.json'),parts=10,legacy_666_reused=True))) + + +if __name__=='__main__':main() diff --git a/tools/run_s2_pilot_preparation.py b/tools/run_s2_pilot_preparation.py new file mode 100644 index 000000000..006cc0c01 --- /dev/null +++ b/tools/run_s2_pilot_preparation.py @@ -0,0 +1,124 @@ +"""16 guarded workers, shared CPU/disk caps, exact first-50 terminal barrier.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import resource +import signal +import subprocess +import sys +import time +import psutil +from freeze_g1_components import sha +from s2_e1_supervisor import supervise,save,disk_usage,stop_group + + +def contract(root): + c=json.loads((root/'contract.json').read_text()) + for p,h in c['code_sha256'].items():assert sha(p)==h,p + return c + + +def guard(root,index): + c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True) + r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker', + dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280, + disk_bytes=100*2**30,max_processes=1),poll=.25) + save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]))) + return 0 if r['status']=='PASS' else 1 + + +def projection(c,done): + by_index={r['index']:r for r in done};observed={False:[0.,0],True:[0.,0]} + for i,r in by_index.items(): + cached=bool(c['cases'][i].get('reuse_preparation'));observed[cached][0]+=r['worker']['cpu_seconds'];observed[cached][1]+=c['cases'][i]['tet_nodes'] + rate={key:(cpu/nodes if nodes else c['initial_cpu_seconds_per_node']) for key,(cpu,nodes) in observed.items()} + spent=sum(r['worker']['cpu_seconds'] for r in done) + remaining=sum(rate[bool(s.get('reuse_preparation'))]*s['tet_nodes'] for i,s in enumerate(c['cases']) if i not in by_index) + return dict(projected_worker_cpu_seconds=spent+remaining,spent_worker_cpu_seconds=spent, + observed_rate_seconds_per_node={str(k):v for k,v in rate.items()},method='Separate reused-complete and newly prepared observed node rates; initial rate only for unseen category') + + +def terminate(active): + for process,_ in active.values(): + try: + for child in psutil.Process(process.pid).children():stop_group(child.pid) + except psutil.NoSuchProcess:pass + + +def run(root): + with (root/'batch.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + return run_locked(root) + + +def run_locked(root): + c=contract(root) + assert not (root/'batch-started.json').exists(),'Batch already attempted; inspect receipts, do not restart' + assert c['initial_projection_cpu_hours']<=60 and c['initial_projection_gib']<=100 + env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') + start=time.monotonic();save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'))) + active={};done=[];next_index=0;stop_reason=None;checkpoint=False;peak_rss=0 + def interrupted(*_): + nonlocal stop_reason + stop_reason='COORDINATOR_INTERRUPTED' + signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) + try: + while active or (next_index=60*3600:stop_reason='GLOBAL_CPU_LIMIT' + if used>=100*2**30:stop_reason='GLOBAL_DISK_LIMIT' + if len(done)==min(50,len(c['cases'])) and not active and not checkpoint and stop_reason is None: + proj=projection(c,done);finished_nodes=sum(c['cases'][r['index']]['tet_nodes'] for r in done) + projected_disk=used/finished_nodes*sum(s['tet_nodes'] for s in c['cases']) + checkpoint=True + if proj['projected_worker_cpu_seconds']>60*3600 or projected_disk>100*2**30:stop_reason='FIRST_50_PROJECTION_OVER_CAP' + save(root/'first-50-checkpoint.json',dict(**proj,projected_disk_bytes=projected_disk,continue_within_cap=stop_reason is None,completed=len(done),failed=sum(r['worker']['status']!='PASS' for r in done))) + print(json.dumps(dict(event='FIRST_50_CHECKPOINT',**proj,projected_disk_bytes=projected_disk,stop_reason=stop_reason)),flush=True) + if stop_reason:terminate(active) + save(root/'progress.json',dict(completed=len(done),active=list(active),next_index=next_index,first50_complete=checkpoint,worker_cpu_seconds=cpu, + live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,elapsed_seconds=time.monotonic()-start,stop_reason=stop_reason)) + if active:time.sleep(.5) + finally: + terminate(active) + for p,log in active.values(): + try:p.wait(timeout=10) + except subprocess.TimeoutExpired:stop_group(p.pid);p.wait() + log.close() + result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) else 'STOPPED',stop_reason=stop_reason,jobs=done, + not_run_indices=list(range(next_index,len(c['cases']))),worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done), + guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), + wall_seconds=time.monotonic()-start,peak_aggregate_rss_bytes=peak_rss,disk_bytes=disk_usage(root), + failed=sum(r['worker']['status']!='PASS' for r in done),retry=False,replacement=False,test_labels_read=False) + save(root/'batch-complete.json',result);print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True) + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args() + sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root)) diff --git a/tools/run_s2_pilot_training.py b/tools/run_s2_pilot_training.py new file mode 100644 index 000000000..b10d24c15 --- /dev/null +++ b/tools/run_s2_pilot_training.py @@ -0,0 +1,126 @@ +"""One fixed pilot arm/width. Must be launched inside the GPU resource guard.""" +import argparse +from collections import Counter +from collections.abc import Mapping +import json +from pathlib import Path +import random +import time +import traceback +import numpy as np +import torch +from freeze_g1_components import sha +from s2_e1_supervisor import save +from s2_models.network import QueryModel,TRAINING_CONTRACT +from s2_models.normalization import BaseStandardizer,fit_base_standardizer +from s2_models.objectives import relative_l2 +from s2_pilot_data import permitted_role +from s2_pilot_train import part_weights,train_part,component_score + + +class LearningBases(Mapping): + def __init__(self,records,ids,roles):self.records=records;self.ids=tuple(ids);self.roles=roles + def __iter__(self):return iter(self.ids) + def __len__(self):return len(self.ids) + def __getitem__(self,case): + if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization') + return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r') + + +def load_part(record,roles,device): + case=record['case'];permitted_role(roles,case) + root=Path(record['preparation_root']);label=Path(record['label_path']) + assert sha(root/'base.npy')==record['input_sha256']['base.npy'] + assert sha(root/'queries.npz')==record['input_sha256']['queries.npz'] + assert sha(label)==record['input_sha256']['labels.npz'] + with np.load(root/'queries.npz',allow_pickle=False) as q: + mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask] + with np.load(label,allow_pickle=False) as z: + assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity']) + target=z['target_t_star'];seconds=z['seconds'] + base=np.load(root/'base.npy',allow_pickle=False) + if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch') + tensor=lambda a,dtype=torch.float32:torch.as_tensor(np.array(a,copy=True),dtype=dtype,device=device) + return dict(base=tensor(base),ids=tensor(ids,torch.long),gaps=tensor(g),edge=tensor(edge),target=tensor(target), + counts=counts,truth_seconds=seconds) + + +def run(root,index): + started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index] + dest=root/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json'))) + try: + for p,h in c['code_sha256'].items():assert sha(p)==h,p + source=Path(c['success_manifest']);assert sha(source)==c['success_manifest_sha256'] + manifest=json.loads(source.read_text());records={r['case']:r for r in manifest['ready_parts']} + roles=json.loads(Path(c['role_manifest']).read_text());assert sha(c['role_manifest'])==c['role_manifest_sha256'] + learning=sorted(p for p in records if permitted_role(roles,p)=='learning') + validation=sorted(p for p in records if permitted_role(roles,p)=='validation') + if not learning or not validation:raise ValueError('Empty successful learning/validation roster') + if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest') + assert run['seed']==0 and c['fold']==0 and c['pool']==100 + torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed']) + if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required') + device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats() + torch.backends.cuda.matmul.allow_tf32=False + if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100') + # Every base statistics calculation is learning-only and inside this run's wall guard. + stats=fit_base_standardizer(LearningBases(records,learning,roles),learning_part_ids=learning,fold='0',pool='100') + save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint)) + shared=root/'shared-standardizer-fingerprint.json' + if shared.exists():assert json.loads(shared.read_text())['fingerprint']==stats.fingerprint + else:save(shared,dict(fingerprint=stats.fingerprint,learning_parts=learning)) + model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device) + optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5) + scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=200) + scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed'])) + group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts) + val_groups={p:records[p]['component_id'] for p in validation} + best=float('inf');best_epoch=None;stale=0;history=[];updates=0 + for epoch in range(200): + model.train();loss_sum=0. + order=[learning[int(j)] for j in rng.permutation(len(learning))] + for j,case in enumerate(order): + record=records[case];data=load_part(record,roles,device) + weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device) + loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True) + loss_sum+=loss/len(learning);updates+=1 + del data,weights + if j%16==0: + torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started, + peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())) + model.eval();errors={} + with torch.no_grad(): + for case in validation: + data=load_part(records[case],roles,device) + with torch.autocast(device_type='cuda',dtype=torch.float16): + pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None) + predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set'] + score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool)) + errors[case]=score['value'];del data,pred + value=component_score(errors,val_groups);torch.cuda.synchronize() + row=dict(epoch=epoch,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started, + peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()) + history.append(row);save(dest/'history.json',history) + if value=20:break + torch.cuda.synchronize() + result=dict(status='COMPLETE',run=run,epochs=len(history),updates=updates,best_epoch=best_epoch,best_validation_score=best, + learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())), + failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint, + wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(), + test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt')) + save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0 + except Exception as exc: + save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started)) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index)) diff --git a/tools/run_s2_ten_part_measurement.py b/tools/run_s2_ten_part_measurement.py new file mode 100644 index 000000000..e0a70ba51 --- /dev/null +++ b/tools/run_s2_ten_part_measurement.py @@ -0,0 +1,49 @@ +"""One sequential, non-resumable ten-part attempt with shared 3h/5GiB limits.""" +import argparse +import fcntl +import json +from pathlib import Path +import sys +import time +from s2_e1_supervisor import supervise,save,disk_usage +from freeze_g1_components import sha + + +def remaining_limits(elapsed, cpu, guard_cpu): + wall=10800-elapsed;total_worker_cpu=10800-guard_cpu + if wall<=0 or total_worker_cpu-cpu<=0:return None + return dict(total_cpu_seconds=total_worker_cpu,job_cpu_seconds=min(1800,total_worker_cpu-cpu), + job_wall_seconds=min(1800,wall),tree_rss_bytes=1342177280,disk_bytes=5*2**30,max_processes=1) + + +def run(root): + contract=json.loads((root/'contract.json').read_text()) + for p,h in contract['code_sha256'].items():assert sha(p)==h,p + with (root/'batch.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + with (root/'batch-started.json').open('x') as f:json.dump(dict(started_unix=time.time(),contract_sha256=sha(root/'contract.json')),f) + start=time.monotonic();cpu_start=time.process_time();worker_cpu=0.;rows=[];stop_reason=None + for i,spec in enumerate(contract['cases']): + limits=remaining_limits(time.monotonic()-start,worker_cpu,time.process_time()-cpu_start) + if limits is None:stop_reason='total_wall_or_cpu_budget_exhausted';break + if disk_usage(root)>=5*2**30:stop_reason='total_disk_budget_exhausted';break + name=f'part-{i:02d}' + r=supervise([sys.executable,'tools/s2_preprocess.py',str(root),str(i)],root,name,limits,poll=.1) + worker_cpu+=r['cpu_seconds'];row=dict(index=i,case=spec['case'],**r);rows.append(row) + print(json.dumps({k:row[k] for k in ['index','case','status','stop_reason','cpu_seconds','wall_seconds','peak_tree_rss_bytes']}),flush=True) + save(root/'batch-progress.json',dict(completed_attempts=len(rows),worker_cpu_seconds=worker_cpu,supervisor_cpu_seconds=time.process_time()-cpu_start, + wall_seconds=time.monotonic()-start,last=row)) + unrun=[dict(index=i,case=contract['cases'][i]['case'],status='NOT_RUN_BUDGET') for i in range(len(rows),10)] + data=dict(status='ATTEMPTS_COMPLETE' if len(rows)==10 else 'STOPPED_AT_STAGE_BUDGET',attempts=rows,not_run=unrun, + stop_reason=stop_reason,failed_attempts=sum(r['status']!='PASS' for r in rows),retry=False,replacement=False, + worker_cpu_seconds=worker_cpu,supervisor_cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start, + peak_worker_tree_rss_bytes=max((r['peak_tree_rss_bytes'] for r in rows),default=0),disk_bytes=disk_usage(root), + contract_sha256=sha(root/'contract.json'),labels_read=False,training_gpu=False) + if data['wall_seconds']>10800:data['stage_wall_budget_exceeded']=True + if worker_cpu+data['supervisor_cpu_seconds']>10800:data['stage_cpu_budget_exceeded']=True + save(root/'batch-complete.json',data) + print(json.dumps({k:v for k,v in data.items() if k not in ['attempts','not_run']}),flush=True) + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root) diff --git a/tools/s2_gpu_process_supervisor.py b/tools/s2_gpu_process_supervisor.py new file mode 100644 index 000000000..5c9c67906 --- /dev/null +++ b/tools/s2_gpu_process_supervisor.py @@ -0,0 +1,165 @@ +"""Pilot process guard derived from frozen E1 guard; explicit CUDA visibility. + +Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot +is measured and fails the job. Run only reviewed label-free worker commands. +""" +import fcntl +import ctypes +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import time + +import psutil + + +def save(path, data): + path = Path(path) + tmp = path.with_suffix(path.suffix + '.tmp') + tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n') + tmp.replace(path) + + +def disk_usage(root): + logical = allocated = 0 + for p in Path(root).rglob('*'): + if p.is_file() and not p.is_symlink(): + s = p.stat() + logical += s.st_size + allocated += s.st_blocks * 512 + return max(logical, allocated) + + +def tree_usage(pid): + try: + root = psutil.Process(pid) + processes = [root] + root.children(recursive=True) + except psutil.NoSuchProcess: + return 0., 0, 0 + cpu = 0. + rss = count = 0 + for p in processes: + try: + t = p.cpu_times() + # Linux child times include reaped descendants, not live children. + cpu += t.user + t.system + t.children_user + t.children_system + rss += p.memory_info().rss + count += 1 + except psutil.NoSuchProcess: + pass + return cpu, rss, count + + +def stop_group(pid): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def supervise(command, root, name, limits, *, poll=.02, gpu_uuid=None): + """One locked attempt; no automatic retry/replacement; account failed CPU.""" + root = Path(root) + root.mkdir(parents=True, exist_ok=True) + with (root/'execution.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + started_path = root/(name+'.started.json') + receipt_path = root/(name+'.receipt.json') + if started_path.exists() or receipt_path.exists(): + raise FileExistsError('Attempt already exists; no silent retry') + if any(not math.isfinite(v) or v <= 0 for v in limits.values()): + raise ValueError('Every limit must be finite and positive') + ledger_path = root/'ledger.json' + ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]} + if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']: + raise RuntimeError('Total CPU exhausted before spawn') + if disk_usage(root) >= limits['disk_bytes']: + raise RuntimeError('Disk exhausted before spawn') + save(started_path, {'command':command,'limits':limits,'started_unix':time.time()}) + start = last = time.monotonic() + cpu_peak = rss_peak = max_processes = max_gap = 0 + reason = None + exit_status = usage = None + env = dict(os.environ, OMP_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', + MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES=str(gpu_uuid or '')) + with (root/(name+'.log')).open('x') as log: + libc = ctypes.CDLL(None, use_errno=True) + previous_subreaper = ctypes.c_int() + if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping') + proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, + env=env, start_new_session=True) + try: + while True: + now = time.monotonic() + max_gap = max(max_gap, now-last) + last = now + cpu, rss, count = tree_usage(proc.pid) + cpu_peak = max(cpu_peak,cpu) + rss_peak = max(rss_peak,rss) + max_processes = max(max_processes,count) + used_disk = disk_usage(root) + if now-start >= limits['job_wall_seconds']: + reason = 'wall_limit' + elif rss > limits['tree_rss_bytes']: + reason = 'tree_rss_limit' + elif count > limits['max_processes']: + reason = 'process_count_limit' + elif used_disk >= limits['disk_bytes']: + reason = 'disk_limit' + elif cpu_peak >= limits['job_cpu_seconds']: + reason = 'job_cpu_limit' + elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']: + reason = 'total_cpu_limit' + if reason: + stop_group(proc.pid) + pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG) + if pid: + exit_status, usage = status, child_usage + break + time.sleep(poll) + finally: + # No detached work may outlive a supervised attempt. + stop_group(proc.pid) + if exit_status is None: + _, exit_status, usage = os.wait4(proc.pid, 0) + proc.returncode = os.waitstatus_to_exitcode(exit_status) + orphan_cpu = 0. + orphan_peak = 0 + # Killed descendants are adopted here rather than losing their + # final CPU usage when the root worker cannot reap them itself. + while True: + try: + _, _, orphan_usage = os.wait4(-proc.pid, 0) + except ChildProcessError: + break + orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime + orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024) + if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state') + elapsed = time.monotonic()-start + charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu) + rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak) + used_disk = disk_usage(root) + if reason is None: + if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit' + elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit' + elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit' + elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit' + elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit' + receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED', + command=command,returncode=proc.returncode,stop_reason=reason, + cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak, + peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk, + max_poll_gap_seconds=max_gap,poll_target_seconds=poll, + reaped_orphan_cpu_seconds=orphan_cpu, + supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure', + gpu_visible=bool(gpu_uuid),gpu_uuid=gpu_uuid,retry=False) + ledger['cpu_seconds_charged'] += charged + ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged}) + save(receipt_path,receipt) + save(ledger_path,ledger) + return receipt diff --git a/tools/s2_pilot_data.py b/tools/s2_pilot_data.py new file mode 100644 index 000000000..3c0256b96 --- /dev/null +++ b/tools/s2_pilot_data.py @@ -0,0 +1,52 @@ +"""Pilot label access is limited to an immutable non-test role manifest.""" +import json +from pathlib import Path +from xml.etree import ElementTree as ET +from zipfile import ZipFile +import numpy as np +from audit_samples import decode_array + + +def permitted_role(manifest, case): + if case in manifest['sealed_test_parts']: + raise PermissionError('Outer test labels are sealed') + if case in manifest['learning_parts']:return 'learning' + if case in manifest['validation_parts']:return 'validation' + raise PermissionError('Case absent from frozen non-test manifest') + + +def read_non_test_fill_time(archive, case, raw_ids, manifest): + # Must reject forbidden cases before opening the archive or any member. + role=permitted_role(manifest,case) + raw_ids=np.asarray(raw_ids) + if raw_ids.ndim!=1 or raw_ids.dtype.kind not in 'iu' or len(raw_ids)==0 or np.any(np.diff(raw_ids)<=0): + raise ValueError('Ascending original raw IDs required') + stack=[];count=None;header=None + with ZipFile(archive) as z,z.open(case+'_sim.vtu') as stream: + for event,e in ET.iterparse(stream,events=('start','end')): + if event=='start': + stack.append(e.tag) + if e.tag=='VTKFile': + if e.get('byte_order')!='LittleEndian' or e.get('compressor')!='vtkZLibDataCompressor':raise ValueError('Unsupported encoding') + header=e.get('header_type','UInt32') + elif e.tag=='Piece': + if count is not None:raise ValueError('Multiple VTU pieces') + count=int(e.get('NumberOfPoints')) + else: + if e.tag=='DataArray': + if stack[-2]=='PointData' and e.get('Name')=='Fill_time': + values=decode_array(e,header) + if values.shape!=(count,) or raw_ids[0]<0 or raw_ids[-1]>=count:raise ValueError('Fill_time size/original IDs mismatch') + return values[raw_ids].copy(),dict(role=role,array_name='Fill_time',association='PointData',decoded_label_arrays=['Fill_time'], + other_field_arrays_decoded=[],archive_member=case+'_sim.vtu',all_raw_nodes=count,retained_tet_nodes=len(raw_ids)) + e.clear() + stack.pop() + raise ValueError('PointData Fill_time missing') + + +def validate_query_targets(fill_time, ids, eligible, t_set): + selected=np.asarray(fill_time)[np.asarray(ids)[np.asarray(eligible,dtype=bool)]] + if not len(selected):raise ValueError('No eligible common queries') + if not np.isfinite(selected).all() or np.any(selected<0):raise ValueError('Invalid Fill_time on eligible common queries') + if not np.isfinite(t_set) or t_set<=0:raise ValueError('Missing process setting') + return selected/t_set diff --git a/tools/s2_pilot_gpu_guard.py b/tools/s2_pilot_gpu_guard.py new file mode 100644 index 000000000..d7503d80e --- /dev/null +++ b/tools/s2_pilot_gpu_guard.py @@ -0,0 +1,106 @@ +"""Sequential four-run pilot guard: process-tree RSS/wall plus device VRAM.""" +import argparse +import fcntl +import json +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +import psutil +from freeze_g1_components import sha +from s2_gpu_process_supervisor import supervise,save,stop_group,disk_usage,tree_usage +from s2_pilot_train import select_capacity + + +def parse_device_usage(text): + lines=text.strip().splitlines() + if len(lines)!=1:raise ValueError('Expected exactly one GPU') + fields=lines[0].split(',') + if len(fields)!=2:raise ValueError('Malformed GPU usage row') + memory,util=map(lambda s:int(s.strip()),fields) + if memory<0 or not 0<=util<=100:raise ValueError('Invalid GPU counters') + return memory*2**20,util + + +def device_usage(uuid): + r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=2) + return parse_device_usage(r.stdout) + + +def active_training_pids(parent,root,index): + ids=[] + try:children=psutil.Process(parent).children(recursive=True) + except psutil.NoSuchProcess:return ids + for p in children: + try: + cmd=p.cmdline() + if 'tools/run_s2_pilot_training.py' in cmd and str(root) in cmd and str(index)==cmd[-1]:ids.append(p.pid) + except psutil.NoSuchProcess:pass + return ids + + +def run(root): + with (root/'pilot.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + return run_locked(root) + + +def run_locked(root): + c=json.loads((root/'training-contract.json').read_text());uuid=c['gpu_uuid'] + assert [(r['arm'],r['width'],r['seed']) for r in c['runs']]==[('N',128,0),('E',128,0),('N',256,0),('E',256,0)] + for p,h in c['code_sha256'].items():assert sha(p)==h,p + assert not (root/'pilot-started.json').exists(),'No automatic pilot retry' + start=time.monotonic();save(root/'pilot-started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json'))) + rows=[];scores={};reason=None + for i,spec in enumerate(c['runs']): + remaining=7200-(time.monotonic()-start) + if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break + run_start=time.monotonic();initial,util=device_usage(uuid) + if initial>128*2**20 or util>0:reason='SELECTED_GPU_BUSY_BEFORE_RUN';break + done=threading.Event();monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,max_poll_gap_seconds=0.,reason=None,samples=0) + def watch(): + previous=time.monotonic() + while not done.is_set(): + now=time.monotonic();monitor['max_poll_gap_seconds']=max(monitor['max_poll_gap_seconds'],now-previous);previous=now + try: + used,_=device_usage(uuid);monitor['samples']+=1;monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used) + if used>15*2**30:monitor['reason']='DEVICE_VRAM_LIMIT' + _,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss) + if rss>16*2**30:monitor['reason']='WHOLE_RUN_HOST_RSS_LIMIT' + if disk_usage(root)>=5*2**30:monitor['reason']='PILOT_DISK_LIMIT' + except Exception as exc:monitor['reason']='GPU_MONITOR_FAILED';monitor['error']=str(exc) + if monitor['reason']: + for pid in active_training_pids(os.getpid(),root,i):stop_group(pid) + done.wait(.5) + thread=threading.Thread(target=watch,daemon=True);thread.start() + try: + wall=min(1800,remaining)-(time.monotonic()-run_start)-2.5 + if wall<=0:raise RuntimeError('No budget left before spawn') + command=[sys.executable,'tools/run_s2_pilot_training.py',str(root),str(i)] + r=supervise(command,root/'guards',spec['name'],dict(total_cpu_seconds=7200,job_cpu_seconds=1800,job_wall_seconds=wall, + tree_rss_bytes=16*2**30,disk_bytes=5*2**30,max_processes=2),poll=.1,gpu_uuid=uuid) + finally: + done.set();thread.join(timeout=3) + total_wall=time.monotonic()-run_start + result_path=root/spec['name']/'result.json' + result=json.loads(result_path.read_text()) if result_path.exists() else None + if monitor['reason']:r.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reason']) + if total_wall>min(1800,remaining):r.update(status='STOPPED_OR_FAILED',stop_reason='TOTAL_RUN_WALL_LIMIT') + if result and max(result['peak_allocated_bytes'],result['peak_reserved_bytes'])>15*2**30:r.update(status='STOPPED_OR_FAILED',stop_reason='RECORDED_VRAM_LIMIT') + r.update(gpu_visible=True,gpu_uuid=uuid,device_monitor=monitor,total_run_wall_seconds=total_wall, + result_sha256=sha(result_path) if result else None) + save(root/(spec['name']+'-guard-receipt.json'),r);rows.append(dict(run=spec,**r)) + if r['status']!='PASS' or not result or result['status']!='COMPLETE':reason='PILOT_RUN_FAILED_NO_RETRY';break + scores[(spec['arm'],spec['width'])]=result['best_validation_score'] + if time.monotonic()-start>7200:reason='TOTAL_PILOT_WALL_LIMIT_AFTER_EXIT' + selected=select_capacity(scores) if len(scores)==4 and reason is None else None + summary=dict(status='FOUR_RUNS_COMPLETE' if selected else 'PILOT_INCOMPLETE',reason=reason,runs=rows,selected_global_width=selected, + selection_rule='mean of N/E validation error at each width, ties smaller; all four runs required', + pilot_wall_seconds=time.monotonic()-start,test_labels_read=False,test_predictions=False,retries=0) + save(root/'pilot-complete.json',summary);print(json.dumps(summary),flush=True) + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root) diff --git a/tools/s2_pilot_prepare.py b/tools/s2_pilot_prepare.py new file mode 100644 index 000000000..4353e28ec --- /dev/null +++ b/tools/s2_pilot_prepare.py @@ -0,0 +1,56 @@ +"""Prepare an authorized non-test part once, then extract only its permitted label.""" +import argparse +import json +from pathlib import Path +import sys +import time +import traceback +import numpy as np +from freeze_g1_components import sha +from s2_e1_supervisor import save +from s2_pilot_data import permitted_role,read_non_test_fill_time,validate_query_targets +import s2_preprocess_pilot as s2_preprocess + + +def run(root,index): + start=time.monotonic();c=json.loads((root/'contract.json').read_text());spec=c['cases'][index];case=spec['case'] + manifest=json.loads((root/'role-manifest.json').read_text());role=permitted_role(manifest,case) + dest=root/'ready'/case;dest.mkdir(parents=True,exist_ok=False) + try: + if spec.get('prior_failed_preparation'): + save(dest/'failure.json',dict(case=case,role=role,status='PRIOR_MEASUREMENT_FAILURE_CARRIED_FORWARD', + prior_artifacts=spec['prior_failed_preparation'],new_scientific_attempt=False,labels_read=False)) + return 3 + for p,h in c['code_sha256'].items():assert sha(p)==h,p + assert sha(root/'role-manifest.json')==c['role_manifest_sha256'] + if spec.get('reuse_preparation'): + source=Path(spec['reuse_preparation']);r=json.loads((source/'result.json').read_text()) + assert sha(source/'result.json')==spec['reuse_result_sha256'] and r['status']=='COMPLETE' + for name,h in r['output_sha256'].items():assert sha(source/name)==h + else: + code=s2_preprocess.run(root,index) + if code:raise ValueError('Full input preparation failed; no labels extracted') + source=root/'parts'/case;r=json.loads((source/'result.json').read_text()) + with np.load(source/'geometry.npz',allow_pickle=False) as z:raw_ids=z['raw_node_ids'] + with np.load(source/'queries.npz',allow_pickle=False) as q: + ids=q['anchor_ids'];eligible=q['eligible_s2'];counts=q['multiplicity'] + if not eligible.any():raise ValueError('No common eligible queries; drop pilot part with reason') + st=Path(c['archive']).stat();ver=c['archive_stat'] + assert (st.st_size,st.st_mtime_ns,st.st_ino)==(ver['bytes'],ver['mtime_ns'],ver['inode']) + values,label_meta=read_non_test_fill_time(c['archive'],case,raw_ids,manifest) + target=validate_query_targets(values,ids,eligible,spec['t_set']) + np.savez_compressed(dest/'labels.npz',anchor_ids=ids[eligible],raw_node_ids=raw_ids[ids[eligible]], + multiplicity=counts[eligible],seconds=values[ids[eligible]],target_t_star=target) + receipt=dict(case=case,role=role,status='READY',preparation_root=str(source),preparation_result_sha256=sha(source/'result.json'), + label_provenance=label_meta,label_sha256=sha(dest/'labels.npz'),t_set=spec['t_set'], + component_id=spec['component_id'],nodes=spec['tet_nodes'],eligible_queries=int(eligible.sum()), + eligible_slots=int(counts[eligible].sum()),reused_preparation=bool(spec.get('reuse_preparation')), + wall_seconds=time.monotonic()-start,test_labels_read=False,model_fit=False,gpu=False) + save(dest/'receipt.json',receipt);print(json.dumps(receipt),flush=True);return 0 + except Exception as exc: + save(dest/'failure.json',dict(case=case,role=role,status='FAILED_NO_REPLACEMENT',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-start)) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();sys.exit(run(a.root,a.index)) diff --git a/tools/s2_pilot_train.py b/tools/s2_pilot_train.py new file mode 100644 index 000000000..adf7809d2 --- /dev/null +++ b/tools/s2_pilot_train.py @@ -0,0 +1,48 @@ +"""Bounded pilot training primitives; corpus runs require a frozen ready manifest.""" +import math +from pathlib import Path +import numpy as np +import torch +from s2_models.network import weighted_mse + + +def part_weights(counts, parts_in_component, component_count, learning_parts): + c=np.asarray(counts,dtype=np.float64) + if c.ndim!=1 or not len(c) or np.any(c<=0) or not np.isfinite(c).all():raise ValueError('Invalid common query counts') + if min(parts_in_component,component_count,learning_parts)<=0:raise ValueError('Empty training hierarchy') + # Uniform shuffled part steps estimate the full component-balanced objective. + return c/c.sum()/parts_in_component/component_count*learning_parts + + +def train_part(model,optimizer,base,ids,gaps,edge,target,weights,*,scaler=None,amp=False): + optimizer.zero_grad(set_to_none=True) + with torch.autocast(device_type=base.device.type,dtype=torch.float16,enabled=amp): + prediction=model(base,ids,gaps,edge if model.arm=='E' else None) + loss=weighted_mse(prediction,target,weights) + if not torch.isfinite(loss):raise FloatingPointError('Nonfinite training loss') + if scaler is None: + loss.backward() + else: + scaler.scale(loss).backward();scaler.unscale_(optimizer) + if any(p.grad is not None and not torch.isfinite(p.grad).all() for p in model.parameters()): + raise FloatingPointError('Nonfinite gradient; no silent retry or precision fallback') + if scaler is None:optimizer.step() + else:scaler.step(optimizer);scaler.update() + if any(not torch.isfinite(p).all() for p in model.parameters()):raise FloatingPointError('Nonfinite updated parameter') + return float(loss.detach()) + + +def component_score(part_errors,component_ids): + if not part_errors or set(part_errors)!=set(component_ids):raise ValueError('Incomplete validation roster') + groups={} + for part,value in part_errors.items(): + if value is None or not math.isfinite(value):raise ValueError('Invalid validation part score') + groups.setdefault(component_ids[part],[]).append(value) + return float(np.mean([np.mean(values) for values in groups.values()])) + + +def select_capacity(scores): + # Exactly four run scores, no choice from an incomplete/failed run matrix. + if set(scores)!={(arm,width) for arm in ['N','E'] for width in [128,256]}:raise ValueError('All four pilot runs required') + if any(v is None or not math.isfinite(v) for v in scores.values()):raise ValueError('All four pilot validation scores must be finite') + return min([128,256],key=lambda w:((scores['N',w]+scores['E',w])/2,w)) diff --git a/tools/s2_preprocess.py b/tools/s2_preprocess.py new file mode 100644 index 000000000..8b477a839 --- /dev/null +++ b/tools/s2_preprocess.py @@ -0,0 +1,214 @@ +"""Label-free full-part S2 preparation; original caches and protocols stay unchanged.""" +import argparse +import json +from pathlib import Path +import resource +import time +import traceback +from zipfile import ZipFile +import numpy as np +import yaml +from boundary_distance import SurfaceDistance, triangle_distance_squared +from g1_candidate_fingerprints import read_tets +import run_e6_real_kernel as e6 +import o2_rep as rep +from s2_models import features +from s2_e1_supervisor import save +from freeze_g1_components import sha + +GEOMETRY_MEMBERS = {'points_raw','tetra','node_volume_raw','boundary_triangles','edges','gate_location_raw'} +O2_MEMBERS = {'tensor','kernel_mass','effective_samples','valid','neff_pass','g','B','mu','status','tau0'} + + +def load_selected(path, members, allowed): + if not set(members) <= allowed: raise ValueError('Array outside explicit whitelist') + with ZipFile(path) as z: + return {name:np.load(z.open(name+'.npy'),allow_pickle=False) for name in members} + + +def scales_and_draw(d, mass, raw_ids, prefix, existing=None): + order=np.argsort(d,kind='stable') + median=float(d[order[np.searchsorted(np.cumsum(mass[order]),mass.sum()/2)]]) + if not np.isfinite(median) or median<=0:raise ValueError('Nonpositive full-node weighted median; no substitute radius') + candidates=np.flatnonzero(d>0) + if existing: + with np.load(existing,allow_pickle=False) as z: + draws=z['draw_ids'];ids=z['unique_ids'];counts=z['multiplicity'];inverse=z['draw_to_unique'] + assert np.array_equal(z['draw_raw_ids'],raw_ids[draws]) + else: + rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence([2026111400,prefix]))) + draws=rng.choice(candidates,size=2000,replace=True,p=mass[candidates]/mass[candidates].sum(dtype=np.float64)) + ids,inverse,counts=np.unique(draws,return_inverse=True,return_counts=True) + assert len(draws)==2000 and np.all(d[draws]>0) and np.array_equal(ids[inverse],draws) and counts.sum()==2000 + return dict(draw_ids=draws,draw_raw_ids=raw_ids[draws],unique_ids=ids,unique_raw_ids=raw_ids[ids], + multiplicity=counts,draw_to_unique=inverse),dict(t_part=4*median,radius=10*median, + positive_mass_fraction=float(mass[d>0].sum()/mass.sum()),weighted_median_distance=median) + + +def edge_summary(x, edges, ids, spectral, length_scale): + lookup=np.full(len(x),-1,np.int64);lookup[ids]=np.arange(len(ids)) + edge_idx,side=np.nonzero(lookup[edges]>=0);local=lookup[edges[edge_idx,side]] + n=len(local);q=np.empty((n,2));valid=np.empty(n,bool);direction=np.empty(n,bool) + for start in range(0,n,4096): + stop=min(start+4096,n);e=edges[edge_idx[start:stop]];j=local[start:stop] + result=rep.contract_direction(spectral['B'][j],spectral['mu'][j],spectral['valid'][j], + x[e[:,1]]-x[e[:,0]],length_scale,'edge') + q[start:stop]=result['q'];valid[start:stop]=result['channel_valid'];direction[start:stop]=result['direction_valid'] + accepted=valid&np.isfinite(q).all(1) + degree=np.bincount(local,minlength=len(ids));good_degree=np.bincount(local[accepted],minlength=len(ids)) + total=np.zeros((len(ids),2));maximum=np.full((len(ids),2),-np.inf) + np.add.at(total,local[accepted],q[accepted]);np.maximum.at(maximum,local[accepted],q[accepted]) + mean=np.full_like(total,np.nan);has=good_degree>0;mean[has]=total[has]/good_degree[has,None];maximum[~has]=np.nan + summary=np.column_stack([mean[:,0],maximum[:,0],mean[:,1],maximum[:,1]]) + complete=(degree==np.bincount(edges.ravel(),minlength=len(x))[ids])&has&np.isfinite(summary).all(1) + return summary,complete,dict(edge_indices=edge_idx,endpoint_side=side,anchor_local=local, + q=q,channel_valid=valid,direction_valid=direction),degree,good_degree + + +def s2_eligibility(distance, spectral, complete, summary): + # Gate q is intentionally absent; original gate geometry remains in outputs. + return (distance>0)&spectral['valid']&spectral['neff_pass']&complete&np.isfinite(spectral['g']).all(1)&np.isfinite(summary).all(1) + + +def run(root,index): + c=json.loads((root/'contract.json').read_text());spec=c['cases'][index];case=spec['case'] + dest=root/'parts'/case;dest.mkdir(parents=True,exist_ok=False) + start=time.monotonic();tick=start;timings={};stage='verify_inputs' + def progress(name,**kw): + nonlocal stage + stage=name;save(dest/'progress.json',dict(stage=name,elapsed_seconds=time.monotonic()-start,**kw)) + def done(name): + nonlocal tick + now=time.monotonic();timings[name]=now-tick;tick=now + try: + for p,h in c['code_sha256'].items():assert sha(p)==h,p + for p,h in spec['input_sha256'].items():assert sha(p)==h,p + progress('geometry') + with np.load(spec['g1_geometry'],allow_pickle=False) as z: + cached_x=z['points'];faces=z['boundary'];center=z['center'];rg=float(z['rg']) + if spec.get('legacy_geometry'): + data=load_selected(spec['legacy_geometry'],sorted(GEOMETRY_MEMBERS),GEOMETRY_MEMBERS) + x=data['points_raw'];tet=data['tetra'];mass=data['node_volume_raw'];edges=data['edges'];gate=data['gate_location_raw'] + raw_ids=np.arange(len(x));assert np.array_equal(data['boundary_triangles'],faces) + geometry_mode='existing_cache_explicit_geometry_members_only' + else: + ver=c['archive_stat'];st=Path(c['archive']).stat() + assert (st.st_size,st.st_mtime_ns,st.st_ino)==(ver['bytes'],ver['mtime_ns'],ver['inode']) + with ZipFile(c['archive']) as z: + with z.open(case+'_sim.vtu') as stream:raw_x,raw_tet=read_tets(stream) + gate=np.asarray(yaml.safe_load(z.read(case+'.yml'))['inlet_location'],dtype=float) + raw_ids=np.unique(raw_tet);x=raw_x[raw_ids];tet=np.searchsorted(raw_ids,raw_tet) + del raw_x,raw_tet + _,_,mat,vol,mass,edges,lengths=features.mesh_geometry(x,tet) + del mat,vol,lengths + geometry_mode='missing_tet_topology_from_geometry_prefix_existing_g1_boundary_reused' + assert np.array_equal(x,cached_x);del cached_x + assert len(x)==spec['tet_nodes'] and len(tet)==spec['tet_cells'] + assert np.isfinite(mass).all() and (mass>0).all() and np.isfinite(gate).all() + center_from_mass=np.sum(mass[:,None]*x,axis=0)/mass.sum() + rg_from_mass=np.sqrt(np.sum(mass*np.sum((x-center_from_mass)**2,axis=1))/mass.sum()) + assert np.allclose(center,center_from_mass,rtol=1e-10,atol=1e-10) and np.isclose(rg,rg_from_mass,rtol=1e-10) + np.savez_compressed(dest/'geometry.npz',points_raw=x,tetra=tet,node_volume_raw=mass,edges=edges, + boundary_triangles=faces,raw_node_ids=raw_ids,gate_location_raw=gate) + done('geometry_load_topology_write') + progress('wall_distance') + boundary=np.zeros(len(x),bool);boundary[np.unique(faces)]=True + if spec.get('existing_distance'): + d=np.load(spec['existing_distance'],allow_pickle=False);distance_mode='existing_verified_distance';brute_error=None + else: + d=np.zeros(len(x));interior=np.flatnonzero(~boundary);surface=SurfaceDistance(x[faces]) + for begin in range(0,len(interior),128): + ids=interior[begin:begin+128];d[ids]=surface.query(x[ids]) + progress('wall_distance',done=min(begin+128,len(interior)),total=len(interior)) + probe=np.random.default_rng(1729).choice(len(x),min(30,len(x)),replace=False) + brute=np.array([np.sqrt(triangle_distance_squared(q,x[faces]).min()) for q in x[probe]]) + brute_error=float(abs(brute-d[probe]).max());assert brute_error<=1e-10*max(1.,np.ptp(x,axis=0).max()) + del surface + distance_mode='new_missing_distance_exact_triangle_with_30_brute_checks' + assert np.isfinite(d).all() and (d>=0).all() and np.array_equal(d==0,boundary) + np.save(dest/'boundary_distance_raw.npy',d);done('wall_distance_load_or_compute_and_write') + progress('fixed_draw') + draw,scales=scales_and_draw(d,mass,raw_ids,spec['numeric_prefix'],spec.get('existing_primary_draw')) + ids=draw['unique_ids'];counts=draw['multiplicity'];np.savez_compressed(dest/'draw.npz',**draw);done('draw_and_scales') + progress('base_features') + geo,gate_node=features.graph_distances(x,tet,d,gate,exclude_wall_edges=False) + base=features.base_features(x,d,geo,gate,center,rg,float(mass.sum()),scales['t_part'],spec['t_set']) + np.save(dest/'base.npy',base);del base,geo;done('unrestricted_geodesic_base_write') + progress('o2') + if spec.get('existing_o2_nodes'): + old=json.loads(Path(spec['existing_o2_result']).read_text()) + assert old['acceptance_status']=='PASS' and old['pose']=='original' and old['nodes']==len(x) + assert np.isclose(old['radius'],scales['radius'],rtol=0,atol=1e-13) + loaded=load_selected(spec['existing_o2_nodes'],sorted(O2_MEMBERS),O2_MEMBERS) + spectral={k:v[ids] for k,v in loaded.items()};del loaded + o2_mode='existing_full_original_O2_v1.1a_subset_no_reintegration';visits=None;raw_meta=None + else: + raw,raw_meta=e6.raw_field(x,tet,d);sx,sm,st=e6.samples(x,tet,mass,'tetra_four_point',raw);del raw + visits=0;real_tree=e6.cKDTree + class ProgressTree: + def __init__(self,*args,**kwargs):self.tree=real_tree(*args,**kwargs);self.count=0 + def query_ball_point(self,*args,**kwargs): + nonlocal visits + result=self.tree.query_ball_point(*args,**kwargs);visits+=len(result);self.count+=1 + if self.count%32==0:progress('o2_kernel',anchors_done=self.count,anchors_total=len(ids),neighbor_visits=visits) + return result + e6.cKDTree=ProgressTree + try:a=e6.integrate(x[ids],sx,sm,st,scales['radius']) + finally:e6.cKDTree=real_tree + del sx,sm,st + spectral=dict(**a,**rep.readout(a['tensor'],a['kernel_mass'],d[ids]>0,a['effective_samples'])) + checks=rep.acceptance(spectral) + if checks['valid_count']: + assert checks['mu_sum_error']<=1e-10 and checks['mu_min']>=-1e-10 and checks['ordering_violation']<=1e-10 + assert checks['B_symmetry_error']<=1e-10 and checks['B_min_eigenvalue']>=-1e-10 and checks['reconstruction_max']<=1e-10 + o2_mode='fixed_full_source_four_point_kernel_at_unique_drawn_anchors' + done('o2_raw_source_kernel_readout_or_cache') + progress('edge_queries_and_s2_eligibility') + length_scale=float(np.median(np.linalg.norm(x[edges[:,1]]-x[edges[:,0]],axis=1))) + summary,complete,edge_data,degree,good_degree=edge_summary(x,edges,ids,spectral,length_scale) + eligible=s2_eligibility(d[ids],spectral,complete,summary) + rejection_masks=dict(nonpositive_distance=d[ids]<=0,spectral_invalid=~spectral['valid'], + neff_failed=~spectral['neff_pass'],edge_incomplete=~complete, + nonfinite_query=~(np.isfinite(spectral['g']).all(1)&np.isfinite(summary).all(1))) + rejections={name:dict(unique=int(mask.sum()),slots=int(counts[mask].sum())) for name,mask in rejection_masks.items()} + np.savez_compressed(dest/'queries.npz',anchor_ids=ids,raw_node_ids=raw_ids[ids],multiplicity=counts, + distance=d[ids],node_mass=mass[ids],g=spectral['g'],edge_summary=summary,eligible_s2=eligible, + spectral_valid=spectral['valid'],neff_pass=spectral['neff_pass'],effective_samples=spectral['effective_samples'], + edge_block_complete=complete,edge_count=degree,valid_edge_count=good_degree,near_wall=d[ids]/scales['t_part']<=.25) + np.savez_compressed(dest/'o2-evidence.npz',**spectral) + np.savez_compressed(dest/'edge-evidence.npz',**edge_data) + del spectral,edge_data + done('edge_queries_eligibility_write') + progress('r2_features') + r2_error=None + try: + r2=features.r2_features(x,tet,d,gate,spec['t_set']) + np.savez_compressed(dest/'r2.npz',features=r2['features'],Di=r2['Di'],Df=r2['Df'],Vf=r2['Vf'],S=r2['S']) + r2_meta=dict(columns=list(r2['columns']),radius=r2['radius'],gate_node=r2['gate_node']);del r2 + except (ValueError,np.linalg.LinAlgError) as exc: + r2_error=dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc());r2_meta=None + save(dest/'r2-failure.json',r2_error) + done('r2_features_and_write') + k=int(counts[eligible].sum());h=scales['positive_mass_fraction'] + result=dict(case=case,status='COMPLETE' if r2_error is None else 'PARTIAL_R2_FAILED',nodes=len(x),tets=len(tet), + geometry_mode=geometry_mode,distance_mode=distance_mode,distance_brute_max_abs_error=brute_error, + o2_mode=o2_mode,raw_field=raw_meta,neighbor_visits=visits,scales=dict(**scales,Rg=rg,volume=float(mass.sum()),L_med=length_scale,t_set=spec['t_set']), + draw_slots=2000,unique_drawn=len(ids),eligible_unique=int(eligible.sum()),eligible_slots=k,conditional_coverage=k/2000, + full_lumped_coverage=h*k/2000,boundary_mass_fraction=1-h,eligible_near_wall_slots=int(counts[eligible&(d[ids]/scales['t_part']<=.25)].sum()), + rejection_counts_overlapping=rejections,total_ineligible_unique=int((~eligible).sum()), + gate_q_used_for_eligibility=False,eligibility_version='S2-v1-no-gate-q',base_columns=list(features.BASE_COLUMNS), + base_normalization='NOT_FITTED; future fold/pool learning only',base_gate_node=gate_node,r2=r2_meta,r2_error=r2_error, + label_arrays_decoded=[],model_fit=False,gpu=False,timings=timings,wall_seconds=time.monotonic()-start, + cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),peak_rss_bytes=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024, + output_sha256={p.name:sha(p) for p in sorted(dest.iterdir()) if p.is_file() and p.name!='progress.json'}) + save(dest/'result.json',result);progress('complete' if not r2_error else 'partial_r2_failed') + print(json.dumps({k:result[k] for k in ['case','status','wall_seconds','peak_rss_bytes','eligible_slots','r2_error']}),flush=True) + return 0 if r2_error is None else 2 + except Exception as exc: + save(dest/'failure.json',dict(stage=stage,error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),timings=timings,wall_seconds=time.monotonic()-start)) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args() + raise SystemExit(run(a.root,a.index)) diff --git a/tools/s2_preprocess_pilot.py b/tools/s2_preprocess_pilot.py new file mode 100644 index 000000000..6af7d86a3 --- /dev/null +++ b/tools/s2_preprocess_pilot.py @@ -0,0 +1,232 @@ +"""Pilot preparation adapter v2: additionally reuse S1 primary sampled O2 caches. + +Based on s2_preprocess.py v1; its ten-part execution remains immutable. +""" +import argparse +import json +from pathlib import Path +import resource +import time +import traceback +from zipfile import ZipFile +import numpy as np +import yaml +from boundary_distance import SurfaceDistance, triangle_distance_squared +from g1_candidate_fingerprints import read_tets +import run_e6_real_kernel as e6 +import o2_rep as rep +from s2_models import features +from s2_e1_supervisor import save +from freeze_g1_components import sha + +GEOMETRY_MEMBERS = {'points_raw','tetra','node_volume_raw','boundary_triangles','edges','gate_location_raw','raw_node_ids'} +O2_MEMBERS = {'tensor','kernel_mass','effective_samples','valid','neff_pass','g','B','mu','status','tau0'} + + +def load_selected(path, members, allowed): + if not set(members) <= allowed: raise ValueError('Array outside explicit whitelist') + with ZipFile(path) as z: + return {name:np.load(z.open(name+'.npy'),allow_pickle=False) for name in members} + + +def scales_and_draw(d, mass, raw_ids, prefix, existing=None): + order=np.argsort(d,kind='stable') + median=float(d[order[np.searchsorted(np.cumsum(mass[order]),mass.sum()/2)]]) + if not np.isfinite(median) or median<=0:raise ValueError('Nonpositive full-node weighted median; no substitute radius') + candidates=np.flatnonzero(d>0) + if existing: + with np.load(existing,allow_pickle=False) as z: + draws=z['draw_ids'];ids=z['unique_ids'];counts=z['multiplicity'];inverse=z['draw_to_unique'] + assert np.array_equal(z['draw_raw_ids'],raw_ids[draws]) + else: + rng=np.random.Generator(np.random.PCG64(np.random.SeedSequence([2026111400,prefix]))) + draws=rng.choice(candidates,size=2000,replace=True,p=mass[candidates]/mass[candidates].sum(dtype=np.float64)) + ids,inverse,counts=np.unique(draws,return_inverse=True,return_counts=True) + assert len(draws)==2000 and np.all(d[draws]>0) and np.array_equal(ids[inverse],draws) and counts.sum()==2000 + return dict(draw_ids=draws,draw_raw_ids=raw_ids[draws],unique_ids=ids,unique_raw_ids=raw_ids[ids], + multiplicity=counts,draw_to_unique=inverse),dict(t_part=4*median,radius=10*median, + positive_mass_fraction=float(mass[d>0].sum()/mass.sum()),weighted_median_distance=median) + + +def edge_summary(x, edges, ids, spectral, length_scale): + lookup=np.full(len(x),-1,np.int64);lookup[ids]=np.arange(len(ids)) + edge_idx,side=np.nonzero(lookup[edges]>=0);local=lookup[edges[edge_idx,side]] + n=len(local);q=np.empty((n,2));valid=np.empty(n,bool);direction=np.empty(n,bool) + for start in range(0,n,4096): + stop=min(start+4096,n);e=edges[edge_idx[start:stop]];j=local[start:stop] + result=rep.contract_direction(spectral['B'][j],spectral['mu'][j],spectral['valid'][j], + x[e[:,1]]-x[e[:,0]],length_scale,'edge') + q[start:stop]=result['q'];valid[start:stop]=result['channel_valid'];direction[start:stop]=result['direction_valid'] + accepted=valid&np.isfinite(q).all(1) + degree=np.bincount(local,minlength=len(ids));good_degree=np.bincount(local[accepted],minlength=len(ids)) + total=np.zeros((len(ids),2));maximum=np.full((len(ids),2),-np.inf) + np.add.at(total,local[accepted],q[accepted]);np.maximum.at(maximum,local[accepted],q[accepted]) + mean=np.full_like(total,np.nan);has=good_degree>0;mean[has]=total[has]/good_degree[has,None];maximum[~has]=np.nan + summary=np.column_stack([mean[:,0],maximum[:,0],mean[:,1],maximum[:,1]]) + complete=(degree==np.bincount(edges.ravel(),minlength=len(x))[ids])&has&np.isfinite(summary).all(1) + return summary,complete,dict(edge_indices=edge_idx,endpoint_side=side,anchor_local=local, + q=q,channel_valid=valid,direction_valid=direction),degree,good_degree + + +def s2_eligibility(distance, spectral, complete, summary): + # Gate q is intentionally absent; original gate geometry remains in outputs. + return (distance>0)&spectral['valid']&spectral['neff_pass']&complete&np.isfinite(spectral['g']).all(1)&np.isfinite(summary).all(1) + + +def load_sampled_o2(spec,ids,counts,raw_ids,radius,nodes): + old=json.loads(Path(spec['existing_sampled_o2_result']).read_text()) + assert old['status']=='COMPLETE' and old['pose']=='original' and old['full_nodes']==nodes + assert np.isclose(old['radius'],radius,rtol=0,atol=1e-13) + with np.load(spec['existing_sampled_o2_nodes'],allow_pickle=False) as z: + assert np.array_equal(z['anchor_ids'],ids) and np.array_equal(z['multiplicity'],counts) + assert np.array_equal(z['raw_node_ids'],raw_ids[ids]) + return load_selected(spec['existing_sampled_o2_nodes'],sorted(O2_MEMBERS),O2_MEMBERS) + + +def run(root,index): + c=json.loads((root/'contract.json').read_text());spec=c['cases'][index];case=spec['case'] + dest=root/'parts'/case;dest.mkdir(parents=True,exist_ok=False) + start=time.monotonic();tick=start;timings={};stage='verify_inputs' + def progress(name,**kw): + nonlocal stage + stage=name;save(dest/'progress.json',dict(stage=name,elapsed_seconds=time.monotonic()-start,**kw)) + def done(name): + nonlocal tick + now=time.monotonic();timings[name]=now-tick;tick=now + try: + for p,h in c['code_sha256'].items():assert sha(p)==h,p + for p,h in spec['input_sha256'].items():assert sha(p)==h,p + progress('geometry') + with np.load(spec['g1_geometry'],allow_pickle=False) as z: + cached_x=z['points'];faces=z['boundary'];center=z['center'];rg=float(z['rg']) + if spec.get('legacy_geometry'): + with ZipFile(spec['legacy_geometry']) as z: has_raw_ids='raw_node_ids.npy' in z.namelist() + members=GEOMETRY_MEMBERS if has_raw_ids else GEOMETRY_MEMBERS-{'raw_node_ids'} + data=load_selected(spec['legacy_geometry'],sorted(members),GEOMETRY_MEMBERS) + x=data['points_raw'];tet=data['tetra'];mass=data['node_volume_raw'];edges=data['edges'];gate=data['gate_location_raw'] + raw_ids=data.get('raw_node_ids',np.arange(len(x)));assert np.array_equal(data['boundary_triangles'],faces) + geometry_mode='existing_cache_explicit_geometry_members_only' + else: + ver=c['archive_stat'];st=Path(c['archive']).stat() + assert (st.st_size,st.st_mtime_ns,st.st_ino)==(ver['bytes'],ver['mtime_ns'],ver['inode']) + with ZipFile(c['archive']) as z: + with z.open(case+'_sim.vtu') as stream:raw_x,raw_tet=read_tets(stream) + gate=np.asarray(yaml.safe_load(z.read(case+'.yml'))['inlet_location'],dtype=float) + raw_ids=np.unique(raw_tet);x=raw_x[raw_ids];tet=np.searchsorted(raw_ids,raw_tet) + del raw_x,raw_tet + _,_,mat,vol,mass,edges,lengths=features.mesh_geometry(x,tet) + del mat,vol,lengths + geometry_mode='missing_tet_topology_from_geometry_prefix_existing_g1_boundary_reused' + assert np.array_equal(x,cached_x);del cached_x + assert len(x)==spec['tet_nodes'] and len(tet)==spec['tet_cells'] + assert np.isfinite(mass).all() and (mass>0).all() and np.isfinite(gate).all() + center_from_mass=np.sum(mass[:,None]*x,axis=0)/mass.sum() + rg_from_mass=np.sqrt(np.sum(mass*np.sum((x-center_from_mass)**2,axis=1))/mass.sum()) + assert np.allclose(center,center_from_mass,rtol=1e-10,atol=1e-10) and np.isclose(rg,rg_from_mass,rtol=1e-10) + np.savez_compressed(dest/'geometry.npz',points_raw=x,tetra=tet,node_volume_raw=mass,edges=edges, + boundary_triangles=faces,raw_node_ids=raw_ids,gate_location_raw=gate) + done('geometry_load_topology_write') + progress('wall_distance') + boundary=np.zeros(len(x),bool);boundary[np.unique(faces)]=True + if spec.get('existing_distance'): + d=np.load(spec['existing_distance'],allow_pickle=False);distance_mode='existing_verified_distance';brute_error=None + else: + d=np.zeros(len(x));interior=np.flatnonzero(~boundary);surface=SurfaceDistance(x[faces]) + for begin in range(0,len(interior),128): + ids=interior[begin:begin+128];d[ids]=surface.query(x[ids]) + progress('wall_distance',done=min(begin+128,len(interior)),total=len(interior)) + probe=np.random.default_rng(1729).choice(len(x),min(30,len(x)),replace=False) + brute=np.array([np.sqrt(triangle_distance_squared(q,x[faces]).min()) for q in x[probe]]) + brute_error=float(abs(brute-d[probe]).max());assert brute_error<=1e-10*max(1.,np.ptp(x,axis=0).max()) + del surface + distance_mode='new_missing_distance_exact_triangle_with_30_brute_checks' + assert np.isfinite(d).all() and (d>=0).all() and np.array_equal(d==0,boundary) + np.save(dest/'boundary_distance_raw.npy',d);done('wall_distance_load_or_compute_and_write') + progress('fixed_draw') + draw,scales=scales_and_draw(d,mass,raw_ids,spec['numeric_prefix'],spec.get('existing_primary_draw')) + ids=draw['unique_ids'];counts=draw['multiplicity'];np.savez_compressed(dest/'draw.npz',**draw);done('draw_and_scales') + progress('base_features') + geo,gate_node=features.graph_distances(x,tet,d,gate,exclude_wall_edges=False) + base=features.base_features(x,d,geo,gate,center,rg,float(mass.sum()),scales['t_part'],spec['t_set']) + np.save(dest/'base.npy',base);del base,geo;done('unrestricted_geodesic_base_write') + progress('o2') + if spec.get('existing_sampled_o2_nodes'): + spectral=load_sampled_o2(spec,ids,counts,raw_ids,scales['radius'],len(x)) + o2_mode='existing_S1_primary_sampled_spectrum_no_reintegration';visits=None;raw_meta=None + elif spec.get('existing_o2_nodes'): + old=json.loads(Path(spec['existing_o2_result']).read_text()) + assert old['acceptance_status']=='PASS' and old['pose']=='original' and old['nodes']==len(x) + assert np.isclose(old['radius'],scales['radius'],rtol=0,atol=1e-13) + loaded=load_selected(spec['existing_o2_nodes'],sorted(O2_MEMBERS),O2_MEMBERS) + spectral={k:v[ids] for k,v in loaded.items()};del loaded + o2_mode='existing_full_original_O2_v1.1a_subset_no_reintegration';visits=None;raw_meta=None + else: + raw,raw_meta=e6.raw_field(x,tet,d);sx,sm,st=e6.samples(x,tet,mass,'tetra_four_point',raw);del raw + visits=0;real_tree=e6.cKDTree + class ProgressTree: + def __init__(self,*args,**kwargs):self.tree=real_tree(*args,**kwargs);self.count=0 + def query_ball_point(self,*args,**kwargs): + nonlocal visits + result=self.tree.query_ball_point(*args,**kwargs);visits+=len(result);self.count+=1 + if self.count%32==0:progress('o2_kernel',anchors_done=self.count,anchors_total=len(ids),neighbor_visits=visits) + return result + e6.cKDTree=ProgressTree + try:a=e6.integrate(x[ids],sx,sm,st,scales['radius']) + finally:e6.cKDTree=real_tree + del sx,sm,st + spectral=dict(**a,**rep.readout(a['tensor'],a['kernel_mass'],d[ids]>0,a['effective_samples'])) + checks=rep.acceptance(spectral) + if checks['valid_count']: + assert checks['mu_sum_error']<=1e-10 and checks['mu_min']>=-1e-10 and checks['ordering_violation']<=1e-10 + assert checks['B_symmetry_error']<=1e-10 and checks['B_min_eigenvalue']>=-1e-10 and checks['reconstruction_max']<=1e-10 + o2_mode='fixed_full_source_four_point_kernel_at_unique_drawn_anchors' + done('o2_raw_source_kernel_readout_or_cache') + progress('edge_queries_and_s2_eligibility') + length_scale=float(np.median(np.linalg.norm(x[edges[:,1]]-x[edges[:,0]],axis=1))) + summary,complete,edge_data,degree,good_degree=edge_summary(x,edges,ids,spectral,length_scale) + eligible=s2_eligibility(d[ids],spectral,complete,summary) + rejection_masks=dict(nonpositive_distance=d[ids]<=0,spectral_invalid=~spectral['valid'], + neff_failed=~spectral['neff_pass'],edge_incomplete=~complete, + nonfinite_query=~(np.isfinite(spectral['g']).all(1)&np.isfinite(summary).all(1))) + rejections={name:dict(unique=int(mask.sum()),slots=int(counts[mask].sum())) for name,mask in rejection_masks.items()} + np.savez_compressed(dest/'queries.npz',anchor_ids=ids,raw_node_ids=raw_ids[ids],multiplicity=counts, + distance=d[ids],node_mass=mass[ids],g=spectral['g'],edge_summary=summary,eligible_s2=eligible, + spectral_valid=spectral['valid'],neff_pass=spectral['neff_pass'],effective_samples=spectral['effective_samples'], + edge_block_complete=complete,edge_count=degree,valid_edge_count=good_degree,near_wall=d[ids]/scales['t_part']<=.25) + np.savez_compressed(dest/'o2-evidence.npz',**spectral) + np.savez_compressed(dest/'edge-evidence.npz',**edge_data) + del spectral,edge_data + done('edge_queries_eligibility_write') + progress('r2_features') + r2_error=None + try: + r2=features.r2_features(x,tet,d,gate,spec['t_set']) + np.savez_compressed(dest/'r2.npz',features=r2['features'],Di=r2['Di'],Df=r2['Df'],Vf=r2['Vf'],S=r2['S']) + r2_meta=dict(columns=list(r2['columns']),radius=r2['radius'],gate_node=r2['gate_node']);del r2 + except (ValueError,np.linalg.LinAlgError) as exc: + r2_error=dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc());r2_meta=None + save(dest/'r2-failure.json',r2_error) + done('r2_features_and_write') + k=int(counts[eligible].sum());h=scales['positive_mass_fraction'] + result=dict(case=case,status='COMPLETE' if r2_error is None else 'PARTIAL_R2_FAILED',nodes=len(x),tets=len(tet), + geometry_mode=geometry_mode,distance_mode=distance_mode,distance_brute_max_abs_error=brute_error, + o2_mode=o2_mode,raw_field=raw_meta,neighbor_visits=visits,scales=dict(**scales,Rg=rg,volume=float(mass.sum()),L_med=length_scale,t_set=spec['t_set']), + draw_slots=2000,unique_drawn=len(ids),eligible_unique=int(eligible.sum()),eligible_slots=k,conditional_coverage=k/2000, + full_lumped_coverage=h*k/2000,boundary_mass_fraction=1-h,eligible_near_wall_slots=int(counts[eligible&(d[ids]/scales['t_part']<=.25)].sum()), + rejection_counts_overlapping=rejections,total_ineligible_unique=int((~eligible).sum()), + gate_q_used_for_eligibility=False,eligibility_version='S2-v1-no-gate-q',base_columns=list(features.BASE_COLUMNS), + base_normalization='NOT_FITTED; future fold/pool learning only',base_gate_node=gate_node,r2=r2_meta,r2_error=r2_error, + label_arrays_decoded=[],model_fit=False,gpu=False,timings=timings,wall_seconds=time.monotonic()-start, + cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),peak_rss_bytes=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024, + output_sha256={p.name:sha(p) for p in sorted(dest.iterdir()) if p.is_file() and p.name!='progress.json'}) + save(dest/'result.json',result);progress('complete' if not r2_error else 'partial_r2_failed') + print(json.dumps({k:result[k] for k in ['case','status','wall_seconds','peak_rss_bytes','eligible_slots','r2_error']}),flush=True) + return 0 if r2_error is None else 2 + except Exception as exc: + save(dest/'failure.json',dict(stage=stage,error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),timings=timings,wall_seconds=time.monotonic()-start)) + raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args() + raise SystemExit(run(a.root,a.index)) diff --git a/tools/summarize_s2_ten_part_measurement.py b/tools/summarize_s2_ten_part_measurement.py new file mode 100644 index 000000000..c74f37e04 --- /dev/null +++ b/tools/summarize_s2_ten_part_measurement.py @@ -0,0 +1,74 @@ +"""Verify measured artifacts and report costs; no recomputation or model fitting.""" +import csv +import json +from pathlib import Path +import numpy as np +from freeze_g1_components import sha +from s2_e1_supervisor import disk_usage + +ROOT=Path('artifacts/s2-ten-part-measurement-2026-11-15') + + +def main(): + batch=json.loads((ROOT/'batch-complete.json').read_text()) + contract=json.loads((ROOT/'contract.json').read_text()) + public=ROOT/'public';rows=[];checks=[] + for attempt in batch['attempts']: + i=attempt['index'];spec=contract['cases'][i];dest=ROOT/'parts'/spec['case'] + row={k:attempt[k] for k in ['index','case','status','stop_reason','cpu_seconds','wall_seconds','peak_tree_rss_bytes']} + row.update(nodes=spec['tet_nodes'],tets=spec['tet_cells'],tertile=spec['tertile'],output_disk_bytes=disk_usage(dest)) + if (dest/'result.json').exists(): + r=json.loads((dest/'result.json').read_text()) + for name,h in r['output_sha256'].items():assert sha(dest/name)==h + row.update(preparation_status=r['status'],eligible_unique=r['eligible_unique'],eligible_slots=r['eligible_slots'], + conditional_coverage=r['conditional_coverage'],full_lumped_coverage=r['full_lumped_coverage'], + distance_mode=r['distance_mode'],o2_mode=r['o2_mode'],r2_error=r['r2_error'],timings=r['timings'],result_sha256=sha(dest/'result.json')) + base=np.load(dest/'base.npy',allow_pickle=False,mmap_mode='r') + assert base.shape==(r['nodes'],10) and np.isfinite(base).all() + with np.load(dest/'queries.npz',allow_pickle=False) as q: + ids=q['anchor_ids'];counts=q['multiplicity'];eligible=q['eligible_s2'] + assert np.all(np.diff(ids)>0) and counts.sum()==2000 and np.all(counts>0) + expected=(q['distance']>0)&q['spectral_valid']&q['neff_pass']&q['edge_block_complete']&np.isfinite(q['g']).all(1)&np.isfinite(q['edge_summary']).all(1) + assert np.array_equal(eligible,expected) + assert int(counts[eligible].sum())==r['eligible_slots'] + with np.load(dest/'draw.npz',allow_pickle=False) as draw: + assert np.array_equal(ids[draw['draw_to_unique']],draw['draw_ids']) + assert np.array_equal(counts,draw['multiplicity']) + if r['r2_error'] is None: + with np.load(dest/'r2.npz',allow_pickle=False) as z:assert z['features'].shape==(r['nodes'],20) and np.isfinite(z['features']).all() + assert r['label_arrays_decoded']==[] and not r['model_fit'] and not r['gpu'] and not r['gate_q_used_for_eligibility'] + checks.append(dict(case=spec['case'],status='PASS',checks=['output hashes','base shape/finiteness','draw identity/multiplicity','S2 common eligibility/coverage','R2 shape/finiteness if completed','no labels/fit/GPU recorded'])) + elif (dest/'failure.json').exists():row['failure']=json.loads((dest/'failure.json').read_text()) + rows.append(row) + for p,h in contract['code_sha256'].items():assert sha(p)==h,p + parts=json.loads(Path('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json').read_text()) + non_test=[p for p in parts if p['assignments'][0]['role']!='test'] + requested={p['case'] for p in non_test};completed={r['case'] for r in rows if r.get('preparation_status')=='COMPLETE' and r['status']=='PASS'} + pilot=dict(proposed_fixed_fold=0,proposed_seed=0,learning_parts=387,validation_parts=112,sealed_test_parts=125, + non_test_parts=len(non_test),non_test_nodes=sum(p['nodes'] for p in non_test), + prepared_from_this_ten_part_stage=len(requested&completed),remaining_after_this_stage=len(requested-completed), + train_loop_and_GPU_supervisor='NOT_YET_ACCEPTED',labels_read=False,pilot_started=False) + # Transparent planning scenarios, not acceptance bounds: compute total roster + # costs using rates from nine newly prepared parts, excluding cached 666. + cold=[r for r in rows if r['case'] in completed and not r['case'].startswith('00000666')] + if cold: + rates=np.array([r['cpu_seconds']/r['nodes'] for r in cold]);bytes_per_node=np.array([r['output_disk_bytes']/r['nodes'] for r in rows if r['case'] in completed]) + missing_nodes=sum(p['nodes'] for p in non_test if p['case'] not in completed) + pilot['missing_non_test_nodes']=missing_nodes + pilot['linear_node_rate_scenarios']={name:dict(cpu_hours=float(np.quantile(rates,q)*missing_nodes/3600),additional_gib=float(np.quantile(bytes_per_node,q)*missing_nodes/2**30)) for name,q in [('median',.5),('p95',.95),('max',1)]} + pilot['scenario_limit']='Shape/neighbourhood-dependent runtime; nine new cases, cache reuse and cold largest-part costs differ; these are not guarantees or authority to exceed caps' + result=dict(status='MEASUREMENT_RECORDED_AWAITING_REVIEW',contract_sha256=sha(ROOT/'contract.json'),batch_sha256=sha(ROOT/'batch-complete.json'), + attempted=len(rows),complete=sum(r.get('preparation_status')=='COMPLETE' and r['status']=='PASS' for r in rows), + failed_or_partial=sum(r.get('preparation_status')!='COMPLETE' or r['status']!='PASS' for r in rows),not_run=batch['not_run'], + worker_cpu_seconds=batch['worker_cpu_seconds'],supervisor_cpu_seconds=batch['supervisor_cpu_seconds'],wall_seconds=batch['wall_seconds'], + peak_worker_tree_rss_bytes=batch['peak_worker_tree_rss_bytes'],disk_bytes_before_publication=disk_usage(ROOT), + checks=checks,rows=rows,pilot_readiness=pilot, + no_repeated_G1_geometry_registration_review=True,no_labels_fit_gpu=True,code_sha256=sha(__file__)) + with (public/'measurement-result.json').open('x') as f:json.dump(result,f,ensure_ascii=False,indent=2);f.write('\n') + with (public/'cost-table.csv').open('x') as f: + fields=['index','case','nodes','tets','tertile','status','preparation_status','cpu_seconds','wall_seconds','peak_tree_rss_bytes','output_disk_bytes','eligible_slots','distance_mode','o2_mode'] + w=csv.DictWriter(f,fields,extrasaction='ignore');w.writeheader();w.writerows(rows) + print(json.dumps({k:v for k,v in result.items() if k not in ['checks','rows']},ensure_ascii=False)) + + +if __name__=='__main__':main() diff --git a/tools/test_s2_pilot_data.py b/tools/test_s2_pilot_data.py new file mode 100644 index 000000000..dc78d41e3 --- /dev/null +++ b/tools/test_s2_pilot_data.py @@ -0,0 +1,37 @@ +import unittest +import tempfile +from pathlib import Path +from zipfile import ZipFile +from unittest.mock import patch +import numpy as np +import s2_pilot_data as d + + +class PilotSealTests(unittest.TestCase): + def setUp(self):self.manifest=dict(learning_parts=['a'],validation_parts=['b'],sealed_test_parts=['c']) + def test_test_unknown_and_conflicting_role_fail_before_io(self): + with patch.object(d,'ZipFile') as opener: + for case in ['c','unknown']: + with self.assertRaises(PermissionError):d.read_non_test_fill_time('not-opened.zip',case,np.arange(3),self.manifest) + conflicting=dict(self.manifest,learning_parts=['a','c']) + with self.assertRaises(PermissionError):d.read_non_test_fill_time('not-opened.zip','c',np.arange(3),conflicting) + opener.assert_not_called() + def test_roles(self): + self.assertEqual(d.permitted_role(self.manifest,'a'),'learning');self.assertEqual(d.permitted_role(self.manifest,'b'),'validation') + def test_only_named_point_field_decoded_and_raw_ids_preserved(self): + xml=b'skipskipchosen' + with tempfile.TemporaryDirectory() as root: + path=Path(root)/'synthetic.zip' + with ZipFile(path,'w') as z:z.writestr('a_sim.vtu',xml) + with patch.object(d,'decode_array',return_value=np.array([0.,1.,2.,3.])) as decode: + values,receipt=d.read_non_test_fill_time(path,'a',np.array([1,3]),self.manifest) + decode.assert_called_once();np.testing.assert_array_equal(values,[1.,3.]) + self.assertEqual(receipt['decoded_label_arrays'],['Fill_time']) + def test_only_common_eligible_targets_are_validated(self): + target=d.validate_query_targets([np.nan,0.,2.],np.array([0,1,2]),np.array([False,True,True]),.5) + np.testing.assert_array_equal(target,[0.,4.]) + with self.assertRaises(ValueError):d.validate_query_targets([np.nan],np.array([0]),np.array([True]),.5) + with self.assertRaises(ValueError):d.validate_query_targets([1.],np.array([0]),np.array([False]),.5) + + +if __name__=='__main__':unittest.main(verbosity=2) diff --git a/tools/test_s2_pilot_entry.py b/tools/test_s2_pilot_entry.py new file mode 100644 index 000000000..a9fc297ab --- /dev/null +++ b/tools/test_s2_pilot_entry.py @@ -0,0 +1,22 @@ +import unittest +from unittest.mock import patch +import numpy as np +import torch +from run_s2_pilot_training import LearningBases,load_part + + +class PilotEntrySealTests(unittest.TestCase): + def setUp(self):self.roles=dict(learning_parts=['a'],validation_parts=['b'],sealed_test_parts=['c']) + def test_normalizer_mapping_rejects_validation_and_test_before_io(self): + for case in ['b','c']: + bases=LearningBases({case:dict(preparation_root='never-open')},[case],self.roles) + with patch('run_s2_pilot_training.np.load') as loader: + with self.assertRaises(PermissionError):bases[case] + loader.assert_not_called() + def test_training_loader_rejects_sealed_test_before_hash_or_arrays(self): + with patch('run_s2_pilot_training.sha') as hasher,patch('run_s2_pilot_training.np.load') as loader: + with self.assertRaises(PermissionError):load_part(dict(case='c'),self.roles,torch.device('cpu')) + hasher.assert_not_called();loader.assert_not_called() + + +if __name__=='__main__':unittest.main(verbosity=2) diff --git a/tools/test_s2_pilot_gpu_guard.py b/tools/test_s2_pilot_gpu_guard.py new file mode 100644 index 000000000..3d6203a0e --- /dev/null +++ b/tools/test_s2_pilot_gpu_guard.py @@ -0,0 +1,49 @@ +import unittest +import tempfile +import sys +import json +import threading +from pathlib import Path +from unittest.mock import patch +import s2_pilot_gpu_guard as guard +from s2_gpu_process_supervisor import supervise +from s2_pilot_gpu_guard import parse_device_usage + + +class GPUUsageTests(unittest.TestCase): + def test_exact_device_memory_units(self): + self.assertEqual(parse_device_usage('15360, 100\n'),(15*2**30,100)) + self.assertEqual(parse_device_usage('5, 0'),(5*2**20,0)) + def test_missing_or_ambiguous_counters_fail_closed(self): + for text in ['', 'N/A, 0','5, 0\n5, 0','-1, 0','2, 101','2']: + with self.assertRaises(ValueError):parse_device_usage(text) + def test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use(self): + with tempfile.TemporaryDirectory() as root: + # wait4 includes transient inherited torch parent RSS before exec. + limits=dict(total_cpu_seconds=5,job_cpu_seconds=2,job_wall_seconds=2,tree_rss_bytes=1024*2**20,disk_bytes=2**20,max_processes=1) + r=supervise([sys.executable,'-c','import os; assert os.environ["CUDA_VISIBLE_DEVICES"]=="synthetic-device"'],root,'synthetic',limits,gpu_uuid='synthetic-device') + self.assertEqual(r['status'],'PASS');self.assertTrue(r['gpu_visible']);self.assertEqual(r['gpu_uuid'],'synthetic-device') + def test_vram_monitor_error_and_whole_host_rss_stop_without_second_run(self): + for trigger in ['vram','monitor','host']: + with self.subTest(trigger=trigger), tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);killed=threading.Event() + c=dict(gpu_uuid='synthetic-device',code_sha256={},runs=[dict(name=f'{arm}{width}',arm=arm,width=width,seed=0) for width in [128,256] for arm in ['N','E']]) + (root/'training-contract.json').write_text(json.dumps(c)) + calls=0 + def usage(uuid): + nonlocal calls + calls+=1 + if calls==1:return 5*2**20,0 + if trigger=='monitor':raise RuntimeError('synthetic monitor failure') + return (16*2**30 if trigger=='vram' else 5*2**20),0 + def fake_supervise(*a,**kw): + self.assertTrue(killed.wait(2));return dict(status='PASS',stop_reason=None) + with patch.object(guard,'device_usage',usage),patch.object(guard,'tree_usage',return_value=(0,17*2**30 if trigger=='host' else 0,1)),patch.object(guard,'active_training_pids',return_value=[123456789]),patch.object(guard,'stop_group',side_effect=lambda pid:killed.set()),patch.object(guard,'supervise',side_effect=fake_supervise) as worker: + guard.run(root) + self.assertEqual(worker.call_count,1) + result=json.loads((root/'pilot-complete.json').read_text()) + self.assertEqual(result['status'],'PILOT_INCOMPLETE');self.assertIsNone(result['selected_global_width']) + self.assertEqual(result['runs'][0]['stop_reason'],{'vram':'DEVICE_VRAM_LIMIT','monitor':'GPU_MONITOR_FAILED','host':'WHOLE_RUN_HOST_RSS_LIMIT'}[trigger]) + + +if __name__=='__main__':unittest.main(verbosity=2) diff --git a/tools/test_s2_pilot_preparation.py b/tools/test_s2_pilot_preparation.py new file mode 100644 index 000000000..e5c824bfd --- /dev/null +++ b/tools/test_s2_pilot_preparation.py @@ -0,0 +1,29 @@ +import tempfile +import unittest +from pathlib import Path +import json +import numpy as np +from s2_preprocess_pilot import load_sampled_o2,O2_MEMBERS +from run_s2_pilot_preparation import projection + + +class PreparationChecks(unittest.TestCase): + def test_cache_requires_exact_original_pose_draw_radius_and_raw_ids(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);p=root/'nodes.npz';r=root/'result.json' + np.savez(p,anchor_ids=[1,3],raw_node_ids=[11,13],multiplicity=[1000,1000],**{k:np.ones(2) for k in O2_MEMBERS}) + r.write_text(json.dumps(dict(status='COMPLETE',pose='original',full_nodes=4,radius=2.))) + spec=dict(existing_sampled_o2_nodes=str(p),existing_sampled_o2_result=str(r)) + result=load_sampled_o2(spec,np.array([1,3]),np.array([1000,1000]),np.arange(10,14),2.,4) + self.assertEqual(set(result),O2_MEMBERS) + for ids,counts,raw,radius in [(np.array([0,3]),[1000,1000],np.arange(10,14),2.),(np.array([1,3]),[1,1999],np.arange(10,14),2.),(np.array([1,3]),[1000,1000],np.arange(4),2.),(np.array([1,3]),[1000,1000],np.arange(10,14),3.)]: + with self.assertRaises(AssertionError):load_sampled_o2(spec,ids,np.array(counts),raw,radius,4) + def test_checkpoint_counts_failed_cpu_and_separates_cached_rate(self): + c=dict(initial_cpu_seconds_per_node=1.,cases=[dict(tet_nodes=100),dict(tet_nodes=200),dict(tet_nodes=100,reuse_preparation='cache'),dict(tet_nodes=300,reuse_preparation='cache')]) + done=[dict(index=0,worker=dict(cpu_seconds=100.,status='STOPPED_OR_FAILED')),dict(index=2,worker=dict(cpu_seconds=10.,status='PASS'))] + p=projection(c,done) + self.assertEqual(p['spent_worker_cpu_seconds'],110.) + self.assertEqual(p['projected_worker_cpu_seconds'],340.) + + +if __name__=='__main__':unittest.main(verbosity=2) diff --git a/tools/test_s2_pilot_train.py b/tools/test_s2_pilot_train.py new file mode 100644 index 000000000..69c07ec7b --- /dev/null +++ b/tools/test_s2_pilot_train.py @@ -0,0 +1,47 @@ +import unittest +import numpy as np +import torch +from torch import nn +from s2_pilot_train import part_weights,train_part,component_score,select_capacity + + +class TinyQueryModel(nn.Module): + arm='N' + def __init__(self):super().__init__();self.linear=nn.Linear(2,1) + def forward(self,base,ids,gaps,edge):return self.linear(base[ids]).squeeze(-1) + + +class PilotTrainChecks(unittest.TestCase): + def test_step_weights_expectation_and_component_balance(self): + # Two components with 1 and 2 parts; total global contribution is one. + weights=[part_weights([1,3],n,2,3) for n in [1,2,2]] + self.assertAlmostEqual(sum(w.sum() for w in weights)/3,1.) + self.assertAlmostEqual(weights[0].sum(),weights[1].sum()+weights[2].sum()) + def test_synthetic_cpu_update_and_nonfinite_no_update(self): + torch.manual_seed(0);m=TinyQueryModel();o=torch.optim.AdamW(m.parameters(),lr=1e-3) + b=torch.tensor([[1.,2.],[2.,3.]]);ids=torch.tensor([0,1]);g=torch.zeros(2,2);w=torch.tensor([.25,.75]);target=torch.tensor([1.,2.]) + before=m.linear.weight.detach().clone();loss=train_part(m,o,b,ids,g,None,target,w) + self.assertTrue(np.isfinite(loss));self.assertFalse(torch.equal(before,m.linear.weight)) + before=m.linear.weight.detach().clone() + with self.assertRaises(ValueError):train_part(m,o,b,ids,g,None,torch.tensor([float('nan'),2.]),w) + self.assertTrue(torch.equal(before,m.linear.weight)) + def test_validation_component_weight_and_capacity_tie(self): + self.assertEqual(component_score({'a':1.,'b':3.,'c':5.},{'a':'x','b':'y','c':'y'}),2.5) + self.assertEqual(select_capacity({('N',128):1.,('E',128):3.,('N',256):2.,('E',256):2.}),128) + with self.assertRaises(ValueError):select_capacity({('N',128):1.}) + with self.assertRaises(ValueError):component_score({'a':1.},{'a':'x','b':'y'}) + def test_actual_four_model_variants_backward_on_synthetic_nodes(self): + from s2_models.normalization import fit_base_standardizer + from s2_models.network import QueryModel + values=np.arange(90,dtype=float).reshape(9,10)/90 + stats=fit_base_standardizer({'synthetic':values},learning_part_ids=['synthetic'],fold='synthetic',pool='100') + for arm in ['N','E']: + for width in [128,256]: + torch.manual_seed(0);m=QueryModel(arm,width,standardizer=stats) + o=torch.optim.AdamW(m.parameters(),lr=1e-3,weight_decay=1e-5) + before=m.decoder[-1].weight.detach().clone() + loss=train_part(m,o,torch.tensor(values,dtype=torch.float32),torch.tensor([1,3]),torch.ones(2,2)*.1,torch.ones(2,4)*.1,torch.tensor([.2,.8]),torch.tensor([.5,.5])) + self.assertTrue(np.isfinite(loss));self.assertFalse(torch.equal(before,m.decoder[-1].weight)) + + +if __name__=='__main__':torch.set_num_threads(1);unittest.main(verbosity=2) diff --git a/tools/test_s2_preprocess.py b/tools/test_s2_preprocess.py new file mode 100644 index 000000000..6d12d067e --- /dev/null +++ b/tools/test_s2_preprocess.py @@ -0,0 +1,55 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +import numpy as np +from s2_preprocess import load_selected,GEOMETRY_MEMBERS,scales_and_draw,s2_eligibility,edge_summary +from run_s2_ten_part_measurement import remaining_limits + + +class PreprocessChecks(unittest.TestCase): + def test_whitelist_never_opens_label_member(self): + from zipfile import ZipFile + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'mixed.npz';np.savez(p,points_raw=np.ones((2,3)),fill_time_raw=np.array([999.])) + opened=[];original=ZipFile.open + def spy(z,name,*a,**kw):opened.append(name);return original(z,name,*a,**kw) + with patch.object(ZipFile,'open',spy):r=load_selected(p,['points_raw'],GEOMETRY_MEMBERS) + self.assertEqual(opened,['points_raw.npy']);self.assertEqual(r['points_raw'].shape,(2,3)) + with self.assertRaises(ValueError):load_selected(p,['fill_time_raw'],GEOMETRY_MEMBERS) + + def test_draw_seed_multiplicity_and_original_identity(self): + d=np.array([0.,1.,2.]);mass=np.array([1.,3.,4.]);raw=np.array([10,20,30]) + a,s=scales_and_draw(d,mass,raw,13);b,_=scales_and_draw(d,mass,raw,13) + np.testing.assert_array_equal(a['draw_ids'],b['draw_ids']) + np.testing.assert_array_equal(a['unique_ids'][a['draw_to_unique']],a['draw_ids']) + np.testing.assert_array_equal(a['draw_raw_ids'],raw[a['draw_ids']]) + self.assertEqual(a['multiplicity'].sum(),2000);self.assertEqual(s['t_part'],4.) + with tempfile.TemporaryDirectory() as t: + p=Path(t)/'draw.npz';np.savez(p,**a) + c,_=scales_and_draw(d,mass,raw,999,existing=p) + np.testing.assert_array_equal(a['draw_ids'],c['draw_ids']) + + def test_nonpositive_median_fails_without_radius_substitute(self): + with self.assertRaises(ValueError):scales_and_draw(np.array([0.,1.]),np.array([3.,1.]),np.arange(2),1) + + def test_s2_eligibility_does_not_reuse_gate_filter(self): + s=dict(valid=np.array([True,True,False,True]),neff_pass=np.array([True,False,True,True]),g=np.ones((4,2)),gate_direction_valid=np.zeros(4,bool)) + np.testing.assert_array_equal(s2_eligibility(np.ones(4),s,np.array([True,True,True,False]),np.ones((4,4))),[True,False,False,False]) + + def test_all_incident_edges_and_anchor_tensor(self): + x=np.array([[0.,0,0],[1.,0,0],[0.,1,0]]);edges=np.array([[0,1],[0,2],[1,2]]) + b=np.zeros((1,2,3,3));b[0,0,0,0]=1 + s=dict(B=b,mu=np.ones((1,3))/3,valid=np.ones(1,bool)) + summary,complete,e,degree,good=edge_summary(x,edges,np.array([0]),s,1.) + np.testing.assert_allclose(summary,[[.5,1.,0,0]]) + self.assertTrue(complete[0]);self.assertEqual(degree[0],2);self.assertEqual(good[0],2) + + def test_stage_deadline_and_supervisor_cpu_count(self): + self.assertEqual(remaining_limits(10799,10,5)['job_wall_seconds'],1) + self.assertEqual(remaining_limits(100,10790,5)['job_cpu_seconds'],5) + self.assertIsNone(remaining_limits(10800,0,0)) + self.assertIsNone(remaining_limits(0,10799,1)) + + +if __name__=='__main__':unittest.main(verbosity=2)