Add independent recorded-peak guard and freeze fresh main-matrix engineering plan

This commit is contained in:
Frank14f committed 2026-11-15 07:29:21 +08:00
1 parent 59fd40adba
commit 0af3dab8d1
16 files changed
+3342

No files matched your search

+2
View File
@@ -24,3 +24,5 @@
- Latest terminal update (2026-11-15 07:14:59 UTC+8): session34632 and all pilot workers have exited. N128/E128 completed42 epochs with guardPASS; N256 failed at epoch2 on4/382 AMP skips exceeding1%, E256 never started. PILOT_INCOMPLETE/no width; cumulative2439.912972s/7200. N256 history/progress also records reserved16416505856B above15GiB; the failed-result path did not inspect recorded peaks. Preserve this resource-boundary gap. Read `docs/s2-v2-pilot-stop-results-2026-11-15.md`; no restart, reprobe, threshold/source change or full matrix is authorized by a polling request.125 tests sealed; evidence submitted toClaude.
- Claude1794698179.161599 records a separate feasibility decision: main-matrix width128, width effect not tested, no use of validation scores to justify selection. Original pilot selected_global_width staysnull. Before any main-matrix execution: fix recorded progress/result VRAM-peak enforcement with synthetic test, allocator configuration per ruling, unchangedAMP1% andE_max42, and budget/execution plan. This read-only stage audit has not implemented or launched that work.
- Main engineering plan update: independent `s2_main_memory_guard.py` checks all progress/result/history peak records during and after failed workers;7 CPU synthetic checks passed. No old pinned source changed. Allocatorexpandable_segments parsedtrue withV100 VMM support, no GPU tensors/workload. Claude1794698734.967459 supersedes pilot-cell reuse: separately prepare6 timeouts at60min and125 fold0 test geometries without label decoding, freeze one final roster, then all90 neural fits fresh; pilot weights remain historical only. Read `docs/s2-main-engineering-and-budget-2026-11-15.md`. Proposed costs/caps are plans; no131 queue or main matrix launched in this engineering turn.
@@ -0,0 +1,42 @@
{
"status": "METADATA_PREFLIGHT_PASS_NOT_ALLOCATION_WORKLOAD",
"source_tool_sessions": [
98992,
87464
],
"torch_version": "2.5.1+cu124",
"torch_cuda_build": "12.4",
"allocator_conf": "expandable_segments:True",
"allocator_backend": "native",
"parsed_allocator_settings_expandable_segments": true,
"torch_cuda_context_initialized": false,
"allocated_GPU_tensors": 0,
"virtual_memory_attribute": {
"id": 102,
"name": "CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED",
"local_header": "/usr/local/cuda/include/cuda.h:786"
},
"devices": [
{
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"name": "Tesla V100-SXM2-16GB",
"virtual_memory_management_supported": true
},
{
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"name": "Tesla V100-SXM2-16GB",
"virtual_memory_management_supported": true
},
{
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"name": "Tesla V100-SXM2-16GB",
"virtual_memory_management_supported": true
},
{
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"name": "Tesla V100-SXM2-16GB",
"virtual_memory_management_supported": true
}
],
"limit": "Configuration parsing and driver VMM capability verified; no successful real expandable CUDA allocation claimed"
}
@@ -0,0 +1,228 @@
{
"status": "PLAN_ONLY_NO_MAIN_EXECUTION",
"neural_runs": 90,
"arms": [
"N",
"E"
],
"folds": 5,
"component_fraction_percent": [
25,
50,
100
],
"seeds": [
0,
1,
2
],
"width": 128,
"width_selection": "feasibility; width effect not tested",
"E_max": 42,
"cosine_T_max": 42,
"baseline_ready_nodes": 14555918,
"baseline_worker_guard_wall_seconds": {
"N": 1129.3405424560187,
"E": 1121.3519870990422
},
"node_scaled_gpu_hours": 19.942144065735626,
"node_scaled_plus25percent_and_idle_hours": 25.42768008216953,
"all90_priced_at_measured_full_pool_hours": 28.133656619438263,
"all90_full_pool_plus25percent_and_idle_hours": 35.66707077429783,
"proposed_per_run_wall_limit_seconds": 1800,
"90_run_caps_plus_expected_idle_gpu_hours": 45.5,
"proposed_global_neural_wall_limit_hours": 46,
"R2_final_fits": 15,
"R2_HPO_configurations_per_fold_pool_max": 12,
"R2_HPO_trials_max": 180,
"R2_CPU_time": "UNMEASURED_NOT_INCLUDED",
"preparation_completion_cost": "NOT_INCLUDED_125_UNPREPARED_AND_6_TIMEOUTS_REQUIRE_SEPARATE_PLAN",
"estimation_limits": [
"Composite per-node scaling cannot separate 128 training/inference/setup rates",
"Measured one fold/full successful pool/seed; no confidence bound",
"Allocator change may change timing; proposed reserve is engineering allowance",
"No early-stopping savings assumed",
"Global proposed cap includes waits/setup, may stop before all90; no automatic extra budget",
"GPU-hours are aggregate device time, not multi-GPU elapsed promise"
],
"rows": [
{
"fold": 0,
"component_fraction_percent": 25,
"learning_parts": 145,
"learning_nodes": 4319375,
"validation_parts": 112,
"validation_nodes": 3330152,
"projected_N_seconds": 593.4988759700324,
"projected_E_seconds": 589.3006749431931
},
{
"fold": 0,
"component_fraction_percent": 50,
"learning_parts": 254,
"learning_nodes": 7524426,
"validation_parts": 112,
"validation_nodes": 3330152,
"projected_N_seconds": 842.1670832888153,
"projected_E_seconds": 836.2098913597581
},
{
"fold": 0,
"component_fraction_percent": 100,
"learning_parts": 387,
"learning_nodes": 11863762,
"validation_parts": 112,
"validation_nodes": 3330152,
"projected_N_seconds": 1178.840323144861,
"projected_E_seconds": 1170.501623855806
},
{
"fold": 1,
"component_fraction_percent": 25,
"learning_parts": 85,
"learning_nodes": 2401741,
"validation_parts": 90,
"validation_nodes": 3242126,
"projected_N_seconds": 437.88703806449195,
"projected_E_seconds": 434.78958011255014
},
{
"fold": 1,
"component_fraction_percent": 50,
"learning_parts": 180,
"learning_nodes": 4549249,
"validation_parts": 90,
"validation_nodes": 3242126,
"projected_N_seconds": 604.5043444857455,
"projected_E_seconds": 600.2282946691372
},
{
"fold": 1,
"component_fraction_percent": 100,
"learning_parts": 409,
"learning_nodes": 11386113,
"validation_parts": 90,
"validation_nodes": 3242126,
"projected_N_seconds": 1134.9516648442434,
"projected_E_seconds": 1126.9234183931035
},
{
"fold": 2,
"component_fraction_percent": 25,
"learning_parts": 90,
"learning_nodes": 2998926,
"validation_parts": 88,
"validation_nodes": 2648316,
"projected_N_seconds": 438.14889199433605,
"projected_E_seconds": 435.0495817803569
},
{
"fold": 2,
"component_fraction_percent": 50,
"learning_parts": 203,
"learning_nodes": 6659657,
"validation_parts": 88,
"validation_nodes": 2648316,
"projected_N_seconds": 722.1716470913051,
"projected_E_seconds": 717.0632604150582
},
{
"fold": 2,
"component_fraction_percent": 100,
"learning_parts": 411,
"learning_nodes": 12229477,
"validation_parts": 88,
"validation_nodes": 2648316,
"projected_N_seconds": 1154.3136487281913,
"projected_E_seconds": 1146.1484424546923
},
{
"fold": 3,
"component_fraction_percent": 25,
"learning_parts": 128,
"learning_nodes": 4348385,
"validation_parts": 144,
"validation_nodes": 3976319,
"projected_N_seconds": 645.8833947227367,
"projected_E_seconds": 641.3146441475793
},
{
"fold": 3,
"component_fraction_percent": 50,
"learning_parts": 236,
"learning_nodes": 7631328,
"validation_parts": 144,
"validation_nodes": 3976319,
"projected_N_seconds": 900.5949579832738,
"projected_E_seconds": 894.224467944532
},
{
"fold": 3,
"component_fraction_percent": 100,
"learning_parts": 355,
"learning_nodes": 11031937,
"validation_parts": 144,
"validation_nodes": 3976319,
"projected_N_seconds": 1164.435796653897,
"projected_E_seconds": 1156.1989898879015
},
{
"fold": 4,
"component_fraction_percent": 25,
"learning_parts": 58,
"learning_nodes": 1748016,
"validation_parts": 114,
"validation_nodes": 3401388,
"projected_N_seconds": 399.52345889041095,
"projected_E_seconds": 396.69737132180575
},
{
"fold": 4,
"component_fraction_percent": 50,
"learning_parts": 144,
"learning_nodes": 4743744,
"validation_parts": 114,
"validation_nodes": 3401388,
"projected_N_seconds": 631.9510587553376,
"projected_E_seconds": 627.4808605945702
},
{
"fold": 4,
"component_fraction_percent": 100,
"learning_parts": 386,
"learning_nodes": 11535306,
"validation_parts": 114,
"validation_nodes": 3401388,
"projected_N_seconds": 1158.883562305006,
"projected_E_seconds": 1150.6860300800224
}
],
"preparation_proposal": {
"new_parts_without_S2_ready": 125,
"their_nodes": 3467310,
"retained_true_timeout_parts": 6,
"conservative_observed_cpu_seconds_per_original_planned_node": 0.008235846461945216,
"rate_includes_prior_aborted_and_recovery_cost": true,
"new125_projected_worker_hours": 7.932286887768686,
"six_timeout_repreparations_separate_60min_each_cap_hours": 6,
"sum_with25percent_reserve_hours": 17.415358609710857,
"proposed_new_worker_cpu_cap_hours": 20,
"proposed_cumulative_output_disk_GiB": 100,
"proposal_only_no_execution": true,
"no_labels_from_current_sealed_test_parts_read": true,
"label_policy": "125 fold0 tests: geometry/features only; labels decoded only at custody step; no label reads in this planning turn"
},
"R2_proposal": {
"status": "UNMEASURED_COST_CAP_PROPOSAL_ONLY",
"single_thread": true,
"fits_max_including_HPO_and_final": 195,
"proposed_per_fit_cpu_seconds": 1800,
"proposed_total_worker_cpu_hours": 100,
"initial_checkpoint": "Measure one authorized fixed fit before dispatching full search; no current fit executed",
"no_completion_guarantee": true
},
"final_roster_ruling": "1794698734.967459",
"neural_runs_fresh": 90,
"pilot_cells_reused": 0,
"reuse_policy": "Pilot cells remain historical pilot evidence; no pilot weights reused or bound into matrix"
}
@@ -0,0 +1,27 @@
{
"status": "INDEPENDENT_ENGINEERING_ACCEPTANCE_AND_PLAN_ONLY",
"observed_unix": 1794698943.6114888,
"original_source_pins_unchanged": true,
"GPU_training_or_tensor_work_executed": false,
"main_matrix_started": false,
"preparation_started": false,
"new_neural_cells_planned": 90,
"pilot_cells_reused": 0,
"roster_ruling": "1794698734.967459",
"guard_tests": 7,
"artifact_sha256": {
"artifacts/s2-main-engineering-2026-11-15/allocator-preflight.json": "85b19a711897dfc7b6979e16c9018a56fa7f97789f5e9a510799779c6b806570",
"artifacts/s2-main-engineering-2026-11-15/budget-estimate.json": "0a2298d37176237ee786511a10bbaa7106f14539d2265dd719fd60321699f22b",
"artifacts/s2-main-engineering-2026-11-15/historical-N256-readonly-memory-audit.json": "71b3d666ff4ac877604d7c0f0cad7b0082960a180c2032ab366507f2f936e1b2",
"artifacts/s2-main-engineering-2026-11-15/planned-fold-pool-costs.csv": "b0a2e1f7b9da4258eecd6ddd60748e94502f0d6e0114253ecdd4ea7667afdc9c",
"artifacts/s2-main-engineering-2026-11-15/planned-neural-cells.json": "4d36ff92a46d7c82b5a3c6fe6c4747eced8bb98e4b0000cda913d792329c5292",
"artifacts/s2-main-engineering-2026-11-15/planned-preparation-131.json": "bd3e5b2a1e56baafd924ebdd9661c35290e53c68f36c26d2becebcd6501a06ca",
"artifacts/s2-main-engineering-2026-11-15/reuse-candidates.json": "4a7d670a6e7e004dd330358a6e6c85a5334dbbfff837182d02a2ef75e17fe619",
"artifacts/s2-main-engineering-2026-11-15/source-hashes.json": "5223fa9423e42aea415ab6756e44e7a11c7bde27e090d2256a64b089b47722d2",
"artifacts/s2-main-engineering-2026-11-15/synthetic-final-tests.txt": "388cada21d5df8240258cfe4fc5a888ece2d8a5bc73d5e7499940e56b5e7e11e",
"artifacts/s2-main-engineering-2026-11-15/test-receipt.json": "18c44e5c612d0a2a7cea3aa7c1dc7fc65c491a59d9406cb9dd4ca53f8924e449",
"docs/s2-main-engineering-and-budget-2026-11-15.md": "7d86331bfbf0040b21dc6455c73926fe01e1537409e9883b62bcfa79735fc110",
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"tools/test_s2_main_memory_guard.py": "510b9d93628c06196d01b8d0c89616070125a0c39546bc96e2f5c732d8c19cc6"
}
}
@@ -0,0 +1,48 @@
{
"maxima": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
},
"witnesses": {
"peak_allocated_bytes": {
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/history.json",
"sha256": "03a086fa5dd18cbae94984cfd1991d98a3e4011105c237fa32ab98c9d29004a4",
"observed_unix": 1794698530.1258473,
"peaks": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
}
},
"peak_reserved_bytes": {
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/history.json",
"sha256": "03a086fa5dd18cbae94984cfd1991d98a3e4011105c237fa32ab98c9d29004a4",
"observed_unix": 1794698530.1258473,
"peaks": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
}
}
},
"observations": [
{
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/history.json",
"sha256": "03a086fa5dd18cbae94984cfd1991d98a3e4011105c237fa32ab98c9d29004a4",
"observed_unix": 1794698530.1258473,
"peaks": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
}
},
{
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/progress.json",
"sha256": "a7bb4a85494c928d0daeacd90ef61a8043d6f4b2c82c362d768517fe4e30ac67",
"observed_unix": 1794698530.1259105,
"peaks": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
}
}
],
"limit_bytes": 16106127360,
"breach": true
}
@@ -0,0 +1,16 @@
fold,component_fraction_percent,learning_parts,learning_nodes,validation_parts,validation_nodes,projected_N_seconds,projected_E_seconds
0,25,145,4319375,112,3330152,593.4988759700324,589.3006749431931
0,50,254,7524426,112,3330152,842.1670832888153,836.2098913597581
0,100,387,11863762,112,3330152,1178.840323144861,1170.501623855806
1,25,85,2401741,90,3242126,437.88703806449195,434.78958011255014
1,50,180,4549249,90,3242126,604.5043444857455,600.2282946691372
1,100,409,11386113,90,3242126,1134.9516648442434,1126.9234183931035
2,25,90,2998926,88,2648316,438.14889199433605,435.0495817803569
2,50,203,6659657,88,2648316,722.1716470913051,717.0632604150582
2,100,411,12229477,88,2648316,1154.3136487281913,1146.1484424546923
3,25,128,4348385,144,3976319,645.8833947227367,641.3146441475793
3,50,236,7631328,144,3976319,900.5949579832738,894.224467944532
3,100,355,11031937,144,3976319,1164.435796653897,1156.1989898879015
4,25,58,1748016,114,3401388,399.52345889041095,396.69737132180575
4,50,144,4743744,114,3401388,631.9510587553376,627.4808605945702
4,100,386,11535306,114,3401388,1158.883562305006,1150.6860300800224
1 fold component_fraction_percent learning_parts learning_nodes validation_parts validation_nodes projected_N_seconds projected_E_seconds
2 0 25 145 4319375 112 3330152 593.4988759700324 589.3006749431931
3 0 50 254 7524426 112 3330152 842.1670832888153 836.2098913597581
4 0 100 387 11863762 112 3330152 1178.840323144861 1170.501623855806
5 1 25 85 2401741 90 3242126 437.88703806449195 434.78958011255014
6 1 50 180 4549249 90 3242126 604.5043444857455 600.2282946691372
7 1 100 409 11386113 90 3242126 1134.9516648442434 1126.9234183931035
8 2 25 90 2998926 88 2648316 438.14889199433605 435.0495817803569
9 2 50 203 6659657 88 2648316 722.1716470913051 717.0632604150582
10 2 100 411 12229477 88 2648316 1154.3136487281913 1146.1484424546923
11 3 25 128 4348385 144 3976319 645.8833947227367 641.3146441475793
12 3 50 236 7631328 144 3976319 900.5949579832738 894.224467944532
13 3 100 355 11031937 144 3976319 1164.435796653897 1156.1989898879015
14 4 25 58 1748016 114 3401388 399.52345889041095 396.69737132180575
15 4 50 144 4743744 114 3401388 631.9510587553376 627.4808605945702
16 4 100 386 11535306 114 3401388 1158.883562305006 1150.6860300800224
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,63 @@
{
"status": "SUPERSEDED_NO_REUSE_BY_1794698734.967459",
"candidates": [
{
"matrix_cell": {
"fold": 0,
"component_fraction_percent": 100,
"seed": 0,
"arm": "N",
"width": 128
},
"source_run": "artifacts/s2-pilot-v2-continuation-2026-11-15/N128-seed0",
"result_sha256": "f75ded80d92c4e9835ca8bcda6b9772fd1fe56b131c9964097fdfea7ea8f1a44",
"checkpoint_sha256": "ca2eb128116e877e2ffed65f4c1c5864e090beab9b7c0615f117e613599ccd34",
"guard_sha256": "ab7b44f144d5563cb8ffcc91c62f7b4452c6db645adf1f6068985c5a11ce98eb",
"training_contract_sha256": "f6eaa3cb2fd6918e5bd940826a2cc954ffcee0a9a2d1578b2da1e5e1c72abd5c",
"success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
"standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03",
"learning_parts": 382,
"validation_parts": 111,
"E_max": 42,
"cosine_T_max": 42,
"completed_epochs": 42,
"reuse_authorization": "1794698584.380959",
"metadata_eligibility": "EXACT_ROLE_AND_POOL_ROSTER_MUST_REMAIN_IDENTICAL",
"allocator_difference": "historical accepted fit used default allocator; new fits require expandable_segments, no numerical change",
"test_predictions_produced": false,
"test_predictions_policy": "only with rest of matrix at custody step",
"original_guard_recorded_peaks_readonly_accepted": true,
"saved_training_wall_seconds": 1129.3405424560187
},
{
"matrix_cell": {
"fold": 0,
"component_fraction_percent": 100,
"seed": 0,
"arm": "E",
"width": 128
},
"source_run": "artifacts/s2-pilot-v2-continuation-2026-11-15/E128-seed0",
"result_sha256": "2e1271dbcc00ba4ee4889aa2c9e192283c06a9bd3eabc10a973dabba2f7dad78",
"checkpoint_sha256": "910437bd645a965241786a90ad14325fe7af09092238bfd0be5ee5bddf8a9a24",
"guard_sha256": "39d3c063ab4ac979add7150e374ad1531f6621b49c3fe5653d443e999e57624b",
"training_contract_sha256": "f6eaa3cb2fd6918e5bd940826a2cc954ffcee0a9a2d1578b2da1e5e1c72abd5c",
"success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
"standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03",
"learning_parts": 382,
"validation_parts": 111,
"E_max": 42,
"cosine_T_max": 42,
"completed_epochs": 42,
"reuse_authorization": "1794698584.380959",
"metadata_eligibility": "EXACT_ROLE_AND_POOL_ROSTER_MUST_REMAIN_IDENTICAL",
"allocator_difference": "historical accepted fit used default allocator; new fits require expandable_segments, no numerical change",
"test_predictions_produced": false,
"test_predictions_policy": "only with rest of matrix at custody step",
"original_guard_recorded_peaks_readonly_accepted": true,
"saved_training_wall_seconds": 1121.3519870990422
}
],
"saved_measured_training_seconds": 2250.692529555061,
"final_decision": "All90 neural fits fresh on one post-preparation roster; these entries retained as historical proposal only"
}
@@ -0,0 +1,7 @@
{
"artifacts/s2-pilot-v2-continuation-2026-11-15/N128-seed0/result.json": "f75ded80d92c4e9835ca8bcda6b9772fd1fe56b131c9964097fdfea7ea8f1a44",
"artifacts/s2-pilot-v2-continuation-2026-11-15/E128-seed0/result.json": "2e1271dbcc00ba4ee4889aa2c9e192283c06a9bd3eabc10a973dabba2f7dad78",
"artifacts/s2-pilot-v2-continuation-2026-11-15/N128-seed0-guard-receipt.json": "ab7b44f144d5563cb8ffcc91c62f7b4452c6db645adf1f6068985c5a11ce98eb",
"artifacts/s2-pilot-v2-continuation-2026-11-15/E128-seed0-guard-receipt.json": "39d3c063ab4ac979add7150e374ad1531f6621b49c3fe5653d443e999e57624b",
"artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json": "53f2fa6f79e368a340100a9166b0125fe57614cdd3f22a73822dba2a9697c6fd"
}
@@ -0,0 +1,12 @@
test_allocator_is_required_before_device_or_worker_call (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_allocator_reaches_supervised_cpu_child_without_cuda_work (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_failed_worker_final_scan_catches_breach_without_result (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_invalid_recorded_peak_rejected (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_maximum_survives_lower_replacement_and_deletion (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_nested_history_and_result_both_contribute (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_only_progress_reserved_breach_stops_live_worker (test_s2_main_memory_guard.RecordedPeakTests) ... ok
----------------------------------------------------------------------
Ran 7 tests in 0.124s
OK
@@ -0,0 +1,19 @@
{
"status": "PASS",
"initial_checks": {
"tests": 6,
"seconds": 0.016,
"tool_session": 22962
},
"final_checks": {
"tests": 7,
"CPU_only": true,
"GPU_work": false
},
"parse_check": "PASS",
"observed_unix": 1794698644.070644,
"code_sha256": {
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"tools/test_s2_main_memory_guard.py": "510b9d93628c06196d01b8d0c89616070125a0c39546bc96e2f5c732d8c19cc6"
}
}
@@ -0,0 +1,47 @@
# S2 主矩阵前工程验收与预算/执行计划
当前只完成独立工程验收及计划。没有新的GPU训练、标签解码、准备或主矩阵执行。原pilot、诊断、v2零run停止和接续失败及旧源码全部保留。Claude1794698179.161599决定宽128按可行性采用,宽度效应未检验;原pilot自动选宽仍null,验证分数不用于解释该决定。
## 已完成条件与独立监督器
Claude1794698584.380959接受128显存条件:N128/E128的整run reserved峰值7.679688/7.677734GiB,allocated均约4.3834GiB;两个run都完整42轮覆盖最大187345节点件。这是包含最大件执行的全程上界,没有逐件隔离峰值记录,也没有补跑。
新的tools/s2_main_memory_guard.py是独立模块,没有矩阵启动CLI,不改旧监督器。对当前run目录所有JSON(progress/result/failure/history/epoch等)递归提取peak_reserved_bytes和peak_allocated_bytes,保留跨覆盖/删除的累计最大值、来源文件SHA和观察时间。运行时每约0.5秒扫描记录并结合设备抽样/主机RSS;任何记录峰值超过15GiB就停止本任务的匹配命令子进程。无result或worker失败退出也强制再扫描;保存原worker状态、原停止原因及所有监督原因。无效峰值/损坏记录失败关闭。其他进程不终止。
监督器要求启动环境PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,记录该值并由现有进程监督器传给子进程。只有完整结果且两类峰值存在才可能PASS。单run上限沿提案≤1800秒、15GiB VRAM、16GiB主机RSS、5GiB输出;调用者必须显式给累计worker CPU限额,未来矩阵入口仍须单独维护全局预算。此模块没有把90-run执行器或托管评估器实现完成。
7/7 CPU合成检查通过:仅progress reserved超限触发停止;失败无result的最终扫描;峰值跨覆盖/删除保留;history/result两类记录取最大;非法峰值拒绝;缺allocator环境在spawn前拒绝;真实受监督CPU子进程收到allocator/GPU可见性环境(未调用CUDA)。真实N256历史记录只读扫描重现15.2890625GiB上限违反。保留原guard缺口,不回写过去为已按新规则执行。检测发生在记录落盘/设备采样后,不宣称连续无采样延迟。
## Allocator配置核查
PyTorch2.5.1+cu124、CUDA12.4、native allocator。设置后读取allocator snapshot,expandable_segments=true;四张Tesla V100-SXM2-16GB的CUDA属性102(VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED)均true。只查元数据,没有分配GPU张量,torch.cuda.is_initialized=false;这证明配置解析/驱动能力,不冒称实际分配工作负载验收。未来获准主矩阵必须在启动前设置并在实际运行回执中记录;数值计算、AMP1%和E_max42不变。
## 名单与复用裁量
1794698584曾允许在名单完全相同的条件下复用两个pilot cell。核查指出旧cell使用382学习/111验证,回填6件后会变更名单。Claude1794698734.967459最终裁定:补齐6件、统一完整名单、90run全部fresh;两个pilot只作历史证据,不复用权重,不将其checkpoint SHA绑定为主矩阵初始化。历史reuse-candidates明确标为已被取代,预算回到90新run。
准备计划精确列131项:6个真实超时件独立标记再准备,单件60min/1.25GiB、0重试;125个fold0测试件首次S2几何/特征准备,单件30min/1.25GiB,标签只在托管步骤解码。不能直接用旧会解码标签的pilot ready入口处理这125件。主矩阵开始前,根据所有准备终态生成唯一名单SHA;仍失败件在所有五折一致排除,保留原组件身份和逐件失败/节点/质量报告,不重抽或重排原组件/池。现有493成功准备可复用几何/特征,不能把准备复用与pilot权重复用混淆。
原125测试标签在本次验收仍封存。主矩阵各fold角色必须由已固定split映射明确授权;不得以元数据规划为已解封标签。测试预测仅在其余矩阵一起进入托管评估时生成,不提前补pilot测试预测。
## 成本依据与提案(不是已经放行的预算)
N128/E128实测guard总wall1129.340542/1121.351987秒,均42轮。固定矩阵5外折×3组件比例档(25/50/100%)×3匹配种子×2臂=90个fresh神经fit;宽128,E_max=cosine_T_max=42,AMP首轮比例豁免/后续1%及最多20连续跳步规则保持。15行逐折/池件数、节点数与估时在planned-fold-pool-costs.csv,90个明确run ID在planned-neural-cells.json(不可执行计划)。
以两臂实测wall按(目标学习节点+验证节点)/(成功pilot学习节点+验证节点14,555,918)同比例外推,90run约19.94聚合GPU小时;加25%工程余量和每run20秒空闲确认约25.43小时。此为混合训练/验证/设置速率估计,没有假称掌握128分阶段速率。
较保守情景把90run全部按本次完整池实测计价:28.13 GPU小时;加25%余量及空闲确认35.67小时。两情景均不是统计置信上限,allocator、其他fold大小和池分布可能改变耗时,不计早停节约。提议单run≤30min、全部90run聚合墙钟硬限46GPU小时(90×30min=45h,再给正常门控/设置余量);门控等待也记账,到总限可能未完成,不能自动加时。四卡并发只有在对应卡空闲、每卡隔离监督及聚合计账实现验收后再定,不能把聚合GPU小时直接承诺为多卡日历时长。
准备另账:125件合计3,467,310节点;用历史包含中止/恢复成本的125134.74286CPU秒÷15,193,914原计划节点作保守率,预计约7.93 worker CPU小时。另给6件各60分钟最多6小时,加25%余量约17.42小时,提议新增20 worker CPU小时/累计100GiB输出,仍是规划、未启动。
R2-G24为15个fold×pool最终fit;每格最多12个HPO配置,共最多180 HPO+15最终fit。尚无拟合实测,不能给可信耗时预测。提议单线程每fit≤30 CPU分钟、合计100 worker CPU小时作为硬上限,首个获准固定fit后做成本检查再派发后续;不保证全部195 fit能完成。B0确定性计算/托管预测/统计以及缺失准备的实际实现与开销另列,不能将上述神经估时称为项目全成本。
## 进入顺序与待办
1. 固定本工程包及Claude名单/预算裁量;原失败证据永久保留。
2. 独立实现并冻结131件不解码测试标签的准备队列与账本;按上述名单规则产出唯一最终roster SHA。当前只列计划,不宣称已实现或启动队列。
3. 依据roster生成每fold/pool的角色白名单与学习统计、共同查询及目标;固定90fresh神经cell和15 R2/HPO清单。准备失败一致剔除、组件配对及覆盖报告沿既定政策,不凭pilot部分分数改变科学设计。
4. 将本监督器与allocator设置接入新主矩阵入口,验收全局预算、失败终态、不重试/不替补、无提前测试访问及托管预测输出;不复用旧硬编码fold0/seed0/pool100的pilot训练入口当作完整矩阵执行器。
5. 在已确认的执行范围和预算内按固定清单推进,所有失败保留、AMP阈值不变,测试评分/bootstrap待托管阶段;当前没有主矩阵启动回执。
研究目标仍是同主干N/E边信息增量、跨来源泛化与样本效率;容量可行性裁量与部分pilot分数不回答这些科学问题。6件超时含其他折测试件,主审要求先尝试补齐,避免按准备成本静默排除大件。资源与准备失败覆盖都需随主结果公开;O1类型/连通路线仍是独立未检验问题。
+2
View File
@@ -60,3 +60,5 @@ Claude 1794684155.730259已接受接续入口,并要求合法单件FAIL保持
夜间核查补充(Claude 1794684781.751169):首50已正式接受,原60h队列继续,不启用80h续队列。pilot准备仍守单件1800s;失败件报告必须补case/role/nodes/停止原因及原回执hash。探针VRAM结论仅覆盖其实际最大成功学习件;不得把20件probe推到被剔除或更大未测件。当前固定池最大学习件为666,187,345节点,但尚无本次GPU测量。已追加artifacts/s2-pilot-continuation-2026-11-15/probe-coverage-boundary.json;probe实际回执产生后,另建与其hash关联的覆盖附件,列实际最大节点与失败件节点数,保留原回执/运行源码pin不改。主矩阵之前,超时大件须按另行标记的更长单件上限准备,并在真实最大件核VRAM;这不是pilot静默重试,本轮未启动该工作。
最新主审裁量1794698179.161599:主矩阵宽128按可行性选取,宽度效应未检验;原pilot自动选宽仍null。主矩阵前guard记录峰值修复、allocator设置、预算/执行计划待办;本轮未启动。详细依据与128整run显存上界见s2-v2-pilot-stop-results-2026-11-15.md。
主矩阵前工程验收已完成独立guard7/7合成测试、旧N256记录只读检测及allocator元数据核查;未改旧源码。Claude1794698734.967459最终要求6件60min独立再准备+125件不解码标签准备、统一roster、90神经run全fresh,否决旧pilot cell复用。预算与分阶段执行计划见s2-main-engineering-and-budget-2026-11-15.md;131准备及主矩阵尚未启动。
+133
View File
@@ -0,0 +1,133 @@
"""Independent future-run guard; no matrix launcher or training entry point.
Preserves the old pilot guard and its receipts. Checks recorded cumulative CUDA
peaks while the worker is alive and again after any exit, including failure.
"""
import hashlib
import json
import math
import os
from pathlib import Path
import threading
import time
import psutil
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise, save, stop_group, disk_usage, tree_usage
ALLOCATOR_CONF='expandable_segments:True'
VRAM_BYTES=15*2**30
HOST_BYTES=16*2**30
DISK_BYTES=5*2**30
PEAK_KEYS=('peak_allocated_bytes','peak_reserved_bytes')
def peaks_in(value):
found={}
if isinstance(value,dict):
for key,child in value.items():
if key in PEAK_KEYS:
if isinstance(child,bool) or not isinstance(child,(int,float)) or not math.isfinite(child) or child<0:
raise ValueError('Invalid recorded CUDA peak: '+key)
found[key]=child
nested=peaks_in(child)
for k,v in nested.items():found[k]=max(found.get(k,0),v)
elif isinstance(value,list):
for child in value:
for k,v in peaks_in(child).items():found[k]=max(found.get(k,0),v)
return found
class PeakRecords:
def __init__(self):
self.maxima={k:0 for k in PEAK_KEYS};self.witnesses={};self.cache={};self.observations=[]
def scan(self,root):
# Includes progress, result and failure records, plus epoch/history JSON.
# Running maxima survive file replacement, regression and deletion.
for path in sorted(Path(root).rglob('*.json')):
try:
stat=path.stat();version=(stat.st_mtime_ns,stat.st_size,stat.st_ino)
if self.cache.get(str(path))==version:continue
raw=path.read_bytes()
except FileNotFoundError:continue
peaks=peaks_in(json.loads(raw));self.cache[str(path)]=version
if not peaks:continue
record=dict(path=str(path),sha256=hashlib.sha256(raw).hexdigest(),observed_unix=time.time(),peaks=peaks)
self.observations.append(record)
for key,value in peaks.items():
if key not in self.witnesses or value>self.maxima[key]:
self.maxima[key]=value;self.witnesses[key]=record
return max(self.maxima.values())>VRAM_BYTES
def as_dict(self):
return dict(maxima=self.maxima,witnesses=self.witnesses,observations=self.observations,
limit_bytes=VRAM_BYTES,breach=max(self.maxima.values())>VRAM_BYTES)
def active_command_pids(command):
result=[]
try:children=psutil.Process(os.getpid()).children(recursive=True)
except psutil.NoSuchProcess:return result
for p in children:
try:
if p.cmdline()==command:result.append(p.pid)
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
return result
def guarded_worker(root,name,uuid,wall_limit,command,*,cumulative_worker_cpu_limit_seconds):
root=Path(root)
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!=ALLOCATOR_CONF:
raise ValueError('Explicit PYTORCH_CUDA_ALLOC_CONF='+ALLOCATOR_CONF+' required before spawn')
assert 0<wall_limit<=1800 and cumulative_worker_cpu_limit_seconds>0
begin=time.monotonic();initial,util=device_usage(uuid)
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
tracker=PeakRecords();done=threading.Event()
monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,reasons=[],errors=[],samples=0)
def reason(value):
if value not in monitor['reasons']:monitor['reasons'].append(value)
def scan_records():
try:
if tracker.scan(root/name):reason('RECORDED_VRAM_LIMIT')
except Exception as exc:
reason('RECORDED_MEMORY_READ_FAILED');monitor['errors'].append(str(exc))
def watch():
while not done.is_set():
scan_records()
try:
used,_=device_usage(uuid);monitor['samples']+=1
monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
if used>VRAM_BYTES:reason('DEVICE_VRAM_LIMIT')
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
if rss>HOST_BYTES:reason('WHOLE_RUN_HOST_RSS_LIMIT')
if disk_usage(root)>=DISK_BYTES:reason('RUN_DISK_LIMIT')
except Exception as exc:
reason('GPU_MONITOR_FAILED');monitor['errors'].append(str(exc))
if monitor['reasons']:
for pid in active_command_pids(command):stop_group(pid)
done.wait(.5)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
remaining=wall_limit-(time.monotonic()-begin)-2.5
if remaining<=0:raise RuntimeError('No wall budget left before spawn')
receipt=supervise(command,root/'guards',name,dict(total_cpu_seconds=cumulative_worker_cpu_limit_seconds,
job_cpu_seconds=wall_limit,job_wall_seconds=remaining,tree_rss_bytes=HOST_BYTES,
disk_bytes=DISK_BYTES,max_processes=2),poll=.1,gpu_uuid=uuid)
finally:
done.set();thread.join(timeout=3)
if thread.is_alive():reason('GPU_MONITOR_DID_NOT_STOP')
else:scan_records() # Mandatory even if the worker failed or wrote no result.
result_path=root/name/'result.json';result=None
if result_path.exists():
try:result=json.loads(result_path.read_text())
except Exception as exc:reason('RESULT_READ_FAILED');monitor['errors'].append(str(exc))
if receipt['status']=='PASS' and (not result or result.get('status')!='COMPLETE'):reason('MISSING_COMPLETE_RESULT')
if receipt['status']=='PASS' and not all(k in tracker.witnesses for k in PEAK_KEYS):reason('MISSING_RECORDED_MEMORY_PEAKS')
elapsed=time.monotonic()-begin
if elapsed>wall_limit:reason('TOTAL_RUN_WALL_LIMIT')
receipt['underlying_worker_status']=receipt['status'];receipt['underlying_stop_reason']=receipt.get('stop_reason')
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
receipt.update(device_monitor=monitor,recorded_memory=tracker.as_dict(),total_run_wall_seconds=elapsed,
allocator_conf=os.environ['PYTORCH_CUDA_ALLOC_CONF'],gpu_uuid=uuid,
result_sha256=hashlib.sha256(result_path.read_bytes()).hexdigest() if result else None)
save(root/(name+'-guard-receipt.json'),receipt)
return receipt,result
+61
View File
@@ -0,0 +1,61 @@
import json
import os
import sys
from pathlib import Path
import tempfile
import threading
import unittest
from unittest.mock import patch
import s2_main_memory_guard as guard
class RecordedPeakTests(unittest.TestCase):
def test_only_progress_reserved_breach_stops_live_worker(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'run').mkdir();killed=threading.Event()
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+1)))
def supervise(*args,**kwargs):
self.assertTrue(killed.wait(2));return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=-9)
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[123456789]),patch.object(guard,'stop_group',side_effect=lambda pid:killed.set()):
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
self.assertIsNone(result);self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT')
self.assertEqual(receipt['recorded_memory']['maxima']['peak_reserved_bytes'],guard.VRAM_BYTES+1)
def test_failed_worker_final_scan_catches_breach_without_result(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'run').mkdir()
def supervise(*args,**kwargs):
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+3)))
return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=1)
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[]):
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT');self.assertEqual(receipt['returncode'],1)
self.assertIsNone(result)
def test_maximum_survives_lower_replacement_and_deletion(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'progress.json';tracker=guard.PeakRecords()
p.write_text(json.dumps(dict(peak_reserved_bytes=guard.VRAM_BYTES+1)));self.assertTrue(tracker.scan(tmp))
p.write_text(json.dumps(dict(peak_reserved_bytes=7)));self.assertTrue(tracker.scan(tmp))
p.unlink();self.assertTrue(tracker.scan(tmp));self.assertEqual(tracker.maxima['peak_reserved_bytes'],guard.VRAM_BYTES+1)
def test_nested_history_and_result_both_contribute(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'history.json').write_text(json.dumps([dict(peak_reserved_bytes=20)]))
(root/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=30,peak_reserved_bytes=10)))
tracker=guard.PeakRecords();self.assertFalse(tracker.scan(root));self.assertEqual(tracker.maxima,dict(peak_allocated_bytes=30,peak_reserved_bytes=20))
def test_invalid_recorded_peak_rejected(self):
for value in [float('nan'),-1,True,'100']:
with self.subTest(value=value),self.assertRaises(ValueError):guard.peaks_in(dict(peak_reserved_bytes=value))
def test_allocator_is_required_before_device_or_worker_call(self):
with patch.dict(os.environ,{},clear=True),patch.object(guard,'device_usage') as device:
with self.assertRaises(ValueError):guard.guarded_worker(Path('/unused'),'run','synthetic',20,[],cumulative_worker_cpu_limit_seconds=20)
device.assert_not_called()
def test_allocator_reaches_supervised_cpu_child_without_cuda_work(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp)
code="import os,json,pathlib; assert os.environ['PYTORCH_CUDA_ALLOC_CONF']=='expandable_segments:True'; assert os.environ['CUDA_VISIBLE_DEVICES']=='synthetic'; p=pathlib.Path('"+str(root/ 'run')+"'); p.mkdir(); (p/'result.json').write_text(json.dumps(dict(status='COMPLETE',peak_allocated_bytes=0,peak_reserved_bytes=0)))"
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)):
receipt,result=guard.guarded_worker(root,'run','synthetic',20,[sys.executable,'-c',code],cumulative_worker_cpu_limit_seconds=20)
self.assertEqual(receipt['status'],'PASS');self.assertEqual(result['status'],'COMPLETE')
self.assertEqual(receipt['allocator_conf'],guard.ALLOCATOR_CONF)
if __name__=='__main__':unittest.main(verbosity=2)