Add independent recorded-peak guard and freeze fresh main-matrix engineering plan

This commit is contained in:
Frank14f committed 2026-11-15 07:29:21 +08:00
1 parent 59fd40adba
commit 0af3dab8d1
16 files changed
+3342

No files matched your search

@@ -0,0 +1,42 @@
{
"status": "METADATA_PREFLIGHT_PASS_NOT_ALLOCATION_WORKLOAD",
"source_tool_sessions": [
98992,
87464
],
"torch_version": "2.5.1+cu124",
"torch_cuda_build": "12.4",
"allocator_conf": "expandable_segments:True",
"allocator_backend": "native",
"parsed_allocator_settings_expandable_segments": true,
"torch_cuda_context_initialized": false,
"allocated_GPU_tensors": 0,
"virtual_memory_attribute": {
"id": 102,
"name": "CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED",
"local_header": "/usr/local/cuda/include/cuda.h:786"
},
"devices": [
{
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"name": "Tesla V100-SXM2-16GB",
"virtual_memory_management_supported": true
},
{
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"name": "Tesla V100-SXM2-16GB",
"virtual_memory_management_supported": true
},
{
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"name": "Tesla V100-SXM2-16GB",
"virtual_memory_management_supported": true
},
{
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"name": "Tesla V100-SXM2-16GB",
"virtual_memory_management_supported": true
}
],
"limit": "Configuration parsing and driver VMM capability verified; no successful real expandable CUDA allocation claimed"
}
@@ -0,0 +1,228 @@
{
"status": "PLAN_ONLY_NO_MAIN_EXECUTION",
"neural_runs": 90,
"arms": [
"N",
"E"
],
"folds": 5,
"component_fraction_percent": [
25,
50,
100
],
"seeds": [
0,
1,
2
],
"width": 128,
"width_selection": "feasibility; width effect not tested",
"E_max": 42,
"cosine_T_max": 42,
"baseline_ready_nodes": 14555918,
"baseline_worker_guard_wall_seconds": {
"N": 1129.3405424560187,
"E": 1121.3519870990422
},
"node_scaled_gpu_hours": 19.942144065735626,
"node_scaled_plus25percent_and_idle_hours": 25.42768008216953,
"all90_priced_at_measured_full_pool_hours": 28.133656619438263,
"all90_full_pool_plus25percent_and_idle_hours": 35.66707077429783,
"proposed_per_run_wall_limit_seconds": 1800,
"90_run_caps_plus_expected_idle_gpu_hours": 45.5,
"proposed_global_neural_wall_limit_hours": 46,
"R2_final_fits": 15,
"R2_HPO_configurations_per_fold_pool_max": 12,
"R2_HPO_trials_max": 180,
"R2_CPU_time": "UNMEASURED_NOT_INCLUDED",
"preparation_completion_cost": "NOT_INCLUDED_125_UNPREPARED_AND_6_TIMEOUTS_REQUIRE_SEPARATE_PLAN",
"estimation_limits": [
"Composite per-node scaling cannot separate 128 training/inference/setup rates",
"Measured one fold/full successful pool/seed; no confidence bound",
"Allocator change may change timing; proposed reserve is engineering allowance",
"No early-stopping savings assumed",
"Global proposed cap includes waits/setup, may stop before all90; no automatic extra budget",
"GPU-hours are aggregate device time, not multi-GPU elapsed promise"
],
"rows": [
{
"fold": 0,
"component_fraction_percent": 25,
"learning_parts": 145,
"learning_nodes": 4319375,
"validation_parts": 112,
"validation_nodes": 3330152,
"projected_N_seconds": 593.4988759700324,
"projected_E_seconds": 589.3006749431931
},
{
"fold": 0,
"component_fraction_percent": 50,
"learning_parts": 254,
"learning_nodes": 7524426,
"validation_parts": 112,
"validation_nodes": 3330152,
"projected_N_seconds": 842.1670832888153,
"projected_E_seconds": 836.2098913597581
},
{
"fold": 0,
"component_fraction_percent": 100,
"learning_parts": 387,
"learning_nodes": 11863762,
"validation_parts": 112,
"validation_nodes": 3330152,
"projected_N_seconds": 1178.840323144861,
"projected_E_seconds": 1170.501623855806
},
{
"fold": 1,
"component_fraction_percent": 25,
"learning_parts": 85,
"learning_nodes": 2401741,
"validation_parts": 90,
"validation_nodes": 3242126,
"projected_N_seconds": 437.88703806449195,
"projected_E_seconds": 434.78958011255014
},
{
"fold": 1,
"component_fraction_percent": 50,
"learning_parts": 180,
"learning_nodes": 4549249,
"validation_parts": 90,
"validation_nodes": 3242126,
"projected_N_seconds": 604.5043444857455,
"projected_E_seconds": 600.2282946691372
},
{
"fold": 1,
"component_fraction_percent": 100,
"learning_parts": 409,
"learning_nodes": 11386113,
"validation_parts": 90,
"validation_nodes": 3242126,
"projected_N_seconds": 1134.9516648442434,
"projected_E_seconds": 1126.9234183931035
},
{
"fold": 2,
"component_fraction_percent": 25,
"learning_parts": 90,
"learning_nodes": 2998926,
"validation_parts": 88,
"validation_nodes": 2648316,
"projected_N_seconds": 438.14889199433605,
"projected_E_seconds": 435.0495817803569
},
{
"fold": 2,
"component_fraction_percent": 50,
"learning_parts": 203,
"learning_nodes": 6659657,
"validation_parts": 88,
"validation_nodes": 2648316,
"projected_N_seconds": 722.1716470913051,
"projected_E_seconds": 717.0632604150582
},
{
"fold": 2,
"component_fraction_percent": 100,
"learning_parts": 411,
"learning_nodes": 12229477,
"validation_parts": 88,
"validation_nodes": 2648316,
"projected_N_seconds": 1154.3136487281913,
"projected_E_seconds": 1146.1484424546923
},
{
"fold": 3,
"component_fraction_percent": 25,
"learning_parts": 128,
"learning_nodes": 4348385,
"validation_parts": 144,
"validation_nodes": 3976319,
"projected_N_seconds": 645.8833947227367,
"projected_E_seconds": 641.3146441475793
},
{
"fold": 3,
"component_fraction_percent": 50,
"learning_parts": 236,
"learning_nodes": 7631328,
"validation_parts": 144,
"validation_nodes": 3976319,
"projected_N_seconds": 900.5949579832738,
"projected_E_seconds": 894.224467944532
},
{
"fold": 3,
"component_fraction_percent": 100,
"learning_parts": 355,
"learning_nodes": 11031937,
"validation_parts": 144,
"validation_nodes": 3976319,
"projected_N_seconds": 1164.435796653897,
"projected_E_seconds": 1156.1989898879015
},
{
"fold": 4,
"component_fraction_percent": 25,
"learning_parts": 58,
"learning_nodes": 1748016,
"validation_parts": 114,
"validation_nodes": 3401388,
"projected_N_seconds": 399.52345889041095,
"projected_E_seconds": 396.69737132180575
},
{
"fold": 4,
"component_fraction_percent": 50,
"learning_parts": 144,
"learning_nodes": 4743744,
"validation_parts": 114,
"validation_nodes": 3401388,
"projected_N_seconds": 631.9510587553376,
"projected_E_seconds": 627.4808605945702
},
{
"fold": 4,
"component_fraction_percent": 100,
"learning_parts": 386,
"learning_nodes": 11535306,
"validation_parts": 114,
"validation_nodes": 3401388,
"projected_N_seconds": 1158.883562305006,
"projected_E_seconds": 1150.6860300800224
}
],
"preparation_proposal": {
"new_parts_without_S2_ready": 125,
"their_nodes": 3467310,
"retained_true_timeout_parts": 6,
"conservative_observed_cpu_seconds_per_original_planned_node": 0.008235846461945216,
"rate_includes_prior_aborted_and_recovery_cost": true,
"new125_projected_worker_hours": 7.932286887768686,
"six_timeout_repreparations_separate_60min_each_cap_hours": 6,
"sum_with25percent_reserve_hours": 17.415358609710857,
"proposed_new_worker_cpu_cap_hours": 20,
"proposed_cumulative_output_disk_GiB": 100,
"proposal_only_no_execution": true,
"no_labels_from_current_sealed_test_parts_read": true,
"label_policy": "125 fold0 tests: geometry/features only; labels decoded only at custody step; no label reads in this planning turn"
},
"R2_proposal": {
"status": "UNMEASURED_COST_CAP_PROPOSAL_ONLY",
"single_thread": true,
"fits_max_including_HPO_and_final": 195,
"proposed_per_fit_cpu_seconds": 1800,
"proposed_total_worker_cpu_hours": 100,
"initial_checkpoint": "Measure one authorized fixed fit before dispatching full search; no current fit executed",
"no_completion_guarantee": true
},
"final_roster_ruling": "1794698734.967459",
"neural_runs_fresh": 90,
"pilot_cells_reused": 0,
"reuse_policy": "Pilot cells remain historical pilot evidence; no pilot weights reused or bound into matrix"
}
@@ -0,0 +1,27 @@
{
"status": "INDEPENDENT_ENGINEERING_ACCEPTANCE_AND_PLAN_ONLY",
"observed_unix": 1794698943.6114888,
"original_source_pins_unchanged": true,
"GPU_training_or_tensor_work_executed": false,
"main_matrix_started": false,
"preparation_started": false,
"new_neural_cells_planned": 90,
"pilot_cells_reused": 0,
"roster_ruling": "1794698734.967459",
"guard_tests": 7,
"artifact_sha256": {
"artifacts/s2-main-engineering-2026-11-15/allocator-preflight.json": "85b19a711897dfc7b6979e16c9018a56fa7f97789f5e9a510799779c6b806570",
"artifacts/s2-main-engineering-2026-11-15/budget-estimate.json": "0a2298d37176237ee786511a10bbaa7106f14539d2265dd719fd60321699f22b",
"artifacts/s2-main-engineering-2026-11-15/historical-N256-readonly-memory-audit.json": "71b3d666ff4ac877604d7c0f0cad7b0082960a180c2032ab366507f2f936e1b2",
"artifacts/s2-main-engineering-2026-11-15/planned-fold-pool-costs.csv": "b0a2e1f7b9da4258eecd6ddd60748e94502f0d6e0114253ecdd4ea7667afdc9c",
"artifacts/s2-main-engineering-2026-11-15/planned-neural-cells.json": "4d36ff92a46d7c82b5a3c6fe6c4747eced8bb98e4b0000cda913d792329c5292",
"artifacts/s2-main-engineering-2026-11-15/planned-preparation-131.json": "bd3e5b2a1e56baafd924ebdd9661c35290e53c68f36c26d2becebcd6501a06ca",
"artifacts/s2-main-engineering-2026-11-15/reuse-candidates.json": "4a7d670a6e7e004dd330358a6e6c85a5334dbbfff837182d02a2ef75e17fe619",
"artifacts/s2-main-engineering-2026-11-15/source-hashes.json": "5223fa9423e42aea415ab6756e44e7a11c7bde27e090d2256a64b089b47722d2",
"artifacts/s2-main-engineering-2026-11-15/synthetic-final-tests.txt": "388cada21d5df8240258cfe4fc5a888ece2d8a5bc73d5e7499940e56b5e7e11e",
"artifacts/s2-main-engineering-2026-11-15/test-receipt.json": "18c44e5c612d0a2a7cea3aa7c1dc7fc65c491a59d9406cb9dd4ca53f8924e449",
"docs/s2-main-engineering-and-budget-2026-11-15.md": "7d86331bfbf0040b21dc6455c73926fe01e1537409e9883b62bcfa79735fc110",
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"tools/test_s2_main_memory_guard.py": "510b9d93628c06196d01b8d0c89616070125a0c39546bc96e2f5c732d8c19cc6"
}
}
@@ -0,0 +1,48 @@
{
"maxima": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
},
"witnesses": {
"peak_allocated_bytes": {
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/history.json",
"sha256": "03a086fa5dd18cbae94984cfd1991d98a3e4011105c237fa32ab98c9d29004a4",
"observed_unix": 1794698530.1258473,
"peaks": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
}
},
"peak_reserved_bytes": {
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/history.json",
"sha256": "03a086fa5dd18cbae94984cfd1991d98a3e4011105c237fa32ab98c9d29004a4",
"observed_unix": 1794698530.1258473,
"peaks": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
}
}
},
"observations": [
{
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/history.json",
"sha256": "03a086fa5dd18cbae94984cfd1991d98a3e4011105c237fa32ab98c9d29004a4",
"observed_unix": 1794698530.1258473,
"peaks": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
}
},
{
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/progress.json",
"sha256": "a7bb4a85494c928d0daeacd90ef61a8043d6f4b2c82c362d768517fe4e30ac67",
"observed_unix": 1794698530.1259105,
"peaks": {
"peak_allocated_bytes": 7525132800,
"peak_reserved_bytes": 16416505856
}
}
],
"limit_bytes": 16106127360,
"breach": true
}
@@ -0,0 +1,16 @@
fold,component_fraction_percent,learning_parts,learning_nodes,validation_parts,validation_nodes,projected_N_seconds,projected_E_seconds
0,25,145,4319375,112,3330152,593.4988759700324,589.3006749431931
0,50,254,7524426,112,3330152,842.1670832888153,836.2098913597581
0,100,387,11863762,112,3330152,1178.840323144861,1170.501623855806
1,25,85,2401741,90,3242126,437.88703806449195,434.78958011255014
1,50,180,4549249,90,3242126,604.5043444857455,600.2282946691372
1,100,409,11386113,90,3242126,1134.9516648442434,1126.9234183931035
2,25,90,2998926,88,2648316,438.14889199433605,435.0495817803569
2,50,203,6659657,88,2648316,722.1716470913051,717.0632604150582
2,100,411,12229477,88,2648316,1154.3136487281913,1146.1484424546923
3,25,128,4348385,144,3976319,645.8833947227367,641.3146441475793
3,50,236,7631328,144,3976319,900.5949579832738,894.224467944532
3,100,355,11031937,144,3976319,1164.435796653897,1156.1989898879015
4,25,58,1748016,114,3401388,399.52345889041095,396.69737132180575
4,50,144,4743744,114,3401388,631.9510587553376,627.4808605945702
4,100,386,11535306,114,3401388,1158.883562305006,1150.6860300800224
1 fold component_fraction_percent learning_parts learning_nodes validation_parts validation_nodes projected_N_seconds projected_E_seconds
2 0 25 145 4319375 112 3330152 593.4988759700324 589.3006749431931
3 0 50 254 7524426 112 3330152 842.1670832888153 836.2098913597581
4 0 100 387 11863762 112 3330152 1178.840323144861 1170.501623855806
5 1 25 85 2401741 90 3242126 437.88703806449195 434.78958011255014
6 1 50 180 4549249 90 3242126 604.5043444857455 600.2282946691372
7 1 100 409 11386113 90 3242126 1134.9516648442434 1126.9234183931035
8 2 25 90 2998926 88 2648316 438.14889199433605 435.0495817803569
9 2 50 203 6659657 88 2648316 722.1716470913051 717.0632604150582
10 2 100 411 12229477 88 2648316 1154.3136487281913 1146.1484424546923
11 3 25 128 4348385 144 3976319 645.8833947227367 641.3146441475793
12 3 50 236 7631328 144 3976319 900.5949579832738 894.224467944532
13 3 100 355 11031937 144 3976319 1164.435796653897 1156.1989898879015
14 4 25 58 1748016 114 3401388 399.52345889041095 396.69737132180575
15 4 50 144 4743744 114 3401388 631.9510587553376 627.4808605945702
16 4 100 386 11535306 114 3401388 1158.883562305006 1150.6860300800224
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,63 @@
{
"status": "SUPERSEDED_NO_REUSE_BY_1794698734.967459",
"candidates": [
{
"matrix_cell": {
"fold": 0,
"component_fraction_percent": 100,
"seed": 0,
"arm": "N",
"width": 128
},
"source_run": "artifacts/s2-pilot-v2-continuation-2026-11-15/N128-seed0",
"result_sha256": "f75ded80d92c4e9835ca8bcda6b9772fd1fe56b131c9964097fdfea7ea8f1a44",
"checkpoint_sha256": "ca2eb128116e877e2ffed65f4c1c5864e090beab9b7c0615f117e613599ccd34",
"guard_sha256": "ab7b44f144d5563cb8ffcc91c62f7b4452c6db645adf1f6068985c5a11ce98eb",
"training_contract_sha256": "f6eaa3cb2fd6918e5bd940826a2cc954ffcee0a9a2d1578b2da1e5e1c72abd5c",
"success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
"standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03",
"learning_parts": 382,
"validation_parts": 111,
"E_max": 42,
"cosine_T_max": 42,
"completed_epochs": 42,
"reuse_authorization": "1794698584.380959",
"metadata_eligibility": "EXACT_ROLE_AND_POOL_ROSTER_MUST_REMAIN_IDENTICAL",
"allocator_difference": "historical accepted fit used default allocator; new fits require expandable_segments, no numerical change",
"test_predictions_produced": false,
"test_predictions_policy": "only with rest of matrix at custody step",
"original_guard_recorded_peaks_readonly_accepted": true,
"saved_training_wall_seconds": 1129.3405424560187
},
{
"matrix_cell": {
"fold": 0,
"component_fraction_percent": 100,
"seed": 0,
"arm": "E",
"width": 128
},
"source_run": "artifacts/s2-pilot-v2-continuation-2026-11-15/E128-seed0",
"result_sha256": "2e1271dbcc00ba4ee4889aa2c9e192283c06a9bd3eabc10a973dabba2f7dad78",
"checkpoint_sha256": "910437bd645a965241786a90ad14325fe7af09092238bfd0be5ee5bddf8a9a24",
"guard_sha256": "39d3c063ab4ac979add7150e374ad1531f6621b49c3fe5653d443e999e57624b",
"training_contract_sha256": "f6eaa3cb2fd6918e5bd940826a2cc954ffcee0a9a2d1578b2da1e5e1c72abd5c",
"success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
"standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03",
"learning_parts": 382,
"validation_parts": 111,
"E_max": 42,
"cosine_T_max": 42,
"completed_epochs": 42,
"reuse_authorization": "1794698584.380959",
"metadata_eligibility": "EXACT_ROLE_AND_POOL_ROSTER_MUST_REMAIN_IDENTICAL",
"allocator_difference": "historical accepted fit used default allocator; new fits require expandable_segments, no numerical change",
"test_predictions_produced": false,
"test_predictions_policy": "only with rest of matrix at custody step",
"original_guard_recorded_peaks_readonly_accepted": true,
"saved_training_wall_seconds": 1121.3519870990422
}
],
"saved_measured_training_seconds": 2250.692529555061,
"final_decision": "All90 neural fits fresh on one post-preparation roster; these entries retained as historical proposal only"
}
@@ -0,0 +1,7 @@
{
"artifacts/s2-pilot-v2-continuation-2026-11-15/N128-seed0/result.json": "f75ded80d92c4e9835ca8bcda6b9772fd1fe56b131c9964097fdfea7ea8f1a44",
"artifacts/s2-pilot-v2-continuation-2026-11-15/E128-seed0/result.json": "2e1271dbcc00ba4ee4889aa2c9e192283c06a9bd3eabc10a973dabba2f7dad78",
"artifacts/s2-pilot-v2-continuation-2026-11-15/N128-seed0-guard-receipt.json": "ab7b44f144d5563cb8ffcc91c62f7b4452c6db645adf1f6068985c5a11ce98eb",
"artifacts/s2-pilot-v2-continuation-2026-11-15/E128-seed0-guard-receipt.json": "39d3c063ab4ac979add7150e374ad1531f6621b49c3fe5653d443e999e57624b",
"artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json": "53f2fa6f79e368a340100a9166b0125fe57614cdd3f22a73822dba2a9697c6fd"
}
@@ -0,0 +1,12 @@
test_allocator_is_required_before_device_or_worker_call (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_allocator_reaches_supervised_cpu_child_without_cuda_work (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_failed_worker_final_scan_catches_breach_without_result (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_invalid_recorded_peak_rejected (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_maximum_survives_lower_replacement_and_deletion (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_nested_history_and_result_both_contribute (test_s2_main_memory_guard.RecordedPeakTests) ... ok
test_only_progress_reserved_breach_stops_live_worker (test_s2_main_memory_guard.RecordedPeakTests) ... ok
----------------------------------------------------------------------
Ran 7 tests in 0.124s
OK
@@ -0,0 +1,19 @@
{
"status": "PASS",
"initial_checks": {
"tests": 6,
"seconds": 0.016,
"tool_session": 22962
},
"final_checks": {
"tests": 7,
"CPU_only": true,
"GPU_work": false
},
"parse_check": "PASS",
"observed_unix": 1794698644.070644,
"code_sha256": {
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"tools/test_s2_main_memory_guard.py": "510b9d93628c06196d01b8d0c89616070125a0c39546bc96e2f5c732d8c19cc6"
}
}