Add independent recorded-peak guard and freeze fresh main-matrix engineering plan
This commit is contained in:
1 parent
59fd40adba
commit
0af3dab8d1
16 files changed
+3342
No files matched your search
@@ -24,3 +24,5 @@
|
||||
- Latest terminal update (2026-11-15 07:14:59 UTC+8): session34632 and all pilot workers have exited. N128/E128 completed42 epochs with guardPASS; N256 failed at epoch2 on4/382 AMP skips exceeding1%, E256 never started. PILOT_INCOMPLETE/no width; cumulative2439.912972s/7200. N256 history/progress also records reserved16416505856B above15GiB; the failed-result path did not inspect recorded peaks. Preserve this resource-boundary gap. Read `docs/s2-v2-pilot-stop-results-2026-11-15.md`; no restart, reprobe, threshold/source change or full matrix is authorized by a polling request.125 tests sealed; evidence submitted toClaude.
|
||||
|
||||
- Claude1794698179.161599 records a separate feasibility decision: main-matrix width128, width effect not tested, no use of validation scores to justify selection. Original pilot selected_global_width staysnull. Before any main-matrix execution: fix recorded progress/result VRAM-peak enforcement with synthetic test, allocator configuration per ruling, unchangedAMP1% andE_max42, and budget/execution plan. This read-only stage audit has not implemented or launched that work.
|
||||
|
||||
- Main engineering plan update: independent `s2_main_memory_guard.py` checks all progress/result/history peak records during and after failed workers;7 CPU synthetic checks passed. No old pinned source changed. Allocatorexpandable_segments parsedtrue withV100 VMM support, no GPU tensors/workload. Claude1794698734.967459 supersedes pilot-cell reuse: separately prepare6 timeouts at60min and125 fold0 test geometries without label decoding, freeze one final roster, then all90 neural fits fresh; pilot weights remain historical only. Read `docs/s2-main-engineering-and-budget-2026-11-15.md`. Proposed costs/caps are plans; no131 queue or main matrix launched in this engineering turn.
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"status": "METADATA_PREFLIGHT_PASS_NOT_ALLOCATION_WORKLOAD",
|
||||
"source_tool_sessions": [
|
||||
98992,
|
||||
87464
|
||||
],
|
||||
"torch_version": "2.5.1+cu124",
|
||||
"torch_cuda_build": "12.4",
|
||||
"allocator_conf": "expandable_segments:True",
|
||||
"allocator_backend": "native",
|
||||
"parsed_allocator_settings_expandable_segments": true,
|
||||
"torch_cuda_context_initialized": false,
|
||||
"allocated_GPU_tensors": 0,
|
||||
"virtual_memory_attribute": {
|
||||
"id": 102,
|
||||
"name": "CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED",
|
||||
"local_header": "/usr/local/cuda/include/cuda.h:786"
|
||||
},
|
||||
"devices": [
|
||||
{
|
||||
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"name": "Tesla V100-SXM2-16GB",
|
||||
"virtual_memory_management_supported": true
|
||||
},
|
||||
{
|
||||
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"name": "Tesla V100-SXM2-16GB",
|
||||
"virtual_memory_management_supported": true
|
||||
},
|
||||
{
|
||||
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"name": "Tesla V100-SXM2-16GB",
|
||||
"virtual_memory_management_supported": true
|
||||
},
|
||||
{
|
||||
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"name": "Tesla V100-SXM2-16GB",
|
||||
"virtual_memory_management_supported": true
|
||||
}
|
||||
],
|
||||
"limit": "Configuration parsing and driver VMM capability verified; no successful real expandable CUDA allocation claimed"
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
{
|
||||
"status": "PLAN_ONLY_NO_MAIN_EXECUTION",
|
||||
"neural_runs": 90,
|
||||
"arms": [
|
||||
"N",
|
||||
"E"
|
||||
],
|
||||
"folds": 5,
|
||||
"component_fraction_percent": [
|
||||
25,
|
||||
50,
|
||||
100
|
||||
],
|
||||
"seeds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"width": 128,
|
||||
"width_selection": "feasibility; width effect not tested",
|
||||
"E_max": 42,
|
||||
"cosine_T_max": 42,
|
||||
"baseline_ready_nodes": 14555918,
|
||||
"baseline_worker_guard_wall_seconds": {
|
||||
"N": 1129.3405424560187,
|
||||
"E": 1121.3519870990422
|
||||
},
|
||||
"node_scaled_gpu_hours": 19.942144065735626,
|
||||
"node_scaled_plus25percent_and_idle_hours": 25.42768008216953,
|
||||
"all90_priced_at_measured_full_pool_hours": 28.133656619438263,
|
||||
"all90_full_pool_plus25percent_and_idle_hours": 35.66707077429783,
|
||||
"proposed_per_run_wall_limit_seconds": 1800,
|
||||
"90_run_caps_plus_expected_idle_gpu_hours": 45.5,
|
||||
"proposed_global_neural_wall_limit_hours": 46,
|
||||
"R2_final_fits": 15,
|
||||
"R2_HPO_configurations_per_fold_pool_max": 12,
|
||||
"R2_HPO_trials_max": 180,
|
||||
"R2_CPU_time": "UNMEASURED_NOT_INCLUDED",
|
||||
"preparation_completion_cost": "NOT_INCLUDED_125_UNPREPARED_AND_6_TIMEOUTS_REQUIRE_SEPARATE_PLAN",
|
||||
"estimation_limits": [
|
||||
"Composite per-node scaling cannot separate 128 training/inference/setup rates",
|
||||
"Measured one fold/full successful pool/seed; no confidence bound",
|
||||
"Allocator change may change timing; proposed reserve is engineering allowance",
|
||||
"No early-stopping savings assumed",
|
||||
"Global proposed cap includes waits/setup, may stop before all90; no automatic extra budget",
|
||||
"GPU-hours are aggregate device time, not multi-GPU elapsed promise"
|
||||
],
|
||||
"rows": [
|
||||
{
|
||||
"fold": 0,
|
||||
"component_fraction_percent": 25,
|
||||
"learning_parts": 145,
|
||||
"learning_nodes": 4319375,
|
||||
"validation_parts": 112,
|
||||
"validation_nodes": 3330152,
|
||||
"projected_N_seconds": 593.4988759700324,
|
||||
"projected_E_seconds": 589.3006749431931
|
||||
},
|
||||
{
|
||||
"fold": 0,
|
||||
"component_fraction_percent": 50,
|
||||
"learning_parts": 254,
|
||||
"learning_nodes": 7524426,
|
||||
"validation_parts": 112,
|
||||
"validation_nodes": 3330152,
|
||||
"projected_N_seconds": 842.1670832888153,
|
||||
"projected_E_seconds": 836.2098913597581
|
||||
},
|
||||
{
|
||||
"fold": 0,
|
||||
"component_fraction_percent": 100,
|
||||
"learning_parts": 387,
|
||||
"learning_nodes": 11863762,
|
||||
"validation_parts": 112,
|
||||
"validation_nodes": 3330152,
|
||||
"projected_N_seconds": 1178.840323144861,
|
||||
"projected_E_seconds": 1170.501623855806
|
||||
},
|
||||
{
|
||||
"fold": 1,
|
||||
"component_fraction_percent": 25,
|
||||
"learning_parts": 85,
|
||||
"learning_nodes": 2401741,
|
||||
"validation_parts": 90,
|
||||
"validation_nodes": 3242126,
|
||||
"projected_N_seconds": 437.88703806449195,
|
||||
"projected_E_seconds": 434.78958011255014
|
||||
},
|
||||
{
|
||||
"fold": 1,
|
||||
"component_fraction_percent": 50,
|
||||
"learning_parts": 180,
|
||||
"learning_nodes": 4549249,
|
||||
"validation_parts": 90,
|
||||
"validation_nodes": 3242126,
|
||||
"projected_N_seconds": 604.5043444857455,
|
||||
"projected_E_seconds": 600.2282946691372
|
||||
},
|
||||
{
|
||||
"fold": 1,
|
||||
"component_fraction_percent": 100,
|
||||
"learning_parts": 409,
|
||||
"learning_nodes": 11386113,
|
||||
"validation_parts": 90,
|
||||
"validation_nodes": 3242126,
|
||||
"projected_N_seconds": 1134.9516648442434,
|
||||
"projected_E_seconds": 1126.9234183931035
|
||||
},
|
||||
{
|
||||
"fold": 2,
|
||||
"component_fraction_percent": 25,
|
||||
"learning_parts": 90,
|
||||
"learning_nodes": 2998926,
|
||||
"validation_parts": 88,
|
||||
"validation_nodes": 2648316,
|
||||
"projected_N_seconds": 438.14889199433605,
|
||||
"projected_E_seconds": 435.0495817803569
|
||||
},
|
||||
{
|
||||
"fold": 2,
|
||||
"component_fraction_percent": 50,
|
||||
"learning_parts": 203,
|
||||
"learning_nodes": 6659657,
|
||||
"validation_parts": 88,
|
||||
"validation_nodes": 2648316,
|
||||
"projected_N_seconds": 722.1716470913051,
|
||||
"projected_E_seconds": 717.0632604150582
|
||||
},
|
||||
{
|
||||
"fold": 2,
|
||||
"component_fraction_percent": 100,
|
||||
"learning_parts": 411,
|
||||
"learning_nodes": 12229477,
|
||||
"validation_parts": 88,
|
||||
"validation_nodes": 2648316,
|
||||
"projected_N_seconds": 1154.3136487281913,
|
||||
"projected_E_seconds": 1146.1484424546923
|
||||
},
|
||||
{
|
||||
"fold": 3,
|
||||
"component_fraction_percent": 25,
|
||||
"learning_parts": 128,
|
||||
"learning_nodes": 4348385,
|
||||
"validation_parts": 144,
|
||||
"validation_nodes": 3976319,
|
||||
"projected_N_seconds": 645.8833947227367,
|
||||
"projected_E_seconds": 641.3146441475793
|
||||
},
|
||||
{
|
||||
"fold": 3,
|
||||
"component_fraction_percent": 50,
|
||||
"learning_parts": 236,
|
||||
"learning_nodes": 7631328,
|
||||
"validation_parts": 144,
|
||||
"validation_nodes": 3976319,
|
||||
"projected_N_seconds": 900.5949579832738,
|
||||
"projected_E_seconds": 894.224467944532
|
||||
},
|
||||
{
|
||||
"fold": 3,
|
||||
"component_fraction_percent": 100,
|
||||
"learning_parts": 355,
|
||||
"learning_nodes": 11031937,
|
||||
"validation_parts": 144,
|
||||
"validation_nodes": 3976319,
|
||||
"projected_N_seconds": 1164.435796653897,
|
||||
"projected_E_seconds": 1156.1989898879015
|
||||
},
|
||||
{
|
||||
"fold": 4,
|
||||
"component_fraction_percent": 25,
|
||||
"learning_parts": 58,
|
||||
"learning_nodes": 1748016,
|
||||
"validation_parts": 114,
|
||||
"validation_nodes": 3401388,
|
||||
"projected_N_seconds": 399.52345889041095,
|
||||
"projected_E_seconds": 396.69737132180575
|
||||
},
|
||||
{
|
||||
"fold": 4,
|
||||
"component_fraction_percent": 50,
|
||||
"learning_parts": 144,
|
||||
"learning_nodes": 4743744,
|
||||
"validation_parts": 114,
|
||||
"validation_nodes": 3401388,
|
||||
"projected_N_seconds": 631.9510587553376,
|
||||
"projected_E_seconds": 627.4808605945702
|
||||
},
|
||||
{
|
||||
"fold": 4,
|
||||
"component_fraction_percent": 100,
|
||||
"learning_parts": 386,
|
||||
"learning_nodes": 11535306,
|
||||
"validation_parts": 114,
|
||||
"validation_nodes": 3401388,
|
||||
"projected_N_seconds": 1158.883562305006,
|
||||
"projected_E_seconds": 1150.6860300800224
|
||||
}
|
||||
],
|
||||
"preparation_proposal": {
|
||||
"new_parts_without_S2_ready": 125,
|
||||
"their_nodes": 3467310,
|
||||
"retained_true_timeout_parts": 6,
|
||||
"conservative_observed_cpu_seconds_per_original_planned_node": 0.008235846461945216,
|
||||
"rate_includes_prior_aborted_and_recovery_cost": true,
|
||||
"new125_projected_worker_hours": 7.932286887768686,
|
||||
"six_timeout_repreparations_separate_60min_each_cap_hours": 6,
|
||||
"sum_with25percent_reserve_hours": 17.415358609710857,
|
||||
"proposed_new_worker_cpu_cap_hours": 20,
|
||||
"proposed_cumulative_output_disk_GiB": 100,
|
||||
"proposal_only_no_execution": true,
|
||||
"no_labels_from_current_sealed_test_parts_read": true,
|
||||
"label_policy": "125 fold0 tests: geometry/features only; labels decoded only at custody step; no label reads in this planning turn"
|
||||
},
|
||||
"R2_proposal": {
|
||||
"status": "UNMEASURED_COST_CAP_PROPOSAL_ONLY",
|
||||
"single_thread": true,
|
||||
"fits_max_including_HPO_and_final": 195,
|
||||
"proposed_per_fit_cpu_seconds": 1800,
|
||||
"proposed_total_worker_cpu_hours": 100,
|
||||
"initial_checkpoint": "Measure one authorized fixed fit before dispatching full search; no current fit executed",
|
||||
"no_completion_guarantee": true
|
||||
},
|
||||
"final_roster_ruling": "1794698734.967459",
|
||||
"neural_runs_fresh": 90,
|
||||
"pilot_cells_reused": 0,
|
||||
"reuse_policy": "Pilot cells remain historical pilot evidence; no pilot weights reused or bound into matrix"
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "INDEPENDENT_ENGINEERING_ACCEPTANCE_AND_PLAN_ONLY",
|
||||
"observed_unix": 1794698943.6114888,
|
||||
"original_source_pins_unchanged": true,
|
||||
"GPU_training_or_tensor_work_executed": false,
|
||||
"main_matrix_started": false,
|
||||
"preparation_started": false,
|
||||
"new_neural_cells_planned": 90,
|
||||
"pilot_cells_reused": 0,
|
||||
"roster_ruling": "1794698734.967459",
|
||||
"guard_tests": 7,
|
||||
"artifact_sha256": {
|
||||
"artifacts/s2-main-engineering-2026-11-15/allocator-preflight.json": "85b19a711897dfc7b6979e16c9018a56fa7f97789f5e9a510799779c6b806570",
|
||||
"artifacts/s2-main-engineering-2026-11-15/budget-estimate.json": "0a2298d37176237ee786511a10bbaa7106f14539d2265dd719fd60321699f22b",
|
||||
"artifacts/s2-main-engineering-2026-11-15/historical-N256-readonly-memory-audit.json": "71b3d666ff4ac877604d7c0f0cad7b0082960a180c2032ab366507f2f936e1b2",
|
||||
"artifacts/s2-main-engineering-2026-11-15/planned-fold-pool-costs.csv": "b0a2e1f7b9da4258eecd6ddd60748e94502f0d6e0114253ecdd4ea7667afdc9c",
|
||||
"artifacts/s2-main-engineering-2026-11-15/planned-neural-cells.json": "4d36ff92a46d7c82b5a3c6fe6c4747eced8bb98e4b0000cda913d792329c5292",
|
||||
"artifacts/s2-main-engineering-2026-11-15/planned-preparation-131.json": "bd3e5b2a1e56baafd924ebdd9661c35290e53c68f36c26d2becebcd6501a06ca",
|
||||
"artifacts/s2-main-engineering-2026-11-15/reuse-candidates.json": "4a7d670a6e7e004dd330358a6e6c85a5334dbbfff837182d02a2ef75e17fe619",
|
||||
"artifacts/s2-main-engineering-2026-11-15/source-hashes.json": "5223fa9423e42aea415ab6756e44e7a11c7bde27e090d2256a64b089b47722d2",
|
||||
"artifacts/s2-main-engineering-2026-11-15/synthetic-final-tests.txt": "388cada21d5df8240258cfe4fc5a888ece2d8a5bc73d5e7499940e56b5e7e11e",
|
||||
"artifacts/s2-main-engineering-2026-11-15/test-receipt.json": "18c44e5c612d0a2a7cea3aa7c1dc7fc65c491a59d9406cb9dd4ca53f8924e449",
|
||||
"docs/s2-main-engineering-and-budget-2026-11-15.md": "7d86331bfbf0040b21dc6455c73926fe01e1537409e9883b62bcfa79735fc110",
|
||||
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"tools/test_s2_main_memory_guard.py": "510b9d93628c06196d01b8d0c89616070125a0c39546bc96e2f5c732d8c19cc6"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"maxima": {
|
||||
"peak_allocated_bytes": 7525132800,
|
||||
"peak_reserved_bytes": 16416505856
|
||||
},
|
||||
"witnesses": {
|
||||
"peak_allocated_bytes": {
|
||||
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/history.json",
|
||||
"sha256": "03a086fa5dd18cbae94984cfd1991d98a3e4011105c237fa32ab98c9d29004a4",
|
||||
"observed_unix": 1794698530.1258473,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 7525132800,
|
||||
"peak_reserved_bytes": 16416505856
|
||||
}
|
||||
},
|
||||
"peak_reserved_bytes": {
|
||||
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/history.json",
|
||||
"sha256": "03a086fa5dd18cbae94984cfd1991d98a3e4011105c237fa32ab98c9d29004a4",
|
||||
"observed_unix": 1794698530.1258473,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 7525132800,
|
||||
"peak_reserved_bytes": 16416505856
|
||||
}
|
||||
}
|
||||
},
|
||||
"observations": [
|
||||
{
|
||||
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/history.json",
|
||||
"sha256": "03a086fa5dd18cbae94984cfd1991d98a3e4011105c237fa32ab98c9d29004a4",
|
||||
"observed_unix": 1794698530.1258473,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 7525132800,
|
||||
"peak_reserved_bytes": 16416505856
|
||||
}
|
||||
},
|
||||
{
|
||||
"path": "artifacts/s2-pilot-v2-continuation-2026-11-15/N256-seed0/progress.json",
|
||||
"sha256": "a7bb4a85494c928d0daeacd90ef61a8043d6f4b2c82c362d768517fe4e30ac67",
|
||||
"observed_unix": 1794698530.1259105,
|
||||
"peaks": {
|
||||
"peak_allocated_bytes": 7525132800,
|
||||
"peak_reserved_bytes": 16416505856
|
||||
}
|
||||
}
|
||||
],
|
||||
"limit_bytes": 16106127360,
|
||||
"breach": true
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
fold,component_fraction_percent,learning_parts,learning_nodes,validation_parts,validation_nodes,projected_N_seconds,projected_E_seconds
|
||||
0,25,145,4319375,112,3330152,593.4988759700324,589.3006749431931
|
||||
0,50,254,7524426,112,3330152,842.1670832888153,836.2098913597581
|
||||
0,100,387,11863762,112,3330152,1178.840323144861,1170.501623855806
|
||||
1,25,85,2401741,90,3242126,437.88703806449195,434.78958011255014
|
||||
1,50,180,4549249,90,3242126,604.5043444857455,600.2282946691372
|
||||
1,100,409,11386113,90,3242126,1134.9516648442434,1126.9234183931035
|
||||
2,25,90,2998926,88,2648316,438.14889199433605,435.0495817803569
|
||||
2,50,203,6659657,88,2648316,722.1716470913051,717.0632604150582
|
||||
2,100,411,12229477,88,2648316,1154.3136487281913,1146.1484424546923
|
||||
3,25,128,4348385,144,3976319,645.8833947227367,641.3146441475793
|
||||
3,50,236,7631328,144,3976319,900.5949579832738,894.224467944532
|
||||
3,100,355,11031937,144,3976319,1164.435796653897,1156.1989898879015
|
||||
4,25,58,1748016,114,3401388,399.52345889041095,396.69737132180575
|
||||
4,50,144,4743744,114,3401388,631.9510587553376,627.4808605945702
|
||||
4,100,386,11535306,114,3401388,1158.883562305006,1150.6860300800224
|
||||
|
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"status": "SUPERSEDED_NO_REUSE_BY_1794698734.967459",
|
||||
"candidates": [
|
||||
{
|
||||
"matrix_cell": {
|
||||
"fold": 0,
|
||||
"component_fraction_percent": 100,
|
||||
"seed": 0,
|
||||
"arm": "N",
|
||||
"width": 128
|
||||
},
|
||||
"source_run": "artifacts/s2-pilot-v2-continuation-2026-11-15/N128-seed0",
|
||||
"result_sha256": "f75ded80d92c4e9835ca8bcda6b9772fd1fe56b131c9964097fdfea7ea8f1a44",
|
||||
"checkpoint_sha256": "ca2eb128116e877e2ffed65f4c1c5864e090beab9b7c0615f117e613599ccd34",
|
||||
"guard_sha256": "ab7b44f144d5563cb8ffcc91c62f7b4452c6db645adf1f6068985c5a11ce98eb",
|
||||
"training_contract_sha256": "f6eaa3cb2fd6918e5bd940826a2cc954ffcee0a9a2d1578b2da1e5e1c72abd5c",
|
||||
"success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
|
||||
"standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03",
|
||||
"learning_parts": 382,
|
||||
"validation_parts": 111,
|
||||
"E_max": 42,
|
||||
"cosine_T_max": 42,
|
||||
"completed_epochs": 42,
|
||||
"reuse_authorization": "1794698584.380959",
|
||||
"metadata_eligibility": "EXACT_ROLE_AND_POOL_ROSTER_MUST_REMAIN_IDENTICAL",
|
||||
"allocator_difference": "historical accepted fit used default allocator; new fits require expandable_segments, no numerical change",
|
||||
"test_predictions_produced": false,
|
||||
"test_predictions_policy": "only with rest of matrix at custody step",
|
||||
"original_guard_recorded_peaks_readonly_accepted": true,
|
||||
"saved_training_wall_seconds": 1129.3405424560187
|
||||
},
|
||||
{
|
||||
"matrix_cell": {
|
||||
"fold": 0,
|
||||
"component_fraction_percent": 100,
|
||||
"seed": 0,
|
||||
"arm": "E",
|
||||
"width": 128
|
||||
},
|
||||
"source_run": "artifacts/s2-pilot-v2-continuation-2026-11-15/E128-seed0",
|
||||
"result_sha256": "2e1271dbcc00ba4ee4889aa2c9e192283c06a9bd3eabc10a973dabba2f7dad78",
|
||||
"checkpoint_sha256": "910437bd645a965241786a90ad14325fe7af09092238bfd0be5ee5bddf8a9a24",
|
||||
"guard_sha256": "39d3c063ab4ac979add7150e374ad1531f6621b49c3fe5653d443e999e57624b",
|
||||
"training_contract_sha256": "f6eaa3cb2fd6918e5bd940826a2cc954ffcee0a9a2d1578b2da1e5e1c72abd5c",
|
||||
"success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
|
||||
"standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03",
|
||||
"learning_parts": 382,
|
||||
"validation_parts": 111,
|
||||
"E_max": 42,
|
||||
"cosine_T_max": 42,
|
||||
"completed_epochs": 42,
|
||||
"reuse_authorization": "1794698584.380959",
|
||||
"metadata_eligibility": "EXACT_ROLE_AND_POOL_ROSTER_MUST_REMAIN_IDENTICAL",
|
||||
"allocator_difference": "historical accepted fit used default allocator; new fits require expandable_segments, no numerical change",
|
||||
"test_predictions_produced": false,
|
||||
"test_predictions_policy": "only with rest of matrix at custody step",
|
||||
"original_guard_recorded_peaks_readonly_accepted": true,
|
||||
"saved_training_wall_seconds": 1121.3519870990422
|
||||
}
|
||||
],
|
||||
"saved_measured_training_seconds": 2250.692529555061,
|
||||
"final_decision": "All90 neural fits fresh on one post-preparation roster; these entries retained as historical proposal only"
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"artifacts/s2-pilot-v2-continuation-2026-11-15/N128-seed0/result.json": "f75ded80d92c4e9835ca8bcda6b9772fd1fe56b131c9964097fdfea7ea8f1a44",
|
||||
"artifacts/s2-pilot-v2-continuation-2026-11-15/E128-seed0/result.json": "2e1271dbcc00ba4ee4889aa2c9e192283c06a9bd3eabc10a973dabba2f7dad78",
|
||||
"artifacts/s2-pilot-v2-continuation-2026-11-15/N128-seed0-guard-receipt.json": "ab7b44f144d5563cb8ffcc91c62f7b4452c6db645adf1f6068985c5a11ce98eb",
|
||||
"artifacts/s2-pilot-v2-continuation-2026-11-15/E128-seed0-guard-receipt.json": "39d3c063ab4ac979add7150e374ad1531f6621b49c3fe5653d443e999e57624b",
|
||||
"artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json": "53f2fa6f79e368a340100a9166b0125fe57614cdd3f22a73822dba2a9697c6fd"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
test_allocator_is_required_before_device_or_worker_call (test_s2_main_memory_guard.RecordedPeakTests) ... ok
|
||||
test_allocator_reaches_supervised_cpu_child_without_cuda_work (test_s2_main_memory_guard.RecordedPeakTests) ... ok
|
||||
test_failed_worker_final_scan_catches_breach_without_result (test_s2_main_memory_guard.RecordedPeakTests) ... ok
|
||||
test_invalid_recorded_peak_rejected (test_s2_main_memory_guard.RecordedPeakTests) ... ok
|
||||
test_maximum_survives_lower_replacement_and_deletion (test_s2_main_memory_guard.RecordedPeakTests) ... ok
|
||||
test_nested_history_and_result_both_contribute (test_s2_main_memory_guard.RecordedPeakTests) ... ok
|
||||
test_only_progress_reserved_breach_stops_live_worker (test_s2_main_memory_guard.RecordedPeakTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 7 tests in 0.124s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"initial_checks": {
|
||||
"tests": 6,
|
||||
"seconds": 0.016,
|
||||
"tool_session": 22962
|
||||
},
|
||||
"final_checks": {
|
||||
"tests": 7,
|
||||
"CPU_only": true,
|
||||
"GPU_work": false
|
||||
},
|
||||
"parse_check": "PASS",
|
||||
"observed_unix": 1794698644.070644,
|
||||
"code_sha256": {
|
||||
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"tools/test_s2_main_memory_guard.py": "510b9d93628c06196d01b8d0c89616070125a0c39546bc96e2f5c732d8c19cc6"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
# S2 主矩阵前工程验收与预算/执行计划
|
||||
|
||||
当前只完成独立工程验收及计划。没有新的GPU训练、标签解码、准备或主矩阵执行。原pilot、诊断、v2零run停止和接续失败及旧源码全部保留。Claude1794698179.161599决定宽128按可行性采用,宽度效应未检验;原pilot自动选宽仍null,验证分数不用于解释该决定。
|
||||
|
||||
## 已完成条件与独立监督器
|
||||
|
||||
Claude1794698584.380959接受128显存条件:N128/E128的整run reserved峰值7.679688/7.677734GiB,allocated均约4.3834GiB;两个run都完整42轮覆盖最大187345节点件。这是包含最大件执行的全程上界,没有逐件隔离峰值记录,也没有补跑。
|
||||
|
||||
新的tools/s2_main_memory_guard.py是独立模块,没有矩阵启动CLI,不改旧监督器。对当前run目录所有JSON(progress/result/failure/history/epoch等)递归提取peak_reserved_bytes和peak_allocated_bytes,保留跨覆盖/删除的累计最大值、来源文件SHA和观察时间。运行时每约0.5秒扫描记录并结合设备抽样/主机RSS;任何记录峰值超过15GiB就停止本任务的匹配命令子进程。无result或worker失败退出也强制再扫描;保存原worker状态、原停止原因及所有监督原因。无效峰值/损坏记录失败关闭。其他进程不终止。
|
||||
|
||||
监督器要求启动环境PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,记录该值并由现有进程监督器传给子进程。只有完整结果且两类峰值存在才可能PASS。单run上限沿提案≤1800秒、15GiB VRAM、16GiB主机RSS、5GiB输出;调用者必须显式给累计worker CPU限额,未来矩阵入口仍须单独维护全局预算。此模块没有把90-run执行器或托管评估器实现完成。
|
||||
|
||||
7/7 CPU合成检查通过:仅progress reserved超限触发停止;失败无result的最终扫描;峰值跨覆盖/删除保留;history/result两类记录取最大;非法峰值拒绝;缺allocator环境在spawn前拒绝;真实受监督CPU子进程收到allocator/GPU可见性环境(未调用CUDA)。真实N256历史记录只读扫描重现15.2890625GiB上限违反。保留原guard缺口,不回写过去为已按新规则执行。检测发生在记录落盘/设备采样后,不宣称连续无采样延迟。
|
||||
|
||||
## Allocator配置核查
|
||||
|
||||
PyTorch2.5.1+cu124、CUDA12.4、native allocator。设置后读取allocator snapshot,expandable_segments=true;四张Tesla V100-SXM2-16GB的CUDA属性102(VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED)均true。只查元数据,没有分配GPU张量,torch.cuda.is_initialized=false;这证明配置解析/驱动能力,不冒称实际分配工作负载验收。未来获准主矩阵必须在启动前设置并在实际运行回执中记录;数值计算、AMP1%和E_max42不变。
|
||||
|
||||
## 名单与复用裁量
|
||||
|
||||
1794698584曾允许在名单完全相同的条件下复用两个pilot cell。核查指出旧cell使用382学习/111验证,回填6件后会变更名单。Claude1794698734.967459最终裁定:补齐6件、统一完整名单、90run全部fresh;两个pilot只作历史证据,不复用权重,不将其checkpoint SHA绑定为主矩阵初始化。历史reuse-candidates明确标为已被取代,预算回到90新run。
|
||||
|
||||
准备计划精确列131项:6个真实超时件独立标记再准备,单件60min/1.25GiB、0重试;125个fold0测试件首次S2几何/特征准备,单件30min/1.25GiB,标签只在托管步骤解码。不能直接用旧会解码标签的pilot ready入口处理这125件。主矩阵开始前,根据所有准备终态生成唯一名单SHA;仍失败件在所有五折一致排除,保留原组件身份和逐件失败/节点/质量报告,不重抽或重排原组件/池。现有493成功准备可复用几何/特征,不能把准备复用与pilot权重复用混淆。
|
||||
|
||||
原125测试标签在本次验收仍封存。主矩阵各fold角色必须由已固定split映射明确授权;不得以元数据规划为已解封标签。测试预测仅在其余矩阵一起进入托管评估时生成,不提前补pilot测试预测。
|
||||
|
||||
## 成本依据与提案(不是已经放行的预算)
|
||||
|
||||
N128/E128实测guard总wall1129.340542/1121.351987秒,均42轮。固定矩阵5外折×3组件比例档(25/50/100%)×3匹配种子×2臂=90个fresh神经fit;宽128,E_max=cosine_T_max=42,AMP首轮比例豁免/后续1%及最多20连续跳步规则保持。15行逐折/池件数、节点数与估时在planned-fold-pool-costs.csv,90个明确run ID在planned-neural-cells.json(不可执行计划)。
|
||||
|
||||
以两臂实测wall按(目标学习节点+验证节点)/(成功pilot学习节点+验证节点14,555,918)同比例外推,90run约19.94聚合GPU小时;加25%工程余量和每run20秒空闲确认约25.43小时。此为混合训练/验证/设置速率估计,没有假称掌握128分阶段速率。
|
||||
|
||||
较保守情景把90run全部按本次完整池实测计价:28.13 GPU小时;加25%余量及空闲确认35.67小时。两情景均不是统计置信上限,allocator、其他fold大小和池分布可能改变耗时,不计早停节约。提议单run≤30min、全部90run聚合墙钟硬限46GPU小时(90×30min=45h,再给正常门控/设置余量);门控等待也记账,到总限可能未完成,不能自动加时。四卡并发只有在对应卡空闲、每卡隔离监督及聚合计账实现验收后再定,不能把聚合GPU小时直接承诺为多卡日历时长。
|
||||
|
||||
准备另账:125件合计3,467,310节点;用历史包含中止/恢复成本的125134.74286CPU秒÷15,193,914原计划节点作保守率,预计约7.93 worker CPU小时。另给6件各60分钟最多6小时,加25%余量约17.42小时,提议新增20 worker CPU小时/累计100GiB输出,仍是规划、未启动。
|
||||
|
||||
R2-G24为15个fold×pool最终fit;每格最多12个HPO配置,共最多180 HPO+15最终fit。尚无拟合实测,不能给可信耗时预测。提议单线程每fit≤30 CPU分钟、合计100 worker CPU小时作为硬上限,首个获准固定fit后做成本检查再派发后续;不保证全部195 fit能完成。B0确定性计算/托管预测/统计以及缺失准备的实际实现与开销另列,不能将上述神经估时称为项目全成本。
|
||||
|
||||
## 进入顺序与待办
|
||||
|
||||
1. 固定本工程包及Claude名单/预算裁量;原失败证据永久保留。
|
||||
2. 独立实现并冻结131件不解码测试标签的准备队列与账本;按上述名单规则产出唯一最终roster SHA。当前只列计划,不宣称已实现或启动队列。
|
||||
3. 依据roster生成每fold/pool的角色白名单与学习统计、共同查询及目标;固定90fresh神经cell和15 R2/HPO清单。准备失败一致剔除、组件配对及覆盖报告沿既定政策,不凭pilot部分分数改变科学设计。
|
||||
4. 将本监督器与allocator设置接入新主矩阵入口,验收全局预算、失败终态、不重试/不替补、无提前测试访问及托管预测输出;不复用旧硬编码fold0/seed0/pool100的pilot训练入口当作完整矩阵执行器。
|
||||
5. 在已确认的执行范围和预算内按固定清单推进,所有失败保留、AMP阈值不变,测试评分/bootstrap待托管阶段;当前没有主矩阵启动回执。
|
||||
|
||||
研究目标仍是同主干N/E边信息增量、跨来源泛化与样本效率;容量可行性裁量与部分pilot分数不回答这些科学问题。6件超时含其他折测试件,主审要求先尝试补齐,避免按准备成本静默排除大件。资源与准备失败覆盖都需随主结果公开;O1类型/连通路线仍是独立未检验问题。
|
||||
@@ -60,3 +60,5 @@ Claude 1794684155.730259已接受接续入口,并要求合法单件FAIL保持
|
||||
夜间核查补充(Claude 1794684781.751169):首50已正式接受,原60h队列继续,不启用80h续队列。pilot准备仍守单件1800s;失败件报告必须补case/role/nodes/停止原因及原回执hash。探针VRAM结论仅覆盖其实际最大成功学习件;不得把20件probe推到被剔除或更大未测件。当前固定池最大学习件为666,187,345节点,但尚无本次GPU测量。已追加artifacts/s2-pilot-continuation-2026-11-15/probe-coverage-boundary.json;probe实际回执产生后,另建与其hash关联的覆盖附件,列实际最大节点与失败件节点数,保留原回执/运行源码pin不改。主矩阵之前,超时大件须按另行标记的更长单件上限准备,并在真实最大件核VRAM;这不是pilot静默重试,本轮未启动该工作。
|
||||
|
||||
最新主审裁量1794698179.161599:主矩阵宽128按可行性选取,宽度效应未检验;原pilot自动选宽仍null。主矩阵前guard记录峰值修复、allocator设置、预算/执行计划待办;本轮未启动。详细依据与128整run显存上界见s2-v2-pilot-stop-results-2026-11-15.md。
|
||||
|
||||
主矩阵前工程验收已完成独立guard7/7合成测试、旧N256记录只读检测及allocator元数据核查;未改旧源码。Claude1794698734.967459最终要求6件60min独立再准备+125件不解码标签准备、统一roster、90神经run全fresh,否决旧pilot cell复用。预算与分阶段执行计划见s2-main-engineering-and-budget-2026-11-15.md;131准备及主矩阵尚未启动。
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Independent future-run guard; no matrix launcher or training entry point.
|
||||
|
||||
Preserves the old pilot guard and its receipts. Checks recorded cumulative CUDA
|
||||
peaks while the worker is alive and again after any exit, including failure.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise, save, stop_group, disk_usage, tree_usage
|
||||
|
||||
ALLOCATOR_CONF='expandable_segments:True'
|
||||
VRAM_BYTES=15*2**30
|
||||
HOST_BYTES=16*2**30
|
||||
DISK_BYTES=5*2**30
|
||||
PEAK_KEYS=('peak_allocated_bytes','peak_reserved_bytes')
|
||||
|
||||
|
||||
def peaks_in(value):
|
||||
found={}
|
||||
if isinstance(value,dict):
|
||||
for key,child in value.items():
|
||||
if key in PEAK_KEYS:
|
||||
if isinstance(child,bool) or not isinstance(child,(int,float)) or not math.isfinite(child) or child<0:
|
||||
raise ValueError('Invalid recorded CUDA peak: '+key)
|
||||
found[key]=child
|
||||
nested=peaks_in(child)
|
||||
for k,v in nested.items():found[k]=max(found.get(k,0),v)
|
||||
elif isinstance(value,list):
|
||||
for child in value:
|
||||
for k,v in peaks_in(child).items():found[k]=max(found.get(k,0),v)
|
||||
return found
|
||||
|
||||
|
||||
class PeakRecords:
|
||||
def __init__(self):
|
||||
self.maxima={k:0 for k in PEAK_KEYS};self.witnesses={};self.cache={};self.observations=[]
|
||||
def scan(self,root):
|
||||
# Includes progress, result and failure records, plus epoch/history JSON.
|
||||
# Running maxima survive file replacement, regression and deletion.
|
||||
for path in sorted(Path(root).rglob('*.json')):
|
||||
try:
|
||||
stat=path.stat();version=(stat.st_mtime_ns,stat.st_size,stat.st_ino)
|
||||
if self.cache.get(str(path))==version:continue
|
||||
raw=path.read_bytes()
|
||||
except FileNotFoundError:continue
|
||||
peaks=peaks_in(json.loads(raw));self.cache[str(path)]=version
|
||||
if not peaks:continue
|
||||
record=dict(path=str(path),sha256=hashlib.sha256(raw).hexdigest(),observed_unix=time.time(),peaks=peaks)
|
||||
self.observations.append(record)
|
||||
for key,value in peaks.items():
|
||||
if key not in self.witnesses or value>self.maxima[key]:
|
||||
self.maxima[key]=value;self.witnesses[key]=record
|
||||
return max(self.maxima.values())>VRAM_BYTES
|
||||
def as_dict(self):
|
||||
return dict(maxima=self.maxima,witnesses=self.witnesses,observations=self.observations,
|
||||
limit_bytes=VRAM_BYTES,breach=max(self.maxima.values())>VRAM_BYTES)
|
||||
|
||||
|
||||
def active_command_pids(command):
|
||||
result=[]
|
||||
try:children=psutil.Process(os.getpid()).children(recursive=True)
|
||||
except psutil.NoSuchProcess:return result
|
||||
for p in children:
|
||||
try:
|
||||
if p.cmdline()==command:result.append(p.pid)
|
||||
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
|
||||
return result
|
||||
|
||||
|
||||
def guarded_worker(root,name,uuid,wall_limit,command,*,cumulative_worker_cpu_limit_seconds):
|
||||
root=Path(root)
|
||||
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!=ALLOCATOR_CONF:
|
||||
raise ValueError('Explicit PYTORCH_CUDA_ALLOC_CONF='+ALLOCATOR_CONF+' required before spawn')
|
||||
assert 0<wall_limit<=1800 and cumulative_worker_cpu_limit_seconds>0
|
||||
begin=time.monotonic();initial,util=device_usage(uuid)
|
||||
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
|
||||
tracker=PeakRecords();done=threading.Event()
|
||||
monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,reasons=[],errors=[],samples=0)
|
||||
def reason(value):
|
||||
if value not in monitor['reasons']:monitor['reasons'].append(value)
|
||||
def scan_records():
|
||||
try:
|
||||
if tracker.scan(root/name):reason('RECORDED_VRAM_LIMIT')
|
||||
except Exception as exc:
|
||||
reason('RECORDED_MEMORY_READ_FAILED');monitor['errors'].append(str(exc))
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
scan_records()
|
||||
try:
|
||||
used,_=device_usage(uuid);monitor['samples']+=1
|
||||
monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
|
||||
if used>VRAM_BYTES:reason('DEVICE_VRAM_LIMIT')
|
||||
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
|
||||
if rss>HOST_BYTES:reason('WHOLE_RUN_HOST_RSS_LIMIT')
|
||||
if disk_usage(root)>=DISK_BYTES:reason('RUN_DISK_LIMIT')
|
||||
except Exception as exc:
|
||||
reason('GPU_MONITOR_FAILED');monitor['errors'].append(str(exc))
|
||||
if monitor['reasons']:
|
||||
for pid in active_command_pids(command):stop_group(pid)
|
||||
done.wait(.5)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
remaining=wall_limit-(time.monotonic()-begin)-2.5
|
||||
if remaining<=0:raise RuntimeError('No wall budget left before spawn')
|
||||
receipt=supervise(command,root/'guards',name,dict(total_cpu_seconds=cumulative_worker_cpu_limit_seconds,
|
||||
job_cpu_seconds=wall_limit,job_wall_seconds=remaining,tree_rss_bytes=HOST_BYTES,
|
||||
disk_bytes=DISK_BYTES,max_processes=2),poll=.1,gpu_uuid=uuid)
|
||||
finally:
|
||||
done.set();thread.join(timeout=3)
|
||||
if thread.is_alive():reason('GPU_MONITOR_DID_NOT_STOP')
|
||||
else:scan_records() # Mandatory even if the worker failed or wrote no result.
|
||||
result_path=root/name/'result.json';result=None
|
||||
if result_path.exists():
|
||||
try:result=json.loads(result_path.read_text())
|
||||
except Exception as exc:reason('RESULT_READ_FAILED');monitor['errors'].append(str(exc))
|
||||
if receipt['status']=='PASS' and (not result or result.get('status')!='COMPLETE'):reason('MISSING_COMPLETE_RESULT')
|
||||
if receipt['status']=='PASS' and not all(k in tracker.witnesses for k in PEAK_KEYS):reason('MISSING_RECORDED_MEMORY_PEAKS')
|
||||
elapsed=time.monotonic()-begin
|
||||
if elapsed>wall_limit:reason('TOTAL_RUN_WALL_LIMIT')
|
||||
receipt['underlying_worker_status']=receipt['status'];receipt['underlying_stop_reason']=receipt.get('stop_reason')
|
||||
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
|
||||
receipt.update(device_monitor=monitor,recorded_memory=tracker.as_dict(),total_run_wall_seconds=elapsed,
|
||||
allocator_conf=os.environ['PYTORCH_CUDA_ALLOC_CONF'],gpu_uuid=uuid,
|
||||
result_sha256=hashlib.sha256(result_path.read_bytes()).hexdigest() if result else None)
|
||||
save(root/(name+'-guard-receipt.json'),receipt)
|
||||
return receipt,result
|
||||
@@ -0,0 +1,61 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import s2_main_memory_guard as guard
|
||||
|
||||
|
||||
class RecordedPeakTests(unittest.TestCase):
|
||||
def test_only_progress_reserved_breach_stops_live_worker(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);(root/'run').mkdir();killed=threading.Event()
|
||||
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+1)))
|
||||
def supervise(*args,**kwargs):
|
||||
self.assertTrue(killed.wait(2));return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=-9)
|
||||
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[123456789]),patch.object(guard,'stop_group',side_effect=lambda pid:killed.set()):
|
||||
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
|
||||
self.assertIsNone(result);self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT')
|
||||
self.assertEqual(receipt['recorded_memory']['maxima']['peak_reserved_bytes'],guard.VRAM_BYTES+1)
|
||||
def test_failed_worker_final_scan_catches_breach_without_result(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);(root/'run').mkdir()
|
||||
def supervise(*args,**kwargs):
|
||||
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+3)))
|
||||
return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=1)
|
||||
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[]):
|
||||
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
|
||||
self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT');self.assertEqual(receipt['returncode'],1)
|
||||
self.assertIsNone(result)
|
||||
def test_maximum_survives_lower_replacement_and_deletion(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp)/'progress.json';tracker=guard.PeakRecords()
|
||||
p.write_text(json.dumps(dict(peak_reserved_bytes=guard.VRAM_BYTES+1)));self.assertTrue(tracker.scan(tmp))
|
||||
p.write_text(json.dumps(dict(peak_reserved_bytes=7)));self.assertTrue(tracker.scan(tmp))
|
||||
p.unlink();self.assertTrue(tracker.scan(tmp));self.assertEqual(tracker.maxima['peak_reserved_bytes'],guard.VRAM_BYTES+1)
|
||||
def test_nested_history_and_result_both_contribute(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);(root/'history.json').write_text(json.dumps([dict(peak_reserved_bytes=20)]))
|
||||
(root/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=30,peak_reserved_bytes=10)))
|
||||
tracker=guard.PeakRecords();self.assertFalse(tracker.scan(root));self.assertEqual(tracker.maxima,dict(peak_allocated_bytes=30,peak_reserved_bytes=20))
|
||||
def test_invalid_recorded_peak_rejected(self):
|
||||
for value in [float('nan'),-1,True,'100']:
|
||||
with self.subTest(value=value),self.assertRaises(ValueError):guard.peaks_in(dict(peak_reserved_bytes=value))
|
||||
def test_allocator_is_required_before_device_or_worker_call(self):
|
||||
with patch.dict(os.environ,{},clear=True),patch.object(guard,'device_usage') as device:
|
||||
with self.assertRaises(ValueError):guard.guarded_worker(Path('/unused'),'run','synthetic',20,[],cumulative_worker_cpu_limit_seconds=20)
|
||||
device.assert_not_called()
|
||||
def test_allocator_reaches_supervised_cpu_child_without_cuda_work(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp)
|
||||
code="import os,json,pathlib; assert os.environ['PYTORCH_CUDA_ALLOC_CONF']=='expandable_segments:True'; assert os.environ['CUDA_VISIBLE_DEVICES']=='synthetic'; p=pathlib.Path('"+str(root/ 'run')+"'); p.mkdir(); (p/'result.json').write_text(json.dumps(dict(status='COMPLETE',peak_allocated_bytes=0,peak_reserved_bytes=0)))"
|
||||
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)):
|
||||
receipt,result=guard.guarded_worker(root,'run','synthetic',20,[sys.executable,'-c',code],cumulative_worker_cpu_limit_seconds=20)
|
||||
self.assertEqual(receipt['status'],'PASS');self.assertEqual(result['status'],'COMPLETE')
|
||||
self.assertEqual(receipt['allocator_conf'],guard.ALLOCATOR_CONF)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user