Publish item b shared-mode budget and wait-accounting delta for review
This commit is contained in:
1 parent
482c490afb
commit
5bfef6a61a
16 files changed
+16580
No files matched your search
@@ -0,0 +1,39 @@
|
||||
test_actual_runaway_stops (test_phase3_item_b.BudgetTests) ... ok
|
||||
test_incomplete_history_rejected (test_phase3_item_b.BudgetTests) ... ok
|
||||
test_initial_budget_includes_postfit (test_phase3_item_b.BudgetTests) ... ok
|
||||
test_late_validation_max_used (test_phase3_item_b.BudgetTests) ... ok
|
||||
test_no_single_fold_projection_gate (test_phase3_item_b.BudgetTests) ... ok
|
||||
test_remaining_validation_charged (test_phase3_item_b.BudgetTests) ... ok
|
||||
test_all_learning_invalid_stops (test_phase3_item_b.ContextTests) ... ok
|
||||
test_infinity_or_wrong_shape_stops (test_phase3_item_b.ContextTests) ... ok
|
||||
test_joint_missing_is_zero_after_normalization (test_phase3_item_b.ContextTests) ... ok
|
||||
test_learning_only_iteration (test_phase3_item_b.ContextTests) ... ok
|
||||
test_original_role_guard_precedes_context_read (test_phase3_item_b.ContextTests) ... ok
|
||||
test_partial_missing_stops (test_phase3_item_b.ContextTests) ... ok
|
||||
test_status_mismatch_not_scientific_missing (test_phase3_item_b.ContextTests) ... ok
|
||||
test_unit_node_moments_not_equal_part_means (test_phase3_item_b.ContextTests) ... ok
|
||||
test_valid_mean_still_has_flag (test_phase3_item_b.ContextTests) ... ok
|
||||
test_zero_variance_scale_one (test_phase3_item_b.ContextTests) ... ok
|
||||
test_all_shared_parameters_and_rng_match (test_phase3_item_b.ModelTests) ... ok
|
||||
test_exactly384_additional_parameters (test_phase3_item_b.ModelTests) ... ok
|
||||
test_guard_accepts_only_complete200_epoch_identity (test_phase3_item_b.ModelTests) ... ok
|
||||
test_new_columns_can_learn_without_changing_recipe (test_phase3_item_b.ModelTests) ... ok
|
||||
test_nonzero_readout_still_matches_original_initial_function (test_phase3_item_b.ModelTests) ... ok
|
||||
test_worker_uses_original200_recipe (test_phase3_item_b.ModelTests) ... ok
|
||||
test_wrong_input_columns_rejected (test_phase3_item_b.ModelTests) ... ok
|
||||
test_failure_no_retry_or_test (test_phase3_item_b.QueueTests) ... ok
|
||||
test_five_fits_stop_before_postfit (test_phase3_item_b.QueueTests) ... ok
|
||||
test_mixed_pending_c_reservation_blocks_b (test_phase3_item_b.SharedWrapperTests) ... ok
|
||||
test_mixed_two_visible_allows_only_one_new (test_phase3_item_b.SharedWrapperTests) ... ok
|
||||
test_shared_c_and_b_registration_independent (test_phase3_item_b.SharedWrapperTests) ... ok
|
||||
test_wrapper_changes_only_key_cap_gpu_set (test_phase3_item_b.SharedWrapperTests) ... ok
|
||||
test_long_wait_does_not_hit_actual_training_runaway_limit (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
|
||||
test_observed_b_epoch_can_raise_shared_bound (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
|
||||
test_pending_wait_is_charged_once (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
|
||||
test_terminal_uses_actual_charge (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
|
||||
test_wait_changes_total_but_not_training_projection (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 34 tests in 1.272s
|
||||
|
||||
OK
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,9 @@
|
||||
# Item b pre-launch shared-mode budget delta
|
||||
|
||||
Claude's `1795021986.507079` and `1795022093.308479` apply to this never-launched entry. The new root is `/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19`. The old implementation, approval and original review remain historical evidence. Only an independent MeltAssist delta PASS on the new exact binding can release this wrapper for launch.
|
||||
|
||||
For each fold, use the maximum retained c epoch whose full interval falls within the same-card overlap witnessed by c assignment, process identity and GPU samples. The planning bound uses 200 such epochs plus the original b bound for 20 diagnostic validations, with 25% margin on both. Setup and cleanup remain 60 and 120 seconds per fold. The validation term is a retained bound, not a measurement of shared b validation. The live queue can increase the estimate as it observes slower b epochs or diagnostic passes.
|
||||
|
||||
The pre-launch cap is the ceiling of the five fold bounds, one explicit aggregate waiting reserve equal to the largest fold bound, the 600-second preparation charge and the 7,200-second postfit reserve. The waiting reserve is a planning assumption. Actual pre-dispatch waits count once toward the queue total and do not inflate per-fold training projections or the actual training runaway limit. The combined phase 3 ledger keeps c and seed 1 at their approved caps and 1b at its actual terminal total including the freeze tail. The recorded cap must fit the user's 50 GPU-hour item b limit and the 720,000 GPU-second phase 3 limit.
|
||||
|
||||
The b comparison contract, context cache, model, initialization, optimization, order, validation recipe and scientific worker logic stay as previously reviewed. The worker's only source delta is its runtime identity import. The shared registry and admission implementation are reused verbatim. This budget lesson also applies when Track B gets its own entry and measurement. It does not create or launch a Track B job.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"reviewed_commit": "961ce62360003e7621f4fad6ae58648ef96d82af",
|
||||
"contract_sha256": "8a51b67c35fe6f0ec201063c15aa758575afc8c14dda4c165b3d83b5e163883c",
|
||||
"implementation_proof_sha256": "421351347255615185d7a307c2b087118bcb67dde385493e3fe0183747e96dba",
|
||||
"comparison_contract_sha256": "06556bf1f7bd5a1a79451bd9eed47fc6ec027e7e06bc63c6f490a50dfbee6d20",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_capacity.py": "3915da37ead45f42fe620cb89bc7394b576f8e113047a4ad78e1bae50ded120c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_common.py": "d00069d2ca5a346d8260dfb1829bf941f3ea09ebd7d75c1fe377120b553e02d2",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_data.py": "74e35b6268462c999078383e49e551c9da2562390d22977414713811bab164b3",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_guard.py": "e95cd5126f403881aebe137ed3f00bb6173881eea15709952142cb51136ad699",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_model.py": "13000ce6eb7cccfbefdc1710e37a59412be0391ae820b243f0d8fe7a9be4e986",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_queue.py": "0c6f5c9bde84f28bfc829701d97701188f43ed59fa0c5286228ca325cf88235c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_worker.py": "d49a5e6f900850aabacc5825168b6f886215aff8c494937bea2238539e58f652",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b.py": "ac46c3c6d01d8023444d2f53f956d94d1d295ae95a3435f355c5c6e0f6ccd962",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b.py": "bed8e21042971bb688b33c13903a28fab1772b980431a80afd205dbe4350dac2",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "08b7d8ab2ac773fab8b749f9f61af259254422128e9634f8b715c81efce2651f",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "31dfd71369c63512c1d6fe176fd22e8db646520c92da533bb5530ca3843c388d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "06cb97594c3a87f42f712dcde6cec8c2d90e0b50a89a6be7954bdd2f5c18d5de",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "b781e56c519401bd38af6a522deb24506cba6993d2af008f85ea4e1715f1d9c6",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "c7bc9f3b7cf4137f52e1efad19dcb7045282d4e9a9c1aae50f066c9e488f7c01",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "b4d78f4384651711fe83ff00eb6719c6c61664699d801ecb34978b9546439da9",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "3115aa9049974eac86657aa4cee7b5d716f6ee93268555cc86242029ebefce4c",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_budget.py": "2b10e7f126d2e617719eb8c20c1d234d83ef2bb77e94fe1932f5fc57501373e6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_capacity.py": "910fd9cff919824ea9742c55ee8b0d12a262f0d42fcd18afa8060334c5fbc58e",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_common.py": "c84f5b795e9354c5e380e8492308fd80d22bf83d4008fe961b1a36ecfba0dab9",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_guard.py": "01963a986eab70140ada7ca70fc7619012fe92ee52c3921e45ae1b75a7ed8357",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_queue.py": "77bbf7c61e9e64c079d398767527e7f87e9d01d4c27df45354ec756042f32fe9",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_worker.py": "363a8667d11b2ef76cc1c23a66d62be8135834f956d1d311df45a1017c8ecb56",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b_shared.py": "8a638b9fecd92c1dfff4ab97857d519308306d589b6940dfd75d6e8e0a702032",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b_shared_budget.py": "f9900b10d27d2f549618c0ad339f7700dba7df494c6ff296513fe7d80b4a622d"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "8a51b67c35fe6f0ec201063c15aa758575afc8c14dda4c165b3d83b5e163883c",
|
||||
"proof_sha256": "421351347255615185d7a307c2b087118bcb67dde385493e3fe0183747e96dba"
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
{
|
||||
"commit": "961ce62360003e7621f4fad6ae58648ef96d82af",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19/contract.json": "8a51b67c35fe6f0ec201063c15aa758575afc8c14dda4c165b3d83b5e163883c",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19/preparation-audit.json": "23a6424f2f4c54de6817a1cb4463a5c8b191f3db76be463a72fa7e9b6e24417a",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_capacity.py": "3915da37ead45f42fe620cb89bc7394b576f8e113047a4ad78e1bae50ded120c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_common.py": "d00069d2ca5a346d8260dfb1829bf941f3ea09ebd7d75c1fe377120b553e02d2",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_data.py": "74e35b6268462c999078383e49e551c9da2562390d22977414713811bab164b3",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_guard.py": "e95cd5126f403881aebe137ed3f00bb6173881eea15709952142cb51136ad699",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_model.py": "13000ce6eb7cccfbefdc1710e37a59412be0391ae820b243f0d8fe7a9be4e986",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_queue.py": "0c6f5c9bde84f28bfc829701d97701188f43ed59fa0c5286228ca325cf88235c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_worker.py": "d49a5e6f900850aabacc5825168b6f886215aff8c494937bea2238539e58f652",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b.py": "ac46c3c6d01d8023444d2f53f956d94d1d295ae95a3435f355c5c6e0f6ccd962",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b.py": "bed8e21042971bb688b33c13903a28fab1772b980431a80afd205dbe4350dac2",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "08b7d8ab2ac773fab8b749f9f61af259254422128e9634f8b715c81efce2651f",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "31dfd71369c63512c1d6fe176fd22e8db646520c92da533bb5530ca3843c388d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "06cb97594c3a87f42f712dcde6cec8c2d90e0b50a89a6be7954bdd2f5c18d5de",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "b781e56c519401bd38af6a522deb24506cba6993d2af008f85ea4e1715f1d9c6",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "c7bc9f3b7cf4137f52e1efad19dcb7045282d4e9a9c1aae50f066c9e488f7c01",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "b4d78f4384651711fe83ff00eb6719c6c61664699d801ecb34978b9546439da9",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "3115aa9049974eac86657aa4cee7b5d716f6ee93268555cc86242029ebefce4c",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_budget.py": "2b10e7f126d2e617719eb8c20c1d234d83ef2bb77e94fe1932f5fc57501373e6",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_capacity.py": "910fd9cff919824ea9742c55ee8b0d12a262f0d42fcd18afa8060334c5fbc58e",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_common.py": "c84f5b795e9354c5e380e8492308fd80d22bf83d4008fe961b1a36ecfba0dab9",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_guard.py": "01963a986eab70140ada7ca70fc7619012fe92ee52c3921e45ae1b75a7ed8357",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_queue.py": "77bbf7c61e9e64c079d398767527e7f87e9d01d4c27df45354ec756042f32fe9",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_worker.py": "363a8667d11b2ef76cc1c23a66d62be8135834f956d1d311df45a1017c8ecb56",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b_shared.py": "8a638b9fecd92c1dfff4ab97857d519308306d589b6940dfd75d6e8e0a702032",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b_shared_budget.py": "f9900b10d27d2f549618c0ad339f7700dba7df494c6ff296513fe7d80b4a622d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-b-shared-budget-2026-11-19/shared-cost-evidence.json": "1abb73b38620e3649f92796fe31962054c9d2fd55092d6eac09403bc5e18d553",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-b-shared-budget-2026-11-19/acceptance.log": "5362d21dc2aff6f2a45aa86821083179cc3e991e71ae165298b4974d6170f0af",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase3-item-b-shared-budget-2026-11-19.md": "a811143aab79bf1ed4b95b22582eecbae0ffd6852eb80011980427f262403b35"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19/contract.json": "home-runs/phase3-item-b-shared-2026-11-19/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19/preparation-audit.json": "home-runs/phase3-item-b-shared-2026-11-19/preparation-audit.json",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "tools/phase2_h2_pilot_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "tools/phase2_h2_s1_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_capacity.py": "tools/phase3_item_b_capacity.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_common.py": "tools/phase3_item_b_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_data.py": "tools/phase3_item_b_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_guard.py": "tools/phase3_item_b_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_model.py": "tools/phase3_item_b_model.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_queue.py": "tools/phase3_item_b_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_worker.py": "tools/phase3_item_b_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "tools/phase3_seed1_v2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "tools/phase3_shared_capacity.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b.py": "tools/prepare_phase3_item_b.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "tools/s2_models/__init__.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "tools/s2_models/attention.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "tools/s2_models/blocks.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "tools/s2_models/features.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "tools/s2_models/network.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "tools/s2_models/normalization.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "tools/s2_models/objectives.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b.py": "tools/test_phase3_item_b.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "tools/track_a_recovery_monitor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "tools/track_a_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "tools/phase3_item1b_capacity.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "tools/phase3_item1b_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "tools/phase3_item1b_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "tools/phase3_item1b_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "tools/phase3_item1b_schedule.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "tools/phase3_item1b_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "tools/prepare_phase3_item1b.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "tools/test_phase3_item1b.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "tools/phase3_item_c_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "tools/phase3_item_c_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "tools/phase3_item_c_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "tools/phase3_item_c_loss.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "tools/phase3_item_c_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "tools/phase3_item_c_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "tools/prepare_phase3_item_c.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "tools/s2_custody_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "tools/s2_custody_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "tools/test_phase3_item_c.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "tools/test_phase2_h2_s2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "tools/phase2_h2_s2_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "tools/phase2_h2_s2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "tools/phase2_fixed_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "tools/phase2_h2_s2_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "tools/phase2_h2_s1_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "tools/phase2_h2_s2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "tools/phase2_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "tools/phase2_h2_s2_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "tools/phase2_h2_s1_launch.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "tools/phase2_h2_s1_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "docs/phase2-h2-s2-execution-contract-2026-11-17.md",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_budget.py": "tools/phase3_item_b_shared_budget.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_capacity.py": "tools/phase3_item_b_shared_capacity.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_common.py": "tools/phase3_item_b_shared_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_guard.py": "tools/phase3_item_b_shared_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_queue.py": "tools/phase3_item_b_shared_queue.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_worker.py": "tools/phase3_item_b_shared_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b_shared.py": "tools/prepare_phase3_item_b_shared.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b_shared_budget.py": "tools/test_phase3_item_b_shared_budget.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-b-shared-budget-2026-11-19/shared-cost-evidence.json": "artifacts/phase3-item-b-shared-budget-2026-11-19/shared-cost-evidence.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-b-shared-budget-2026-11-19/acceptance.log": "artifacts/phase3-item-b-shared-budget-2026-11-19/acceptance.log",
|
||||
"/home/frank14f/optane/MeltMind/docs/phase3-item-b-shared-budget-2026-11-19.md": "docs/phase3-item-b-shared-budget-2026-11-19.md"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"status": "WAIT_INDEPENDENT_DELTA_REVIEW",
|
||||
"base_implementation": "b2a9dfdc4a2c237416452ff6b4c32bdbf34e172e",
|
||||
"gpu_cap": 172908,
|
||||
"phase_reservations": {
|
||||
"item1": 72421.31599938683,
|
||||
"seed1": 167200.0,
|
||||
"c": 205200.0,
|
||||
"item1b": 19262.057604262838,
|
||||
"b": 172908
|
||||
},
|
||||
"initial_projection": {
|
||||
"status": "PASS",
|
||||
"projected_gpu_seconds": 143203.1576138339,
|
||||
"actual_training_and_preparation_gpu_seconds": 900.0,
|
||||
"actual_waiting_gpu_seconds": 0.0,
|
||||
"folds": {
|
||||
"0": {
|
||||
"charged": 60.0,
|
||||
"waiting_gpu_seconds": 0.0,
|
||||
"actual_training_with_setup_gpu_seconds": 60.0,
|
||||
"projected_training_gpu_seconds": 25870.330589201767,
|
||||
"projected": 25870.330589201767
|
||||
},
|
||||
"1": {
|
||||
"charged": 60.0,
|
||||
"waiting_gpu_seconds": 0.0,
|
||||
"actual_training_with_setup_gpu_seconds": 60.0,
|
||||
"projected_training_gpu_seconds": 28917.776216800557,
|
||||
"projected": 28917.776216800557
|
||||
},
|
||||
"2": {
|
||||
"charged": 60.0,
|
||||
"waiting_gpu_seconds": 0.0,
|
||||
"actual_training_with_setup_gpu_seconds": 60.0,
|
||||
"projected_training_gpu_seconds": 29704.07792180602,
|
||||
"projected": 29704.07792180602
|
||||
},
|
||||
"3": {
|
||||
"charged": 60.0,
|
||||
"waiting_gpu_seconds": 0.0,
|
||||
"actual_training_with_setup_gpu_seconds": 60.0,
|
||||
"projected_training_gpu_seconds": 24445.709940344095,
|
||||
"projected": 24445.709940344095
|
||||
},
|
||||
"4": {
|
||||
"charged": 60.0,
|
||||
"waiting_gpu_seconds": 0.0,
|
||||
"actual_training_with_setup_gpu_seconds": 60.0,
|
||||
"projected_training_gpu_seconds": 26465.26294568146,
|
||||
"projected": 26465.26294568146
|
||||
}
|
||||
},
|
||||
"postfit_gpu_reserve": 7200,
|
||||
"safety_factor": 1.25
|
||||
},
|
||||
"code_count": 77,
|
||||
"source_count": 1309,
|
||||
"metadata_cpu_seconds": 0.31858634399999997,
|
||||
"comparison_unchanged": true,
|
||||
"scientific_worker_unchanged": true,
|
||||
"started": false,
|
||||
"test_targets_read": false,
|
||||
"review_adopted": false
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Shared-mode extrapolation with queue waiting charged separately from training."""
|
||||
|
||||
import math
|
||||
|
||||
|
||||
MARGIN = 1.25
|
||||
SETUP = 60
|
||||
CLEANUP = 120
|
||||
USER_CAP = 180000
|
||||
RULING = '1795022093.308479'
|
||||
|
||||
|
||||
def full_fold_bound(epoch_seconds, validation_seconds):
|
||||
if not math.isfinite(epoch_seconds) or epoch_seconds <= 0:
|
||||
raise ValueError('Positive measured epoch time required')
|
||||
if not math.isfinite(validation_seconds) or validation_seconds < 0:
|
||||
raise ValueError('Invalid validation time')
|
||||
return SETUP + MARGIN * (200 * epoch_seconds + 20 * validation_seconds) + CLEANUP
|
||||
|
||||
|
||||
def project(contract, charges, waits, histories, terminal):
|
||||
folds = set(range(5))
|
||||
if set(charges) != folds or set(waits) != folds or set(histories) - folds or set(terminal) - folds:
|
||||
raise ValueError('Cost ledger fold identity')
|
||||
rows = {}
|
||||
for i, charged in charges.items():
|
||||
waiting = waits[i]
|
||||
if not math.isfinite(charged) or not math.isfinite(waiting) or not 0 <= waiting <= charged:
|
||||
raise ValueError('Invalid queue and waiting charges')
|
||||
elapsed = charged - waiting
|
||||
job = contract['jobs'][i]
|
||||
history = histories.get(i, [])
|
||||
if i in terminal:
|
||||
training = elapsed
|
||||
elif history:
|
||||
epochs = sorted(r['epoch'] for r in history)
|
||||
finished = epochs[-1]
|
||||
if epochs != list(range(1, finished + 1)) or finished > 200:
|
||||
raise ValueError('Incomplete epoch history')
|
||||
observed = [r['train_epoch_seconds'] for r in history]
|
||||
diagnostics = [r.get('validation_seconds', 0) for r in history]
|
||||
if any(not math.isfinite(x) or x <= 0 for x in observed):
|
||||
raise ValueError('Invalid measured epoch')
|
||||
if any(not math.isfinite(x) or x < 0 for x in diagnostics):
|
||||
raise ValueError('Invalid validation time')
|
||||
epoch_time = max(job['shared_epoch_seconds_bound'], *observed)
|
||||
validation_time = max(job['validation_seconds_bound'], *diagnostics)
|
||||
training = elapsed + MARGIN * ((200-finished)*epoch_time + (20-finished//10)*validation_time) + CLEANUP
|
||||
else:
|
||||
training = max(elapsed, job['full_training_gpu_seconds_bound'])
|
||||
rows[str(i)] = dict(charged=charged, waiting_gpu_seconds=waiting,
|
||||
actual_training_with_setup_gpu_seconds=elapsed,
|
||||
projected_training_gpu_seconds=training, projected=waiting+training)
|
||||
|
||||
actual = contract['preparation_charge_gpu_seconds'] + sum(charges.values())
|
||||
projected = (contract['preparation_charge_gpu_seconds'] + contract['postfit_gpu_reserve']
|
||||
+ sum(row['projected'] for row in rows.values()))
|
||||
ok = projected <= contract['gpu_cap'] and actual + contract['postfit_gpu_reserve'] <= contract['gpu_cap']
|
||||
ok &= all(charges[i] - waits[i] < contract['runaway_gpu_seconds'] for i in folds - set(terminal))
|
||||
return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected,
|
||||
actual_training_and_preparation_gpu_seconds=actual, actual_waiting_gpu_seconds=sum(waits.values()),
|
||||
folds=rows, postfit_gpu_reserve=contract['postfit_gpu_reserve'], safety_factor=MARGIN)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""b registration wrapper; unchanged reviewed shared admission and leases.
|
||||
|
||||
Only b's registered cap/ledger changes; the existing admission and reservations remain in use.
|
||||
The live c module is imported without mutation, never patched or rewritten.
|
||||
"""
|
||||
import os,time,uuid
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from phase3_shared_capacity import (registry,process_identity,alive,descendants,
|
||||
host_snapshot,admission,GPUS,GIB,transfer,release,validate_lease)
|
||||
|
||||
def register_entry(root, digest, contract):
|
||||
with registry() as d:
|
||||
if 'b' in d['entries']:
|
||||
raise FileExistsError('Sole item b entry already registered; no resume')
|
||||
reservations = contract['phase_reservations']
|
||||
cap = contract['gpu_cap']
|
||||
if reservations['b'] != cap or sum(reservations.values()) > 720000:
|
||||
raise RuntimeError('Whole phase3 reserved cap')
|
||||
d['entries']['b'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
|
||||
cap=cap, phase_reservations=reservations, started_unix=time.time())
|
||||
|
||||
|
||||
def acquire(root, fold, digest):
|
||||
with registry() as d:
|
||||
entry = d['entries']['b']
|
||||
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
|
||||
raise PermissionError('Entry owner/identity')
|
||||
leases = []
|
||||
for l in d['leases'].values():
|
||||
# Normal guard exit can race this observation. Block admission until
|
||||
# its owning coordinator collects it; never reclaim or kill peers.
|
||||
if not alive(l['owner']):
|
||||
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
if l['state']=='RESERVED':
|
||||
# owner is a queue with other children, not this job's guard.
|
||||
# One independent slot and full VRAM/RSS remain reserved.
|
||||
leases.append(dict(l,observed_pids=[],observed_rss=0))
|
||||
continue
|
||||
if l['state']!='GUARD_BOUND':
|
||||
raise ValueError('Unknown lease state')
|
||||
try:
|
||||
ids = descendants(l['owner'])
|
||||
except (psutil.NoSuchProcess,RuntimeError):
|
||||
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
|
||||
rss=0
|
||||
for pid in ids:
|
||||
try:rss+=psutil.Process(pid).memory_info().rss
|
||||
except psutil.NoSuchProcess:pass
|
||||
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
|
||||
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
|
||||
used_cpu = {x for l in leases for x in l['worker_cpus']}
|
||||
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
|
||||
if not cpus:
|
||||
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
|
||||
for gpu in GPUS[:3]:
|
||||
ok, reason = admission(s, leases, gpu)
|
||||
if not ok:
|
||||
continue
|
||||
token = uuid.uuid4().hex
|
||||
queue_owner=process_identity(os.getpid())
|
||||
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
|
||||
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
|
||||
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
||||
d['leases'][token] = l
|
||||
return l, dict(reason='ADMITTED', snapshot=s)
|
||||
return None, dict(reason='NO_CAPACITY', snapshot=s)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Item b immutable scope, assignments and conservative total-cost projection."""
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_seed1_v2_common import fold_view
|
||||
|
||||
from phase3_item_b_shared_budget import RULING, USER_CAP, full_fold_bound, project
|
||||
|
||||
ROOT = Path('/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19')
|
||||
|
||||
|
||||
def read(p):
|
||||
return json.loads(Path(p).read_text())
|
||||
|
||||
|
||||
def config(root):
|
||||
if root.resolve() != ROOT:
|
||||
raise PermissionError('Sole shared-mode b root')
|
||||
c = read(root/'contract.json')
|
||||
verify(root, 'implementation')
|
||||
expected = dict(approval='1795011607.907609', seed=0, epochs=200, new_fits=5, shared_budget_ruling=RULING,
|
||||
postfit_gpu_reserve=7200, postfit_cpu_cap=3600,
|
||||
phase3_gpu_cap=720000, no_retry=True)
|
||||
if any(c.get(k) != v for k,v in expected.items()):
|
||||
raise PermissionError('Fixed item_b scope/budgets')
|
||||
for p,h in {**c['source_sha256'], **c['code_sha256']}.items():
|
||||
checked(p,h)
|
||||
bounds = read(c['shared_budget_evidence'])
|
||||
if bounds['ruling'] != RULING or c['phase_reservations'] != bounds['phase_reservations']:
|
||||
raise ValueError('Fixed shared budget identity')
|
||||
full_bounds = []
|
||||
for job, row in zip(c['jobs'], bounds['folds'], strict=True):
|
||||
epoch = row['max_train_epoch_seconds']
|
||||
full = full_fold_bound(epoch, job['validation_seconds_bound'])
|
||||
if (job['id'] != row['fold'] or job['shared_epoch_seconds_bound'] != epoch
|
||||
or job['full_training_gpu_seconds_bound'] != full):
|
||||
raise ValueError('Fixed fold timing bounds')
|
||||
full_bounds.append(full)
|
||||
cap = math.ceil(c['preparation_charge_gpu_seconds'] + c['postfit_gpu_reserve']
|
||||
+ sum(full_bounds) + max(full_bounds))
|
||||
if (c['gpu_cap'] != cap or c['phase_reservations']['b'] != cap or cap > USER_CAP
|
||||
or sum(c['phase_reservations'].values()) > c['phase3_gpu_cap']
|
||||
or c['phase_reservations']['item1b'] != bounds['parked_1b_actual_including_tail']):
|
||||
raise ValueError('User or phase 3 reservation exceeded')
|
||||
pilot = read(c['pilot_contract'])
|
||||
import numpy as np
|
||||
import torch
|
||||
if np.__version__ != pilot['versions']['numpy'] or torch.__version__ != pilot['versions']['torch']:
|
||||
raise ValueError('Original environment versions')
|
||||
if [j['id'] for j in c['jobs']] != list(range(5)):
|
||||
raise ValueError('Exactly five folds')
|
||||
tests = []
|
||||
for job in c['jobs']:
|
||||
f = fold_view(pilot, job['id'])['folds'][0]
|
||||
tests.extend(f['test'])
|
||||
if job['target_steps'] != 200*len(f['learning']):
|
||||
raise ValueError('Full fixed epochs')
|
||||
for k in ('learning','validation','test','c','clock_sha256','learning_nodes'):
|
||||
if job[k] != f[k]:
|
||||
raise ValueError('Original fold identity changed')
|
||||
if len(tests) != 552 or len(set(tests)) != 552:
|
||||
raise ValueError('Original552 roster')
|
||||
rows=read(c['query_roster_contract'])['rows']
|
||||
if len(rows)!=552 or {r['case'] for r in rows}!=set(tests) or len({r['component_id'] for r in rows})!=204:
|
||||
raise ValueError('552 parts /204 components query roster')
|
||||
for r in rows:
|
||||
p=pilot['parts'][r['case']]
|
||||
if r['v5_cache']['output_sha256']!=p['output_sha256'] or r['nodes']!=p['nodes'] or r['component_id']!=p['clock_group']:
|
||||
raise ValueError('Learning query/geometry/cache identity')
|
||||
if c['inputs']!=22 or c['added_parameters']!=384 or c['budget_approval']!='1795007580.895539' or not c['decision_requires_seed_reference']:
|
||||
raise PermissionError('Fixed regional context scope')
|
||||
checked(c['comparison_contract'], c['comparison_contract_sha256'])
|
||||
return c,pilot
|
||||
|
||||
|
||||
def assignment(root,c,index,digest):
|
||||
a = read(checked(root/'jobs'/str(index)/'assignment.json',digest))
|
||||
if a['id'] != index or a['contract_sha256'] != sha(root/'contract.json'):
|
||||
raise PermissionError('Assignment identity')
|
||||
return dict(c['jobs'][index], **{k:a[k] for k in ('gpu_uuid','worker_cpus','token','queue_identity')})
|
||||
|
||||
|
||||
def first_epoch_release(root,index):
|
||||
if (root/'global-stop.json').exists():
|
||||
raise RuntimeError('Global stop')
|
||||
p = root/'jobs'/str(index)/'first-epoch-projection.json'
|
||||
if not p.exists():
|
||||
return False
|
||||
d = read(p)
|
||||
if d['status'] != 'PASS' or d['contract_sha256'] != sha(root/'contract.json'):
|
||||
raise RuntimeError('First-epoch gate not passed')
|
||||
return True
|
||||
@@ -0,0 +1,136 @@
|
||||
"""One item b attempt, scoped process memory plus failed-worker peak checks."""
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase3_item_b_shared_common import config,assignment,read
|
||||
from phase3_shared_capacity import validate_lease,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
|
||||
|
||||
def run(queue,index,digest):
|
||||
parent=os.getppid()
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
|
||||
raise OSError('PDEATHSIG')
|
||||
requested=[]
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):
|
||||
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
|
||||
if os.getppid()!=parent:
|
||||
raise RuntimeError('Queue lost before setup')
|
||||
c,_=config(queue);job=assignment(queue,c,index,digest)
|
||||
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
|
||||
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
|
||||
remaining=c['runaway_gpu_seconds']-60
|
||||
if remaining<90:
|
||||
raise RuntimeError('No cleanup window before worker')
|
||||
deadline=start+remaining
|
||||
# Queue writes ready only after atomically transferring reservation ownership.
|
||||
while not (root/'lease-ready.json').exists():
|
||||
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
|
||||
raise RuntimeError('Lease transfer timeout')
|
||||
time.sleep(.1)
|
||||
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
|
||||
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
|
||||
raise RuntimeError('Pinned queue lost before guard import completed')
|
||||
os.sched_setaffinity(0,[26,27])
|
||||
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
|
||||
reasons=[];receipt=None
|
||||
command=[sys.executable,'tools/phase3_item_b_shared_worker.py',str(queue),str(index),digest]
|
||||
def reason(s):
|
||||
if s not in reasons:reasons.append(s)
|
||||
def workers():
|
||||
result=[]
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:result.append(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
return result
|
||||
def kill():
|
||||
for pid in workers():stop_group(pid)
|
||||
def scan():
|
||||
tracker.scan(root/'worker')
|
||||
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
|
||||
def local_watch():
|
||||
while not done.is_set():
|
||||
monitor.sample('recorded_peaks',scan)
|
||||
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
|
||||
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
|
||||
disk,ok=monitor.sample('disk',lambda:disk_usage(root))
|
||||
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
|
||||
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
|
||||
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
|
||||
if ok and not queue_live:reason('QUEUE_LOST')
|
||||
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
def gpu_watch():
|
||||
def sample():
|
||||
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
|
||||
rows=[]
|
||||
for line in text.strip().splitlines():
|
||||
if not line:continue
|
||||
u,p,m=[x.strip() for x in line.split(',')]
|
||||
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
|
||||
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
|
||||
own=set(workers())
|
||||
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
|
||||
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
|
||||
while not done.is_set():
|
||||
monitor.sample('gpu_processes',sample)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(1)
|
||||
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
|
||||
try:
|
||||
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
|
||||
for t in threads:t.start()
|
||||
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
|
||||
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,
|
||||
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
|
||||
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
|
||||
except BaseException as exc:
|
||||
reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
for t in threads:
|
||||
if t.ident is not None:t.join(timeout=12)
|
||||
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
|
||||
try:monitor.retry('final_peaks',scan,.1)
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
p=root/'worker/result.json';result=read(p) if p.exists() else None
|
||||
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
|
||||
if not result or result.get('status')!='COMPLETE' or result.get('fold')!=index or result.get('completed_epochs')!=200 or result.get('optimizer_steps')!=job['target_steps']:reason('NO_COMPLETE_IDENTITY')
|
||||
if result:
|
||||
from phase2_h1_worker_v2 import checked
|
||||
checked(result['checkpoint'],result['checkpoint_sha256'])
|
||||
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
|
||||
elapsed=time.monotonic()-start
|
||||
child=resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
|
||||
if 60+elapsed>c['runaway_gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(queue)
|
||||
except Exception as exc:reason('FINAL_PINS: '+str(exc))
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
|
||||
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,
|
||||
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
|
||||
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
|
||||
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Review-gated five-fold b training. Stops at frozen models, never postfit."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase3_item_b_shared_common import ROOT,config,read,project
|
||||
from phase3_item_b_shared_capacity import register_entry,acquire,transfer,release,process_identity,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
|
||||
|
||||
def run(root):
|
||||
c,_=config(root);validate_review(root)
|
||||
with (root/'queue.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('Sole b attempt, no resume')
|
||||
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
|
||||
raise ValueError('Explicit allocator required')
|
||||
if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free')
|
||||
register_entry(root,sha(root/'contract.json'),c)
|
||||
start=time.monotonic();identity=process_identity(os.getpid())
|
||||
save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json')))
|
||||
(root/'jobs').mkdir(exist_ok=False)
|
||||
active={};terminal={};pending=[0,1,2,3,4];histories={};waits={};stops=[];failure=None
|
||||
monitor=Monitor(root/'admission-errors.jsonl')
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n)))
|
||||
def charge(i):
|
||||
return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start
|
||||
def waiting_charges(charges):
|
||||
return {i: waits.get(i, max(0, charged - 60)) for i, charged in charges.items()}
|
||||
|
||||
def projection_at_current_time():
|
||||
charges = {i: charge(i) for i in range(5)}
|
||||
return project(c, charges, waiting_charges(charges), histories, terminal)
|
||||
|
||||
def stop(reason):
|
||||
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
|
||||
try:
|
||||
while active or pending:
|
||||
if stops:raise RuntimeError(stops[0])
|
||||
if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP')
|
||||
# Collect exits before granting any further admission.
|
||||
for i,(p,lease,log,owner) in list(active.items()):
|
||||
if p.poll() is None:continue
|
||||
log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
worker_identity=jr/'worker/worker-identity.json'
|
||||
if worker_identity.exists() and alive(read(worker_identity)):
|
||||
raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,
|
||||
gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],guard_sha256=sha(cp) if cp.exists() else None)
|
||||
release(lease['token'],owner);del active[i]
|
||||
if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i))
|
||||
for i in active:
|
||||
folder=root/'jobs'/str(i)/'worker/history'
|
||||
histories[i]=[read(p) for p in sorted(folder.glob('*.json'))]
|
||||
projection=projection_at_current_time()
|
||||
save(root/'latest-cost-projection.json',projection)
|
||||
if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP')
|
||||
for i in active:
|
||||
gate=root/'jobs'/str(i)/'first-epoch-projection.json'
|
||||
if histories.get(i) and not gate.exists():
|
||||
save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection))
|
||||
save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_B_TRAINING',active=list(active),pending=pending,
|
||||
terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False))
|
||||
if pending:
|
||||
i=pending[0]
|
||||
decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json')))
|
||||
if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES')
|
||||
if not ok:
|
||||
time.sleep(1)
|
||||
continue
|
||||
lease,observation=decision
|
||||
save(root/'latest-admission.json',observation)
|
||||
# Acquisition includes bounded GPU queries; recheck cost before Popen.
|
||||
fresh=projection_at_current_time()
|
||||
if fresh['status']!='PASS':
|
||||
if lease:release(lease['token'])
|
||||
raise RuntimeError('CAP_DURING_ADMISSION')
|
||||
if lease:
|
||||
jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False)
|
||||
waits[i]=time.monotonic()-start
|
||||
save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'),
|
||||
gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'],
|
||||
queue_identity=identity,pre_dispatch_wait_seconds=waits[i]))
|
||||
log=(jr/'guard.log').open('x')
|
||||
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
|
||||
p=subprocess.Popen([sys.executable,'tools/phase3_item_b_shared_guard.py',str(root),str(i),sha(jr/'assignment.json')],
|
||||
cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
owner=process_identity(p.pid)
|
||||
active[i]=(p,lease,log,owner);pending.remove(i)
|
||||
transfer(lease['token'],owner)
|
||||
save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner))
|
||||
time.sleep(1)
|
||||
except BaseException as exc:
|
||||
failure=type(exc).__name__+': '+str(exc);stop(failure)
|
||||
finally:
|
||||
if active:
|
||||
stop(failure or 'QUEUE_CLEANUP')
|
||||
for p,lease,log,owner in active.values():
|
||||
if p.poll() is None:p.terminate()
|
||||
deadline=time.monotonic()+45
|
||||
while time.monotonic()<deadline and any(p.poll() is None for p,_,_,_ in active.values()):time.sleep(.2)
|
||||
for i,(p,lease,log,owner) in active.items():
|
||||
if p.poll() is None:p.kill()
|
||||
p.wait();log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
|
||||
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
|
||||
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],
|
||||
guard_sha256=sha(cp) if cp.exists() else None)
|
||||
wp=jr/'worker/worker-identity.json'
|
||||
try:
|
||||
if wp.exists() and alive(read(wp)):raise RuntimeError('Live worker; lease retained')
|
||||
release(lease['token'],owner)
|
||||
except Exception as exc:
|
||||
terminal[i]['lease_release_failure']=str(exc);failure=failure or str(exc)
|
||||
final_charges={i:charge(i) for i in range(5)}
|
||||
final_cost=c['preparation_charge_gpu_seconds']+sum(final_charges.values())
|
||||
if final_cost+c['postfit_gpu_reserve']>c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP'
|
||||
complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values())
|
||||
files=[root/'contract.json']
|
||||
if complete:
|
||||
models=[]
|
||||
for i in range(5):
|
||||
r=read(root/'jobs'/str(i)/'worker/result.json')
|
||||
models.append(dict(r));files.append(Path(r['checkpoint']))
|
||||
files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json'])
|
||||
save(root/'models.json',models);files.append(root/'models.json')
|
||||
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING' if complete else 'STOPPED_INCOMPLETE',
|
||||
failure=failure,entries=list(terminal.values()),never_attempted=pending,
|
||||
never_attempted_wait_charges={str(i):final_charges[i] for i in pending},
|
||||
cumulative_gpu_seconds=final_cost,actual_waiting_gpu_seconds=sum(waiting_charges(final_charges).values()),
|
||||
postfit_gpu_reserve=c['postfit_gpu_reserve'],
|
||||
no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False))
|
||||
files.append(root/'complete.json')
|
||||
freeze_start=time.monotonic()
|
||||
freeze(root,'training-terminal' if complete else 'training-stop',files)
|
||||
final_cost+=time.monotonic()-freeze_start
|
||||
within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap']
|
||||
save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within,
|
||||
freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False,
|
||||
training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True))
|
||||
return 0 if complete and within else 1
|
||||
|
||||
|
||||
if __name__=='__main__':raise SystemExit(run(ROOT))
|
||||
@@ -0,0 +1,79 @@
|
||||
"""One new S2 fold, fixed original200epoch schedule with seed0; no test prediction or target reads."""
|
||||
import json,os,sys,time,traceback,random
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import torch
|
||||
from phase2_h2_s1_common import selected_checkpoint
|
||||
from phase2_h2_model import lr_epoch
|
||||
from phase3_item_b_shared_common import config,fold_view,first_epoch_release,assignment as assigned_job,read
|
||||
from phase3_shared_capacity import process_identity
|
||||
from phase3_item_b_data import standardization,load_part
|
||||
from phase2_h2_data import to_device,arguments,cached
|
||||
from phase3_item_b_model import H2Context as H2Geo
|
||||
from phase2_h2_model import optimizer
|
||||
from phase2_h2_pilot_worker import peaks,train_step
|
||||
from phase2_residual_math import residual_target
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def run(root,fold_id,assignment_sha):
|
||||
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
|
||||
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
|
||||
save(out/'worker-identity.json',process_identity(os.getpid()))
|
||||
def record(item,epoch_file=False):
|
||||
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
|
||||
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
|
||||
save(out/'progress.json',item)
|
||||
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
|
||||
return item
|
||||
try:
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
|
||||
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,c,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
|
||||
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
|
||||
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
|
||||
for epoch in range(1,201):
|
||||
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
|
||||
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
|
||||
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
|
||||
for index in order:
|
||||
case=fold['learning'][int(index)];data=to_device(load_part(old,c,case,stats,'learning',out),'cuda')
|
||||
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
|
||||
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
|
||||
del data
|
||||
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
|
||||
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
|
||||
if epoch%10==0:
|
||||
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
|
||||
with torch.no_grad():
|
||||
for case in fold['validation']:
|
||||
data=to_device(load_part(old,c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
|
||||
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
|
||||
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
|
||||
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
|
||||
torch.cuda.synchronize()
|
||||
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
|
||||
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
|
||||
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
|
||||
if epoch==1:
|
||||
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
|
||||
while not first_epoch_release(root,fold_id):time.sleep(.5)
|
||||
if epoch<200:
|
||||
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
|
||||
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
|
||||
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
|
||||
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
|
||||
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
|
||||
if __name__=='__main__':
|
||||
import ctypes,signal
|
||||
from phase3_shared_capacity import alive
|
||||
expected=read(Path(sys.argv[1])/'jobs'/sys.argv[2]/'lease-ready.json')['owner']
|
||||
parent=os.getppid()
|
||||
if parent!=expected['pid'] or not alive(expected):raise RuntimeError('Pinned guard lost before worker import completed')
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:raise OSError('PDEATHSIG')
|
||||
if os.getppid()!=parent:raise RuntimeError('Guard exited before worker setup')
|
||||
run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Freeze b's pre-launch budget/queue delta using metadata from the live c run."""
|
||||
|
||||
import ast
|
||||
import fcntl
|
||||
import math
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze, verify
|
||||
from phase3_item_b_common import ROOT as ORIGINAL
|
||||
from phase3_item_b_shared_budget import MARGIN, RULING, USER_CAP, full_fold_bound
|
||||
from phase3_item_b_shared_common import ROOT, config, project, read
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
|
||||
BASE = Path(__file__).resolve().parents[1]
|
||||
RECORDS = BASE / 'artifacts/phase3-item-b-shared-budget-2026-11-19'
|
||||
C_ROOT = ORIGINAL.with_name('phase3-item-c-2026-11-18')
|
||||
PARKED = ORIGINAL.with_name('phase3-item-1b-2026-11-18')
|
||||
|
||||
|
||||
def measured_bounds(old_jobs):
|
||||
c = read(C_ROOT / 'contract.json')
|
||||
if sha(C_ROOT / 'contract.json') != read(C_ROOT / 'started.json')['contract_sha256']:
|
||||
raise ValueError('c training identity changed')
|
||||
for p, h in c['code_sha256'].items():
|
||||
checked(p, h)
|
||||
jobs = {i: C_ROOT / 'jobs' / str(i) for i in range(5)}
|
||||
assignments = {i: read(job / 'assignment.json') for i, job in jobs.items()}
|
||||
workers = {i: read(job / 'worker/worker-identity.json') for i, job in jobs.items()}
|
||||
samples = {i: read(job / 'gpu-sample.json') for i, job in jobs.items()}
|
||||
rows = []
|
||||
for i, old in enumerate(old_jobs):
|
||||
assignment = assignments[i]
|
||||
if assignment['id'] != i or assignment['contract_sha256'] != sha(C_ROOT / 'contract.json'):
|
||||
raise ValueError('c fold assignment identity')
|
||||
sample = samples[i]
|
||||
seen = {r['pid'] for r in sample['rows']}
|
||||
worker = workers[i]
|
||||
if worker['pid'] not in seen or worker['pid'] not in sample['own_pids']:
|
||||
raise ValueError('c worker missing from GPU sample')
|
||||
peers = [j for j in jobs if j != i and assignments[j]['gpu_uuid'] == assignment['gpu_uuid']
|
||||
and workers[j]['pid'] in seen]
|
||||
if not peers:
|
||||
raise ValueError('A live same-card c peer must corroborate the overlap')
|
||||
overlap_start = max(worker['created'], min(workers[j]['created'] for j in peers))
|
||||
candidates = []
|
||||
for p in sorted((jobs[i] / 'worker/history').glob('*.json')):
|
||||
mtime = p.stat().st_mtime
|
||||
if mtime > sample['unix']:
|
||||
continue
|
||||
row = read(p)
|
||||
seconds = row['train_epoch_seconds']
|
||||
if (row['fold'] != i or row['epoch_nodes'] != old['learning_nodes']
|
||||
or row['epoch_parts'] != len(old['learning']) or not math.isfinite(seconds) or seconds <= 0):
|
||||
raise ValueError('c and b fold workload or epoch identity differs')
|
||||
if mtime - seconds > overlap_start + 2:
|
||||
candidates.append(dict(epoch=row['epoch'], epoch_seconds=seconds, file=str(p),
|
||||
sha256=sha(p), mtime=mtime))
|
||||
if not candidates:
|
||||
raise ValueError('No c epoch lies fully inside the same-card window')
|
||||
maximum = max(candidates, key=lambda t: t['epoch_seconds'])
|
||||
rows.append(dict(fold=i, gpu_uuid=assignment['gpu_uuid'], worker=worker,
|
||||
shared_peers={str(j): workers[j] for j in peers}, gpu_sample=sample,
|
||||
overlap_start=overlap_start, candidates=candidates, worst_epoch=maximum,
|
||||
max_train_epoch_seconds=maximum['epoch_seconds'],
|
||||
validation_seconds_bound=old['validation_seconds_bound']))
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
start = time.process_time()
|
||||
if ROOT.exists() or (ORIGINAL / 'started.json').exists() or (ORIGINAL / 'launch-attempt.json').exists():
|
||||
raise FileExistsError('Only a never-launched b may be rebound')
|
||||
validate_review(ORIGINAL)
|
||||
old = read(ORIGINAL / 'contract.json')
|
||||
stop = read(PARKED / 'complete.json')
|
||||
verify(PARKED, 'training-stop')
|
||||
tail = read(PARKED / 'post-terminal-cost.json')
|
||||
if (stop['status'] != 'STOPPED_INCOMPLETE' or stop['prediction_started'] or stop['scoring_started']
|
||||
or any(e['status'] == 'PASS' for e in stop['entries'])
|
||||
or tail['training_complete_sha256'] != sha(PARKED / 'complete.json')
|
||||
or tail['postfit_started'] or not tail['freeze_tail_charged_once']):
|
||||
raise ValueError('1b terminal identity changed')
|
||||
rows = measured_bounds(old['jobs'])
|
||||
jobs = []
|
||||
for job, row in zip(old['jobs'], rows, strict=True):
|
||||
epoch = row['max_train_epoch_seconds']
|
||||
jobs.append(dict(job, shared_epoch_seconds_bound=epoch,
|
||||
full_training_gpu_seconds_bound=full_fold_bound(epoch, job['validation_seconds_bound'])))
|
||||
bounds = [j['full_training_gpu_seconds_bound'] for j in jobs]
|
||||
wait_reserve = max(bounds)
|
||||
cap = math.ceil(old['preparation_charge_gpu_seconds'] + old['postfit_gpu_reserve'] + sum(bounds) + wait_reserve)
|
||||
reserves = dict(old['phase_reservations'], item1b=tail['cumulative_gpu_seconds'], b=cap)
|
||||
if cap > USER_CAP or sum(reserves.values()) > old['phase3_gpu_cap']:
|
||||
raise ValueError('Shared-mode cap does not fit the approved GPU limits')
|
||||
original_worker = BASE / 'tools/phase3_item_b_worker.py'
|
||||
new_worker = BASE / 'tools/phase3_item_b_shared_worker.py'
|
||||
normalized = new_worker.read_text().replace('from phase3_item_b_shared_common import',
|
||||
'from phase3_item_b_common import')
|
||||
if ast.dump(ast.parse(normalized)) != ast.dump(ast.parse(original_worker.read_text())):
|
||||
raise ValueError('b worker differs beyond the run identity import')
|
||||
files = [Path(__file__), *sorted((BASE / 'tools').glob('phase3_item_b_shared_*.py')),
|
||||
BASE / 'tools/test_phase3_item_b_shared_budget.py']
|
||||
codes = {**old['code_sha256'], **local_code_closure(files)}
|
||||
proof_sources = [ORIGINAL / x for x in ('contract.json', 'expected-review-binding.json',
|
||||
'implementation-git-freeze.json', 'review.json')]
|
||||
proof_sources += [PARKED / x for x in ('complete.json', 'post-terminal-cost.json', 'training-stop-git-freeze.json')]
|
||||
proof_sources += [C_ROOT / 'contract.json', C_ROOT / 'started.json']
|
||||
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
evidence = RECORDS / 'shared-cost-evidence.json'
|
||||
if evidence.exists():
|
||||
raise FileExistsError('A shared calibration receipt already exists')
|
||||
save(evidence, dict(ruling=RULING, c_root=str(C_ROOT), c_contract_sha256=sha(C_ROOT / 'contract.json'),
|
||||
folds=rows, margin=MARGIN, full_fold_training_bounds=bounds,
|
||||
aggregate_wait_reserve_gpu_seconds=wait_reserve,
|
||||
wait_reserve_basis='one longest full-fold bound across all waiting charges',
|
||||
gpu_cap=cap, user_cap=USER_CAP, phase_reservations=reserves,
|
||||
parked_1b_actual_including_tail=tail['cumulative_gpu_seconds'],
|
||||
phase3_unreserved_gpu_seconds=old['phase3_gpu_cap']-sum(reserves.values()),
|
||||
validation_bound_basis='original b diagnostic bounds; no shared diagnostic measurement',
|
||||
worker_same_except_rebind_import=True))
|
||||
sources = dict(old['source_sha256'])
|
||||
sources.update({str(p): sha(p) for p in proof_sources + [evidence]})
|
||||
contract = dict(old, version='phase3-item-b-shared-budget', shared_budget_ruling=RULING,
|
||||
shared_budget_evidence=str(evidence), gpu_cap=cap, phase_reservations=reserves,
|
||||
waiting_charge='Pre-dispatch waiting charged once to total; excluded from per-fold training',
|
||||
projection='Measured c shared-mode per-fold times +25%; actual b maxima may raise projections',
|
||||
jobs=jobs, code_sha256=codes, source_sha256=sources)
|
||||
ROOT.mkdir()
|
||||
save(ROOT / 'contract.json', contract)
|
||||
initial = project(contract, dict.fromkeys(range(5), 60.), dict.fromkeys(range(5), 0.), {}, {})
|
||||
if initial['status'] != 'PASS':
|
||||
raise ValueError('Initial shared-mode projection cannot fit')
|
||||
save(ROOT / 'preparation-audit.json', dict(status='WAIT_INDEPENDENT_DELTA_REVIEW',
|
||||
base_implementation=verify(ORIGINAL, 'implementation')['commit'],
|
||||
gpu_cap=cap, phase_reservations=reserves, initial_projection=initial,
|
||||
code_count=len(codes), source_count=len(sources), metadata_cpu_seconds=time.process_time()-start,
|
||||
comparison_unchanged=True, scientific_worker_unchanged=True,
|
||||
started=False, test_targets_read=False, review_adopted=False))
|
||||
freeze(ROOT, 'implementation', [ROOT / 'contract.json', ROOT / 'preparation-audit.json',
|
||||
*(Path(p) for p in codes), evidence, RECORDS / 'acceptance.log',
|
||||
BASE / 'docs/phase3-item-b-shared-budget-2026-11-19.md'])
|
||||
frozen = verify(ROOT, 'implementation')
|
||||
config(ROOT)
|
||||
save(ROOT / 'expected-review-binding.json', dict(
|
||||
reviewed_commit=frozen['commit'], contract_sha256=sha(ROOT / 'contract.json'),
|
||||
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
|
||||
comparison_contract_sha256=contract['comparison_contract_sha256'], code_sha256=codes))
|
||||
print(dict(implementation=frozen['commit'], contract=sha(ROOT / 'contract.json'),
|
||||
binding_sha256=sha(ROOT / 'expected-review-binding.json'), gpu_cap=cap,
|
||||
initial_projected_gpu_seconds=initial['projected_gpu_seconds'],
|
||||
phase3_reserved_gpu_seconds=sum(reserves.values()), code_count=len(codes)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,59 @@
|
||||
"""CPU-only checks for the b wrapper's wait and measured-cost delta."""
|
||||
|
||||
from unittest import TestCase, main
|
||||
|
||||
from phase3_item_b_shared_budget import full_fold_bound, project
|
||||
|
||||
|
||||
class SharedBudgetTests(TestCase):
|
||||
def setUp(self):
|
||||
self.contract = dict(
|
||||
jobs=[dict(shared_epoch_seconds_bound=100, validation_seconds_bound=10,
|
||||
full_training_gpu_seconds_bound=full_fold_bound(100, 10)) for _ in range(5)],
|
||||
gpu_cap=170000, preparation_charge_gpu_seconds=600, postfit_gpu_reserve=7200,
|
||||
runaway_gpu_seconds=60000,
|
||||
)
|
||||
|
||||
def test_wait_changes_total_but_not_training_projection(self):
|
||||
base = dict.fromkeys(range(5), 60)
|
||||
waits = dict.fromkeys(range(5), 0)
|
||||
measured = {0: [dict(epoch=1, train_epoch_seconds=103)]}
|
||||
baseline = project(self.contract, base | {0: 185}, waits, measured, {})
|
||||
result = project(self.contract, base | {0: 17185}, waits | {0: 17000}, measured, {})
|
||||
self.assertEqual(baseline['folds']['0']['projected_training_gpu_seconds'],
|
||||
result['folds']['0']['projected_training_gpu_seconds'])
|
||||
self.assertEqual(result['projected_gpu_seconds']-baseline['projected_gpu_seconds'], 17000)
|
||||
self.assertEqual(result['actual_waiting_gpu_seconds'], 17000)
|
||||
|
||||
def test_pending_wait_is_charged_once(self):
|
||||
cost = dict.fromkeys(range(5), 3060)
|
||||
wait = dict.fromkeys(range(5), 3000)
|
||||
result = project(self.contract, cost, wait, {}, {})
|
||||
bound = full_fold_bound(100, 10)
|
||||
self.assertEqual(result['folds']['2']['projected_training_gpu_seconds'], bound)
|
||||
self.assertEqual(result['projected_gpu_seconds'], 7800 + 5*(bound+3000))
|
||||
|
||||
def test_terminal_uses_actual_charge(self):
|
||||
result = project(self.contract, dict.fromkeys(range(5), 60) | {4: 5000},
|
||||
dict.fromkeys(range(5), 0) | {4: 2500}, {}, {4: {}})
|
||||
self.assertEqual(result['folds']['4']['projected_training_gpu_seconds'], 2500)
|
||||
self.assertEqual(result['folds']['4']['projected'], 5000)
|
||||
|
||||
def test_long_wait_does_not_hit_actual_training_runaway_limit(self):
|
||||
costs = dict.fromkeys(range(5), 60)
|
||||
waits = dict.fromkeys(range(5), 0)
|
||||
contract = dict(self.contract, gpu_cap=1000000)
|
||||
self.assertEqual(project(contract, costs | {0: 80000}, waits | {0: 79940}, {}, {})['status'], 'PASS')
|
||||
self.assertEqual(project(contract, costs | {0: 80000}, waits, {}, {})['status'], 'STOP_BUDGET')
|
||||
self.assertEqual(project(self.contract, costs | {0: 80000}, waits | {0: 79940}, {}, {})['status'], 'STOP_BUDGET')
|
||||
|
||||
def test_observed_b_epoch_can_raise_shared_bound(self):
|
||||
costs = dict.fromkeys(range(5), 60) | {0: 220}
|
||||
waits = dict.fromkeys(range(5), 0)
|
||||
faster = project(self.contract, costs, waits, {0: [dict(epoch=1, train_epoch_seconds=90)]}, {})
|
||||
slower = project(self.contract, costs, waits, {0: [dict(epoch=1, train_epoch_seconds=130)]}, {})
|
||||
self.assertEqual(slower['projected_gpu_seconds']-faster['projected_gpu_seconds'], 1.25*199*30)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user