Publish item b shared-mode budget and wait-accounting delta for review

This commit is contained in:
NewMeltAssist committed 2026-11-19 01:33:52 +08:00
1 parent 482c490afb
commit 5bfef6a61a
16 files changed
+16580

No files matched your search

@@ -0,0 +1,39 @@
test_actual_runaway_stops (test_phase3_item_b.BudgetTests) ... ok
test_incomplete_history_rejected (test_phase3_item_b.BudgetTests) ... ok
test_initial_budget_includes_postfit (test_phase3_item_b.BudgetTests) ... ok
test_late_validation_max_used (test_phase3_item_b.BudgetTests) ... ok
test_no_single_fold_projection_gate (test_phase3_item_b.BudgetTests) ... ok
test_remaining_validation_charged (test_phase3_item_b.BudgetTests) ... ok
test_all_learning_invalid_stops (test_phase3_item_b.ContextTests) ... ok
test_infinity_or_wrong_shape_stops (test_phase3_item_b.ContextTests) ... ok
test_joint_missing_is_zero_after_normalization (test_phase3_item_b.ContextTests) ... ok
test_learning_only_iteration (test_phase3_item_b.ContextTests) ... ok
test_original_role_guard_precedes_context_read (test_phase3_item_b.ContextTests) ... ok
test_partial_missing_stops (test_phase3_item_b.ContextTests) ... ok
test_status_mismatch_not_scientific_missing (test_phase3_item_b.ContextTests) ... ok
test_unit_node_moments_not_equal_part_means (test_phase3_item_b.ContextTests) ... ok
test_valid_mean_still_has_flag (test_phase3_item_b.ContextTests) ... ok
test_zero_variance_scale_one (test_phase3_item_b.ContextTests) ... ok
test_all_shared_parameters_and_rng_match (test_phase3_item_b.ModelTests) ... ok
test_exactly384_additional_parameters (test_phase3_item_b.ModelTests) ... ok
test_guard_accepts_only_complete200_epoch_identity (test_phase3_item_b.ModelTests) ... ok
test_new_columns_can_learn_without_changing_recipe (test_phase3_item_b.ModelTests) ... ok
test_nonzero_readout_still_matches_original_initial_function (test_phase3_item_b.ModelTests) ... ok
test_worker_uses_original200_recipe (test_phase3_item_b.ModelTests) ... ok
test_wrong_input_columns_rejected (test_phase3_item_b.ModelTests) ... ok
test_failure_no_retry_or_test (test_phase3_item_b.QueueTests) ... ok
test_five_fits_stop_before_postfit (test_phase3_item_b.QueueTests) ... ok
test_mixed_pending_c_reservation_blocks_b (test_phase3_item_b.SharedWrapperTests) ... ok
test_mixed_two_visible_allows_only_one_new (test_phase3_item_b.SharedWrapperTests) ... ok
test_shared_c_and_b_registration_independent (test_phase3_item_b.SharedWrapperTests) ... ok
test_wrapper_changes_only_key_cap_gpu_set (test_phase3_item_b.SharedWrapperTests) ... ok
test_long_wait_does_not_hit_actual_training_runaway_limit (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
test_observed_b_epoch_can_raise_shared_bound (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
test_pending_wait_is_charged_once (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
test_terminal_uses_actual_charge (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
test_wait_changes_total_but_not_training_projection (test_phase3_item_b_shared_budget.SharedBudgetTests) ... ok
----------------------------------------------------------------------
Ran 34 tests in 1.272s
OK
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,9 @@
# Item b pre-launch shared-mode budget delta
Claude's `1795021986.507079` and `1795022093.308479` apply to this never-launched entry. The new root is `/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19`. The old implementation, approval and original review remain historical evidence. Only an independent MeltAssist delta PASS on the new exact binding can release this wrapper for launch.
For each fold, use the maximum retained c epoch whose full interval falls within the same-card overlap witnessed by c assignment, process identity and GPU samples. The planning bound uses 200 such epochs plus the original b bound for 20 diagnostic validations, with 25% margin on both. Setup and cleanup remain 60 and 120 seconds per fold. The validation term is a retained bound, not a measurement of shared b validation. The live queue can increase the estimate as it observes slower b epochs or diagnostic passes.
The pre-launch cap is the ceiling of the five fold bounds, one explicit aggregate waiting reserve equal to the largest fold bound, the 600-second preparation charge and the 7,200-second postfit reserve. The waiting reserve is a planning assumption. Actual pre-dispatch waits count once toward the queue total and do not inflate per-fold training projections or the actual training runaway limit. The combined phase 3 ledger keeps c and seed 1 at their approved caps and 1b at its actual terminal total including the freeze tail. The recorded cap must fit the user's 50 GPU-hour item b limit and the 720,000 GPU-second phase 3 limit.
The b comparison contract, context cache, model, initialization, optimization, order, validation recipe and scientific worker logic stay as previously reviewed. The worker's only source delta is its runtime identity import. The shared registry and admission implementation are reused verbatim. This budget lesson also applies when Track B gets its own entry and measurement. It does not create or launch a Track B job.
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,85 @@
{
"reviewed_commit": "961ce62360003e7621f4fad6ae58648ef96d82af",
"contract_sha256": "8a51b67c35fe6f0ec201063c15aa758575afc8c14dda4c165b3d83b5e163883c",
"implementation_proof_sha256": "421351347255615185d7a307c2b087118bcb67dde385493e3fe0183747e96dba",
"comparison_contract_sha256": "06556bf1f7bd5a1a79451bd9eed47fc6ec027e7e06bc63c6f490a50dfbee6d20",
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_capacity.py": "3915da37ead45f42fe620cb89bc7394b576f8e113047a4ad78e1bae50ded120c",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_common.py": "d00069d2ca5a346d8260dfb1829bf941f3ea09ebd7d75c1fe377120b553e02d2",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_data.py": "74e35b6268462c999078383e49e551c9da2562390d22977414713811bab164b3",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_guard.py": "e95cd5126f403881aebe137ed3f00bb6173881eea15709952142cb51136ad699",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_model.py": "13000ce6eb7cccfbefdc1710e37a59412be0391ae820b243f0d8fe7a9be4e986",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_queue.py": "0c6f5c9bde84f28bfc829701d97701188f43ed59fa0c5286228ca325cf88235c",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_worker.py": "d49a5e6f900850aabacc5825168b6f886215aff8c494937bea2238539e58f652",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b.py": "ac46c3c6d01d8023444d2f53f956d94d1d295ae95a3435f355c5c6e0f6ccd962",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b.py": "bed8e21042971bb688b33c13903a28fab1772b980431a80afd205dbe4350dac2",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "08b7d8ab2ac773fab8b749f9f61af259254422128e9634f8b715c81efce2651f",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "31dfd71369c63512c1d6fe176fd22e8db646520c92da533bb5530ca3843c388d",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "06cb97594c3a87f42f712dcde6cec8c2d90e0b50a89a6be7954bdd2f5c18d5de",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "b781e56c519401bd38af6a522deb24506cba6993d2af008f85ea4e1715f1d9c6",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "c7bc9f3b7cf4137f52e1efad19dcb7045282d4e9a9c1aae50f066c9e488f7c01",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "b4d78f4384651711fe83ff00eb6719c6c61664699d801ecb34978b9546439da9",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "3115aa9049974eac86657aa4cee7b5d716f6ee93268555cc86242029ebefce4c",
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_budget.py": "2b10e7f126d2e617719eb8c20c1d234d83ef2bb77e94fe1932f5fc57501373e6",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_capacity.py": "910fd9cff919824ea9742c55ee8b0d12a262f0d42fcd18afa8060334c5fbc58e",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_common.py": "c84f5b795e9354c5e380e8492308fd80d22bf83d4008fe961b1a36ecfba0dab9",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_guard.py": "01963a986eab70140ada7ca70fc7619012fe92ee52c3921e45ae1b75a7ed8357",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_queue.py": "77bbf7c61e9e64c079d398767527e7f87e9d01d4c27df45354ec756042f32fe9",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_worker.py": "363a8667d11b2ef76cc1c23a66d62be8135834f956d1d311df45a1017c8ecb56",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b_shared.py": "8a638b9fecd92c1dfff4ab97857d519308306d589b6940dfd75d6e8e0a702032",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b_shared_budget.py": "f9900b10d27d2f549618c0ad339f7700dba7df494c6ff296513fe7d80b4a622d"
}
}
@@ -0,0 +1,4 @@
{
"contract_sha256": "8a51b67c35fe6f0ec201063c15aa758575afc8c14dda4c165b3d83b5e163883c",
"proof_sha256": "421351347255615185d7a307c2b087118bcb67dde385493e3fe0183747e96dba"
}
@@ -0,0 +1,173 @@
{
"commit": "961ce62360003e7621f4fad6ae58648ef96d82af",
"files": {
"/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19/contract.json": "8a51b67c35fe6f0ec201063c15aa758575afc8c14dda4c165b3d83b5e163883c",
"/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19/preparation-audit.json": "23a6424f2f4c54de6817a1cb4463a5c8b191f3db76be463a72fa7e9b6e24417a",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "14651ca402e6d02466d8afdb77bc9e62fb2e752f0c0c3d20d98593157d5ebe81",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "38a29a34663a5d6310571eeff53b5d18a0355e51b9fa58a08033f64fa379a9df",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_capacity.py": "3915da37ead45f42fe620cb89bc7394b576f8e113047a4ad78e1bae50ded120c",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_common.py": "d00069d2ca5a346d8260dfb1829bf941f3ea09ebd7d75c1fe377120b553e02d2",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_data.py": "74e35b6268462c999078383e49e551c9da2562390d22977414713811bab164b3",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_guard.py": "e95cd5126f403881aebe137ed3f00bb6173881eea15709952142cb51136ad699",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_model.py": "13000ce6eb7cccfbefdc1710e37a59412be0391ae820b243f0d8fe7a9be4e986",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_queue.py": "0c6f5c9bde84f28bfc829701d97701188f43ed59fa0c5286228ca325cf88235c",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_worker.py": "d49a5e6f900850aabacc5825168b6f886215aff8c494937bea2238539e58f652",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b.py": "ac46c3c6d01d8023444d2f53f956d94d1d295ae95a3435f355c5c6e0f6ccd962",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b.py": "bed8e21042971bb688b33c13903a28fab1772b980431a80afd205dbe4350dac2",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "771faad0095712fd4fc8131ddcfabaa3901141592d2b055ea2faadf8ea3d3edf",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "08b7d8ab2ac773fab8b749f9f61af259254422128e9634f8b715c81efce2651f",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "31dfd71369c63512c1d6fe176fd22e8db646520c92da533bb5530ca3843c388d",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "06cb97594c3a87f42f712dcde6cec8c2d90e0b50a89a6be7954bdd2f5c18d5de",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "b781e56c519401bd38af6a522deb24506cba6993d2af008f85ea4e1715f1d9c6",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "c7bc9f3b7cf4137f52e1efad19dcb7045282d4e9a9c1aae50f066c9e488f7c01",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "b4d78f4384651711fe83ff00eb6719c6c61664699d801ecb34978b9546439da9",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "4e85f5d211c4d3e2188422aa4c0cefa40dcfa14ef32e90894ed3412de1fd1d12",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "fe3aa3e023f22f4fe55cd47e7ffa677b58f52a041867ac05620cbd329d913cb6",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "7144fa77ffe61a2a3c92c050234e370ef4dfeb0e31fafb419f752f969737ff1a",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "79e291866bda300ecdc5577c62903d34abeec14f5cc36db05e346f9ffd64c24b",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "a04879cc0169ecb523a88360cddca1e234d91b3adbc20b3ec31d02461ed767eb",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "0f8f35e1422021b2a4f097c2e62267fb935fe4658fd0ee8ccd251609a4062ea7",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "782c415b1f536eb2b85aa2c9c4a9a0dc278f9b7134e8a83e2a6af8199396c53a",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "588c646101be0d50f505f9a12a1c98361aa3437c8c594eecb7321831f717a741",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "188565801f6901d0192f79aec0889e632501e672a4149d76d1835efbb9b45447",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "3115aa9049974eac86657aa4cee7b5d716f6ee93268555cc86242029ebefce4c",
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "4235890fd194b7778e937f0be9245b964512df2c75d962ef703e3a988c5b8805",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "6bc7624b0cfe850591f77153cd2034eefdd62f7e63b93211ccdb87eb8aba346c",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "308486507ae3589a0e6cc40565f6e60e6d2f63ae74c4d828bdacf33b3b8b8062",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "ea17fdfc7508f3f980e3cdf69534b1eb8ceb8a8e469c32451b2d5e16d0ada575",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "37dfd75fff50b3d65f4ef37f4255536616582fe5f3d01155a0e037325ab5c7cd",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "ecee04b183ef2b93b7b39a447d48c40a5843d3190e48d2d7eb4939811a5db9b0",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "964fae13ac95e4de4dc50b3c0a80791d86349ce3a9f3c5a6e7dc3cf8ca041fad",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "3a31af0766f628a80f38e0b7286beb11ca3109188eabc4adfdb2ecb0eef3dc27",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "d82dfb365956e08250758bb4c651a19fbecce92fe209072f5f8ed35970450735",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "480fac25dea5a2006bb84130a4b0a0b253b3040b2150a56c38eada62b2c259e6",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "ae94711adc17a7630f398e8d4120fa3cc8f60979a00122c4cd2dc364c382ced5",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "1e8e5ef31650519737cd6cd247abc0bd04744dd93ffb013eff88866d42f03073",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_budget.py": "2b10e7f126d2e617719eb8c20c1d234d83ef2bb77e94fe1932f5fc57501373e6",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_capacity.py": "910fd9cff919824ea9742c55ee8b0d12a262f0d42fcd18afa8060334c5fbc58e",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_common.py": "c84f5b795e9354c5e380e8492308fd80d22bf83d4008fe961b1a36ecfba0dab9",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_guard.py": "01963a986eab70140ada7ca70fc7619012fe92ee52c3921e45ae1b75a7ed8357",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_queue.py": "77bbf7c61e9e64c079d398767527e7f87e9d01d4c27df45354ec756042f32fe9",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_worker.py": "363a8667d11b2ef76cc1c23a66d62be8135834f956d1d311df45a1017c8ecb56",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b_shared.py": "8a638b9fecd92c1dfff4ab97857d519308306d589b6940dfd75d6e8e0a702032",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b_shared_budget.py": "f9900b10d27d2f549618c0ad339f7700dba7df494c6ff296513fe7d80b4a622d",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-b-shared-budget-2026-11-19/shared-cost-evidence.json": "1abb73b38620e3649f92796fe31962054c9d2fd55092d6eac09403bc5e18d553",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-b-shared-budget-2026-11-19/acceptance.log": "5362d21dc2aff6f2a45aa86821083179cc3e991e71ae165298b4974d6170f0af",
"/home/frank14f/optane/MeltMind/docs/phase3-item-b-shared-budget-2026-11-19.md": "a811143aab79bf1ed4b95b22582eecbae0ffd6852eb80011980427f262403b35"
},
"git_paths": {
"/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19/contract.json": "home-runs/phase3-item-b-shared-2026-11-19/contract.json",
"/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19/preparation-audit.json": "home-runs/phase3-item-b-shared-2026-11-19/preparation-audit.json",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_pilot_worker.py": "tools/phase2_h2_pilot_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_common.py": "tools/phase2_h2_s1_common.py",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_capacity.py": "tools/phase3_item_b_capacity.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_common.py": "tools/phase3_item_b_common.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_data.py": "tools/phase3_item_b_data.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_guard.py": "tools/phase3_item_b_guard.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_model.py": "tools/phase3_item_b_model.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_queue.py": "tools/phase3_item_b_queue.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_worker.py": "tools/phase3_item_b_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "tools/phase3_seed1_v2_common.py",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "tools/phase3_shared_capacity.py",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b.py": "tools/prepare_phase3_item_b.py",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/__init__.py": "tools/s2_models/__init__.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/attention.py": "tools/s2_models/attention.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/blocks.py": "tools/s2_models/blocks.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/features.py": "tools/s2_models/features.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/network.py": "tools/s2_models/network.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/normalization.py": "tools/s2_models/normalization.py",
"/home/frank14f/optane/MeltMind/tools/s2_models/objectives.py": "tools/s2_models/objectives.py",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b.py": "tools/test_phase3_item_b.py",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "tools/track_a_recovery_monitor.py",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "tools/track_a_recovery_supervisor.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_capacity.py": "tools/phase3_item1b_capacity.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "tools/phase3_item1b_common.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_guard.py": "tools/phase3_item1b_guard.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_queue.py": "tools/phase3_item1b_queue.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_schedule.py": "tools/phase3_item1b_schedule.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_worker.py": "tools/phase3_item1b_worker.py",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item1b.py": "tools/prepare_phase3_item1b.py",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item1b.py": "tools/test_phase3_item1b.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_common.py": "tools/phase3_item_c_common.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_data.py": "tools/phase3_item_c_data.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_guard.py": "tools/phase3_item_c_guard.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_loss.py": "tools/phase3_item_c_loss.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_queue.py": "tools/phase3_item_c_queue.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_c_worker.py": "tools/phase3_item_c_worker.py",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_c.py": "tools/prepare_phase3_item_c.py",
"/home/frank14f/optane/MeltMind/tools/s2_custody_common.py": "tools/s2_custody_common.py",
"/home/frank14f/optane/MeltMind/tools/s2_custody_statistics.py": "tools/s2_custody_statistics.py",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_c.py": "tools/test_phase3_item_c.py",
"/home/frank14f/optane/MeltMind/tools/test_phase2_h2_s2.py": "tools/test_phase2_h2_s2.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_guard.py": "tools/phase2_h2_s2_guard.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_common.py": "tools/phase2_h2_s2_common.py",
"/home/frank14f/optane/MeltMind/tools/phase2_fixed_io.py": "tools/phase2_fixed_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_worker.py": "tools/phase2_h2_s2_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_worker.py": "tools/phase2_h2_s1_worker.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_prepare.py": "tools/phase2_h2_s2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_prepare.py": "tools/phase2_h2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s2_queue.py": "tools/phase2_h2_s2_queue.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_launch.py": "tools/phase2_h2_s1_launch.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_s1_guard.py": "tools/phase2_h2_s1_guard.py",
"/home/frank14f/optane/MeltMind/docs/phase2-h2-s2-execution-contract-2026-11-17.md": "docs/phase2-h2-s2-execution-contract-2026-11-17.md",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_budget.py": "tools/phase3_item_b_shared_budget.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_capacity.py": "tools/phase3_item_b_shared_capacity.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_common.py": "tools/phase3_item_b_shared_common.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_guard.py": "tools/phase3_item_b_shared_guard.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_queue.py": "tools/phase3_item_b_shared_queue.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item_b_shared_worker.py": "tools/phase3_item_b_shared_worker.py",
"/home/frank14f/optane/MeltMind/tools/prepare_phase3_item_b_shared.py": "tools/prepare_phase3_item_b_shared.py",
"/home/frank14f/optane/MeltMind/tools/test_phase3_item_b_shared_budget.py": "tools/test_phase3_item_b_shared_budget.py",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-b-shared-budget-2026-11-19/shared-cost-evidence.json": "artifacts/phase3-item-b-shared-budget-2026-11-19/shared-cost-evidence.json",
"/home/frank14f/optane/MeltMind/artifacts/phase3-item-b-shared-budget-2026-11-19/acceptance.log": "artifacts/phase3-item-b-shared-budget-2026-11-19/acceptance.log",
"/home/frank14f/optane/MeltMind/docs/phase3-item-b-shared-budget-2026-11-19.md": "docs/phase3-item-b-shared-budget-2026-11-19.md"
},
"remote_published": false,
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
}
@@ -0,0 +1,65 @@
{
"status": "WAIT_INDEPENDENT_DELTA_REVIEW",
"base_implementation": "b2a9dfdc4a2c237416452ff6b4c32bdbf34e172e",
"gpu_cap": 172908,
"phase_reservations": {
"item1": 72421.31599938683,
"seed1": 167200.0,
"c": 205200.0,
"item1b": 19262.057604262838,
"b": 172908
},
"initial_projection": {
"status": "PASS",
"projected_gpu_seconds": 143203.1576138339,
"actual_training_and_preparation_gpu_seconds": 900.0,
"actual_waiting_gpu_seconds": 0.0,
"folds": {
"0": {
"charged": 60.0,
"waiting_gpu_seconds": 0.0,
"actual_training_with_setup_gpu_seconds": 60.0,
"projected_training_gpu_seconds": 25870.330589201767,
"projected": 25870.330589201767
},
"1": {
"charged": 60.0,
"waiting_gpu_seconds": 0.0,
"actual_training_with_setup_gpu_seconds": 60.0,
"projected_training_gpu_seconds": 28917.776216800557,
"projected": 28917.776216800557
},
"2": {
"charged": 60.0,
"waiting_gpu_seconds": 0.0,
"actual_training_with_setup_gpu_seconds": 60.0,
"projected_training_gpu_seconds": 29704.07792180602,
"projected": 29704.07792180602
},
"3": {
"charged": 60.0,
"waiting_gpu_seconds": 0.0,
"actual_training_with_setup_gpu_seconds": 60.0,
"projected_training_gpu_seconds": 24445.709940344095,
"projected": 24445.709940344095
},
"4": {
"charged": 60.0,
"waiting_gpu_seconds": 0.0,
"actual_training_with_setup_gpu_seconds": 60.0,
"projected_training_gpu_seconds": 26465.26294568146,
"projected": 26465.26294568146
}
},
"postfit_gpu_reserve": 7200,
"safety_factor": 1.25
},
"code_count": 77,
"source_count": 1309,
"metadata_cpu_seconds": 0.31858634399999997,
"comparison_unchanged": true,
"scientific_worker_unchanged": true,
"started": false,
"test_targets_read": false,
"review_adopted": false
}
+62
View File
@@ -0,0 +1,62 @@
"""Shared-mode extrapolation with queue waiting charged separately from training."""
import math
MARGIN = 1.25
SETUP = 60
CLEANUP = 120
USER_CAP = 180000
RULING = '1795022093.308479'
def full_fold_bound(epoch_seconds, validation_seconds):
if not math.isfinite(epoch_seconds) or epoch_seconds <= 0:
raise ValueError('Positive measured epoch time required')
if not math.isfinite(validation_seconds) or validation_seconds < 0:
raise ValueError('Invalid validation time')
return SETUP + MARGIN * (200 * epoch_seconds + 20 * validation_seconds) + CLEANUP
def project(contract, charges, waits, histories, terminal):
folds = set(range(5))
if set(charges) != folds or set(waits) != folds or set(histories) - folds or set(terminal) - folds:
raise ValueError('Cost ledger fold identity')
rows = {}
for i, charged in charges.items():
waiting = waits[i]
if not math.isfinite(charged) or not math.isfinite(waiting) or not 0 <= waiting <= charged:
raise ValueError('Invalid queue and waiting charges')
elapsed = charged - waiting
job = contract['jobs'][i]
history = histories.get(i, [])
if i in terminal:
training = elapsed
elif history:
epochs = sorted(r['epoch'] for r in history)
finished = epochs[-1]
if epochs != list(range(1, finished + 1)) or finished > 200:
raise ValueError('Incomplete epoch history')
observed = [r['train_epoch_seconds'] for r in history]
diagnostics = [r.get('validation_seconds', 0) for r in history]
if any(not math.isfinite(x) or x <= 0 for x in observed):
raise ValueError('Invalid measured epoch')
if any(not math.isfinite(x) or x < 0 for x in diagnostics):
raise ValueError('Invalid validation time')
epoch_time = max(job['shared_epoch_seconds_bound'], *observed)
validation_time = max(job['validation_seconds_bound'], *diagnostics)
training = elapsed + MARGIN * ((200-finished)*epoch_time + (20-finished//10)*validation_time) + CLEANUP
else:
training = max(elapsed, job['full_training_gpu_seconds_bound'])
rows[str(i)] = dict(charged=charged, waiting_gpu_seconds=waiting,
actual_training_with_setup_gpu_seconds=elapsed,
projected_training_gpu_seconds=training, projected=waiting+training)
actual = contract['preparation_charge_gpu_seconds'] + sum(charges.values())
projected = (contract['preparation_charge_gpu_seconds'] + contract['postfit_gpu_reserve']
+ sum(row['projected'] for row in rows.values()))
ok = projected <= contract['gpu_cap'] and actual + contract['postfit_gpu_reserve'] <= contract['gpu_cap']
ok &= all(charges[i] - waits[i] < contract['runaway_gpu_seconds'] for i in folds - set(terminal))
return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected,
actual_training_and_preparation_gpu_seconds=actual, actual_waiting_gpu_seconds=sum(waits.values()),
folds=rows, postfit_gpu_reserve=contract['postfit_gpu_reserve'], safety_factor=MARGIN)
+67
View File
@@ -0,0 +1,67 @@
"""b registration wrapper; unchanged reviewed shared admission and leases.
Only b's registered cap/ledger changes; the existing admission and reservations remain in use.
The live c module is imported without mutation, never patched or rewritten.
"""
import os,time,uuid
from pathlib import Path
import psutil
from phase3_shared_capacity import (registry,process_identity,alive,descendants,
host_snapshot,admission,GPUS,GIB,transfer,release,validate_lease)
def register_entry(root, digest, contract):
with registry() as d:
if 'b' in d['entries']:
raise FileExistsError('Sole item b entry already registered; no resume')
reservations = contract['phase_reservations']
cap = contract['gpu_cap']
if reservations['b'] != cap or sum(reservations.values()) > 720000:
raise RuntimeError('Whole phase3 reserved cap')
d['entries']['b'] = dict(root=str(root), contract_sha256=digest, owner=process_identity(os.getpid()),
cap=cap, phase_reservations=reservations, started_unix=time.time())
def acquire(root, fold, digest):
with registry() as d:
entry = d['entries']['b']
if entry['root'] != str(root) or entry['contract_sha256'] != digest or not alive(entry['owner']):
raise PermissionError('Entry owner/identity')
leases = []
for l in d['leases'].values():
# Normal guard exit can race this observation. Block admission until
# its owning coordinator collects it; never reclaim or kill peers.
if not alive(l['owner']):
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION',token=l['token'])
if l['state']=='RESERVED':
# owner is a queue with other children, not this job's guard.
# One independent slot and full VRAM/RSS remain reserved.
leases.append(dict(l,observed_pids=[],observed_rss=0))
continue
if l['state']!='GUARD_BOUND':
raise ValueError('Unknown lease state')
try:
ids = descendants(l['owner'])
except (psutil.NoSuchProcess,RuntimeError):
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION',token=l['token'])
rss=0
for pid in ids:
try:rss+=psutil.Process(pid).memory_info().rss
except psutil.NoSuchProcess:pass
leases.append(dict(l, observed_pids=list(ids), observed_rss=rss))
s = host_snapshot() # fresh inside inter-entry lock, before making reservation
used_cpu = {x for l in leases for x in l['worker_cpus']}
cpus = [x for x in range(16,26) if x not in used_cpu and x in os.sched_getaffinity(0)]
if not cpus:
return None, dict(reason='CPU_RESERVATIONS', snapshot=s)
for gpu in GPUS[:3]:
ok, reason = admission(s, leases, gpu)
if not ok:
continue
token = uuid.uuid4().hex
queue_owner=process_identity(os.getpid())
l = dict(token=token, state='RESERVED',root=str(root), fold=fold, contract_sha256=digest, gpu_uuid=gpu,
worker_cpus=[cpus[0]], owner=queue_owner,queue_owner=queue_owner,
vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
d['leases'][token] = l
return l, dict(reason='ADMITTED', snapshot=s)
return None, dict(reason='NO_CAPACITY', snapshot=s)
+95
View File
@@ -0,0 +1,95 @@
"""Item b immutable scope, assignments and conservative total-cost projection."""
import json
import math
import statistics
from pathlib import Path
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
from phase3_seed1_v2_common import fold_view
from phase3_item_b_shared_budget import RULING, USER_CAP, full_fold_bound, project
ROOT = Path('/home/frank14f/MeltMind-runs/phase3-item-b-shared-2026-11-19')
def read(p):
return json.loads(Path(p).read_text())
def config(root):
if root.resolve() != ROOT:
raise PermissionError('Sole shared-mode b root')
c = read(root/'contract.json')
verify(root, 'implementation')
expected = dict(approval='1795011607.907609', seed=0, epochs=200, new_fits=5, shared_budget_ruling=RULING,
postfit_gpu_reserve=7200, postfit_cpu_cap=3600,
phase3_gpu_cap=720000, no_retry=True)
if any(c.get(k) != v for k,v in expected.items()):
raise PermissionError('Fixed item_b scope/budgets')
for p,h in {**c['source_sha256'], **c['code_sha256']}.items():
checked(p,h)
bounds = read(c['shared_budget_evidence'])
if bounds['ruling'] != RULING or c['phase_reservations'] != bounds['phase_reservations']:
raise ValueError('Fixed shared budget identity')
full_bounds = []
for job, row in zip(c['jobs'], bounds['folds'], strict=True):
epoch = row['max_train_epoch_seconds']
full = full_fold_bound(epoch, job['validation_seconds_bound'])
if (job['id'] != row['fold'] or job['shared_epoch_seconds_bound'] != epoch
or job['full_training_gpu_seconds_bound'] != full):
raise ValueError('Fixed fold timing bounds')
full_bounds.append(full)
cap = math.ceil(c['preparation_charge_gpu_seconds'] + c['postfit_gpu_reserve']
+ sum(full_bounds) + max(full_bounds))
if (c['gpu_cap'] != cap or c['phase_reservations']['b'] != cap or cap > USER_CAP
or sum(c['phase_reservations'].values()) > c['phase3_gpu_cap']
or c['phase_reservations']['item1b'] != bounds['parked_1b_actual_including_tail']):
raise ValueError('User or phase 3 reservation exceeded')
pilot = read(c['pilot_contract'])
import numpy as np
import torch
if np.__version__ != pilot['versions']['numpy'] or torch.__version__ != pilot['versions']['torch']:
raise ValueError('Original environment versions')
if [j['id'] for j in c['jobs']] != list(range(5)):
raise ValueError('Exactly five folds')
tests = []
for job in c['jobs']:
f = fold_view(pilot, job['id'])['folds'][0]
tests.extend(f['test'])
if job['target_steps'] != 200*len(f['learning']):
raise ValueError('Full fixed epochs')
for k in ('learning','validation','test','c','clock_sha256','learning_nodes'):
if job[k] != f[k]:
raise ValueError('Original fold identity changed')
if len(tests) != 552 or len(set(tests)) != 552:
raise ValueError('Original552 roster')
rows=read(c['query_roster_contract'])['rows']
if len(rows)!=552 or {r['case'] for r in rows}!=set(tests) or len({r['component_id'] for r in rows})!=204:
raise ValueError('552 parts /204 components query roster')
for r in rows:
p=pilot['parts'][r['case']]
if r['v5_cache']['output_sha256']!=p['output_sha256'] or r['nodes']!=p['nodes'] or r['component_id']!=p['clock_group']:
raise ValueError('Learning query/geometry/cache identity')
if c['inputs']!=22 or c['added_parameters']!=384 or c['budget_approval']!='1795007580.895539' or not c['decision_requires_seed_reference']:
raise PermissionError('Fixed regional context scope')
checked(c['comparison_contract'], c['comparison_contract_sha256'])
return c,pilot
def assignment(root,c,index,digest):
a = read(checked(root/'jobs'/str(index)/'assignment.json',digest))
if a['id'] != index or a['contract_sha256'] != sha(root/'contract.json'):
raise PermissionError('Assignment identity')
return dict(c['jobs'][index], **{k:a[k] for k in ('gpu_uuid','worker_cpus','token','queue_identity')})
def first_epoch_release(root,index):
if (root/'global-stop.json').exists():
raise RuntimeError('Global stop')
p = root/'jobs'/str(index)/'first-epoch-projection.json'
if not p.exists():
return False
d = read(p)
if d['status'] != 'PASS' or d['contract_sha256'] != sha(root/'contract.json'):
raise RuntimeError('First-epoch gate not passed')
return True
+136
View File
@@ -0,0 +1,136 @@
"""One item b attempt, scoped process memory plus failed-worker peak checks."""
import ctypes
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import threading
import time
import psutil
from phase2_h1_worker_v2 import save,sha
from phase3_item_b_shared_common import config,assignment,read
from phase3_shared_capacity import validate_lease,alive
from track_a_recovery_monitor import Monitor
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
from s2_main_memory_guard import PeakRecords
def run(queue,index,digest):
parent=os.getppid()
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
raise OSError('PDEATHSIG')
requested=[]
for sig in (signal.SIGTERM,signal.SIGINT):
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
if os.getppid()!=parent:
raise RuntimeError('Queue lost before setup')
c,_=config(queue);job=assignment(queue,c,index,digest)
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
remaining=c['runaway_gpu_seconds']-60
if remaining<90:
raise RuntimeError('No cleanup window before worker')
deadline=start+remaining
# Queue writes ready only after atomically transferring reservation ownership.
while not (root/'lease-ready.json').exists():
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
raise RuntimeError('Lease transfer timeout')
time.sleep(.1)
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
raise RuntimeError('Pinned queue lost before guard import completed')
os.sched_setaffinity(0,[26,27])
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
reasons=[];receipt=None
command=[sys.executable,'tools/phase3_item_b_shared_worker.py',str(queue),str(index),digest]
def reason(s):
if s not in reasons:reasons.append(s)
def workers():
result=[]
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:result.append(p.pid)
except psutil.NoSuchProcess:pass
return result
def kill():
for pid in workers():stop_group(pid)
def scan():
tracker.scan(root/'worker')
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
def local_watch():
while not done.is_set():
monitor.sample('recorded_peaks',scan)
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
disk,ok=monitor.sample('disk',lambda:disk_usage(root))
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
if ok and not queue_live:reason('QUEUE_LOST')
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(.5)
def gpu_watch():
def sample():
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
rows=[]
for line in text.strip().splitlines():
if not line:continue
u,p,m=[x.strip() for x in line.split(',')]
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
own=set(workers())
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
while not done.is_set():
monitor.sample('gpu_processes',sample)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(1)
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
try:
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
for t in threads:t.start()
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
except BaseException as exc:
reason(type(exc).__name__+': '+str(exc));kill()
finally:
done.set()
for t in threads:
if t.ident is not None:t.join(timeout=12)
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
try:monitor.retry('final_peaks',scan,.1)
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
p=root/'worker/result.json';result=read(p) if p.exists() else None
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
if not result or result.get('status')!='COMPLETE' or result.get('fold')!=index or result.get('completed_epochs')!=200 or result.get('optimizer_steps')!=job['target_steps']:reason('NO_COMPLETE_IDENTITY')
if result:
from phase2_h1_worker_v2 import checked
checked(result['checkpoint'],result['checkpoint_sha256'])
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
elapsed=time.monotonic()-start
child=resource.getrusage(resource.RUSAGE_CHILDREN)
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
if 60+elapsed>c['runaway_gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
try:config(queue)
except Exception as exc:reason('FINAL_PINS: '+str(exc))
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
return 0 if not reasons else 1
if __name__=='__main__':
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
+152
View File
@@ -0,0 +1,152 @@
"""Review-gated five-fold b training. Stops at frozen models, never postfit."""
import fcntl
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from phase2_h1_worker_v2 import save,sha
from phase2_home_freeze_io import freeze
from phase2_gate_outcome_review import validate_review
from phase3_item_b_shared_common import ROOT,config,read,project
from phase3_item_b_shared_capacity import register_entry,acquire,transfer,release,process_identity,alive
from track_a_recovery_monitor import Monitor
def run(root):
c,_=config(root);validate_review(root)
with (root/'queue.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Sole b attempt, no resume')
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
raise ValueError('Explicit allocator required')
if psutil.disk_usage(root).free<100*2**30:raise RuntimeError('Less than100GiB free')
register_entry(root,sha(root/'contract.json'),c)
start=time.monotonic();identity=process_identity(os.getpid())
save(root/'started.json',dict(unix=time.time(),identity=identity,contract_sha256=sha(root/'contract.json')))
(root/'jobs').mkdir(exist_ok=False)
active={};terminal={};pending=[0,1,2,3,4];histories={};waits={};stops=[];failure=None
monitor=Monitor(root/'admission-errors.jsonl')
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda n,f:stops.append('SIGNAL_'+str(n)))
def charge(i):
return terminal[i]['gpu_seconds_charged'] if i in terminal else 60+time.monotonic()-start
def waiting_charges(charges):
return {i: waits.get(i, max(0, charged - 60)) for i, charged in charges.items()}
def projection_at_current_time():
charges = {i: charge(i) for i in range(5)}
return project(c, charges, waiting_charges(charges), histories, terminal)
def stop(reason):
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
try:
while active or pending:
if stops:raise RuntimeError(stops[0])
if (root/'global-stop.json').exists():raise RuntimeError('EXTERNAL_GLOBAL_STOP')
# Collect exits before granting any further admission.
for i,(p,lease,log,owner) in list(active.items()):
if p.poll() is None:continue
log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
worker_identity=jr/'worker/worker-identity.json'
if worker_identity.exists() and alive(read(worker_identity)):
raise RuntimeError('WORKER_OUTLIVED_GUARD_LEASE_HELD')
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,
gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],guard_sha256=sha(cp) if cp.exists() else None)
release(lease['token'],owner);del active[i]
if result['status']!='PASS' or p.returncode!=0:raise RuntimeError('FOLD_FAILED_'+str(i))
for i in active:
folder=root/'jobs'/str(i)/'worker/history'
histories[i]=[read(p) for p in sorted(folder.glob('*.json'))]
projection=projection_at_current_time()
save(root/'latest-cost-projection.json',projection)
if projection['status']!='PASS':raise RuntimeError('TOTAL_PROJECTION_OR_ACTUAL_CAP')
for i in active:
gate=root/'jobs'/str(i)/'first-epoch-projection.json'
if histories.get(i) and not gate.exists():
save(gate,dict(status='PASS',contract_sha256=sha(root/'contract.json'),projection=projection))
save(root/'live-progress.json',dict(unix=time.time(),stage='ITEM_B_TRAINING',active=list(active),pending=pending,
terminal=list(terminal.values()),charged_training_gpu_seconds=projection['actual_training_and_preparation_gpu_seconds'],postfit_started=False))
if pending:
i=pending[0]
decision,ok=monitor.sample('admission',lambda:acquire(root,i,sha(root/'contract.json')))
if monitor.failed:raise RuntimeError('THREE_ADMISSION_FAILURES')
if not ok:
time.sleep(1)
continue
lease,observation=decision
save(root/'latest-admission.json',observation)
# Acquisition includes bounded GPU queries; recheck cost before Popen.
fresh=projection_at_current_time()
if fresh['status']!='PASS':
if lease:release(lease['token'])
raise RuntimeError('CAP_DURING_ADMISSION')
if lease:
jr=root/'jobs'/str(i);jr.mkdir(exist_ok=False)
waits[i]=time.monotonic()-start
save(jr/'assignment.json',dict(id=i,contract_sha256=sha(root/'contract.json'),
gpu_uuid=lease['gpu_uuid'],worker_cpus=lease['worker_cpus'],token=lease['token'],
queue_identity=identity,pre_dispatch_wait_seconds=waits[i]))
log=(jr/'guard.log').open('x')
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
p=subprocess.Popen([sys.executable,'tools/phase3_item_b_shared_guard.py',str(root),str(i),sha(jr/'assignment.json')],
cwd=str(Path(__file__).resolve().parents[1]),env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
owner=process_identity(p.pid)
active[i]=(p,lease,log,owner);pending.remove(i)
transfer(lease['token'],owner)
save(jr/'lease-ready.json',dict(token=lease['token'],owner=owner))
time.sleep(1)
except BaseException as exc:
failure=type(exc).__name__+': '+str(exc);stop(failure)
finally:
if active:
stop(failure or 'QUEUE_CLEANUP')
for p,lease,log,owner in active.values():
if p.poll() is None:p.terminate()
deadline=time.monotonic()+45
while time.monotonic()<deadline and any(p.poll() is None for p,_,_,_ in active.values()):time.sleep(.2)
for i,(p,lease,log,owner) in active.items():
if p.poll() is None:p.kill()
p.wait();log.close();jr=root/'jobs'/str(i);cp=jr/'complete.json'
result=read(cp) if cp.exists() else dict(status='MISSING_GUARD_RECEIPT')
terminal[i]=dict(id=i,status=result['status'],returncode=p.returncode,gpu_seconds_charged=charge(i),pre_dispatch_wait_seconds=waits[i],
guard_sha256=sha(cp) if cp.exists() else None)
wp=jr/'worker/worker-identity.json'
try:
if wp.exists() and alive(read(wp)):raise RuntimeError('Live worker; lease retained')
release(lease['token'],owner)
except Exception as exc:
terminal[i]['lease_release_failure']=str(exc);failure=failure or str(exc)
final_charges={i:charge(i) for i in range(5)}
final_cost=c['preparation_charge_gpu_seconds']+sum(final_charges.values())
if final_cost+c['postfit_gpu_reserve']>c['gpu_cap']:failure=failure or 'FINAL_CUMULATIVE_CAP'
complete=not failure and len(terminal)==5 and all(r['status']=='PASS' and r['returncode']==0 for r in terminal.values())
files=[root/'contract.json']
if complete:
models=[]
for i in range(5):
r=read(root/'jobs'/str(i)/'worker/result.json')
models.append(dict(r));files.append(Path(r['checkpoint']))
files.extend([root/'jobs'/str(i)/'complete.json',root/'jobs'/str(i)/'worker/result.json',root/'jobs'/str(i)/'worker/standardization.json'])
save(root/'models.json',models);files.append(root/'models.json')
save(root/'complete.json',dict(status='TRAIN_COMPLETE_POSTFIT_PENDING' if complete else 'STOPPED_INCOMPLETE',
failure=failure,entries=list(terminal.values()),never_attempted=pending,
never_attempted_wait_charges={str(i):final_charges[i] for i in pending},
cumulative_gpu_seconds=final_cost,actual_waiting_gpu_seconds=sum(waiting_charges(final_charges).values()),
postfit_gpu_reserve=c['postfit_gpu_reserve'],
no_retry=True,test_targets_read=False,prediction_started=False,scoring_started=False))
files.append(root/'complete.json')
freeze_start=time.monotonic()
freeze(root,'training-terminal' if complete else 'training-stop',files)
final_cost+=time.monotonic()-freeze_start
within=final_cost+c['postfit_gpu_reserve']<=c['gpu_cap']
save(root/'post-terminal-cost.json',dict(cumulative_gpu_seconds=final_cost,within_cap=within,
freeze_tail_charged_once=True,postfit_gpu_reserve=c['postfit_gpu_reserve'],postfit_started=False,
training_complete_sha256=sha(root/'complete.json'),required_before_any_postfit=True))
return 0 if complete and within else 1
if __name__=='__main__':raise SystemExit(run(ROOT))
+79
View File
@@ -0,0 +1,79 @@
"""One new S2 fold, fixed original200epoch schedule with seed0; no test prediction or target reads."""
import json,os,sys,time,traceback,random
from pathlib import Path
import numpy as np
import torch
from phase2_h2_s1_common import selected_checkpoint
from phase2_h2_model import lr_epoch
from phase3_item_b_shared_common import config,fold_view,first_epoch_release,assignment as assigned_job,read
from phase3_shared_capacity import process_identity
from phase3_item_b_data import standardization,load_part
from phase2_h2_data import to_device,arguments,cached
from phase3_item_b_model import H2Context as H2Geo
from phase2_h2_model import optimizer
from phase2_h2_pilot_worker import peaks,train_step
from phase2_residual_math import residual_target
from phase2_h1_worker_v2 import save,sha
def run(root,fold_id,assignment_sha):
c,pilot=config(root);old=fold_view(pilot,fold_id);job=assigned_job(root,c,fold_id,assignment_sha);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
save(out/'worker-identity.json',process_identity(os.getpid()))
def record(item,epoch_file=False):
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
save(out/'progress.json',item)
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
return item
try:
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,c,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();random.seed(0);np.random.seed(0);torch.manual_seed(0);torch.cuda.manual_seed_all(0)
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
for epoch in range(1,201):
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
for index in order:
case=fold['learning'][int(index)];data=to_device(load_part(old,c,case,stats,'learning',out),'cuda')
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
del data
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
if epoch%10==0:
stage='VALIDATION_DIAGNOSTIC';model.eval();val_loss=0.;val_nodes=0;vstart=time.monotonic()
with torch.no_grad():
for case in fold['validation']:
data=to_device(load_part(old,c,case,stats,'validation',out),'cuda');p=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(p).all()):raise ValueError('Nonfinite validation output')
with (out/'validation-target-access.jsonl').open('a') as f:f.write(json.dumps(dict(case=case,epoch=epoch,role='validation_diagnostic_only',target_sha256=old['parts'][case]['output_sha256']['target.npy']))+'\n')
target=residual_target(cached(old,case,'target'),data['F'],fold['c'],fit_dtype='float32')
delta=p.cpu().numpy().astype(np.float64)-target;val_loss+=float(delta@delta);val_nodes+=data['nodes'];del data,p
torch.cuda.synchronize()
if val_nodes!=fold['validation_nodes']:raise ValueError('Validation coverage')
metrics.update(validation_mse=val_loss/val_nodes,validation_nodes=val_nodes,validation_seconds=time.monotonic()-vstart,validation_used_for_selection=False)
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
if epoch==1:
stage='WAIT_THIS_FOLD_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
while not first_epoch_release(root,fold_id):time.sleep(.5)
if epoch<200:
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
save(out/'result.json',dict(status='COMPLETE',fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
if __name__=='__main__':
import ctypes,signal
from phase3_shared_capacity import alive
expected=read(Path(sys.argv[1])/'jobs'/sys.argv[2]/'lease-ready.json')['owner']
parent=os.getppid()
if parent!=expected['pid'] or not alive(expected):raise RuntimeError('Pinned guard lost before worker import completed')
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:raise OSError('PDEATHSIG')
if os.getppid()!=parent:raise RuntimeError('Guard exited before worker setup')
run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3])
+159
View File
@@ -0,0 +1,159 @@
"""Freeze b's pre-launch budget/queue delta using metadata from the live c run."""
import ast
import fcntl
import math
import time
from pathlib import Path
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze, verify
from phase3_item_b_common import ROOT as ORIGINAL
from phase3_item_b_shared_budget import MARGIN, RULING, USER_CAP, full_fold_bound
from phase3_item_b_shared_common import ROOT, config, project, read
from track_a_h2_prepare import local_code_closure
BASE = Path(__file__).resolve().parents[1]
RECORDS = BASE / 'artifacts/phase3-item-b-shared-budget-2026-11-19'
C_ROOT = ORIGINAL.with_name('phase3-item-c-2026-11-18')
PARKED = ORIGINAL.with_name('phase3-item-1b-2026-11-18')
def measured_bounds(old_jobs):
c = read(C_ROOT / 'contract.json')
if sha(C_ROOT / 'contract.json') != read(C_ROOT / 'started.json')['contract_sha256']:
raise ValueError('c training identity changed')
for p, h in c['code_sha256'].items():
checked(p, h)
jobs = {i: C_ROOT / 'jobs' / str(i) for i in range(5)}
assignments = {i: read(job / 'assignment.json') for i, job in jobs.items()}
workers = {i: read(job / 'worker/worker-identity.json') for i, job in jobs.items()}
samples = {i: read(job / 'gpu-sample.json') for i, job in jobs.items()}
rows = []
for i, old in enumerate(old_jobs):
assignment = assignments[i]
if assignment['id'] != i or assignment['contract_sha256'] != sha(C_ROOT / 'contract.json'):
raise ValueError('c fold assignment identity')
sample = samples[i]
seen = {r['pid'] for r in sample['rows']}
worker = workers[i]
if worker['pid'] not in seen or worker['pid'] not in sample['own_pids']:
raise ValueError('c worker missing from GPU sample')
peers = [j for j in jobs if j != i and assignments[j]['gpu_uuid'] == assignment['gpu_uuid']
and workers[j]['pid'] in seen]
if not peers:
raise ValueError('A live same-card c peer must corroborate the overlap')
overlap_start = max(worker['created'], min(workers[j]['created'] for j in peers))
candidates = []
for p in sorted((jobs[i] / 'worker/history').glob('*.json')):
mtime = p.stat().st_mtime
if mtime > sample['unix']:
continue
row = read(p)
seconds = row['train_epoch_seconds']
if (row['fold'] != i or row['epoch_nodes'] != old['learning_nodes']
or row['epoch_parts'] != len(old['learning']) or not math.isfinite(seconds) or seconds <= 0):
raise ValueError('c and b fold workload or epoch identity differs')
if mtime - seconds > overlap_start + 2:
candidates.append(dict(epoch=row['epoch'], epoch_seconds=seconds, file=str(p),
sha256=sha(p), mtime=mtime))
if not candidates:
raise ValueError('No c epoch lies fully inside the same-card window')
maximum = max(candidates, key=lambda t: t['epoch_seconds'])
rows.append(dict(fold=i, gpu_uuid=assignment['gpu_uuid'], worker=worker,
shared_peers={str(j): workers[j] for j in peers}, gpu_sample=sample,
overlap_start=overlap_start, candidates=candidates, worst_epoch=maximum,
max_train_epoch_seconds=maximum['epoch_seconds'],
validation_seconds_bound=old['validation_seconds_bound']))
return rows
def main():
start = time.process_time()
if ROOT.exists() or (ORIGINAL / 'started.json').exists() or (ORIGINAL / 'launch-attempt.json').exists():
raise FileExistsError('Only a never-launched b may be rebound')
validate_review(ORIGINAL)
old = read(ORIGINAL / 'contract.json')
stop = read(PARKED / 'complete.json')
verify(PARKED, 'training-stop')
tail = read(PARKED / 'post-terminal-cost.json')
if (stop['status'] != 'STOPPED_INCOMPLETE' or stop['prediction_started'] or stop['scoring_started']
or any(e['status'] == 'PASS' for e in stop['entries'])
or tail['training_complete_sha256'] != sha(PARKED / 'complete.json')
or tail['postfit_started'] or not tail['freeze_tail_charged_once']):
raise ValueError('1b terminal identity changed')
rows = measured_bounds(old['jobs'])
jobs = []
for job, row in zip(old['jobs'], rows, strict=True):
epoch = row['max_train_epoch_seconds']
jobs.append(dict(job, shared_epoch_seconds_bound=epoch,
full_training_gpu_seconds_bound=full_fold_bound(epoch, job['validation_seconds_bound'])))
bounds = [j['full_training_gpu_seconds_bound'] for j in jobs]
wait_reserve = max(bounds)
cap = math.ceil(old['preparation_charge_gpu_seconds'] + old['postfit_gpu_reserve'] + sum(bounds) + wait_reserve)
reserves = dict(old['phase_reservations'], item1b=tail['cumulative_gpu_seconds'], b=cap)
if cap > USER_CAP or sum(reserves.values()) > old['phase3_gpu_cap']:
raise ValueError('Shared-mode cap does not fit the approved GPU limits')
original_worker = BASE / 'tools/phase3_item_b_worker.py'
new_worker = BASE / 'tools/phase3_item_b_shared_worker.py'
normalized = new_worker.read_text().replace('from phase3_item_b_shared_common import',
'from phase3_item_b_common import')
if ast.dump(ast.parse(normalized)) != ast.dump(ast.parse(original_worker.read_text())):
raise ValueError('b worker differs beyond the run identity import')
files = [Path(__file__), *sorted((BASE / 'tools').glob('phase3_item_b_shared_*.py')),
BASE / 'tools/test_phase3_item_b_shared_budget.py']
codes = {**old['code_sha256'], **local_code_closure(files)}
proof_sources = [ORIGINAL / x for x in ('contract.json', 'expected-review-binding.json',
'implementation-git-freeze.json', 'review.json')]
proof_sources += [PARKED / x for x in ('complete.json', 'post-terminal-cost.json', 'training-stop-git-freeze.json')]
proof_sources += [C_ROOT / 'contract.json', C_ROOT / 'started.json']
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
evidence = RECORDS / 'shared-cost-evidence.json'
if evidence.exists():
raise FileExistsError('A shared calibration receipt already exists')
save(evidence, dict(ruling=RULING, c_root=str(C_ROOT), c_contract_sha256=sha(C_ROOT / 'contract.json'),
folds=rows, margin=MARGIN, full_fold_training_bounds=bounds,
aggregate_wait_reserve_gpu_seconds=wait_reserve,
wait_reserve_basis='one longest full-fold bound across all waiting charges',
gpu_cap=cap, user_cap=USER_CAP, phase_reservations=reserves,
parked_1b_actual_including_tail=tail['cumulative_gpu_seconds'],
phase3_unreserved_gpu_seconds=old['phase3_gpu_cap']-sum(reserves.values()),
validation_bound_basis='original b diagnostic bounds; no shared diagnostic measurement',
worker_same_except_rebind_import=True))
sources = dict(old['source_sha256'])
sources.update({str(p): sha(p) for p in proof_sources + [evidence]})
contract = dict(old, version='phase3-item-b-shared-budget', shared_budget_ruling=RULING,
shared_budget_evidence=str(evidence), gpu_cap=cap, phase_reservations=reserves,
waiting_charge='Pre-dispatch waiting charged once to total; excluded from per-fold training',
projection='Measured c shared-mode per-fold times +25%; actual b maxima may raise projections',
jobs=jobs, code_sha256=codes, source_sha256=sources)
ROOT.mkdir()
save(ROOT / 'contract.json', contract)
initial = project(contract, dict.fromkeys(range(5), 60.), dict.fromkeys(range(5), 0.), {}, {})
if initial['status'] != 'PASS':
raise ValueError('Initial shared-mode projection cannot fit')
save(ROOT / 'preparation-audit.json', dict(status='WAIT_INDEPENDENT_DELTA_REVIEW',
base_implementation=verify(ORIGINAL, 'implementation')['commit'],
gpu_cap=cap, phase_reservations=reserves, initial_projection=initial,
code_count=len(codes), source_count=len(sources), metadata_cpu_seconds=time.process_time()-start,
comparison_unchanged=True, scientific_worker_unchanged=True,
started=False, test_targets_read=False, review_adopted=False))
freeze(ROOT, 'implementation', [ROOT / 'contract.json', ROOT / 'preparation-audit.json',
*(Path(p) for p in codes), evidence, RECORDS / 'acceptance.log',
BASE / 'docs/phase3-item-b-shared-budget-2026-11-19.md'])
frozen = verify(ROOT, 'implementation')
config(ROOT)
save(ROOT / 'expected-review-binding.json', dict(
reviewed_commit=frozen['commit'], contract_sha256=sha(ROOT / 'contract.json'),
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
comparison_contract_sha256=contract['comparison_contract_sha256'], code_sha256=codes))
print(dict(implementation=frozen['commit'], contract=sha(ROOT / 'contract.json'),
binding_sha256=sha(ROOT / 'expected-review-binding.json'), gpu_cap=cap,
initial_projected_gpu_seconds=initial['projected_gpu_seconds'],
phase3_reserved_gpu_seconds=sum(reserves.values()), code_count=len(codes)))
if __name__ == '__main__':
main()
+59
View File
@@ -0,0 +1,59 @@
"""CPU-only checks for the b wrapper's wait and measured-cost delta."""
from unittest import TestCase, main
from phase3_item_b_shared_budget import full_fold_bound, project
class SharedBudgetTests(TestCase):
def setUp(self):
self.contract = dict(
jobs=[dict(shared_epoch_seconds_bound=100, validation_seconds_bound=10,
full_training_gpu_seconds_bound=full_fold_bound(100, 10)) for _ in range(5)],
gpu_cap=170000, preparation_charge_gpu_seconds=600, postfit_gpu_reserve=7200,
runaway_gpu_seconds=60000,
)
def test_wait_changes_total_but_not_training_projection(self):
base = dict.fromkeys(range(5), 60)
waits = dict.fromkeys(range(5), 0)
measured = {0: [dict(epoch=1, train_epoch_seconds=103)]}
baseline = project(self.contract, base | {0: 185}, waits, measured, {})
result = project(self.contract, base | {0: 17185}, waits | {0: 17000}, measured, {})
self.assertEqual(baseline['folds']['0']['projected_training_gpu_seconds'],
result['folds']['0']['projected_training_gpu_seconds'])
self.assertEqual(result['projected_gpu_seconds']-baseline['projected_gpu_seconds'], 17000)
self.assertEqual(result['actual_waiting_gpu_seconds'], 17000)
def test_pending_wait_is_charged_once(self):
cost = dict.fromkeys(range(5), 3060)
wait = dict.fromkeys(range(5), 3000)
result = project(self.contract, cost, wait, {}, {})
bound = full_fold_bound(100, 10)
self.assertEqual(result['folds']['2']['projected_training_gpu_seconds'], bound)
self.assertEqual(result['projected_gpu_seconds'], 7800 + 5*(bound+3000))
def test_terminal_uses_actual_charge(self):
result = project(self.contract, dict.fromkeys(range(5), 60) | {4: 5000},
dict.fromkeys(range(5), 0) | {4: 2500}, {}, {4: {}})
self.assertEqual(result['folds']['4']['projected_training_gpu_seconds'], 2500)
self.assertEqual(result['folds']['4']['projected'], 5000)
def test_long_wait_does_not_hit_actual_training_runaway_limit(self):
costs = dict.fromkeys(range(5), 60)
waits = dict.fromkeys(range(5), 0)
contract = dict(self.contract, gpu_cap=1000000)
self.assertEqual(project(contract, costs | {0: 80000}, waits | {0: 79940}, {}, {})['status'], 'PASS')
self.assertEqual(project(contract, costs | {0: 80000}, waits, {}, {})['status'], 'STOP_BUDGET')
self.assertEqual(project(self.contract, costs | {0: 80000}, waits | {0: 79940}, {}, {})['status'], 'STOP_BUDGET')
def test_observed_b_epoch_can_raise_shared_bound(self):
costs = dict.fromkeys(range(5), 60) | {0: 220}
waits = dict.fromkeys(range(5), 0)
faster = project(self.contract, costs, waits, {0: [dict(epoch=1, train_epoch_seconds=90)]}, {})
slower = project(self.contract, costs, waits, {0: [dict(epoch=1, train_epoch_seconds=130)]}, {})
self.assertEqual(slower['projected_gpu_seconds']-faster['projected_gpu_seconds'], 1.25*199*30)
if __name__ == '__main__':
main()