Publish Track A shared-slot dispatch delta for narrow independent review
This commit is contained in:
1 parent
3438020be4
commit
3e62e08c76
15 files changed
+10123
No files matched your search
@@ -0,0 +1,142 @@
|
||||
{
|
||||
"ruling": "1795024014.778499",
|
||||
"measured_shared_reference": [
|
||||
{
|
||||
"fold": 0,
|
||||
"shared_epoch_seconds": 101.52302173001226,
|
||||
"learning_support": {
|
||||
"parts": 320,
|
||||
"nodes": 8742060,
|
||||
"edges": 115060278
|
||||
},
|
||||
"witness": {
|
||||
"epoch": 42,
|
||||
"epoch_seconds": 101.52302173001226,
|
||||
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/0/worker/history/042.json",
|
||||
"sha256": "188e83bce645c07326df46e56b7e912aba25c220d8db066b3342a8df5decb344",
|
||||
"mtime": 1795014328.5125103
|
||||
},
|
||||
"samples": 136
|
||||
},
|
||||
{
|
||||
"fold": 1,
|
||||
"shared_epoch_seconds": 113.8008639710024,
|
||||
"learning_support": {
|
||||
"parts": 374,
|
||||
"nodes": 9731293,
|
||||
"edges": 128102636
|
||||
},
|
||||
"witness": {
|
||||
"epoch": 37,
|
||||
"epoch_seconds": 113.8008639710024,
|
||||
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/1/worker/history/037.json",
|
||||
"sha256": "3aa3dc428558f95e73eb5f865f784fa50fda106fe0de2c75b34919945d0698b7",
|
||||
"mtime": 1795014343.7204428
|
||||
},
|
||||
"samples": 120
|
||||
},
|
||||
{
|
||||
"fold": 2,
|
||||
"shared_epoch_seconds": 117.12941370601766,
|
||||
"learning_support": {
|
||||
"parts": 366,
|
||||
"nodes": 9875407,
|
||||
"edges": 130004422
|
||||
},
|
||||
"witness": {
|
||||
"epoch": 34,
|
||||
"epoch_seconds": 117.12941370601766,
|
||||
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/2/worker/history/034.json",
|
||||
"sha256": "18b246591839acfaf38cdc88a108f9f46aebea6690626da185feb2d6c443f7b2",
|
||||
"mtime": 1795014267.8009477
|
||||
},
|
||||
"samples": 120
|
||||
},
|
||||
{
|
||||
"fold": 3,
|
||||
"shared_epoch_seconds": 95.61896364297718,
|
||||
"learning_support": {
|
||||
"parts": 291,
|
||||
"nodes": 8118914,
|
||||
"edges": 106967636
|
||||
},
|
||||
"witness": {
|
||||
"epoch": 42,
|
||||
"epoch_seconds": 95.61896364297718,
|
||||
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/3/worker/history/042.json",
|
||||
"sha256": "549f9350857aeb0fdd65dda4959e711084a3a89b54c34dd4c525287d9ae6dfb9",
|
||||
"mtime": 1795014267.823946
|
||||
},
|
||||
"samples": 148
|
||||
},
|
||||
{
|
||||
"fold": 4,
|
||||
"shared_epoch_seconds": 104.39079825102817,
|
||||
"learning_support": {
|
||||
"parts": 350,
|
||||
"nodes": 9749420,
|
||||
"edges": 128488900
|
||||
},
|
||||
"witness": {
|
||||
"epoch": 48,
|
||||
"epoch_seconds": 104.39079825102817,
|
||||
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/4/worker/history/048.json",
|
||||
"sha256": "2f394e76e2a8355ab7b495d647c1795ccc1d205c569476e9a0ddae1681e39a66",
|
||||
"mtime": 1795021270.2891376
|
||||
},
|
||||
"samples": 68
|
||||
}
|
||||
],
|
||||
"splits": [
|
||||
{
|
||||
"split": 0,
|
||||
"evaluation_support": {
|
||||
"parts": 304,
|
||||
"nodes": 8773925,
|
||||
"edges": 115624384
|
||||
},
|
||||
"proxy_epoch_seconds": 117.12941370601766,
|
||||
"cap_gpu_seconds": 702
|
||||
},
|
||||
{
|
||||
"split": 1,
|
||||
"evaluation_support": {
|
||||
"parts": 311,
|
||||
"nodes": 8993731,
|
||||
"edges": 118567416
|
||||
},
|
||||
"proxy_epoch_seconds": 117.12941370601766,
|
||||
"cap_gpu_seconds": 702
|
||||
},
|
||||
{
|
||||
"split": 2,
|
||||
"evaluation_support": {
|
||||
"parts": 308,
|
||||
"nodes": 8433445,
|
||||
"edges": 111118302
|
||||
},
|
||||
"proxy_epoch_seconds": 117.12941370601766,
|
||||
"cap_gpu_seconds": 702
|
||||
},
|
||||
{
|
||||
"split": 4,
|
||||
"evaluation_support": {
|
||||
"parts": 305,
|
||||
"nodes": 8899385,
|
||||
"edges": 117282304
|
||||
},
|
||||
"proxy_epoch_seconds": 117.12941370601766,
|
||||
"cap_gpu_seconds": 702
|
||||
}
|
||||
],
|
||||
"margin": 1.25,
|
||||
"method": "Conservative full c training-epoch proxy scaled by the maximum of part, node and edge ratios",
|
||||
"proxy_limit": "Prediction and compression not timed on c; original 300 CPU-s per-fold allowance added",
|
||||
"wait_reserve": 702,
|
||||
"postfit_gpu_cap": 3690,
|
||||
"Track_A_total_gpu_seconds_with_planned_postfit": 82269.79901044304,
|
||||
"one_slot": true,
|
||||
"case_data_decoded": false,
|
||||
"score_ast_identical": true,
|
||||
"live_other_pins_verified": 180
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
test_all_node_proxy_not_relative_l2 (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_bootstrap_order_invariant_and_deterministic (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_bootstrap_zero_difference_degenerate_interval (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_fixed_last_epoch_identity (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_missing_visit_not_silently_averaged (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_no_target_io_before_actual_review (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_no_target_io_before_predictions_frozen (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_postfit_reserve_and_training_charged_once (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_prediction_manifest_native_id_and_duplicate_rejection (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_same_paired_support_five_summary_means (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_test_loader_rejects_foreign_case_before_io (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_undefined_candidates_and_visit_counts_retained (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_undefined_pairs_excluded_symmetrically (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_unique_part_bootstrap_not_visit_weighted (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_zero_truth_is_explicitly_undefined (test_track_a_postfit.PostfitTests) ... ok
|
||||
test_exact_split_ids_and_visit_counts (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_four_split_summary_uses_original_part_bootstrap (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_incomplete_duplicate_or_extra_visits_rejected (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_mixed_source_checkpoint_identity (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_scoring_requires_new_review_before_target_access (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_scoring_requires_prediction_freeze_before_target_access (test_track_a_postfit_rebind.RebindTests) ... ok
|
||||
test_available_checks_only_gpu0_through_gpu2 (test_track_a_postfit_shared.SharedTrackATests) ... ok
|
||||
test_older_pending_has_priority_before_any_admission (test_track_a_postfit_shared.SharedTrackATests) ... ok
|
||||
test_queue_wait_is_charged_only_to_overall_cap (test_track_a_postfit_shared.SharedTrackATests) ... ok
|
||||
test_score_function_ast_unchanged (test_track_a_postfit_shared.SharedTrackATests) ... ok
|
||||
test_score_refuses_missing_dispatch_pass_before_targets (test_track_a_postfit_shared.SharedTrackATests) ... ok
|
||||
test_score_refuses_missing_prediction_freeze_before_targets (test_track_a_postfit_shared.SharedTrackATests) ... ok
|
||||
test_uses_one_available_cpu_and_card (test_track_a_postfit_shared.SharedTrackATests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 28 tests in 0.044s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,9 @@
|
||||
# Track A shared-slot dispatch delta
|
||||
|
||||
Claude `1795024014.778499` replaces the whole-GPU2 idle condition with one shared slot on GPU0–2. This entry remains a serial four-split prediction and CPU scoring job. It uses the existing c/b registry, upstream admission and original guard transfer and release operations. Earlier pending folds keep dispatch priority, including the already running b queue. GPU3 stays outside this entry. Reservations hold 4 GiB each of VRAM and host RAM, enforced by the b guard's own-process/recorded-peak and process-tree monitoring. A lease transfers to one live guard and is released only after its own guard and worker exit. No launch occurs as part of preparing the binding.
|
||||
|
||||
The scientific review is MeltAssist's `a5fa4a6e` frozen at `958abd8c`. The original metric, A predictions, H2 checkpoints and normalization, four selected test lists, split 3 exclusion and paired bootstrap still come from that binding. The scoring function is AST-identical. The prediction worker adds lease and GPU assignment identity before using the same H2 inference and write path. Both the original scientific review and a new independent PASS tied to the exact shared dispatch binding are required. The complete frozen prediction manifest remains mandatory before any test target access.
|
||||
|
||||
The timing proxy uses c's recorded same-card epochs, scaled by the largest relative part, node or edge workload across the four selected Track A test lists and the c learning workloads. Each split uses the largest such proxy, an additional 300 seconds for the previous per-fold CPU allowance, a 25% margin on those two terms, and the fixed 60-second setup and 120-second cleanup allowances. The postfit budget reserves one largest split envelope for aggregate queue waiting, plus coordinator setup/cleanup. This is a conservative planning proxy, not a shared measurement of V4 predictions or compression. Real waiting is charged to the postfit total, outside each guard's prediction cap. The former 7,200 GPU-second and separate 162,000 GPU-second limits are upper bounds, and the 3,600 CPU-second limit remains in force. Original training cost 78,579.79901044304 GPU seconds is counted once.
|
||||
|
||||
The entry root is `/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19`. The former unused four-split GPU2 binding remains historical evidence. Live c, seed 1 and b use their own frozen roots.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"reviewed_commit": "d71fe9c54566693a4292f0187bd23b0e468566aa",
|
||||
"contract_sha256": "595d44ce4f90075f042692bf098905c687c2f2a9606c9f625a4eea3de15cab52",
|
||||
"implementation_proof_sha256": "f433fe2b8366ed8ae6eca9b4eec76cd3ccb83e4cd4a5072e3f0edb52928078ef",
|
||||
"comparison_contract_sha256": "8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_rebind.py": "e5fcb0c1ca594aa74e6c3cb70dada60f79baa0d8bcaa0622b19da191ad596536",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_rebind.py": "aefdab5d2bcb0e4cf63b7ebe4e736307726f2bd8f98dc2985cc85b0d814315b3",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "8b7d425ab702eb033d4027d24d2812042b66411943b004d6be08a13d4ffcd1c9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind.py": "54389c854f8ee43bdf66834675c22b5e705446cab363ebf55d340d8e3c79381b",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_common.py": "b0f283577da25142201c6d2c7fbfdc486e0f21a75a554a6dd8c2a1da58d29778",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_guard.py": "0aaacc9518041235eecb7f078cebba6d0bc62dadfbd4951f0066bd8271db14a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_statistics.py": "07b1d26dbdb898d3ffe26bec497204a59226749d299d35ad55b074a4e0ad03e9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_worker.py": "e34dbcd50bd91b9f84367b61f7568e0cdac471e9e1262a082bb9cfba80b79751",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "b7d346d1762d7737eb1ebd723989a71f8c65b597d4cdb8c5d512b48a0fd984ce",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a932d069340909465a5a1e66be1504e28c8df159a5ce304b0586295003d2e734",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_shared.py": "9aa00647d1ff79f75be665383557c746d5b424621a32b9c5b4f148c5c2a441e8",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_shared.py": "29fb4b535f41f562cac6e1b9f97c7ebf1248e6ec49f25c0792a8b516f8ed8955",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared.py": "96e226914f5ad2e1a0cc500657420d788f8aead04dad8c00c91fbc9ba967a86c",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_capacity.py": "b1b6c108586cb66dd9facd0b2fcdb864082da37cfc9920b517718994e9a58e67",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_common.py": "da6a6c0d2c4b3849f823159d0a6ea97b8f79778146102385f7ec3632c609bc5c",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_guard.py": "6a250dd2879d45c9379c03253b8a5920c60a1355768ea199437702633bd8fed6",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_worker.py": "ffa9c730d1662ff18d667ccf7c0dd20f5797d5e23eecdceb06ba8ebdf4505b53",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d"
|
||||
}
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"contract_sha256": "595d44ce4f90075f042692bf098905c687c2f2a9606c9f625a4eea3de15cab52",
|
||||
"proof_sha256": "f433fe2b8366ed8ae6eca9b4eec76cd3ccb83e4cd4a5072e3f0edb52928078ef"
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
{
|
||||
"commit": "d71fe9c54566693a4292f0187bd23b0e468566aa",
|
||||
"files": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19/contract.json": "595d44ce4f90075f042692bf098905c687c2f2a9606c9f625a4eea3de15cab52",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19/preparation-receipt.json": "abef135b3c11837f5859668ca0629cd1506c9798c90c76f0116a3a5fe3269804",
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_rebind.py": "e5fcb0c1ca594aa74e6c3cb70dada60f79baa0d8bcaa0622b19da191ad596536",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_rebind.py": "aefdab5d2bcb0e4cf63b7ebe4e736307726f2bd8f98dc2985cc85b0d814315b3",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "8b7d425ab702eb033d4027d24d2812042b66411943b004d6be08a13d4ffcd1c9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind.py": "54389c854f8ee43bdf66834675c22b5e705446cab363ebf55d340d8e3c79381b",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_common.py": "b0f283577da25142201c6d2c7fbfdc486e0f21a75a554a6dd8c2a1da58d29778",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_guard.py": "0aaacc9518041235eecb7f078cebba6d0bc62dadfbd4951f0066bd8271db14a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_statistics.py": "07b1d26dbdb898d3ffe26bec497204a59226749d299d35ad55b074a4e0ad03e9",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_worker.py": "e34dbcd50bd91b9f84367b61f7568e0cdac471e9e1262a082bb9cfba80b79751",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "b7d346d1762d7737eb1ebd723989a71f8c65b597d4cdb8c5d512b48a0fd984ce",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a932d069340909465a5a1e66be1504e28c8df159a5ce304b0586295003d2e734",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_shared.py": "9aa00647d1ff79f75be665383557c746d5b424621a32b9c5b4f148c5c2a441e8",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_shared.py": "29fb4b535f41f562cac6e1b9f97c7ebf1248e6ec49f25c0792a8b516f8ed8955",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared.py": "96e226914f5ad2e1a0cc500657420d788f8aead04dad8c00c91fbc9ba967a86c",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_capacity.py": "b1b6c108586cb66dd9facd0b2fcdb864082da37cfc9920b517718994e9a58e67",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_common.py": "da6a6c0d2c4b3849f823159d0a6ea97b8f79778146102385f7ec3632c609bc5c",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_guard.py": "6a250dd2879d45c9379c03253b8a5920c60a1355768ea199437702633bd8fed6",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_worker.py": "ffa9c730d1662ff18d667ccf7c0dd20f5797d5e23eecdceb06ba8ebdf4505b53",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-shared-2026-11-19/shared-cost-evidence.json": "4b74d29cd9a82d41db2eadc3c0a24ee5cfc762c4163c3ff522a8b72a96b819fd",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-shared-2026-11-19/synthetic-tests.log": "17d87b07efa8c3c3dab4b84de5ce9fbedd85bc8ab4f774dae8004291539644e2",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-shared-postfit-2026-11-19.md": "7c37a24a9599433ea60c0793af001cba1f0ba246e1c0c0e26e30ece0f19d5c79"
|
||||
},
|
||||
"git_paths": {
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19/contract.json": "home-runs/track-a-postfit-four-split-shared-2026-11-19/contract.json",
|
||||
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19/preparation-receipt.json": "home-runs/track-a-postfit-four-split-shared-2026-11-19/preparation-receipt.json",
|
||||
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "tools/boundary_distance.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "tools/gre24_features_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "tools/phase3_item1_postfit_data.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_rebind.py": "tools/prepare_track_a_postfit_rebind.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "tools/r4bc_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "tools/r4bc_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "tools/s2_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_rebind.py": "tools/test_track_a_postfit_rebind.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "tools/track_a_postfit_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind.py": "tools/track_a_postfit_rebind.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_common.py": "tools/track_a_postfit_rebind_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_guard.py": "tools/track_a_postfit_rebind_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_statistics.py": "tools/track_a_postfit_rebind_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_worker.py": "tools/track_a_postfit_rebind_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "tools/track_a_postfit_statistics.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "tools/track_a_recovery_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "tools/phase3_shared_capacity.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_shared.py": "tools/prepare_track_a_postfit_shared.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_shared.py": "tools/test_track_a_postfit_shared.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared.py": "tools/track_a_postfit_shared.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_capacity.py": "tools/track_a_postfit_shared_capacity.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_common.py": "tools/track_a_postfit_shared_common.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_guard.py": "tools/track_a_postfit_shared_guard.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_worker.py": "tools/track_a_postfit_shared_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "tools/track_a_recovery_monitor.py",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "tools/track_a_recovery_supervisor.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-shared-2026-11-19/shared-cost-evidence.json": "artifacts/track-a-postfit-shared-2026-11-19/shared-cost-evidence.json",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-shared-2026-11-19/synthetic-tests.log": "artifacts/track-a-postfit-shared-2026-11-19/synthetic-tests.log",
|
||||
"/home/frank14f/optane/MeltMind/docs/track-a-shared-postfit-2026-11-19.md": "docs/track-a-shared-postfit-2026-11-19.md"
|
||||
},
|
||||
"remote_published": false,
|
||||
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"status": "PENDING_EXACT_DISPATCH_DELTA_REVIEW",
|
||||
"science_review_sha256": "a5fa4a6e040493ff187b084e07cba0d95e3cae8f0fc0fa0d571f4af9efd30fce",
|
||||
"science_review_freeze": "958abd8c916614c5366b5f0216958d46e00b8504",
|
||||
"shared_gpu_cap": 3690,
|
||||
"split_caps": [
|
||||
702,
|
||||
702,
|
||||
702,
|
||||
702
|
||||
],
|
||||
"prior_gpu_seconds": 78579.79901044304,
|
||||
"code_pins": 43,
|
||||
"source_pins": 49,
|
||||
"metadata_cpu_seconds": 3.6802666669999997,
|
||||
"launched": false,
|
||||
"test_targets_read": false
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Freeze the reviewed-scope Track A shared-slot delta using only metadata."""
|
||||
|
||||
import ast
|
||||
import fcntl
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze, verify
|
||||
from phase3_shared_capacity import GIB, GPUS
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
from track_a_postfit_rebind_common import config as reviewed_config, selected_training, terminal
|
||||
from track_a_postfit_shared_common import (ROOT, PREVIOUS, RULING, SCIENCE_REVIEW, SCIENCE_FREEZE,
|
||||
GUARD_CPU, MARGIN, config, fold_cap, postfit_cap, read)
|
||||
|
||||
|
||||
BASE = Path(__file__).resolve().parents[1]
|
||||
AUDIT = BASE/'artifacts/track-a-postfit-shared-2026-11-19'
|
||||
REFERENCE = BASE/'artifacts/phase3-item-b-shared-budget-2026-11-19/shared-cost-evidence.json'
|
||||
C_ROOT = ROOT.with_name('phase3-item-c-2026-11-18')
|
||||
B_ROOT = ROOT.with_name('phase3-item-b-shared-2026-11-19')
|
||||
|
||||
|
||||
def counts(contract, cases):
|
||||
return dict(parts=len(cases), nodes=sum(contract['parts'][case]['nodes'] for case in cases),
|
||||
edges=sum(contract['geometry'][case]['directed_edges'] for case in cases))
|
||||
|
||||
|
||||
def workload_projections(tc):
|
||||
reference = read(REFERENCE)
|
||||
c = read(C_ROOT/'contract.json')
|
||||
if sha(C_ROOT/'contract.json') != reference['c_contract_sha256']:
|
||||
raise ValueError('Original c timing configuration changed')
|
||||
pilot = read(checked(c['pilot_contract'], c['source_sha256'][c['pilot_contract']]))
|
||||
measured = []
|
||||
for row in reference['folds']:
|
||||
index = row['fold']
|
||||
witness = row['worst_epoch']
|
||||
original = read(checked(witness['file'], witness['sha256']))
|
||||
if original['train_epoch_seconds'] != row['max_train_epoch_seconds'] or original['fold'] != index:
|
||||
raise ValueError('Same-card epoch witness changed')
|
||||
support = counts(pilot, c['jobs'][index]['learning'])
|
||||
measured.append(dict(fold=index, shared_epoch_seconds=row['max_train_epoch_seconds'],
|
||||
learning_support=support, witness=witness, samples=len(row['candidates'])))
|
||||
folds = []
|
||||
for job in selected_training(tc)['jobs']:
|
||||
n = counts(tc, job['test'])
|
||||
proxy = max(x['shared_epoch_seconds'] * max(1., *(n[k]/x['learning_support'][k] for k in n))
|
||||
for x in measured)
|
||||
folds.append(dict(split=job['id'], evaluation_support=n, proxy_epoch_seconds=proxy,
|
||||
cap_gpu_seconds=fold_cap(proxy)))
|
||||
return measured, folds
|
||||
|
||||
|
||||
def main():
|
||||
start = time.process_time()
|
||||
if ROOT.exists():
|
||||
raise FileExistsError('Already prepared shared Track A entry')
|
||||
prior, tc, _ = reviewed_config(PREVIOUS)
|
||||
validate_review(PREVIOUS)
|
||||
if (sha(PREVIOUS/'review.json') != SCIENCE_REVIEW or
|
||||
verify(PREVIOUS, 'implementation-review')['commit'] != SCIENCE_FREEZE or
|
||||
terminal(tc) != prior['training_evidence']):
|
||||
raise PermissionError('Scientific pre-score review lineage changed')
|
||||
measured, folds = workload_projections(tc)
|
||||
jobs = [dict(id=x['split'], gpu_seconds=x['cap_gpu_seconds'], cpu_seconds=GUARD_CPU,
|
||||
rss_bytes=4*GIB, recorded_vram_bytes=4*GIB, process_vram_bytes=4*GIB,
|
||||
disk_bytes=20*GIB) for x in folds]
|
||||
cap = postfit_cap(jobs)
|
||||
training = prior['training_evidence']['training_gpu_seconds_once']
|
||||
if cap > prior['postfit_gpu_cap'] or training + cap > prior['gpu_cap']:
|
||||
raise ValueError('Shared planning exceeds the reviewed Track A allowance')
|
||||
old_worker = ast.parse((BASE/'tools/track_a_postfit_rebind_worker.py').read_text())
|
||||
new_worker = ast.parse((BASE/'tools/track_a_postfit_shared_worker.py').read_text())
|
||||
old_score = next(x for x in old_worker.body if isinstance(x, ast.FunctionDef) and x.name=='score')
|
||||
new_score = next(x for x in new_worker.body if isinstance(x, ast.FunctionDef) and x.name=='score')
|
||||
if ast.dump(old_score) != ast.dump(new_score):
|
||||
raise ValueError('Scoring logic differs')
|
||||
code = {**prior['code_sha256'], **local_code_closure([Path(__file__),
|
||||
*sorted((BASE/'tools').glob('track_a_postfit_shared*.py')),
|
||||
BASE/'tools/test_track_a_postfit_shared.py'])}
|
||||
live_pins = 0
|
||||
for root in (C_ROOT, B_ROOT, ROOT.with_name('phase3-seed1-v2-2026-11-18')):
|
||||
for p,h in read(root/'contract.json')['code_sha256'].items():
|
||||
checked(p,h)
|
||||
live_pins += 1
|
||||
context = [PREVIOUS/n for n in ('contract.json', 'review.json', 'expected-review-binding.json',
|
||||
'implementation-review-git-freeze.json', 'implementation-review-authorization.json')]
|
||||
c_contract = read(C_ROOT/'contract.json')
|
||||
context.extend([REFERENCE, C_ROOT/'contract.json', Path(c_contract['pilot_contract'])])
|
||||
context.extend(Path(m['witness']['file']) for m in measured)
|
||||
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
evidence = AUDIT/'shared-cost-evidence.json'
|
||||
if evidence.exists():
|
||||
raise FileExistsError('Prior shared Track A cost evidence exists')
|
||||
save(evidence, dict(ruling=RULING, measured_shared_reference=measured, splits=folds, margin=MARGIN,
|
||||
method='Conservative full c training-epoch proxy scaled by the maximum of part, node and edge ratios',
|
||||
proxy_limit='Prediction and compression not timed on c; original 300 CPU-s per-fold allowance added',
|
||||
wait_reserve=max(x['gpu_seconds'] for x in jobs), postfit_gpu_cap=cap,
|
||||
Track_A_total_gpu_seconds_with_planned_postfit=training+cap,
|
||||
one_slot=True, case_data_decoded=False, score_ast_identical=True, live_other_pins_verified=live_pins))
|
||||
sources = {**prior['source_sha256'], **{str(p):sha(p) for p in context+[evidence]}}
|
||||
contract = dict(prior, version='track-a-four-split-shared-postfit', shared_ruling=RULING,
|
||||
shared_budget_evidence=str(evidence), shared_cards=GPUS[:3], predictions=jobs,
|
||||
postfit_gpu_cap=cap, code_sha256=code, source_sha256=sources)
|
||||
for key in ('gpu_uuid', 'worker_cpus', 'idle_wait_seconds'):
|
||||
del contract[key]
|
||||
ROOT.mkdir()
|
||||
save(ROOT/'contract.json',contract)
|
||||
save(ROOT/'preparation-receipt.json',dict(status='PENDING_EXACT_DISPATCH_DELTA_REVIEW',
|
||||
science_review_sha256=SCIENCE_REVIEW, science_review_freeze=SCIENCE_FREEZE,
|
||||
shared_gpu_cap=cap, split_caps=[x['gpu_seconds'] for x in jobs],
|
||||
prior_gpu_seconds=training, code_pins=len(code), source_pins=len(sources),
|
||||
metadata_cpu_seconds=time.process_time()-start, launched=False, test_targets_read=False))
|
||||
freeze(ROOT,'implementation',[ROOT/'contract.json',ROOT/'preparation-receipt.json',
|
||||
*(Path(p) for p in code),evidence,AUDIT/'synthetic-tests.log',
|
||||
BASE/'docs/track-a-shared-postfit-2026-11-19.md'])
|
||||
final=verify(ROOT,'implementation')
|
||||
config(ROOT)
|
||||
save(ROOT/'expected-review-binding.json',dict(reviewed_commit=final['commit'],
|
||||
contract_sha256=sha(ROOT/'contract.json'),implementation_proof_sha256=sha(ROOT/'implementation-git-freeze.json'),
|
||||
comparison_contract_sha256=contract['comparison_contract_sha256'],code_sha256=code))
|
||||
print(dict(implementation=final['commit'],contract=sha(ROOT/'contract.json'),
|
||||
binding=sha(ROOT/'expected-review-binding.json'),gpu_cap=cap,code_pins=len(code)))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Synthetic capacity, cost and custody checks for the Track A dispatch delta."""
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
from unittest import TestCase, main
|
||||
from unittest.mock import patch
|
||||
|
||||
from phase3_shared_capacity import GPUS
|
||||
from track_a_postfit_shared_capacity import available
|
||||
from track_a_postfit_shared_common import budget, fold_cap, postfit_cap
|
||||
|
||||
|
||||
class SharedTrackATests(TestCase):
|
||||
def test_older_pending_has_priority_before_any_admission(self):
|
||||
with patch('track_a_postfit_shared_capacity.older_shared_pending', return_value=['b']), \
|
||||
patch('track_a_postfit_shared_capacity.host_snapshot') as host:
|
||||
selected, result = available({'leases': {}}, {16})
|
||||
self.assertIsNone(selected)
|
||||
self.assertEqual(result['reason'], 'OLDER_SHARED_FOLDS_PENDING')
|
||||
host.assert_not_called()
|
||||
|
||||
def test_available_checks_only_gpu0_through_gpu2(self):
|
||||
with patch('track_a_postfit_shared_capacity.older_shared_pending', return_value=[]), \
|
||||
patch('track_a_postfit_shared_capacity.host_snapshot', return_value={'old_pending': False}), \
|
||||
patch('track_a_postfit_shared_capacity.admission', return_value=(False, 'GPU_SLOT_LIMIT')) as admit:
|
||||
self.assertIsNone(available({'leases': {}}, {19})[0])
|
||||
self.assertEqual([call.args[2] for call in admit.call_args_list], GPUS[:3])
|
||||
|
||||
def test_uses_one_available_cpu_and_card(self):
|
||||
with patch('track_a_postfit_shared_capacity.older_shared_pending', return_value=[]), \
|
||||
patch('track_a_postfit_shared_capacity.host_snapshot', return_value={'old_pending': False}), \
|
||||
patch('track_a_postfit_shared_capacity.admission', side_effect=[(False, 'GPU_SLOT_LIMIT'), (True, 'ADMITTED')]):
|
||||
selected, _ = available({'leases': {}}, {12,21,22})
|
||||
self.assertEqual(selected, (GPUS[1], 21))
|
||||
|
||||
def test_queue_wait_is_charged_only_to_overall_cap(self):
|
||||
jobs = [{'gpu_seconds': fold_cap(120)} for _ in range(4)]
|
||||
c = dict(postfit_gpu_cap=postfit_cap(jobs), postfit_cpu_cap=3600, gpu_cap=162000,
|
||||
training_evidence={'training_gpu_seconds_once': 78579.79901044304})
|
||||
self.assertEqual(jobs[0]['gpu_seconds'], 705)
|
||||
budget(c, 765, 180, jobs)
|
||||
with self.assertRaises(RuntimeError):
|
||||
budget(c, c['postfit_gpu_cap'], 180, jobs)
|
||||
with self.assertRaises(RuntimeError):
|
||||
budget(dict(c, gpu_cap=80000), 0, 180, jobs)
|
||||
|
||||
def test_score_function_ast_unchanged(self):
|
||||
here = Path(__file__).parent
|
||||
def score_source(p):
|
||||
return next(n for n in ast.parse(p.read_text()).body if isinstance(n, ast.FunctionDef) and n.name=='score')
|
||||
old, new = (here/n for n in ('track_a_postfit_rebind_worker.py','track_a_postfit_shared_worker.py'))
|
||||
self.assertEqual(ast.dump(score_source(old)), ast.dump(score_source(new)))
|
||||
|
||||
def test_score_refuses_missing_dispatch_pass_before_targets(self):
|
||||
from track_a_postfit_shared_worker import score
|
||||
with patch('track_a_postfit_shared_worker.config', return_value=({}, {}, {})), \
|
||||
patch('track_a_postfit_shared_worker.validate_review', side_effect=PermissionError), \
|
||||
patch('phase2_h2_data.cached') as targets:
|
||||
with self.assertRaises(PermissionError):
|
||||
score(Path('/synthetic'))
|
||||
targets.assert_not_called()
|
||||
|
||||
def test_score_refuses_missing_prediction_freeze_before_targets(self):
|
||||
from track_a_postfit_shared_worker import score
|
||||
with patch('track_a_postfit_shared_worker.config', return_value=({}, {}, {})), \
|
||||
patch('track_a_postfit_shared_worker.validate_review'), \
|
||||
patch('track_a_postfit_shared_worker.adopted'), \
|
||||
patch('track_a_postfit_shared_worker.verify', side_effect=PermissionError), \
|
||||
patch('phase2_h2_data.cached') as targets:
|
||||
with self.assertRaises(PermissionError):
|
||||
score(Path('/synthetic'))
|
||||
targets.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Serial, shared-slot prediction followed by the original paired scoring barrier."""
|
||||
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import save, sha
|
||||
from phase2_home_freeze_io import freeze, verify
|
||||
from phase3_shared_capacity import alive, process_identity, release, transfer
|
||||
from r4bc_guard import supervise
|
||||
from track_a_postfit_common import prediction_identity
|
||||
from track_a_postfit_rebind_common import RECOVERY, SPLITS, selected_training, terminal
|
||||
from track_a_postfit_shared_capacity import acquire, register
|
||||
from track_a_postfit_shared_common import ROOT, SETUP, budget, config, read
|
||||
|
||||
|
||||
def stop_guard(guard):
|
||||
if guard.poll() is not None:
|
||||
return
|
||||
guard.terminate()
|
||||
try:
|
||||
guard.wait(timeout=45)
|
||||
except subprocess.TimeoutExpired:
|
||||
guard.kill()
|
||||
guard.wait()
|
||||
|
||||
|
||||
def predict_fold(root, job, reservation, queue_identity, wait_seconds, resource_check):
|
||||
folder = root/'jobs'/str(job['id'])
|
||||
token = reservation['token']
|
||||
guard = guard_id = None
|
||||
transferred = False
|
||||
try:
|
||||
folder.mkdir(exist_ok=False)
|
||||
path = folder/'assignment.json'
|
||||
save(path, dict(id=job['id'], contract_sha256=sha(root/'contract.json'), token=token,
|
||||
gpu_uuid=reservation['gpu_uuid'], worker_cpus=reservation['worker_cpus'],
|
||||
queue_identity=queue_identity, pre_dispatch_wait_seconds=wait_seconds))
|
||||
with (folder/'guard.log').open('x') as log:
|
||||
guard = subprocess.Popen([sys.executable, 'tools/track_a_postfit_shared_guard.py',
|
||||
str(root), str(job['id']), sha(path)],
|
||||
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
guard_id = process_identity(guard.pid)
|
||||
transfer(token, guard_id)
|
||||
transferred = True
|
||||
save(folder/'lease-ready.json', dict(token=token, owner=guard_id))
|
||||
while guard.poll() is None:
|
||||
resource_check()
|
||||
time.sleep(0.5)
|
||||
receipt = read(folder/'complete.json')
|
||||
if (guard.returncode != 0 or receipt['status'] != 'PASS'
|
||||
or receipt['gpu_seconds_charged'] > job['gpu_seconds'] or receipt['monitor_policy']['failed']):
|
||||
raise RuntimeError(f'Shared prediction guard rejected fold {job["id"]}')
|
||||
return receipt
|
||||
finally:
|
||||
if guard is not None:
|
||||
stop_guard(guard)
|
||||
worker = folder/'prediction-records/worker-identity.json'
|
||||
if worker.exists() and alive(read(worker)):
|
||||
raise RuntimeError('Prediction worker outlived guard; retained own lease')
|
||||
release(token, guard_id if transferred else None)
|
||||
|
||||
|
||||
def run(root):
|
||||
gpu_clock = time.monotonic()
|
||||
started_cpu = time.process_time()
|
||||
initial_children = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
c, tc, _ = config(root)
|
||||
validate_review(root)
|
||||
adopted = terminal(tc)
|
||||
if adopted != c['training_evidence']:
|
||||
raise ValueError('Mixed-source training evidence changed')
|
||||
prior = adopted['training_gpu_seconds_once']
|
||||
charged_cpu = 0.
|
||||
frozen_gpu = None
|
||||
attempted = 0
|
||||
records = []
|
||||
stop_requests = []
|
||||
original_affinity = os.sched_getaffinity(0)
|
||||
|
||||
def gpu():
|
||||
return (SETUP + attempted*SETUP + time.monotonic()-gpu_clock
|
||||
if frozen_gpu is None else frozen_gpu)
|
||||
|
||||
def cpu():
|
||||
children = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
child_time = children.ru_utime + children.ru_stime - initial_children.ru_utime - initial_children.ru_stime
|
||||
return 180 + time.process_time() - started_cpu + max(charged_cpu, child_time)
|
||||
|
||||
def resource_check(pending):
|
||||
if stop_requests:
|
||||
raise RuntimeError(stop_requests[0])
|
||||
budget(c, gpu(), cpu(), pending)
|
||||
|
||||
with (root/'run.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():
|
||||
raise FileExistsError('One Track A shared postfit, zero retries')
|
||||
identity = process_identity(os.getpid())
|
||||
digest = sha(root/'contract.json')
|
||||
register(root, digest, c)
|
||||
os.sched_setaffinity(0, c['coordinator_cpus'])
|
||||
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
signal.signal(sig, lambda n, f: stop_requests.append(f'COORDINATOR_SIGNAL_{n}'))
|
||||
cpu_counters = dict(self_process_time=started_cpu, children_user=initial_children.ru_utime,
|
||||
children_system=initial_children.ru_stime)
|
||||
save(root/'started.json', dict(identity=identity, unix=time.time(), contract_sha256=digest,
|
||||
review_sha256=sha(root/'review.json'), training_gpu_seconds_once=prior, shared_slot_limit=1,
|
||||
postfit_gpu_cap=c['postfit_gpu_cap'], track_a_gpu_cap=c['gpu_cap'],
|
||||
cpu_start_counters=cpu_counters, cpu_setup_allowance=180, new_fits=0, new_decodes=0))
|
||||
try:
|
||||
save(root/'models-adopted.json', adopted)
|
||||
train = Path(c['training_root'])
|
||||
freeze(root, 'models', [root/'contract.json', root/'models-adopted.json', train/'complete.json',
|
||||
RECOVERY/'complete.json', RECOVERY/'available-models-incomplete.json',
|
||||
RECOVERY/'incomplete-models-git-freeze.json', RECOVERY/'terminal-git-freeze.json',
|
||||
*[Path(m['path']) for m in adopted['models']],
|
||||
*[Path(m['standardization']) for m in adopted['models']]])
|
||||
entries = []
|
||||
(root/'jobs').mkdir(exist_ok=False)
|
||||
jobs = c['predictions']
|
||||
for pos, job in enumerate(jobs):
|
||||
wait_started = time.monotonic()
|
||||
while True:
|
||||
resource_check(jobs[pos:])
|
||||
lease, observation = acquire(root, job['id'], digest, original_affinity)
|
||||
if lease is not None:
|
||||
break
|
||||
save(root/'latest-admission.json', observation)
|
||||
time.sleep(1)
|
||||
attempted += 1
|
||||
receipt = predict_fold(root, job, lease, identity, time.monotonic()-wait_started,
|
||||
lambda: resource_check(jobs[pos+1:]))
|
||||
charged_cpu += receipt['worker']['cpu_seconds']
|
||||
records.append(dict(mode='predict', split=job['id'], gpu_uuid=lease['gpu_uuid'], guard=receipt))
|
||||
save(root/'resource-progress.json', dict(records=records, prediction_gpu_seconds=gpu(),
|
||||
postfit_cpu_seconds=cpu(), cumulative_track_a_gpu_seconds=prior+gpu()))
|
||||
entries.extend(read(root/'jobs'/str(job['id'])/'prediction-records/manifest.json')['entries'])
|
||||
prediction_identity(root, c, selected_training(tc), entries)
|
||||
save(root/'predictions-manifest.json', dict(entries=entries, A_predictions_reused=c['expected_pair_visits'],
|
||||
evaluated_splits=list(SPLITS), excluded_split=c['excluded_split'], new_fits=0, targets_read=False))
|
||||
freeze(root, 'predictions', [root/'contract.json', root/'models-adopted.json',
|
||||
root/'predictions-manifest.json', root/'resource-progress.json', *map(Path,c['code_sha256'])])
|
||||
resource_check([])
|
||||
frozen_gpu = gpu()
|
||||
validate_review(root)
|
||||
verify(root, 'models')
|
||||
verify(root, 'predictions')
|
||||
score = supervise([sys.executable, 'tools/track_a_postfit_shared_worker.py', 'score', str(root)],
|
||||
root/'score-guard', dict(cpu_seconds=900, wall_seconds=1200,
|
||||
rss_bytes=16*2**30, disk_bytes=20*2**30))
|
||||
charged_cpu += score['cpu_seconds']
|
||||
records.append(dict(mode='score', guard=score))
|
||||
if score['status'] != 'PASS' or cpu()+120 > c['postfit_cpu_cap']:
|
||||
raise RuntimeError('Original Track A scoring cap or guard')
|
||||
save(root/'complete.json', dict(status='COMPLETE_FOUR_OF_FIVE', evaluated_splits=list(SPLITS),
|
||||
excluded_split=c['excluded_split'], records=records, prediction_gpu_seconds=gpu(),
|
||||
postfit_cpu_seconds=cpu(), training_gpu_seconds_once=prior,
|
||||
cumulative_track_a_gpu_seconds=prior+gpu(), track_a_gpu_cap=c['gpu_cap'],
|
||||
new_fits=0, new_decodes=0, no_retry=True))
|
||||
freeze(root, 'terminal', [root/'contract.json', root/'complete.json', root/'score/result.json', root/'review.json'])
|
||||
within = cpu()<=c['postfit_cpu_cap'] and gpu()<=c['postfit_gpu_cap'] and prior+gpu()<=c['gpu_cap']
|
||||
save(root/'post-terminal-cost.json', dict(postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
|
||||
cumulative_track_a_gpu_seconds=prior+gpu(), within_cap=within, cpu_start_counters=cpu_counters))
|
||||
if not within:
|
||||
raise RuntimeError('Shared Track A final resource cap')
|
||||
except BaseException as exc:
|
||||
if (root/'complete.json').exists():
|
||||
save(root/'pre-failure-complete.json', read(root/'complete.json'))
|
||||
save(root/'complete.json', dict(status='STOPPED', reason=f'{type(exc).__name__}: {exc}',
|
||||
records=records, postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
|
||||
training_gpu_seconds_once=prior, cumulative_track_a_gpu_seconds=prior+gpu(), new_fits=0, no_retry=True))
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(ROOT)
|
||||
@@ -0,0 +1,97 @@
|
||||
"""One Track A slot in the same registry and admission policy used by item b."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import psutil
|
||||
from phase3_shared_capacity import GIB, GPUS, admission, alive, descendants, host_snapshot, process_identity, registry
|
||||
from track_a_postfit_shared_common import ROOT, read
|
||||
|
||||
|
||||
KEY = 'track_a_postfit_shared'
|
||||
OLDER = ('phase3-item-c-2026-11-18', 'phase3-item-b-shared-2026-11-19')
|
||||
|
||||
|
||||
def older_shared_pending(base, now):
|
||||
pending = []
|
||||
for name in OLDER:
|
||||
root = base/name
|
||||
if (root/'complete.json').exists():
|
||||
read(root/'complete.json')
|
||||
continue
|
||||
live = read(root/'live-progress.json')
|
||||
if now-live['unix'] > 120:
|
||||
raise RuntimeError(f'Stale earlier queue receipt {name}')
|
||||
if live['pending']:
|
||||
pending.append(name)
|
||||
return pending
|
||||
|
||||
|
||||
def available(data, allowed_cpus):
|
||||
pending = older_shared_pending(ROOT.parent, time.time())
|
||||
if pending:
|
||||
return None, dict(reason='OLDER_SHARED_FOLDS_PENDING', queues=pending)
|
||||
leases = []
|
||||
for lease in data['leases'].values():
|
||||
if not alive(lease['owner']):
|
||||
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION')
|
||||
if lease['state'] == 'RESERVED':
|
||||
leases.append(dict(lease, observed_pids=[], observed_rss=0))
|
||||
continue
|
||||
if lease['state'] != 'GUARD_BOUND':
|
||||
raise ValueError('Unknown lease state')
|
||||
try:
|
||||
ids = descendants(lease['owner'])
|
||||
except (psutil.NoSuchProcess, RuntimeError):
|
||||
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION')
|
||||
rss = 0
|
||||
for pid in ids:
|
||||
try:
|
||||
rss += psutil.Process(pid).memory_info().rss
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
leases.append(dict(lease, observed_pids=sorted(ids), observed_rss=rss))
|
||||
snapshot = host_snapshot()
|
||||
if snapshot['old_pending']:
|
||||
return None, dict(reason='OLDER_FIXED_FOLDS_PENDING', snapshot=snapshot)
|
||||
occupied = {cpu for lease in leases for cpu in lease['worker_cpus']}
|
||||
cpus = [cpu for cpu in range(16,26) if cpu in allowed_cpus and cpu not in occupied]
|
||||
if not cpus:
|
||||
return None, dict(reason='CPU_RESERVATIONS', snapshot=snapshot)
|
||||
decisions = {}
|
||||
for gpu in GPUS[:3]:
|
||||
decisions[gpu] = admission(snapshot, leases, gpu)
|
||||
if decisions[gpu][0]:
|
||||
return (gpu, cpus[0]), dict(reason='ADMITTED', snapshot=snapshot, decisions=decisions)
|
||||
return None, dict(reason='NO_CAPACITY', snapshot=snapshot, decisions=decisions)
|
||||
|
||||
|
||||
def register(root, digest, contract):
|
||||
with registry() as data:
|
||||
if KEY in data['entries']:
|
||||
raise FileExistsError('Only one shared Track A attempt')
|
||||
data['entries'][KEY] = dict(root=str(root), contract_sha256=digest,
|
||||
owner=process_identity(os.getpid()), cap=contract['postfit_gpu_cap'],
|
||||
track_a_total_cap=contract['gpu_cap'], started_unix=time.time())
|
||||
|
||||
|
||||
def acquire(root, fold, digest, allowed_cpus):
|
||||
with registry() as data:
|
||||
queue = data['entries'][KEY]
|
||||
owner = process_identity(os.getpid())
|
||||
if queue['root'] != str(root) or queue['contract_sha256'] != digest or queue['owner'] != owner:
|
||||
raise PermissionError('Only the exact shared Track A queue may reserve')
|
||||
if any(lease['root'] == str(root) for lease in data['leases'].values()):
|
||||
raise RuntimeError('Track A already owns one shared slot')
|
||||
selected, observation = available(data, allowed_cpus)
|
||||
if selected is None:
|
||||
return None, observation
|
||||
gpu, cpu = selected
|
||||
token = uuid.uuid4().hex
|
||||
lease = dict(token=token, state='RESERVED', root=str(root), fold=fold,
|
||||
contract_sha256=digest, gpu_uuid=gpu, worker_cpus=[cpu], owner=owner,
|
||||
queue_owner=owner, vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
|
||||
data['leases'][token] = lease
|
||||
return lease, observation
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Execution-only shared-slot binding layered on the approved four-split science."""
|
||||
|
||||
import fcntl
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_shared_capacity import GIB, GPUS, SHARED, alive
|
||||
from track_a_postfit_rebind_common import ROOT as PREVIOUS
|
||||
from track_a_postfit_rebind_common import SPLITS, adopted, config as reviewed_config, read
|
||||
|
||||
|
||||
ROOT = PREVIOUS.with_name('track-a-postfit-four-split-shared-2026-11-19')
|
||||
RULING = '1795024014.778499'
|
||||
SCIENCE_REVIEW = 'a5fa4a6e040493ff187b084e07cba0d95e3cae8f0fc0fa0d571f4af9efd30fce'
|
||||
SCIENCE_FREEZE = '958abd8c916614c5366b5f0216958d46e00b8504'
|
||||
GUARD_CPU = 300
|
||||
SETUP = 60
|
||||
CLEANUP = 120
|
||||
MARGIN = 1.25
|
||||
EXECUTION_KEYS = {'version', 'shared_ruling', 'shared_budget_evidence', 'shared_cards',
|
||||
'gpu_uuid', 'worker_cpus', 'idle_wait_seconds', 'postfit_gpu_cap',
|
||||
'predictions', 'code_sha256', 'source_sha256'}
|
||||
|
||||
|
||||
def fold_cap(proxy_epoch):
|
||||
if not math.isfinite(proxy_epoch) or proxy_epoch <= 0:
|
||||
raise ValueError('Positive shared timeframe proxy required')
|
||||
return math.ceil(SETUP + CLEANUP + MARGIN * (proxy_epoch + GUARD_CPU))
|
||||
|
||||
|
||||
def postfit_cap(predictions):
|
||||
caps = [p['gpu_seconds'] for p in predictions]
|
||||
return SETUP + CLEANUP + sum(caps) + max(caps)
|
||||
|
||||
|
||||
def config(root):
|
||||
if root.resolve() != ROOT:
|
||||
raise PermissionError('Exact shared Track A root')
|
||||
c = read(root/'contract.json')
|
||||
verify(root, 'implementation')
|
||||
original, tc, entry = reviewed_config(PREVIOUS)
|
||||
validate_review(PREVIOUS)
|
||||
if (sha(PREVIOUS/'review.json') != SCIENCE_REVIEW or
|
||||
read(PREVIOUS/'implementation-review-git-freeze.json')['commit'] != SCIENCE_FREEZE or
|
||||
any((PREVIOUS / name).exists() for name in ('started.json', 'launch-attempt.json', 'complete.json'))):
|
||||
raise PermissionError('Original science review or superseded-entry identity')
|
||||
if ({k:v for k,v in c.items() if k not in EXECUTION_KEYS}
|
||||
!= {k:v for k,v in original.items() if k not in EXECUTION_KEYS}):
|
||||
raise ValueError('Track A scoring scope changed')
|
||||
if c['shared_ruling'] != RULING or c['shared_cards'] != GPUS[:3]:
|
||||
raise PermissionError('Authorized GPU0–2 shared entry')
|
||||
for path, digest in {**c['source_sha256'], **c['code_sha256']}.items():
|
||||
checked(path, digest)
|
||||
evidence = read(c['shared_budget_evidence'])
|
||||
expected = [dict(id=row['split'], gpu_seconds=fold_cap(row['proxy_epoch_seconds']),
|
||||
cpu_seconds=GUARD_CPU, rss_bytes=4*GIB, recorded_vram_bytes=4*GIB,
|
||||
process_vram_bytes=4*GIB, disk_bytes=20*GIB)
|
||||
for row in evidence['splits']]
|
||||
if (tuple(x['id'] for x in expected) != SPLITS or c['predictions'] != expected
|
||||
or c['postfit_gpu_cap'] != postfit_cap(expected) or c['postfit_gpu_cap'] > original['postfit_gpu_cap']
|
||||
or c['training_evidence']['training_gpu_seconds_once'] + c['postfit_gpu_cap'] > c['gpu_cap']):
|
||||
raise ValueError('Track A projected resource caps differ')
|
||||
return c, tc, entry
|
||||
|
||||
|
||||
def assignment(root, contract, fold, digest):
|
||||
path = root/'jobs'/str(fold)/'assignment.json'
|
||||
a = read(checked(path, digest))
|
||||
if a['id'] != fold or a['contract_sha256'] != sha(root/'contract.json') or a['gpu_uuid'] not in GPUS[:3]:
|
||||
raise PermissionError('Shared prediction assignment identity')
|
||||
job = next(x for x in contract['predictions'] if x['id'] == fold)
|
||||
return dict(job, **{k:a[k] for k in ('gpu_uuid', 'worker_cpus', 'token', 'queue_identity')})
|
||||
|
||||
|
||||
def worker_lease(root, fold, job):
|
||||
with (SHARED/'capacity.lock').open('r') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_SH)
|
||||
lease = read(SHARED/'registry.json')['leases'][job['token']]
|
||||
if (lease['state'] != 'GUARD_BOUND' or lease['root'] != str(root) or lease['fold'] != fold
|
||||
or lease['owner']['pid'] != os.getppid() or not alive(lease['owner'])
|
||||
or lease['gpu_uuid'] != job['gpu_uuid'] or lease['worker_cpus'] != job['worker_cpus']
|
||||
or lease['contract_sha256'] != sha(root/'contract.json')):
|
||||
raise PermissionError('Prediction worker must be the shared guard child')
|
||||
|
||||
|
||||
def budget(c, gpu_elapsed, cpu_elapsed, pending):
|
||||
if any(not math.isfinite(x) or x < 0 for x in (gpu_elapsed, cpu_elapsed)):
|
||||
raise ValueError('Invalid execution ledger')
|
||||
predicted = gpu_elapsed + sum(p['gpu_seconds'] for p in pending)
|
||||
if predicted > c['postfit_gpu_cap'] or predicted + c['training_evidence']['training_gpu_seconds_once'] > c['gpu_cap']:
|
||||
raise RuntimeError('Track A shared postfit total budget gate')
|
||||
if cpu_elapsed + GUARD_CPU*len(pending) + 900 + CLEANUP > c['postfit_cpu_cap']:
|
||||
raise RuntimeError('Track A original postfit CPU guard')
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Shared Track A prediction guard with b’s lease, monitoring and process limits."""
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
from track_a_postfit_shared_common import config,assignment,read
|
||||
from phase3_shared_capacity import validate_lease,alive
|
||||
from track_a_recovery_monitor import Monitor
|
||||
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
|
||||
|
||||
def run(queue,index,digest):
|
||||
parent=os.getppid()
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
|
||||
raise OSError('PDEATHSIG')
|
||||
requested=[]
|
||||
for sig in (signal.SIGTERM,signal.SIGINT):
|
||||
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
|
||||
if os.getppid()!=parent:
|
||||
raise RuntimeError('Queue lost before setup')
|
||||
c,_,_=config(queue);job=assignment(queue,c,index,digest)
|
||||
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
|
||||
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
|
||||
remaining=job['gpu_seconds']-60
|
||||
if remaining<90:
|
||||
raise RuntimeError('No cleanup window before worker')
|
||||
deadline=start+remaining
|
||||
# Queue writes ready only after atomically transferring reservation ownership.
|
||||
while not (root/'lease-ready.json').exists():
|
||||
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
|
||||
raise RuntimeError('Lease transfer timeout')
|
||||
time.sleep(.1)
|
||||
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
|
||||
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
|
||||
raise RuntimeError('Pinned queue lost before guard import completed')
|
||||
os.sched_setaffinity(0,[26,27])
|
||||
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
|
||||
reasons=[];receipt=None
|
||||
command=[sys.executable,'tools/track_a_postfit_shared_worker.py','predict',str(queue),str(index),digest]
|
||||
def reason(s):
|
||||
if s not in reasons:reasons.append(s)
|
||||
def workers():
|
||||
result=[]
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:result.append(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
return result
|
||||
def kill():
|
||||
for pid in workers():stop_group(pid)
|
||||
def scan():
|
||||
tracker.scan(root/'prediction-records')
|
||||
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
|
||||
def local_watch():
|
||||
while not done.is_set():
|
||||
monitor.sample('recorded_peaks',scan)
|
||||
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
|
||||
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
|
||||
disk,ok=monitor.sample('disk',lambda:disk_usage(queue))
|
||||
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
|
||||
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
|
||||
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
|
||||
if ok and not queue_live:reason('QUEUE_LOST')
|
||||
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
def gpu_watch():
|
||||
def sample():
|
||||
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
|
||||
rows=[]
|
||||
for line in text.strip().splitlines():
|
||||
if not line:continue
|
||||
u,p,m=[x.strip() for x in line.split(',')]
|
||||
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
|
||||
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
|
||||
own=set(workers())
|
||||
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
|
||||
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
|
||||
while not done.is_set():
|
||||
monitor.sample('gpu_processes',sample)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
if reasons:kill()
|
||||
done.wait(1)
|
||||
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
|
||||
try:
|
||||
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
|
||||
for t in threads:t.start()
|
||||
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
|
||||
receipt=supervise(command,root/'guards','predict',dict(total_cpu_seconds=job['cpu_seconds']-60,
|
||||
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
|
||||
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
|
||||
except BaseException as exc:
|
||||
reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
for t in threads:
|
||||
if t.ident is not None:t.join(timeout=12)
|
||||
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
|
||||
try:monitor.retry('final_peaks',scan,.1)
|
||||
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
|
||||
for s in requested:reason(s)
|
||||
if monitor.failed:reason('MONITOR_FAILED')
|
||||
p=root/'prediction-records/result.json';result=read(p) if p.exists() else None
|
||||
expected=next(j['expected_parts'] for j in c['jobs'] if j['id']==index)
|
||||
model=next(m for m in c['training_evidence']['models'] if m['job']==index)
|
||||
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
|
||||
if (not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected
|
||||
or result.get('checkpoint_sha256')!=model['sha256'] or result.get('test_targets_read') is not False):
|
||||
reason('INCOMPLETE_PREDICTION_IDENTITY')
|
||||
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
|
||||
elapsed=time.monotonic()-start
|
||||
child=resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
|
||||
if 60+elapsed>job['gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||||
try:config(queue)
|
||||
except Exception as exc:reason('FINAL_PINS: '+str(exc))
|
||||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
|
||||
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,gpu_seconds_charged=60+elapsed,
|
||||
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
|
||||
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
|
||||
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
|
||||
return 0 if not reasons else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Four fixed test split predictions then independently gated paired full-node scoring."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, sha, save
|
||||
from phase2_home_freeze_io import verify
|
||||
from track_a_postfit_common import checkpoint_identity, prediction_identity
|
||||
from track_a_postfit_rebind_common import selected_training, SPLITS
|
||||
from track_a_postfit_shared_common import config, read, adopted, assignment, worker_lease
|
||||
from phase3_shared_capacity import process_identity
|
||||
from track_a_postfit_rebind_statistics import summarize
|
||||
from phase2_residual_math import decode_seconds
|
||||
|
||||
|
||||
def predict(root,index,digest):
|
||||
import torch
|
||||
from phase2_h2_model import H2Geo
|
||||
from phase2_h2_data import to_device,arguments
|
||||
from phase3_item1_postfit_data import load_evaluation_part
|
||||
c,tc,entry=config(root);validate_review(root);a=adopted(root,c)
|
||||
if index not in SPLITS:raise ValueError('Split outside approved four-of-five support')
|
||||
job=assignment(root,c,index,digest);worker_lease(root,index,job)
|
||||
j=tc['jobs'][index];m=next(m for m in a['models'] if m['job']==index)
|
||||
out=root/'jobs'/str(index)/'prediction-records';out.mkdir(exist_ok=False)
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity')
|
||||
if torch.__version__!=tc['versions']['torch'] or np.__version__!=tc['versions']['numpy']:raise ValueError('Library identity')
|
||||
save(out/'worker-identity.json',process_identity(os.getpid()))
|
||||
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
|
||||
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
||||
parts=0;entries=[];start=time.monotonic();cpu=time.process_time()
|
||||
try:
|
||||
blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False)
|
||||
checkpoint_identity(blob,j,m['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
|
||||
stats=read(checked(m['standardization'],m['standardization_sha256']));view=dict(tc,folds=[j])
|
||||
with torch.no_grad():
|
||||
for case in j['test']:
|
||||
data=to_device(load_evaluation_part(view,case,stats,'test'),'cuda')
|
||||
residual=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite test residual')
|
||||
seconds=decode_seconds(residual.cpu().numpy(),data['F'],j['c'],data['t_set']);torch.cuda.synchronize()
|
||||
if seconds.shape!=(tc['parts'][case]['nodes'],) or not np.isfinite(seconds).all():raise ValueError('Full-node prediction invalid')
|
||||
p=root/'predictions'/str(index)/(case+'.npz');p.parent.mkdir(parents=True,exist_ok=True)
|
||||
if p.exists():raise FileExistsError('No prediction overwrite/retry')
|
||||
np.savez_compressed(p,seconds=seconds,raw_ids=data['raw_ids'])
|
||||
entries.append(dict(split=index,case=case,path=str(p),sha256=sha(p),nodes=len(seconds),
|
||||
raw_ids_sha256=tc['parts'][case]['output_sha256']['raw_ids.npy'],negative_predictions=int(np.sum(seconds<0))))
|
||||
parts+=1;save(out/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',parts=parts,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
del data,residual,seconds
|
||||
save(out/'manifest.json',dict(entries=entries,test_targets_read=False))
|
||||
save(out/'result.json',dict(status='COMPLETE',parts=parts,checkpoint_sha256=m['sha256'],test_targets_read=False,**peaks()))
|
||||
except BaseException as exc:
|
||||
save(out/'failure.json',dict(status='FAILED',parts=parts,reason=type(exc).__name__+': '+str(exc),**peaks()));raise
|
||||
|
||||
|
||||
def score(root):
|
||||
import json
|
||||
from phase2_h2_data import cached
|
||||
from gre24_features_v2 import per_geometry_metrics
|
||||
c,tc,entry=config(root);validate_review(root);adopted(root,c);verify(root,'predictions')
|
||||
selected=selected_training(tc)
|
||||
entries=read(root/'predictions-manifest.json')['entries'];prediction_identity(root,c,selected,entries)
|
||||
out=root/'score';out.mkdir(exist_ok=False);bycase={}
|
||||
save(out/'review-consumed.json',dict(review_sha256=sha(root/'review.json'),prediction_manifest_sha256=sha(root/'predictions-manifest.json'),before_target_values=True))
|
||||
for e in entries:bycase.setdefault(e['case'],[]).append(e)
|
||||
expected=set().union(*(set(j['test']) for j in selected['jobs']))
|
||||
if set(bycase)!=expected or len(expected)!=c['expected_unique_target_reads']:raise ValueError('Frozen unique test domain')
|
||||
rows=[];count=0
|
||||
for case in sorted(bycase):
|
||||
raw=cached(tc,case,'raw_ids')
|
||||
with (out/'target-access.jsonl').open('a') as log:
|
||||
log.write(json.dumps(dict(case=case,role='test',source_sha256=tc['parts'][case]['output_sha256']['seconds.npy'],unix=time.time()))+'\n')
|
||||
truth=cached(tc,case,'seconds');count+=1
|
||||
if truth.shape!=raw.shape:raise ValueError('Target native node domain')
|
||||
for e in bycase[case]:
|
||||
ref=c['A_predictions'][str(e['split'])][case]
|
||||
A=np.load(checked(ref['path'],ref['sha256']),allow_pickle=False)
|
||||
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
|
||||
if not np.array_equal(z['raw_ids'],raw):raise ValueError('H2 native raw-ID ordering')
|
||||
H2=z['seconds'].copy()
|
||||
am=per_geometry_metrics(truth,A);hm=per_geometry_metrics(truth,H2)
|
||||
if am['nodes']!=e['nodes'] or hm['nodes']!=e['nodes'] or int(np.sum(H2<0))!=e['negative_predictions']:raise ValueError('Paired full-node coverage')
|
||||
av=am['rrmse_maximum'];hv=hm['rrmse_maximum']
|
||||
rows.append(dict(split=e['split'],case=case,nodes=e['nodes'],A=av,H2=hv,
|
||||
delta_H2_minus_A=hv-av if av is not None and hv is not None else None,
|
||||
truth_maximum=am['denominators']['maximum'],undefined=av is None or hv is None,
|
||||
A_rmse=am['rmse'],H2_rmse=hm['rmse'],A_negative=int(np.sum(A<0)),H2_negative=e['negative_predictions']))
|
||||
del truth
|
||||
save(out/'progress.json',dict(unique_test_target_reads=count,paired_rows=len(rows)))
|
||||
if len(rows)!=c['expected_pair_visits'] or count!=c['expected_unique_target_reads']:raise ValueError('Complete paired scoring required')
|
||||
save(out/'result.json',dict(status='COMPLETE_FOUR_OF_FIVE',rows=rows,summary=summarize(rows,selected['jobs']),
|
||||
evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],
|
||||
unique_test_target_reads=count,paired_visits=len(rows),original_failed=tc['original_failed'],
|
||||
A_refit=False,H2_refit=False,clipped=False,new_decodes=0,comparison_contract_sha256=c['comparison_contract_sha256'],
|
||||
review_sha256=sha(root/'review.json'),limitations=['Four of five fixed splits; split 3 has no completed training checkpoint after the budget stop',
|
||||
'Protocol-level re-creation; original author split IDs unavailable',
|
||||
'Random-part splits can share geometry families','Original14 preparation failures are retained, not resampled',
|
||||
'RMSE/max(full truth) is the frozen documented proxy; paper numbers are context only']))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
if sys.argv[1]=='predict' and len(sys.argv)==5:predict(Path(sys.argv[2]),int(sys.argv[3]),sys.argv[4])
|
||||
elif sys.argv[1]=='score' and len(sys.argv)==3:score(Path(sys.argv[2]))
|
||||
else:raise ValueError('predict root index digest or score root')
|
||||
Reference in new issue
Block a user