Publish Track A shared-slot dispatch delta for narrow independent review

This commit is contained in:
NewMeltAssist committed 2026-11-19 02:09:07 +08:00
1 parent 3438020be4
commit 3e62e08c76
15 files changed
+10123

No files matched your search

@@ -0,0 +1,142 @@
{
"ruling": "1795024014.778499",
"measured_shared_reference": [
{
"fold": 0,
"shared_epoch_seconds": 101.52302173001226,
"learning_support": {
"parts": 320,
"nodes": 8742060,
"edges": 115060278
},
"witness": {
"epoch": 42,
"epoch_seconds": 101.52302173001226,
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/0/worker/history/042.json",
"sha256": "188e83bce645c07326df46e56b7e912aba25c220d8db066b3342a8df5decb344",
"mtime": 1795014328.5125103
},
"samples": 136
},
{
"fold": 1,
"shared_epoch_seconds": 113.8008639710024,
"learning_support": {
"parts": 374,
"nodes": 9731293,
"edges": 128102636
},
"witness": {
"epoch": 37,
"epoch_seconds": 113.8008639710024,
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/1/worker/history/037.json",
"sha256": "3aa3dc428558f95e73eb5f865f784fa50fda106fe0de2c75b34919945d0698b7",
"mtime": 1795014343.7204428
},
"samples": 120
},
{
"fold": 2,
"shared_epoch_seconds": 117.12941370601766,
"learning_support": {
"parts": 366,
"nodes": 9875407,
"edges": 130004422
},
"witness": {
"epoch": 34,
"epoch_seconds": 117.12941370601766,
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/2/worker/history/034.json",
"sha256": "18b246591839acfaf38cdc88a108f9f46aebea6690626da185feb2d6c443f7b2",
"mtime": 1795014267.8009477
},
"samples": 120
},
{
"fold": 3,
"shared_epoch_seconds": 95.61896364297718,
"learning_support": {
"parts": 291,
"nodes": 8118914,
"edges": 106967636
},
"witness": {
"epoch": 42,
"epoch_seconds": 95.61896364297718,
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/3/worker/history/042.json",
"sha256": "549f9350857aeb0fdd65dda4959e711084a3a89b54c34dd4c525287d9ae6dfb9",
"mtime": 1795014267.823946
},
"samples": 148
},
{
"fold": 4,
"shared_epoch_seconds": 104.39079825102817,
"learning_support": {
"parts": 350,
"nodes": 9749420,
"edges": 128488900
},
"witness": {
"epoch": 48,
"epoch_seconds": 104.39079825102817,
"file": "/home/frank14f/MeltMind-runs/phase3-item-c-2026-11-18/jobs/4/worker/history/048.json",
"sha256": "2f394e76e2a8355ab7b495d647c1795ccc1d205c569476e9a0ddae1681e39a66",
"mtime": 1795021270.2891376
},
"samples": 68
}
],
"splits": [
{
"split": 0,
"evaluation_support": {
"parts": 304,
"nodes": 8773925,
"edges": 115624384
},
"proxy_epoch_seconds": 117.12941370601766,
"cap_gpu_seconds": 702
},
{
"split": 1,
"evaluation_support": {
"parts": 311,
"nodes": 8993731,
"edges": 118567416
},
"proxy_epoch_seconds": 117.12941370601766,
"cap_gpu_seconds": 702
},
{
"split": 2,
"evaluation_support": {
"parts": 308,
"nodes": 8433445,
"edges": 111118302
},
"proxy_epoch_seconds": 117.12941370601766,
"cap_gpu_seconds": 702
},
{
"split": 4,
"evaluation_support": {
"parts": 305,
"nodes": 8899385,
"edges": 117282304
},
"proxy_epoch_seconds": 117.12941370601766,
"cap_gpu_seconds": 702
}
],
"margin": 1.25,
"method": "Conservative full c training-epoch proxy scaled by the maximum of part, node and edge ratios",
"proxy_limit": "Prediction and compression not timed on c; original 300 CPU-s per-fold allowance added",
"wait_reserve": 702,
"postfit_gpu_cap": 3690,
"Track_A_total_gpu_seconds_with_planned_postfit": 82269.79901044304,
"one_slot": true,
"case_data_decoded": false,
"score_ast_identical": true,
"live_other_pins_verified": 180
}
@@ -0,0 +1,33 @@
test_all_node_proxy_not_relative_l2 (test_track_a_postfit.PostfitTests) ... ok
test_bootstrap_order_invariant_and_deterministic (test_track_a_postfit.PostfitTests) ... ok
test_bootstrap_zero_difference_degenerate_interval (test_track_a_postfit.PostfitTests) ... ok
test_fixed_last_epoch_identity (test_track_a_postfit.PostfitTests) ... ok
test_missing_visit_not_silently_averaged (test_track_a_postfit.PostfitTests) ... ok
test_no_target_io_before_actual_review (test_track_a_postfit.PostfitTests) ... ok
test_no_target_io_before_predictions_frozen (test_track_a_postfit.PostfitTests) ... ok
test_postfit_reserve_and_training_charged_once (test_track_a_postfit.PostfitTests) ... ok
test_prediction_manifest_native_id_and_duplicate_rejection (test_track_a_postfit.PostfitTests) ... ok
test_same_paired_support_five_summary_means (test_track_a_postfit.PostfitTests) ... ok
test_test_loader_rejects_foreign_case_before_io (test_track_a_postfit.PostfitTests) ... ok
test_undefined_candidates_and_visit_counts_retained (test_track_a_postfit.PostfitTests) ... ok
test_undefined_pairs_excluded_symmetrically (test_track_a_postfit.PostfitTests) ... ok
test_unique_part_bootstrap_not_visit_weighted (test_track_a_postfit.PostfitTests) ... ok
test_zero_truth_is_explicitly_undefined (test_track_a_postfit.PostfitTests) ... ok
test_exact_split_ids_and_visit_counts (test_track_a_postfit_rebind.RebindTests) ... ok
test_four_split_summary_uses_original_part_bootstrap (test_track_a_postfit_rebind.RebindTests) ... ok
test_incomplete_duplicate_or_extra_visits_rejected (test_track_a_postfit_rebind.RebindTests) ... ok
test_mixed_source_checkpoint_identity (test_track_a_postfit_rebind.RebindTests) ... ok
test_scoring_requires_new_review_before_target_access (test_track_a_postfit_rebind.RebindTests) ... ok
test_scoring_requires_prediction_freeze_before_target_access (test_track_a_postfit_rebind.RebindTests) ... ok
test_available_checks_only_gpu0_through_gpu2 (test_track_a_postfit_shared.SharedTrackATests) ... ok
test_older_pending_has_priority_before_any_admission (test_track_a_postfit_shared.SharedTrackATests) ... ok
test_queue_wait_is_charged_only_to_overall_cap (test_track_a_postfit_shared.SharedTrackATests) ... ok
test_score_function_ast_unchanged (test_track_a_postfit_shared.SharedTrackATests) ... ok
test_score_refuses_missing_dispatch_pass_before_targets (test_track_a_postfit_shared.SharedTrackATests) ... ok
test_score_refuses_missing_prediction_freeze_before_targets (test_track_a_postfit_shared.SharedTrackATests) ... ok
test_uses_one_available_cpu_and_card (test_track_a_postfit_shared.SharedTrackATests) ... ok
----------------------------------------------------------------------
Ran 28 tests in 0.044s
OK
@@ -0,0 +1,9 @@
# Track A shared-slot dispatch delta
Claude `1795024014.778499` replaces the whole-GPU2 idle condition with one shared slot on GPU0–2. This entry remains a serial four-split prediction and CPU scoring job. It uses the existing c/b registry, upstream admission and original guard transfer and release operations. Earlier pending folds keep dispatch priority, including the already running b queue. GPU3 stays outside this entry. Reservations hold 4 GiB each of VRAM and host RAM, enforced by the b guard's own-process/recorded-peak and process-tree monitoring. A lease transfers to one live guard and is released only after its own guard and worker exit. No launch occurs as part of preparing the binding.
The scientific review is MeltAssist's `a5fa4a6e` frozen at `958abd8c`. The original metric, A predictions, H2 checkpoints and normalization, four selected test lists, split 3 exclusion and paired bootstrap still come from that binding. The scoring function is AST-identical. The prediction worker adds lease and GPU assignment identity before using the same H2 inference and write path. Both the original scientific review and a new independent PASS tied to the exact shared dispatch binding are required. The complete frozen prediction manifest remains mandatory before any test target access.
The timing proxy uses c's recorded same-card epochs, scaled by the largest relative part, node or edge workload across the four selected Track A test lists and the c learning workloads. Each split uses the largest such proxy, an additional 300 seconds for the previous per-fold CPU allowance, a 25% margin on those two terms, and the fixed 60-second setup and 120-second cleanup allowances. The postfit budget reserves one largest split envelope for aggregate queue waiting, plus coordinator setup/cleanup. This is a conservative planning proxy, not a shared measurement of V4 predictions or compression. Real waiting is charged to the postfit total, outside each guard's prediction cap. The former 7,200 GPU-second and separate 162,000 GPU-second limits are upper bounds, and the 3,600 CPU-second limit remains in force. Original training cost 78,579.79901044304 GPU seconds is counted once.
The entry root is `/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19`. The former unused four-split GPU2 binding remains historical evidence. Live c, seed 1 and b use their own frozen roots.
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,51 @@
{
"reviewed_commit": "d71fe9c54566693a4292f0187bd23b0e468566aa",
"contract_sha256": "595d44ce4f90075f042692bf098905c687c2f2a9606c9f625a4eea3de15cab52",
"implementation_proof_sha256": "f433fe2b8366ed8ae6eca9b4eec76cd3ccb83e4cd4a5072e3f0edb52928078ef",
"comparison_contract_sha256": "8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6",
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_rebind.py": "e5fcb0c1ca594aa74e6c3cb70dada60f79baa0d8bcaa0622b19da191ad596536",
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_rebind.py": "aefdab5d2bcb0e4cf63b7ebe4e736307726f2bd8f98dc2985cc85b0d814315b3",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "8b7d425ab702eb033d4027d24d2812042b66411943b004d6be08a13d4ffcd1c9",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind.py": "54389c854f8ee43bdf66834675c22b5e705446cab363ebf55d340d8e3c79381b",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_common.py": "b0f283577da25142201c6d2c7fbfdc486e0f21a75a554a6dd8c2a1da58d29778",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_guard.py": "0aaacc9518041235eecb7f078cebba6d0bc62dadfbd4951f0066bd8271db14a8",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_statistics.py": "07b1d26dbdb898d3ffe26bec497204a59226749d299d35ad55b074a4e0ad03e9",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_worker.py": "e34dbcd50bd91b9f84367b61f7568e0cdac471e9e1262a082bb9cfba80b79751",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "b7d346d1762d7737eb1ebd723989a71f8c65b597d4cdb8c5d512b48a0fd984ce",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a932d069340909465a5a1e66be1504e28c8df159a5ce304b0586295003d2e734",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_shared.py": "9aa00647d1ff79f75be665383557c746d5b424621a32b9c5b4f148c5c2a441e8",
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_shared.py": "29fb4b535f41f562cac6e1b9f97c7ebf1248e6ec49f25c0792a8b516f8ed8955",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared.py": "96e226914f5ad2e1a0cc500657420d788f8aead04dad8c00c91fbc9ba967a86c",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_capacity.py": "b1b6c108586cb66dd9facd0b2fcdb864082da37cfc9920b517718994e9a58e67",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_common.py": "da6a6c0d2c4b3849f823159d0a6ea97b8f79778146102385f7ec3632c609bc5c",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_guard.py": "6a250dd2879d45c9379c03253b8a5920c60a1355768ea199437702633bd8fed6",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_worker.py": "ffa9c730d1662ff18d667ccf7c0dd20f5797d5e23eecdceb06ba8ebdf4505b53",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d"
}
}
@@ -0,0 +1,4 @@
{
"contract_sha256": "595d44ce4f90075f042692bf098905c687c2f2a9606c9f625a4eea3de15cab52",
"proof_sha256": "f433fe2b8366ed8ae6eca9b4eec76cd3ccb83e4cd4a5072e3f0edb52928078ef"
}
@@ -0,0 +1,105 @@
{
"commit": "d71fe9c54566693a4292f0187bd23b0e468566aa",
"files": {
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19/contract.json": "595d44ce4f90075f042692bf098905c687c2f2a9606c9f625a4eea3de15cab52",
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19/preparation-receipt.json": "abef135b3c11837f5859668ca0629cd1506c9798c90c76f0116a3a5fe3269804",
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "7e03dd7a94e7bb43593277f183c222473d7cb7aaf2f60e53b45ab8d9499fcb09",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "fab2470dd0654ca74c3b7d893f35ad1f3ae6c5c08ffe334ef6312cf6ec87892a",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "de9cbcc147677c9144c706acbe52c0f7d505dad17d741d86dcbd04d052f940e2",
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_rebind.py": "e5fcb0c1ca594aa74e6c3cb70dada60f79baa0d8bcaa0622b19da191ad596536",
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "27543fff987975ece767167f9771e6621c825035a053805604e4a696794dfcbd",
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "437a756ddf10b125dce450f463e76e28df52724f8c5766e0a989dfa536805628",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_rebind.py": "aefdab5d2bcb0e4cf63b7ebe4e736307726f2bd8f98dc2985cc85b0d814315b3",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "8b7d425ab702eb033d4027d24d2812042b66411943b004d6be08a13d4ffcd1c9",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind.py": "54389c854f8ee43bdf66834675c22b5e705446cab363ebf55d340d8e3c79381b",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_common.py": "b0f283577da25142201c6d2c7fbfdc486e0f21a75a554a6dd8c2a1da58d29778",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_guard.py": "0aaacc9518041235eecb7f078cebba6d0bc62dadfbd4951f0066bd8271db14a8",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_statistics.py": "07b1d26dbdb898d3ffe26bec497204a59226749d299d35ad55b074a4e0ad03e9",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_worker.py": "e34dbcd50bd91b9f84367b61f7568e0cdac471e9e1262a082bb9cfba80b79751",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "b7d346d1762d7737eb1ebd723989a71f8c65b597d4cdb8c5d512b48a0fd984ce",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "a932d069340909465a5a1e66be1504e28c8df159a5ce304b0586295003d2e734",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_shared.py": "9aa00647d1ff79f75be665383557c746d5b424621a32b9c5b4f148c5c2a441e8",
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_shared.py": "29fb4b535f41f562cac6e1b9f97c7ebf1248e6ec49f25c0792a8b516f8ed8955",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared.py": "96e226914f5ad2e1a0cc500657420d788f8aead04dad8c00c91fbc9ba967a86c",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_capacity.py": "b1b6c108586cb66dd9facd0b2fcdb864082da37cfc9920b517718994e9a58e67",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_common.py": "da6a6c0d2c4b3849f823159d0a6ea97b8f79778146102385f7ec3632c609bc5c",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_guard.py": "6a250dd2879d45c9379c03253b8a5920c60a1355768ea199437702633bd8fed6",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_worker.py": "ffa9c730d1662ff18d667ccf7c0dd20f5797d5e23eecdceb06ba8ebdf4505b53",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "4c423f8b0dff69304d0046c0a9c319bb31aa0c6d1cbaeb3942726b5e6caa0496",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "5ccb2a6eec95b0700b2063a2f8df414037528e90c206648c1c5e49c5e89e741d",
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-shared-2026-11-19/shared-cost-evidence.json": "4b74d29cd9a82d41db2eadc3c0a24ee5cfc762c4163c3ff522a8b72a96b819fd",
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-shared-2026-11-19/synthetic-tests.log": "17d87b07efa8c3c3dab4b84de5ce9fbedd85bc8ab4f774dae8004291539644e2",
"/home/frank14f/optane/MeltMind/docs/track-a-shared-postfit-2026-11-19.md": "7c37a24a9599433ea60c0793af001cba1f0ba246e1c0c0e26e30ece0f19d5c79"
},
"git_paths": {
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19/contract.json": "home-runs/track-a-postfit-four-split-shared-2026-11-19/contract.json",
"/home/frank14f/MeltMind-runs/track-a-postfit-four-split-shared-2026-11-19/preparation-receipt.json": "home-runs/track-a-postfit-four-split-shared-2026-11-19/preparation-receipt.json",
"/home/frank14f/optane/MeltMind/tools/boundary_distance.py": "tools/boundary_distance.py",
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "tools/freeze_g1_components.py",
"/home/frank14f/optane/MeltMind/tools/gre24_features_v2.py": "tools/gre24_features_v2.py",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "tools/gre24_models.py",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "tools/phase2_gate_outcome_review.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "tools/phase2_h1_worker_v2.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_data.py": "tools/phase2_h2_data.py",
"/home/frank14f/optane/MeltMind/tools/phase2_h2_model.py": "tools/phase2_h2_model.py",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "tools/phase2_home_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "tools/phase2_local_freeze_io.py",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "tools/phase2_residual_math.py",
"/home/frank14f/optane/MeltMind/tools/phase3_item1_postfit_data.py": "tools/phase3_item1_postfit_data.py",
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_rebind.py": "tools/prepare_track_a_postfit_rebind.py",
"/home/frank14f/optane/MeltMind/tools/r4bc_common.py": "tools/r4bc_common.py",
"/home/frank14f/optane/MeltMind/tools/r4bc_guard.py": "tools/r4bc_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "tools/s2_gpu_idle_gate.py",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "tools/s2_main_memory_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "tools/s2_pilot_train.py",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "tools/s2_precision_continuation_guard.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "tools/s2_recovery_gpu_supervisor.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "tools/s2_recovery_monitoring.py",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_supervisor.py": "tools/s2_recovery_supervisor.py",
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_rebind.py": "tools/test_track_a_postfit_rebind.py",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "tools/track_a_h2_common.py",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "tools/track_a_h2_prepare.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_common.py": "tools/track_a_postfit_common.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind.py": "tools/track_a_postfit_rebind.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_common.py": "tools/track_a_postfit_rebind_common.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_guard.py": "tools/track_a_postfit_rebind_guard.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_statistics.py": "tools/track_a_postfit_rebind_statistics.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_rebind_worker.py": "tools/track_a_postfit_rebind_worker.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_statistics.py": "tools/track_a_postfit_statistics.py",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_common.py": "tools/track_a_recovery_common.py",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "tools/phase3_shared_capacity.py",
"/home/frank14f/optane/MeltMind/tools/prepare_track_a_postfit_shared.py": "tools/prepare_track_a_postfit_shared.py",
"/home/frank14f/optane/MeltMind/tools/test_track_a_postfit_shared.py": "tools/test_track_a_postfit_shared.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared.py": "tools/track_a_postfit_shared.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_capacity.py": "tools/track_a_postfit_shared_capacity.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_common.py": "tools/track_a_postfit_shared_common.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_guard.py": "tools/track_a_postfit_shared_guard.py",
"/home/frank14f/optane/MeltMind/tools/track_a_postfit_shared_worker.py": "tools/track_a_postfit_shared_worker.py",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_monitor.py": "tools/track_a_recovery_monitor.py",
"/home/frank14f/optane/MeltMind/tools/track_a_recovery_supervisor.py": "tools/track_a_recovery_supervisor.py",
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-shared-2026-11-19/shared-cost-evidence.json": "artifacts/track-a-postfit-shared-2026-11-19/shared-cost-evidence.json",
"/home/frank14f/optane/MeltMind/artifacts/track-a-postfit-shared-2026-11-19/synthetic-tests.log": "artifacts/track-a-postfit-shared-2026-11-19/synthetic-tests.log",
"/home/frank14f/optane/MeltMind/docs/track-a-shared-postfit-2026-11-19.md": "docs/track-a-shared-postfit-2026-11-19.md"
},
"remote_published": false,
"local_repository": "/tmp/meltmind-phase2-local-fixed-20261117"
}
@@ -0,0 +1,18 @@
{
"status": "PENDING_EXACT_DISPATCH_DELTA_REVIEW",
"science_review_sha256": "a5fa4a6e040493ff187b084e07cba0d95e3cae8f0fc0fa0d571f4af9efd30fce",
"science_review_freeze": "958abd8c916614c5366b5f0216958d46e00b8504",
"shared_gpu_cap": 3690,
"split_caps": [
702,
702,
702,
702
],
"prior_gpu_seconds": 78579.79901044304,
"code_pins": 43,
"source_pins": 49,
"metadata_cpu_seconds": 3.6802666669999997,
"launched": false,
"test_targets_read": false
}
+130
View File
@@ -0,0 +1,130 @@
"""Freeze the reviewed-scope Track A shared-slot delta using only metadata."""
import ast
import fcntl
import time
from pathlib import Path
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze, verify
from phase3_shared_capacity import GIB, GPUS
from track_a_h2_prepare import local_code_closure
from track_a_postfit_rebind_common import config as reviewed_config, selected_training, terminal
from track_a_postfit_shared_common import (ROOT, PREVIOUS, RULING, SCIENCE_REVIEW, SCIENCE_FREEZE,
GUARD_CPU, MARGIN, config, fold_cap, postfit_cap, read)
BASE = Path(__file__).resolve().parents[1]
AUDIT = BASE/'artifacts/track-a-postfit-shared-2026-11-19'
REFERENCE = BASE/'artifacts/phase3-item-b-shared-budget-2026-11-19/shared-cost-evidence.json'
C_ROOT = ROOT.with_name('phase3-item-c-2026-11-18')
B_ROOT = ROOT.with_name('phase3-item-b-shared-2026-11-19')
def counts(contract, cases):
return dict(parts=len(cases), nodes=sum(contract['parts'][case]['nodes'] for case in cases),
edges=sum(contract['geometry'][case]['directed_edges'] for case in cases))
def workload_projections(tc):
reference = read(REFERENCE)
c = read(C_ROOT/'contract.json')
if sha(C_ROOT/'contract.json') != reference['c_contract_sha256']:
raise ValueError('Original c timing configuration changed')
pilot = read(checked(c['pilot_contract'], c['source_sha256'][c['pilot_contract']]))
measured = []
for row in reference['folds']:
index = row['fold']
witness = row['worst_epoch']
original = read(checked(witness['file'], witness['sha256']))
if original['train_epoch_seconds'] != row['max_train_epoch_seconds'] or original['fold'] != index:
raise ValueError('Same-card epoch witness changed')
support = counts(pilot, c['jobs'][index]['learning'])
measured.append(dict(fold=index, shared_epoch_seconds=row['max_train_epoch_seconds'],
learning_support=support, witness=witness, samples=len(row['candidates'])))
folds = []
for job in selected_training(tc)['jobs']:
n = counts(tc, job['test'])
proxy = max(x['shared_epoch_seconds'] * max(1., *(n[k]/x['learning_support'][k] for k in n))
for x in measured)
folds.append(dict(split=job['id'], evaluation_support=n, proxy_epoch_seconds=proxy,
cap_gpu_seconds=fold_cap(proxy)))
return measured, folds
def main():
start = time.process_time()
if ROOT.exists():
raise FileExistsError('Already prepared shared Track A entry')
prior, tc, _ = reviewed_config(PREVIOUS)
validate_review(PREVIOUS)
if (sha(PREVIOUS/'review.json') != SCIENCE_REVIEW or
verify(PREVIOUS, 'implementation-review')['commit'] != SCIENCE_FREEZE or
terminal(tc) != prior['training_evidence']):
raise PermissionError('Scientific pre-score review lineage changed')
measured, folds = workload_projections(tc)
jobs = [dict(id=x['split'], gpu_seconds=x['cap_gpu_seconds'], cpu_seconds=GUARD_CPU,
rss_bytes=4*GIB, recorded_vram_bytes=4*GIB, process_vram_bytes=4*GIB,
disk_bytes=20*GIB) for x in folds]
cap = postfit_cap(jobs)
training = prior['training_evidence']['training_gpu_seconds_once']
if cap > prior['postfit_gpu_cap'] or training + cap > prior['gpu_cap']:
raise ValueError('Shared planning exceeds the reviewed Track A allowance')
old_worker = ast.parse((BASE/'tools/track_a_postfit_rebind_worker.py').read_text())
new_worker = ast.parse((BASE/'tools/track_a_postfit_shared_worker.py').read_text())
old_score = next(x for x in old_worker.body if isinstance(x, ast.FunctionDef) and x.name=='score')
new_score = next(x for x in new_worker.body if isinstance(x, ast.FunctionDef) and x.name=='score')
if ast.dump(old_score) != ast.dump(new_score):
raise ValueError('Scoring logic differs')
code = {**prior['code_sha256'], **local_code_closure([Path(__file__),
*sorted((BASE/'tools').glob('track_a_postfit_shared*.py')),
BASE/'tools/test_track_a_postfit_shared.py'])}
live_pins = 0
for root in (C_ROOT, B_ROOT, ROOT.with_name('phase3-seed1-v2-2026-11-18')):
for p,h in read(root/'contract.json')['code_sha256'].items():
checked(p,h)
live_pins += 1
context = [PREVIOUS/n for n in ('contract.json', 'review.json', 'expected-review-binding.json',
'implementation-review-git-freeze.json', 'implementation-review-authorization.json')]
c_contract = read(C_ROOT/'contract.json')
context.extend([REFERENCE, C_ROOT/'contract.json', Path(c_contract['pilot_contract'])])
context.extend(Path(m['witness']['file']) for m in measured)
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
evidence = AUDIT/'shared-cost-evidence.json'
if evidence.exists():
raise FileExistsError('Prior shared Track A cost evidence exists')
save(evidence, dict(ruling=RULING, measured_shared_reference=measured, splits=folds, margin=MARGIN,
method='Conservative full c training-epoch proxy scaled by the maximum of part, node and edge ratios',
proxy_limit='Prediction and compression not timed on c; original 300 CPU-s per-fold allowance added',
wait_reserve=max(x['gpu_seconds'] for x in jobs), postfit_gpu_cap=cap,
Track_A_total_gpu_seconds_with_planned_postfit=training+cap,
one_slot=True, case_data_decoded=False, score_ast_identical=True, live_other_pins_verified=live_pins))
sources = {**prior['source_sha256'], **{str(p):sha(p) for p in context+[evidence]}}
contract = dict(prior, version='track-a-four-split-shared-postfit', shared_ruling=RULING,
shared_budget_evidence=str(evidence), shared_cards=GPUS[:3], predictions=jobs,
postfit_gpu_cap=cap, code_sha256=code, source_sha256=sources)
for key in ('gpu_uuid', 'worker_cpus', 'idle_wait_seconds'):
del contract[key]
ROOT.mkdir()
save(ROOT/'contract.json',contract)
save(ROOT/'preparation-receipt.json',dict(status='PENDING_EXACT_DISPATCH_DELTA_REVIEW',
science_review_sha256=SCIENCE_REVIEW, science_review_freeze=SCIENCE_FREEZE,
shared_gpu_cap=cap, split_caps=[x['gpu_seconds'] for x in jobs],
prior_gpu_seconds=training, code_pins=len(code), source_pins=len(sources),
metadata_cpu_seconds=time.process_time()-start, launched=False, test_targets_read=False))
freeze(ROOT,'implementation',[ROOT/'contract.json',ROOT/'preparation-receipt.json',
*(Path(p) for p in code),evidence,AUDIT/'synthetic-tests.log',
BASE/'docs/track-a-shared-postfit-2026-11-19.md'])
final=verify(ROOT,'implementation')
config(ROOT)
save(ROOT/'expected-review-binding.json',dict(reviewed_commit=final['commit'],
contract_sha256=sha(ROOT/'contract.json'),implementation_proof_sha256=sha(ROOT/'implementation-git-freeze.json'),
comparison_contract_sha256=contract['comparison_contract_sha256'],code_sha256=code))
print(dict(implementation=final['commit'],contract=sha(ROOT/'contract.json'),
binding=sha(ROOT/'expected-review-binding.json'),gpu_cap=cap,code_pins=len(code)))
if __name__=='__main__':
main()
+76
View File
@@ -0,0 +1,76 @@
"""Synthetic capacity, cost and custody checks for the Track A dispatch delta."""
import ast
from pathlib import Path
from unittest import TestCase, main
from unittest.mock import patch
from phase3_shared_capacity import GPUS
from track_a_postfit_shared_capacity import available
from track_a_postfit_shared_common import budget, fold_cap, postfit_cap
class SharedTrackATests(TestCase):
def test_older_pending_has_priority_before_any_admission(self):
with patch('track_a_postfit_shared_capacity.older_shared_pending', return_value=['b']), \
patch('track_a_postfit_shared_capacity.host_snapshot') as host:
selected, result = available({'leases': {}}, {16})
self.assertIsNone(selected)
self.assertEqual(result['reason'], 'OLDER_SHARED_FOLDS_PENDING')
host.assert_not_called()
def test_available_checks_only_gpu0_through_gpu2(self):
with patch('track_a_postfit_shared_capacity.older_shared_pending', return_value=[]), \
patch('track_a_postfit_shared_capacity.host_snapshot', return_value={'old_pending': False}), \
patch('track_a_postfit_shared_capacity.admission', return_value=(False, 'GPU_SLOT_LIMIT')) as admit:
self.assertIsNone(available({'leases': {}}, {19})[0])
self.assertEqual([call.args[2] for call in admit.call_args_list], GPUS[:3])
def test_uses_one_available_cpu_and_card(self):
with patch('track_a_postfit_shared_capacity.older_shared_pending', return_value=[]), \
patch('track_a_postfit_shared_capacity.host_snapshot', return_value={'old_pending': False}), \
patch('track_a_postfit_shared_capacity.admission', side_effect=[(False, 'GPU_SLOT_LIMIT'), (True, 'ADMITTED')]):
selected, _ = available({'leases': {}}, {12,21,22})
self.assertEqual(selected, (GPUS[1], 21))
def test_queue_wait_is_charged_only_to_overall_cap(self):
jobs = [{'gpu_seconds': fold_cap(120)} for _ in range(4)]
c = dict(postfit_gpu_cap=postfit_cap(jobs), postfit_cpu_cap=3600, gpu_cap=162000,
training_evidence={'training_gpu_seconds_once': 78579.79901044304})
self.assertEqual(jobs[0]['gpu_seconds'], 705)
budget(c, 765, 180, jobs)
with self.assertRaises(RuntimeError):
budget(c, c['postfit_gpu_cap'], 180, jobs)
with self.assertRaises(RuntimeError):
budget(dict(c, gpu_cap=80000), 0, 180, jobs)
def test_score_function_ast_unchanged(self):
here = Path(__file__).parent
def score_source(p):
return next(n for n in ast.parse(p.read_text()).body if isinstance(n, ast.FunctionDef) and n.name=='score')
old, new = (here/n for n in ('track_a_postfit_rebind_worker.py','track_a_postfit_shared_worker.py'))
self.assertEqual(ast.dump(score_source(old)), ast.dump(score_source(new)))
def test_score_refuses_missing_dispatch_pass_before_targets(self):
from track_a_postfit_shared_worker import score
with patch('track_a_postfit_shared_worker.config', return_value=({}, {}, {})), \
patch('track_a_postfit_shared_worker.validate_review', side_effect=PermissionError), \
patch('phase2_h2_data.cached') as targets:
with self.assertRaises(PermissionError):
score(Path('/synthetic'))
targets.assert_not_called()
def test_score_refuses_missing_prediction_freeze_before_targets(self):
from track_a_postfit_shared_worker import score
with patch('track_a_postfit_shared_worker.config', return_value=({}, {}, {})), \
patch('track_a_postfit_shared_worker.validate_review'), \
patch('track_a_postfit_shared_worker.adopted'), \
patch('track_a_postfit_shared_worker.verify', side_effect=PermissionError), \
patch('phase2_h2_data.cached') as targets:
with self.assertRaises(PermissionError):
score(Path('/synthetic'))
targets.assert_not_called()
if __name__ == '__main__':
main()
+184
View File
@@ -0,0 +1,184 @@
"""Serial, shared-slot prediction followed by the original paired scoring barrier."""
import fcntl
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import save, sha
from phase2_home_freeze_io import freeze, verify
from phase3_shared_capacity import alive, process_identity, release, transfer
from r4bc_guard import supervise
from track_a_postfit_common import prediction_identity
from track_a_postfit_rebind_common import RECOVERY, SPLITS, selected_training, terminal
from track_a_postfit_shared_capacity import acquire, register
from track_a_postfit_shared_common import ROOT, SETUP, budget, config, read
def stop_guard(guard):
if guard.poll() is not None:
return
guard.terminate()
try:
guard.wait(timeout=45)
except subprocess.TimeoutExpired:
guard.kill()
guard.wait()
def predict_fold(root, job, reservation, queue_identity, wait_seconds, resource_check):
folder = root/'jobs'/str(job['id'])
token = reservation['token']
guard = guard_id = None
transferred = False
try:
folder.mkdir(exist_ok=False)
path = folder/'assignment.json'
save(path, dict(id=job['id'], contract_sha256=sha(root/'contract.json'), token=token,
gpu_uuid=reservation['gpu_uuid'], worker_cpus=reservation['worker_cpus'],
queue_identity=queue_identity, pre_dispatch_wait_seconds=wait_seconds))
with (folder/'guard.log').open('x') as log:
guard = subprocess.Popen([sys.executable, 'tools/track_a_postfit_shared_guard.py',
str(root), str(job['id']), sha(path)],
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
guard_id = process_identity(guard.pid)
transfer(token, guard_id)
transferred = True
save(folder/'lease-ready.json', dict(token=token, owner=guard_id))
while guard.poll() is None:
resource_check()
time.sleep(0.5)
receipt = read(folder/'complete.json')
if (guard.returncode != 0 or receipt['status'] != 'PASS'
or receipt['gpu_seconds_charged'] > job['gpu_seconds'] or receipt['monitor_policy']['failed']):
raise RuntimeError(f'Shared prediction guard rejected fold {job["id"]}')
return receipt
finally:
if guard is not None:
stop_guard(guard)
worker = folder/'prediction-records/worker-identity.json'
if worker.exists() and alive(read(worker)):
raise RuntimeError('Prediction worker outlived guard; retained own lease')
release(token, guard_id if transferred else None)
def run(root):
gpu_clock = time.monotonic()
started_cpu = time.process_time()
initial_children = resource.getrusage(resource.RUSAGE_CHILDREN)
c, tc, _ = config(root)
validate_review(root)
adopted = terminal(tc)
if adopted != c['training_evidence']:
raise ValueError('Mixed-source training evidence changed')
prior = adopted['training_gpu_seconds_once']
charged_cpu = 0.
frozen_gpu = None
attempted = 0
records = []
stop_requests = []
original_affinity = os.sched_getaffinity(0)
def gpu():
return (SETUP + attempted*SETUP + time.monotonic()-gpu_clock
if frozen_gpu is None else frozen_gpu)
def cpu():
children = resource.getrusage(resource.RUSAGE_CHILDREN)
child_time = children.ru_utime + children.ru_stime - initial_children.ru_utime - initial_children.ru_stime
return 180 + time.process_time() - started_cpu + max(charged_cpu, child_time)
def resource_check(pending):
if stop_requests:
raise RuntimeError(stop_requests[0])
budget(c, gpu(), cpu(), pending)
with (root/'run.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root/'started.json').exists():
raise FileExistsError('One Track A shared postfit, zero retries')
identity = process_identity(os.getpid())
digest = sha(root/'contract.json')
register(root, digest, c)
os.sched_setaffinity(0, c['coordinator_cpus'])
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, lambda n, f: stop_requests.append(f'COORDINATOR_SIGNAL_{n}'))
cpu_counters = dict(self_process_time=started_cpu, children_user=initial_children.ru_utime,
children_system=initial_children.ru_stime)
save(root/'started.json', dict(identity=identity, unix=time.time(), contract_sha256=digest,
review_sha256=sha(root/'review.json'), training_gpu_seconds_once=prior, shared_slot_limit=1,
postfit_gpu_cap=c['postfit_gpu_cap'], track_a_gpu_cap=c['gpu_cap'],
cpu_start_counters=cpu_counters, cpu_setup_allowance=180, new_fits=0, new_decodes=0))
try:
save(root/'models-adopted.json', adopted)
train = Path(c['training_root'])
freeze(root, 'models', [root/'contract.json', root/'models-adopted.json', train/'complete.json',
RECOVERY/'complete.json', RECOVERY/'available-models-incomplete.json',
RECOVERY/'incomplete-models-git-freeze.json', RECOVERY/'terminal-git-freeze.json',
*[Path(m['path']) for m in adopted['models']],
*[Path(m['standardization']) for m in adopted['models']]])
entries = []
(root/'jobs').mkdir(exist_ok=False)
jobs = c['predictions']
for pos, job in enumerate(jobs):
wait_started = time.monotonic()
while True:
resource_check(jobs[pos:])
lease, observation = acquire(root, job['id'], digest, original_affinity)
if lease is not None:
break
save(root/'latest-admission.json', observation)
time.sleep(1)
attempted += 1
receipt = predict_fold(root, job, lease, identity, time.monotonic()-wait_started,
lambda: resource_check(jobs[pos+1:]))
charged_cpu += receipt['worker']['cpu_seconds']
records.append(dict(mode='predict', split=job['id'], gpu_uuid=lease['gpu_uuid'], guard=receipt))
save(root/'resource-progress.json', dict(records=records, prediction_gpu_seconds=gpu(),
postfit_cpu_seconds=cpu(), cumulative_track_a_gpu_seconds=prior+gpu()))
entries.extend(read(root/'jobs'/str(job['id'])/'prediction-records/manifest.json')['entries'])
prediction_identity(root, c, selected_training(tc), entries)
save(root/'predictions-manifest.json', dict(entries=entries, A_predictions_reused=c['expected_pair_visits'],
evaluated_splits=list(SPLITS), excluded_split=c['excluded_split'], new_fits=0, targets_read=False))
freeze(root, 'predictions', [root/'contract.json', root/'models-adopted.json',
root/'predictions-manifest.json', root/'resource-progress.json', *map(Path,c['code_sha256'])])
resource_check([])
frozen_gpu = gpu()
validate_review(root)
verify(root, 'models')
verify(root, 'predictions')
score = supervise([sys.executable, 'tools/track_a_postfit_shared_worker.py', 'score', str(root)],
root/'score-guard', dict(cpu_seconds=900, wall_seconds=1200,
rss_bytes=16*2**30, disk_bytes=20*2**30))
charged_cpu += score['cpu_seconds']
records.append(dict(mode='score', guard=score))
if score['status'] != 'PASS' or cpu()+120 > c['postfit_cpu_cap']:
raise RuntimeError('Original Track A scoring cap or guard')
save(root/'complete.json', dict(status='COMPLETE_FOUR_OF_FIVE', evaluated_splits=list(SPLITS),
excluded_split=c['excluded_split'], records=records, prediction_gpu_seconds=gpu(),
postfit_cpu_seconds=cpu(), training_gpu_seconds_once=prior,
cumulative_track_a_gpu_seconds=prior+gpu(), track_a_gpu_cap=c['gpu_cap'],
new_fits=0, new_decodes=0, no_retry=True))
freeze(root, 'terminal', [root/'contract.json', root/'complete.json', root/'score/result.json', root/'review.json'])
within = cpu()<=c['postfit_cpu_cap'] and gpu()<=c['postfit_gpu_cap'] and prior+gpu()<=c['gpu_cap']
save(root/'post-terminal-cost.json', dict(postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
cumulative_track_a_gpu_seconds=prior+gpu(), within_cap=within, cpu_start_counters=cpu_counters))
if not within:
raise RuntimeError('Shared Track A final resource cap')
except BaseException as exc:
if (root/'complete.json').exists():
save(root/'pre-failure-complete.json', read(root/'complete.json'))
save(root/'complete.json', dict(status='STOPPED', reason=f'{type(exc).__name__}: {exc}',
records=records, postfit_cpu_seconds=cpu(), prediction_gpu_seconds=gpu(),
training_gpu_seconds_once=prior, cumulative_track_a_gpu_seconds=prior+gpu(), new_fits=0, no_retry=True))
raise
if __name__ == '__main__':
run(ROOT)
+97
View File
@@ -0,0 +1,97 @@
"""One Track A slot in the same registry and admission policy used by item b."""
import os
from pathlib import Path
import time
import uuid
import psutil
from phase3_shared_capacity import GIB, GPUS, admission, alive, descendants, host_snapshot, process_identity, registry
from track_a_postfit_shared_common import ROOT, read
KEY = 'track_a_postfit_shared'
OLDER = ('phase3-item-c-2026-11-18', 'phase3-item-b-shared-2026-11-19')
def older_shared_pending(base, now):
pending = []
for name in OLDER:
root = base/name
if (root/'complete.json').exists():
read(root/'complete.json')
continue
live = read(root/'live-progress.json')
if now-live['unix'] > 120:
raise RuntimeError(f'Stale earlier queue receipt {name}')
if live['pending']:
pending.append(name)
return pending
def available(data, allowed_cpus):
pending = older_shared_pending(ROOT.parent, time.time())
if pending:
return None, dict(reason='OLDER_SHARED_FOLDS_PENDING', queues=pending)
leases = []
for lease in data['leases'].values():
if not alive(lease['owner']):
return None, dict(reason='STALE_LEASE_AWAIT_OWNER_COLLECTION')
if lease['state'] == 'RESERVED':
leases.append(dict(lease, observed_pids=[], observed_rss=0))
continue
if lease['state'] != 'GUARD_BOUND':
raise ValueError('Unknown lease state')
try:
ids = descendants(lease['owner'])
except (psutil.NoSuchProcess, RuntimeError):
return None, dict(reason='LEASE_EXIT_RACE_AWAIT_OWNER_COLLECTION')
rss = 0
for pid in ids:
try:
rss += psutil.Process(pid).memory_info().rss
except psutil.NoSuchProcess:
pass
leases.append(dict(lease, observed_pids=sorted(ids), observed_rss=rss))
snapshot = host_snapshot()
if snapshot['old_pending']:
return None, dict(reason='OLDER_FIXED_FOLDS_PENDING', snapshot=snapshot)
occupied = {cpu for lease in leases for cpu in lease['worker_cpus']}
cpus = [cpu for cpu in range(16,26) if cpu in allowed_cpus and cpu not in occupied]
if not cpus:
return None, dict(reason='CPU_RESERVATIONS', snapshot=snapshot)
decisions = {}
for gpu in GPUS[:3]:
decisions[gpu] = admission(snapshot, leases, gpu)
if decisions[gpu][0]:
return (gpu, cpus[0]), dict(reason='ADMITTED', snapshot=snapshot, decisions=decisions)
return None, dict(reason='NO_CAPACITY', snapshot=snapshot, decisions=decisions)
def register(root, digest, contract):
with registry() as data:
if KEY in data['entries']:
raise FileExistsError('Only one shared Track A attempt')
data['entries'][KEY] = dict(root=str(root), contract_sha256=digest,
owner=process_identity(os.getpid()), cap=contract['postfit_gpu_cap'],
track_a_total_cap=contract['gpu_cap'], started_unix=time.time())
def acquire(root, fold, digest, allowed_cpus):
with registry() as data:
queue = data['entries'][KEY]
owner = process_identity(os.getpid())
if queue['root'] != str(root) or queue['contract_sha256'] != digest or queue['owner'] != owner:
raise PermissionError('Only the exact shared Track A queue may reserve')
if any(lease['root'] == str(root) for lease in data['leases'].values()):
raise RuntimeError('Track A already owns one shared slot')
selected, observation = available(data, allowed_cpus)
if selected is None:
return None, observation
gpu, cpu = selected
token = uuid.uuid4().hex
lease = dict(token=token, state='RESERVED', root=str(root), fold=fold,
contract_sha256=digest, gpu_uuid=gpu, worker_cpus=[cpu], owner=owner,
queue_owner=owner, vram_reserve=4*GIB, rss_reserve=4*GIB, created_unix=time.time())
data['leases'][token] = lease
return lease, observation
+97
View File
@@ -0,0 +1,97 @@
"""Execution-only shared-slot binding layered on the approved four-split science."""
import fcntl
import math
import os
from pathlib import Path
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
from phase3_shared_capacity import GIB, GPUS, SHARED, alive
from track_a_postfit_rebind_common import ROOT as PREVIOUS
from track_a_postfit_rebind_common import SPLITS, adopted, config as reviewed_config, read
ROOT = PREVIOUS.with_name('track-a-postfit-four-split-shared-2026-11-19')
RULING = '1795024014.778499'
SCIENCE_REVIEW = 'a5fa4a6e040493ff187b084e07cba0d95e3cae8f0fc0fa0d571f4af9efd30fce'
SCIENCE_FREEZE = '958abd8c916614c5366b5f0216958d46e00b8504'
GUARD_CPU = 300
SETUP = 60
CLEANUP = 120
MARGIN = 1.25
EXECUTION_KEYS = {'version', 'shared_ruling', 'shared_budget_evidence', 'shared_cards',
'gpu_uuid', 'worker_cpus', 'idle_wait_seconds', 'postfit_gpu_cap',
'predictions', 'code_sha256', 'source_sha256'}
def fold_cap(proxy_epoch):
if not math.isfinite(proxy_epoch) or proxy_epoch <= 0:
raise ValueError('Positive shared timeframe proxy required')
return math.ceil(SETUP + CLEANUP + MARGIN * (proxy_epoch + GUARD_CPU))
def postfit_cap(predictions):
caps = [p['gpu_seconds'] for p in predictions]
return SETUP + CLEANUP + sum(caps) + max(caps)
def config(root):
if root.resolve() != ROOT:
raise PermissionError('Exact shared Track A root')
c = read(root/'contract.json')
verify(root, 'implementation')
original, tc, entry = reviewed_config(PREVIOUS)
validate_review(PREVIOUS)
if (sha(PREVIOUS/'review.json') != SCIENCE_REVIEW or
read(PREVIOUS/'implementation-review-git-freeze.json')['commit'] != SCIENCE_FREEZE or
any((PREVIOUS / name).exists() for name in ('started.json', 'launch-attempt.json', 'complete.json'))):
raise PermissionError('Original science review or superseded-entry identity')
if ({k:v for k,v in c.items() if k not in EXECUTION_KEYS}
!= {k:v for k,v in original.items() if k not in EXECUTION_KEYS}):
raise ValueError('Track A scoring scope changed')
if c['shared_ruling'] != RULING or c['shared_cards'] != GPUS[:3]:
raise PermissionError('Authorized GPU0–2 shared entry')
for path, digest in {**c['source_sha256'], **c['code_sha256']}.items():
checked(path, digest)
evidence = read(c['shared_budget_evidence'])
expected = [dict(id=row['split'], gpu_seconds=fold_cap(row['proxy_epoch_seconds']),
cpu_seconds=GUARD_CPU, rss_bytes=4*GIB, recorded_vram_bytes=4*GIB,
process_vram_bytes=4*GIB, disk_bytes=20*GIB)
for row in evidence['splits']]
if (tuple(x['id'] for x in expected) != SPLITS or c['predictions'] != expected
or c['postfit_gpu_cap'] != postfit_cap(expected) or c['postfit_gpu_cap'] > original['postfit_gpu_cap']
or c['training_evidence']['training_gpu_seconds_once'] + c['postfit_gpu_cap'] > c['gpu_cap']):
raise ValueError('Track A projected resource caps differ')
return c, tc, entry
def assignment(root, contract, fold, digest):
path = root/'jobs'/str(fold)/'assignment.json'
a = read(checked(path, digest))
if a['id'] != fold or a['contract_sha256'] != sha(root/'contract.json') or a['gpu_uuid'] not in GPUS[:3]:
raise PermissionError('Shared prediction assignment identity')
job = next(x for x in contract['predictions'] if x['id'] == fold)
return dict(job, **{k:a[k] for k in ('gpu_uuid', 'worker_cpus', 'token', 'queue_identity')})
def worker_lease(root, fold, job):
with (SHARED/'capacity.lock').open('r') as lock:
fcntl.flock(lock, fcntl.LOCK_SH)
lease = read(SHARED/'registry.json')['leases'][job['token']]
if (lease['state'] != 'GUARD_BOUND' or lease['root'] != str(root) or lease['fold'] != fold
or lease['owner']['pid'] != os.getppid() or not alive(lease['owner'])
or lease['gpu_uuid'] != job['gpu_uuid'] or lease['worker_cpus'] != job['worker_cpus']
or lease['contract_sha256'] != sha(root/'contract.json')):
raise PermissionError('Prediction worker must be the shared guard child')
def budget(c, gpu_elapsed, cpu_elapsed, pending):
if any(not math.isfinite(x) or x < 0 for x in (gpu_elapsed, cpu_elapsed)):
raise ValueError('Invalid execution ledger')
predicted = gpu_elapsed + sum(p['gpu_seconds'] for p in pending)
if predicted > c['postfit_gpu_cap'] or predicted + c['training_evidence']['training_gpu_seconds_once'] > c['gpu_cap']:
raise RuntimeError('Track A shared postfit total budget gate')
if cpu_elapsed + GUARD_CPU*len(pending) + 900 + CLEANUP > c['postfit_cpu_cap']:
raise RuntimeError('Track A original postfit CPU guard')
+137
View File
@@ -0,0 +1,137 @@
"""Shared Track A prediction guard with b’s lease, monitoring and process limits."""
import ctypes
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import threading
import time
import psutil
from phase2_h1_worker_v2 import save,sha
from track_a_postfit_shared_common import config,assignment,read
from phase3_shared_capacity import validate_lease,alive
from track_a_recovery_monitor import Monitor
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
from s2_main_memory_guard import PeakRecords
def run(queue,index,digest):
parent=os.getppid()
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
raise OSError('PDEATHSIG')
requested=[]
for sig in (signal.SIGTERM,signal.SIGINT):
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
if os.getppid()!=parent:
raise RuntimeError('Queue lost before setup')
c,_,_=config(queue);job=assignment(queue,c,index,digest)
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
remaining=job['gpu_seconds']-60
if remaining<90:
raise RuntimeError('No cleanup window before worker')
deadline=start+remaining
# Queue writes ready only after atomically transferring reservation ownership.
while not (root/'lease-ready.json').exists():
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
raise RuntimeError('Lease transfer timeout')
time.sleep(.1)
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
raise RuntimeError('Pinned queue lost before guard import completed')
os.sched_setaffinity(0,[26,27])
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
reasons=[];receipt=None
command=[sys.executable,'tools/track_a_postfit_shared_worker.py','predict',str(queue),str(index),digest]
def reason(s):
if s not in reasons:reasons.append(s)
def workers():
result=[]
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:result.append(p.pid)
except psutil.NoSuchProcess:pass
return result
def kill():
for pid in workers():stop_group(pid)
def scan():
tracker.scan(root/'prediction-records')
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
def local_watch():
while not done.is_set():
monitor.sample('recorded_peaks',scan)
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
disk,ok=monitor.sample('disk',lambda:disk_usage(queue))
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
if ok and not queue_live:reason('QUEUE_LOST')
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(.5)
def gpu_watch():
def sample():
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
rows=[]
for line in text.strip().splitlines():
if not line:continue
u,p,m=[x.strip() for x in line.split(',')]
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
own=set(workers())
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
while not done.is_set():
monitor.sample('gpu_processes',sample)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(1)
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
try:
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
for t in threads:t.start()
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
receipt=supervise(command,root/'guards','predict',dict(total_cpu_seconds=job['cpu_seconds']-60,
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
except BaseException as exc:
reason(type(exc).__name__+': '+str(exc));kill()
finally:
done.set()
for t in threads:
if t.ident is not None:t.join(timeout=12)
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
try:monitor.retry('final_peaks',scan,.1)
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
p=root/'prediction-records/result.json';result=read(p) if p.exists() else None
expected=next(j['expected_parts'] for j in c['jobs'] if j['id']==index)
model=next(m for m in c['training_evidence']['models'] if m['job']==index)
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
if (not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected
or result.get('checkpoint_sha256')!=model['sha256'] or result.get('test_targets_read') is not False):
reason('INCOMPLETE_PREDICTION_IDENTITY')
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
elapsed=time.monotonic()-start
child=resource.getrusage(resource.RUSAGE_CHILDREN)
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
if 60+elapsed>job['gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
try:config(queue)
except Exception as exc:reason('FINAL_PINS: '+str(exc))
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,gpu_seconds_charged=60+elapsed,
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
return 0 if not reasons else 1
if __name__=='__main__':
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
+107
View File
@@ -0,0 +1,107 @@
"""Four fixed test split predictions then independently gated paired full-node scoring."""
import os
from pathlib import Path
import sys
import time
import numpy as np
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, sha, save
from phase2_home_freeze_io import verify
from track_a_postfit_common import checkpoint_identity, prediction_identity
from track_a_postfit_rebind_common import selected_training, SPLITS
from track_a_postfit_shared_common import config, read, adopted, assignment, worker_lease
from phase3_shared_capacity import process_identity
from track_a_postfit_rebind_statistics import summarize
from phase2_residual_math import decode_seconds
def predict(root,index,digest):
import torch
from phase2_h2_model import H2Geo
from phase2_h2_data import to_device,arguments
from phase3_item1_postfit_data import load_evaluation_part
c,tc,entry=config(root);validate_review(root);a=adopted(root,c)
if index not in SPLITS:raise ValueError('Split outside approved four-of-five support')
job=assignment(root,c,index,digest);worker_lease(root,index,job)
j=tc['jobs'][index];m=next(m for m in a['models'] if m['job']==index)
out=root/'jobs'/str(index)/'prediction-records';out.mkdir(exist_ok=False)
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity')
if torch.__version__!=tc['versions']['torch'] or np.__version__!=tc['versions']['numpy']:raise ValueError('Library identity')
save(out/'worker-identity.json',process_identity(os.getpid()))
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
parts=0;entries=[];start=time.monotonic();cpu=time.process_time()
try:
blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False)
checkpoint_identity(blob,j,m['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
stats=read(checked(m['standardization'],m['standardization_sha256']));view=dict(tc,folds=[j])
with torch.no_grad():
for case in j['test']:
data=to_device(load_evaluation_part(view,case,stats,'test'),'cuda')
residual=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite test residual')
seconds=decode_seconds(residual.cpu().numpy(),data['F'],j['c'],data['t_set']);torch.cuda.synchronize()
if seconds.shape!=(tc['parts'][case]['nodes'],) or not np.isfinite(seconds).all():raise ValueError('Full-node prediction invalid')
p=root/'predictions'/str(index)/(case+'.npz');p.parent.mkdir(parents=True,exist_ok=True)
if p.exists():raise FileExistsError('No prediction overwrite/retry')
np.savez_compressed(p,seconds=seconds,raw_ids=data['raw_ids'])
entries.append(dict(split=index,case=case,path=str(p),sha256=sha(p),nodes=len(seconds),
raw_ids_sha256=tc['parts'][case]['output_sha256']['raw_ids.npy'],negative_predictions=int(np.sum(seconds<0))))
parts+=1;save(out/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',parts=parts,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
del data,residual,seconds
save(out/'manifest.json',dict(entries=entries,test_targets_read=False))
save(out/'result.json',dict(status='COMPLETE',parts=parts,checkpoint_sha256=m['sha256'],test_targets_read=False,**peaks()))
except BaseException as exc:
save(out/'failure.json',dict(status='FAILED',parts=parts,reason=type(exc).__name__+': '+str(exc),**peaks()));raise
def score(root):
import json
from phase2_h2_data import cached
from gre24_features_v2 import per_geometry_metrics
c,tc,entry=config(root);validate_review(root);adopted(root,c);verify(root,'predictions')
selected=selected_training(tc)
entries=read(root/'predictions-manifest.json')['entries'];prediction_identity(root,c,selected,entries)
out=root/'score';out.mkdir(exist_ok=False);bycase={}
save(out/'review-consumed.json',dict(review_sha256=sha(root/'review.json'),prediction_manifest_sha256=sha(root/'predictions-manifest.json'),before_target_values=True))
for e in entries:bycase.setdefault(e['case'],[]).append(e)
expected=set().union(*(set(j['test']) for j in selected['jobs']))
if set(bycase)!=expected or len(expected)!=c['expected_unique_target_reads']:raise ValueError('Frozen unique test domain')
rows=[];count=0
for case in sorted(bycase):
raw=cached(tc,case,'raw_ids')
with (out/'target-access.jsonl').open('a') as log:
log.write(json.dumps(dict(case=case,role='test',source_sha256=tc['parts'][case]['output_sha256']['seconds.npy'],unix=time.time()))+'\n')
truth=cached(tc,case,'seconds');count+=1
if truth.shape!=raw.shape:raise ValueError('Target native node domain')
for e in bycase[case]:
ref=c['A_predictions'][str(e['split'])][case]
A=np.load(checked(ref['path'],ref['sha256']),allow_pickle=False)
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],raw):raise ValueError('H2 native raw-ID ordering')
H2=z['seconds'].copy()
am=per_geometry_metrics(truth,A);hm=per_geometry_metrics(truth,H2)
if am['nodes']!=e['nodes'] or hm['nodes']!=e['nodes'] or int(np.sum(H2<0))!=e['negative_predictions']:raise ValueError('Paired full-node coverage')
av=am['rrmse_maximum'];hv=hm['rrmse_maximum']
rows.append(dict(split=e['split'],case=case,nodes=e['nodes'],A=av,H2=hv,
delta_H2_minus_A=hv-av if av is not None and hv is not None else None,
truth_maximum=am['denominators']['maximum'],undefined=av is None or hv is None,
A_rmse=am['rmse'],H2_rmse=hm['rmse'],A_negative=int(np.sum(A<0)),H2_negative=e['negative_predictions']))
del truth
save(out/'progress.json',dict(unique_test_target_reads=count,paired_rows=len(rows)))
if len(rows)!=c['expected_pair_visits'] or count!=c['expected_unique_target_reads']:raise ValueError('Complete paired scoring required')
save(out/'result.json',dict(status='COMPLETE_FOUR_OF_FIVE',rows=rows,summary=summarize(rows,selected['jobs']),
evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],
unique_test_target_reads=count,paired_visits=len(rows),original_failed=tc['original_failed'],
A_refit=False,H2_refit=False,clipped=False,new_decodes=0,comparison_contract_sha256=c['comparison_contract_sha256'],
review_sha256=sha(root/'review.json'),limitations=['Four of five fixed splits; split 3 has no completed training checkpoint after the budget stop',
'Protocol-level re-creation; original author split IDs unavailable',
'Random-part splits can share geometry families','Original14 preparation failures are retained, not resampled',
'RMSE/max(full truth) is the frozen documented proxy; paper numbers are context only']))
if __name__=='__main__':
if sys.argv[1]=='predict' and len(sys.argv)==5:predict(Path(sys.argv[2]),int(sys.argv[3]),sys.argv[4])
elif sys.argv[1]=='score' and len(sys.argv)==3:score(Path(sys.argv[2]))
else:raise ValueError('predict root index digest or score root')