diff --git a/artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json b/artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json new file mode 100644 index 000000000..bb48ab771 --- /dev/null +++ b/artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json @@ -0,0 +1,41 @@ +{ + "inventory": [ + [ + "0", + "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "00000000:84:00.0", + "Tesla V100-SXM2-16GB" + ], + [ + "1", + "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36", + "00000000:85:00.0", + "Tesla V100-SXM2-16GB" + ], + [ + "2", + "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "00000000:A4:00.0", + "Tesla V100-SXM2-16GB" + ], + [ + "3", + "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "00000000:A5:00.0", + "Tesla V100-SXM2-16GB" + ] + ], + "occupants": [ + [ + "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "2512405", + "820 MiB" + ], + [ + "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36", + "2517067", + "1708 MiB" + ] + ], + "source": "nvidia-smi actual after the original UUID stop" +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json new file mode 100644 index 000000000..e728c9d46 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json @@ -0,0 +1,779 @@ +{ + "mode": "measurement", + "ruling": "h-f84547d7", + "retry": false, + "evaluate_other_models": false, + "measurement_allowed_heldout_labels": false, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "limits": { + "gpu_wall_seconds": 3600, + "cleanup_seconds": 60, + "idle_wait_seconds": 60, + "cpu_seconds": 3500, + "rss_bytes": 12884901888, + "recorded_vram_bytes": 15032385536, + "device_vram_bytes": 16106127360, + "disk_bytes": 2147483648, + "torch_threads": 1, + "max_processes": 1 + }, + "protocol": { + "model": "NeurIPS 2023 supplement SDFFNOGNO decoder", + "seeds": [ + 0, + 1, + 2 + ], + "grid": [ + 64, + 64, + 64 + ], + "modes": [ + 24, + 24, + 24 + ], + "hidden_channels": 32, + "radius": 0.05, + "epochs_per_seed": 100, + "batch_size": 1, + "learning_rate": 0.001, + "bounds_expansion": 0.01, + "pressure_epsilon": 1e-06, + "optimizer": "Adam", + "weight_decay": 0.0001, + "lr_step_epochs": 50, + "lr_gamma": 0.5, + "pressure_training_transform": "scalar Gaussian on the 500 learning cases", + "metric": "per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)", + "report": "equal mean over 111 cases for each seed, then equal mean over the three seeds", + "acceptance_percent": [ + 6.62, + 7.62 + ], + "paper_reference_percent": 7.12, + "checkpoint_epoch": 100, + "heldout_access_after_all_checkpoints_frozen": true + }, + "environment": { + "python": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "python_version": "3.10", + "torch_cuda": "11.6", + "neuralop_commit": "fbdc7f872792b75b84545c2f374e7a881c72a718", + "neuralop_file_sha256": { + "utils.py": "84554eed45ac415ab52f79e456eab70ba204e7ab7539c2f2841955a9145220b4", + "models/__init__.py": "c67679f1f8f90e1ab4eb4c2aa916aeaedf1b175d795fed03b8d4f7210883ebae", + "models/tfno.py": "d74b5805b69a64bf33e3ff5186709c75bd431a56c9548f06fb9635a859d849e4", + "models/spectral_convolution.py": "ccecf80806d93e8f35713ac6ab2c3b61c11739f5fcb8c0ba7397b7d4eb5a5d45" + }, + "package_versions": { + "torch": "1.13.1+cu116", + "open3d": "0.17.0", + "numpy": "1.24.3", + "torch-scatter": "2.1.1+pt113cu116", + "torch-sparse": "0.6.17+pt113cu116", + "torch-geometric": "2.3.1", + "tensorly": "0.8.1", + "tensorly-torch": "0.4.0" + } + }, + "comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-measurement-2026-11-19/comparison-contract.json", + "comparison_contract_sha256": "503f6c8ab1217d66b44e0d127555cfad5da4cd8a17b336299095c3cfd13041d7", + "code_sha256": { + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8", + "/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_shapenet.py": "557ad56277d7e85c6b5cc358eb151632d1757f5883f429988d36022d0d407550", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_entry.py": "e86d955c36e291e6bbef67947de663b24c8d66c5d72d9d28e4193fb8a3051ee2", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py": "7211be211c29472fd9be35a6bcb1f56025de4a63f36852bfafa10df97c003ff3", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_entry.py": "199d833f40339feb0433611ba27302416a280452f8918715e6ef95cb93c38721", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_worker.py": "3acf742dd4549e9f5491e10af5bb0b4acf00e2f47e91753100b5998cfcc61852", + "/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_import_checked.py": "fef485dd0c233c76a336239d5f29cbf44ac0fb62a6a297dff34293b23a1b75fe", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_paper_keyword_patch.py": "c61e2ed7d099eb125386857bf4d7ccdbfc3af522f80edb4f01b450cee91c0b95", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2.py": "7d3ab2412e5208892cffcfc0e1ec363eb5db2be3e10bf60595356f71e82d8552", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py": "ba1fd5ec837e884b000ba640fceafe30ea40ccf708847733df6a17a8f3599680", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery.py": "1e1827498afab62e2f28efb5d450376f2dca4e8bf6f74114f2914ec771872c86", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py": "7a68c77e9c913ba9161832f513dbf8754ebc5c270c1e742fa5ae97a7b14b9120" + }, + "paper_data": { + "path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-car-pressure-data.zip", + "sha256": "b5f9e2b407d54e034596384253fb58196933c46f6b5b586383508da54475f89d" + }, + "paper_supplement": { + "path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-neurips-supplement.zip", + "sha256": "48cdae80ac2781e27ac00e10a37124711d6bb305593559002bda0527d11a533d" + }, + "paper_code_zip_sha256": "347a6f097e2b9958d1ab87930fa1d6227b793a9d213e12418ce9a157e83dce04", + "split_receipt": { + "path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-split.json", + "sha256": "5a96c25c2ec24cfc1e2264d4ebd0879c658c0f168e22711b576bf303151b8601" + }, + "mapping_audit": { + "path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-mapping-audit.json", + "sha256": "055d008a02494acd29df36c41c8e68db931a027f5635e8f5f765a96a54c62289" + }, + "split": { + "train": [ + 1, + 2, + 4, + 5, + 6, + 7, + 8, + 10, + 12, + 13, + 17, + 18, + 21, + 22, + 23, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 34, + 35, + 39, + 40, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 54, + 55, + 56, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 67, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 83, + 84, + 85, + 86, + 87, + 88, + 90, + 91, + 92, + 94, + 95, + 96, + 97, + 100, + 101, + 102, + 105, + 106, + 107, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 133, + 134, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 165, + 166, + 170, + 172, + 173, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 186, + 190, + 191, + 192, + 193, + 195, + 196, + 198, + 199, + 200, + 201, + 202, + 203, + 205, + 207, + 210, + 211, + 212, + 213, + 214, + 215, + 217, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 241, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 251, + 252, + 253, + 255, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 266, + 267, + 268, + 269, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 285, + 286, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 304, + 305, + 306, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 327, + 328, + 329, + 331, + 332, + 333, + 334, + 335, + 337, + 338, + 339, + 340, + 341, + 344, + 345, + 347, + 348, + 349, + 350, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 360, + 362, + 364, + 365, + 366, + 367, + 369, + 371, + 372, + 373, + 374, + 375, + 376, + 378, + 379, + 380, + 381, + 384, + 385, + 389, + 392, + 393, + 397, + 398, + 399, + 401, + 402, + 403, + 404, + 405, + 407, + 408, + 410, + 412, + 413, + 414, + 415, + 417, + 418, + 419, + 420, + 422, + 424, + 425, + 427, + 430, + 431, + 433, + 435, + 436, + 437, + 439, + 440, + 443, + 444, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 459, + 460, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 472, + 473, + 474, + 475, + 476, + 478, + 479, + 480, + 482, + 483, + 486, + 487, + 488, + 490, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 501, + 502, + 503, + 504, + 505, + 507, + 508, + 509, + 511, + 512, + 513, + 514, + 515, + 516, + 518, + 519, + 521, + 522, + 523, + 524, + 525, + 527, + 529, + 530, + 532, + 533, + 536, + 538, + 539, + 540, + 542, + 543, + 545, + 547, + 548, + 549, + 550, + 551, + 552, + 553, + 554, + 555, + 560, + 561, + 562, + 564, + 565, + 566, + 567, + 568, + 569, + 572, + 573, + 574, + 576, + 577, + 579, + 581, + 582, + 583, + 584, + 587, + 588, + 589, + 591, + 593, + 594, + 595, + 596, + 597, + 598, + 600, + 602, + 604, + 608, + 610, + 611, + 612, + 613, + 615, + 616, + 617, + 618, + 620, + 621, + 622, + 623, + 625, + 626, + 627, + 628, + 629, + 630, + 631, + 632, + 633, + 634, + 635, + 636, + 638, + 639, + 640, + 641, + 642, + 643, + 644, + 645, + 646, + 647, + 648, + 649, + 651, + 652, + 654, + 655, + 656, + 657 + ], + "test": [ + 658, + 659, + 660, + 662, + 663, + 664, + 665, + 666, + 667, + 668, + 672, + 673, + 674, + 675, + 676, + 677, + 678, + 679, + 681, + 683, + 684, + 686, + 687, + 688, + 689, + 690, + 691, + 692, + 693, + 695, + 696, + 697, + 700, + 701, + 702, + 703, + 704, + 705, + 708, + 709, + 710, + 711, + 712, + 713, + 715, + 717, + 718, + 719, + 721, + 722, + 723, + 724, + 725, + 726, + 727, + 729, + 730, + 731, + 733, + 734, + 735, + 736, + 737, + 738, + 739, + 740, + 741, + 743, + 744, + 745, + 746, + 747, + 748, + 749, + 750, + 751, + 752, + 753, + 757, + 758, + 760, + 762, + 763, + 764, + 765, + 766, + 767, + 768, + 769, + 770, + 771, + 772, + 773, + 775, + 776, + 777, + 778, + 779, + 781, + 782, + 783, + 784, + 785, + 787, + 789, + 790, + 791, + 792, + 793, + 795, + 797 + ], + "raw_global_bounds": [ + [ + -0.90252799, + 0.0066688, + -2.85272002 + ], + [ + 0.90253001, + 1.95825005, + 2.86385012 + ] + ] + }, + "environment_ruling": "h-0d21f3bd", + "environment_acceptance": { + "path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-import-checked-2026-11-19/cpu-environment-acceptance.json", + "sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3" + }, + "paper_compatibility": { + "handoff": "h-b8508140", + "identity": "supplement code + 2-keyword patch", + "patch_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/two-keyword-patch.json", + "patch_sha256": "7307ba49f83bfc46c5decf803a993a41ed602d987de65714958e738d2f52f077", + "search_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/keyword-search.json", + "search_sha256": "77a897bf534a9a0e9e6a7c072b4f914168648eadc2af3493b70d63d0b715b3d0" + }, + "gpu_ruling": "h-bc42181f", + "ordinal_recovery": { + "handoff": "h-91e61871", + "previous_terminal_sha256": "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e", + "previous_gpu_wall_seconds": 23.69431395700667 + } +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/ordinal-cpu-replay.json b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/ordinal-cpu-replay.json new file mode 100644 index 000000000..05264a5ef --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/ordinal-cpu-replay.json @@ -0,0 +1,109 @@ +{ + "status": "PASS", + "ruling": "h-91e61871", + "old_terminal_sha256": "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e", + "contract_sha256": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a", + "same_scientific_protocol_and_caps": true, + "hardware_inventory": { + "inventory": [ + [ + "0", + "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "00000000:84:00.0", + "Tesla V100-SXM2-16GB" + ], + [ + "1", + "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36", + "00000000:85:00.0", + "Tesla V100-SXM2-16GB" + ], + [ + "2", + "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "00000000:A4:00.0", + "Tesla V100-SXM2-16GB" + ], + [ + "3", + "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "00000000:A5:00.0", + "Tesla V100-SXM2-16GB" + ] + ], + "occupants": [ + [ + "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "2512405", + "820 MiB" + ], + [ + "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36", + "2517067", + "1708 MiB" + ] + ], + "source": "nvidia-smi actual after the original UUID stop" + }, + "inconsistent_pci_order_rejected": true, + "advance_checks": { + "matching": { + "captured": { + "command": [ + "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py", + "/tmp/track-b-cuda-identity-replay-8m_kv8v_" + ], + "device_order": "PCI_BUS_ID", + "visible_devices": "2", + "start_new_session": true + }, + "verified": true, + "mismatch": false, + "stop_called": false, + "terminal_trigger": "MOCK_WORK_REPLACED", + "budget": { + "status": "STOPPED", + "trigger": "MOCK_WORK_REPLACED", + "previous_gpu_wall_seconds": 23.69431395700667, + "recovery_gpu_wall_seconds": 0.6289700350025669, + "cumulative_measurement_gpu_wall_seconds": 24.323283992009237, + "total_cap_gpu_wall_seconds": 3600, + "independent_attempts": 2, + "no_retry": true, + "within_total_cap": true + } + }, + "mismatch": { + "captured": { + "command": [ + "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py", + "/tmp/track-b-cuda-identity-replay-mh5jfyj7" + ], + "device_order": "PCI_BUS_ID", + "visible_devices": "2", + "start_new_session": true + }, + "verified": false, + "mismatch": true, + "stop_called": true, + "terminal_trigger": "MONITOR_FAILURE: WORKER_GPU_UUID_MISMATCH", + "budget": { + "status": "STOPPED", + "trigger": "MONITOR_FAILURE: WORKER_GPU_UUID_MISMATCH", + "previous_gpu_wall_seconds": 23.69431395700667, + "recovery_gpu_wall_seconds": 0.757885464001447, + "cumulative_measurement_gpu_wall_seconds": 24.452199421008117, + "total_cap_gpu_wall_seconds": 3600, + "independent_attempts": 2, + "no_retry": true, + "within_total_cap": true + } + } + }, + "real_spawns": 0, + "real_cuda_calls": 0, + "heldout_targets_read": 0, + "independent_review": "PENDING" +} diff --git a/docs/track-b-gino-ordinal-recovery-2026-11-19.md b/docs/track-b-gino-ordinal-recovery-2026-11-19.md new file mode 100644 index 000000000..ddeac9f3e --- /dev/null +++ b/docs/track-b-gino-ordinal-recovery-2026-11-19.md @@ -0,0 +1,7 @@ +# Track B|GPU2 单轮测量的设备绑定恢复 + +原 GPU2 唯一尝试通过空闲门并派发。Torch 1.13.1 在以物理 UUID 作为 `CUDA_VISIBLE_DEVICES` 时,Python NVML 枚举将该字符串解析为负序号并返回可见卡数 0,`get_device_name(0)` 抛出 `Invalid device id`。原 worker 未进入样本准备,无优化步或计时产物。原申请、工作者退出、终止日志和 CPU Mock 复现已固定。原终止实付 23.69431395700667 GPU-s。 [原终止及 CPU 复现](https://gitea.frank14f.online/Frank14f/MeltMind/src/commit/17a40a6c1b3f765827b2d7c4950b6e6c5a8d1ad6/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/terminal-uuid-enumeration-audit.json) + +Claude h-91e61871 批准唯一一次环境恢复。在原 UUID 守卫中临派时要求 `nvidia-smi` 的序号与 PCI 总线排序严格一致。物理 GPU2 在实核中为序号 2,PCI 地址 A4:00.0,V100,未见计算进程。只在单个 worker 进程设 `CUDA_DEVICE_ORDER=PCI_BUS_ID`、`CUDA_VISIBLE_DEVICES=2`。在读取任何样本之前,worker 核验只见一张 V100,然后作一字节标记分配。守卫须从实际 worker PID 核实该分配确在固定 UUID,随后才放行原工作体。卡不匹配时由原守卫停止该工作者。 [Claude 裁决](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795072229918399) + +CPU 回放验证了正确卡的环境与放行、错误卡的终止、PCI 序号冲突的拒绝,以及旧账计一次。原单轮训练、学习代理、补丁、环境及选择 UUID 均沿用。总帽为 3,600 GPU-s,独立新根的原守卫实际余量为 3,576.305686 GPU-s,支出含等待与设置。独立审查和阶段门在真实开始之前执行。原 111 个留出标签保持封存。 [恢复合同](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json) · [CPU 回放](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/ordinal-cpu-replay.json) diff --git a/tools/prepare_track_b_gino_ordinal_recovery.py b/tools/prepare_track_b_gino_ordinal_recovery.py new file mode 100644 index 000000000..268009b27 --- /dev/null +++ b/tools/prepare_track_b_gino_ordinal_recovery.py @@ -0,0 +1,132 @@ +"""Bind the one approved recovery and replay ordinal dispatch and mismatch.""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +from tempfile import TemporaryDirectory +import threading +import time +from types import FunctionType, SimpleNamespace +from unittest.mock import patch + +import track_b_gino_import_checked_entry as accepted +import track_b_gino_ordinal_recovery as recovery +import track_b_gino_shapenet_entry as original + + +halt = threading.Event() + + +def fake_supervisor(command, root, mode, limits, *, poll, gpu_uuid): + """Executed in a fixture namespace with the real recovery spawn adapter.""" + process = subprocess.Popen(command, env={"CUDA_VISIBLE_DEVICES": gpu_uuid}, start_new_session=True) + deadline = time.monotonic() + 4 + matched = Path(root) / "worker-device-verified.json" + while not matched.exists() and not halt.is_set() and time.monotonic() < deadline: + time.sleep(.01) + if not (matched.exists() or halt.is_set()): + raise AssertionError("Fixture monitor never verified or stopped the worker") + return dict(status="STOPPED_OR_FAILED", returncode=-9, monitor_errors=0, + stop_reason="MOCK_WORK_REPLACED", pid=process.pid) + + +def replay(inventory, slot): + halt = threading.Event() + pid = 12345 + identity = {"pid": pid, "created": 123.} + captured = {} + + def query(option): + if option == "--query-gpu=index,uuid,pci.bus_id,name": + return inventory + if option != "--query-compute-apps=gpu_uuid,pid": + raise AssertionError("Unexpected inventory query") + return [[slot, str(pid)]] + + def spawn(command, *, env, start_new_session): + captured.update(command=list(command), device_order=env["CUDA_DEVICE_ORDER"], + visible_devices=env["CUDA_VISIBLE_DEVICES"], start_new_session=start_new_session) + original.save(root / "worker-device-witness.json", dict( + process=identity, ordinal="2", device_count=1, device_name="Tesla V100-SXM2-16GB", + physical_uuid_expected=recovery.UUID, first_allocation_bytes=1)) + return SimpleNamespace(pid=pid) + + def stop(pid_to_stop): + if pid_to_stop != pid: + raise AssertionError("Attempted to stop the wrong process") + halt.set() + + with TemporaryDirectory(prefix="track-b-cuda-identity-replay-") as temp: + root = Path(temp) + path = root / "contract.json" + original.save(path, recovery.expected_contract()) + isolated = dict(fake_supervisor.__globals__, halt=halt) + fake = FunctionType(fake_supervisor.__code__, isolated, fake_supervisor.__name__) + with (patch.object(recovery, "query_nvidia", side_effect=query), + patch.object(recovery, "process_identity", return_value=identity), + patch.object(recovery, "subprocess", SimpleNamespace(Popen=spawn, STDOUT=-2)), + patch.object(original, "supervise", fake), + patch.object(original, "snapshot", return_value=dict(idle=True, memory_MiB=5, occupants=[])), + patch.object(original, "wait_idle", return_value=dict(status="IDLE_CONFIRMED")), + patch.object(original, "active_command_pids", return_value=[pid]), + patch.object(original, "stop_group", side_effect=stop)): + code = recovery.dispatch_guard(root, path) + witness, mismatch = root / "worker-device-verified.json", root / "worker-device-mismatch.json" + terminal = original.read(root / "complete.json") + budget = original.read(root / "recovery-settlement.json") + assert code == 1 and bool(captured) + assert captured["device_order"] == "PCI_BUS_ID" and captured["visible_devices"] == "2" + assert captured["command"][1] == str(recovery.WORKER) + assert budget["previous_gpu_wall_seconds"] == 23.69431395700667 + assert budget["cumulative_measurement_gpu_wall_seconds"] == ( + budget["previous_gpu_wall_seconds"] + budget["recovery_gpu_wall_seconds"]) + assert terminal["gpu_wall_cap_seconds"] == 3600 - budget["previous_gpu_wall_seconds"] + if slot == recovery.UUID: + assert witness.exists() and not mismatch.exists() and not halt.is_set() + else: + assert mismatch.exists() and not witness.exists() and halt.is_set() + assert "WORKER_GPU_UUID_MISMATCH" in terminal["trigger"] + return dict(captured=captured, verified=witness.exists(), mismatch=mismatch.exists(), + stop_called=halt.is_set(), terminal_trigger=terminal["trigger"], budget=budget) + + +def prepare(inventory_path): + if os.environ.get("CUDA_VISIBLE_DEVICES") != "": + raise PermissionError("The replay requires an empty device mask") + source = json.loads(inventory_path.read_text()) + rows = source["inventory"] + assert recovery.device_ordinal(rows, recovery.UUID) == "2" + assert not any(card == recovery.UUID for card, *_ in source["occupants"]) + mismatched = [row.copy() for row in rows] + mismatched[0][0], mismatched[2][0] = mismatched[2][0], mismatched[0][0] + try: + recovery.device_ordinal(mismatched, recovery.UUID) + except ValueError: + pass + else: + raise AssertionError("A conflicting PCI order was accepted") + c = recovery.expected_contract() + old = original.read(recovery.previous.ROOT / "contract.json") + assert c["gpu_uuid"] == old["gpu_uuid"] and c["limits"] == old["limits"] + assert set(k for k in c if c[k] != old.get(k)) == {"ordinal_recovery", "code_sha256"} + assert accepted.accepted_environment(c)["heldout_target_reads"] == 0 + checks = {"matching": replay(rows, recovery.UUID), + "mismatch": replay(rows, rows[0][1])} + assert not (recovery.ROOT / "started.json").exists() + recovery.ROOT.mkdir(exist_ok=False) + original.save(recovery.ROOT / "contract.json", c) + original.save(recovery.ROOT / "ordinal-cpu-replay.json", dict( + status="PASS", ruling="h-91e61871", old_terminal_sha256=recovery.OLD_TERMINAL, + contract_sha256=original.sha(recovery.ROOT / "contract.json"), + same_scientific_protocol_and_caps=True, hardware_inventory=source, + inconsistent_pci_order_rejected=True, advance_checks=checks, real_spawns=0, real_cuda_calls=0, + heldout_targets_read=0, independent_review="PENDING")) + print("PASS", original.sha(recovery.ROOT / "contract.json")) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("inventory", type=Path) + prepare(parser.parse_args().inventory) diff --git a/tools/track_b_gino_ordinal_recovery.py b/tools/track_b_gino_ordinal_recovery.py new file mode 100644 index 000000000..c4899859e --- /dev/null +++ b/tools/track_b_gino_ordinal_recovery.py @@ -0,0 +1,173 @@ +"""One labelled recovery for the historical Torch UUID-counting failure.""" + +import argparse +import csv +from copy import deepcopy +from pathlib import Path +import subprocess +import time +from types import FunctionType, SimpleNamespace + +import psutil + +import track_b_gino_import_checked_entry as accepted +import track_b_gino_measurement_gpu2 as previous +import track_b_gino_shapenet_entry as original +from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha + + +ROOT = previous.ROOT.with_name("track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19") +ENTRY = Path(__file__).resolve() +WORKER = ENTRY.with_name("track_b_gino_ordinal_recovery_worker.py") +UUID = previous.UUID +OLD_CONTRACT = "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f" +OLD_TERMINAL = "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e" + + +def expected_contract(): + c = read(checked(previous.ROOT / "contract.json", OLD_CONTRACT)) + stopped = read(checked(previous.ROOT / "complete.json", OLD_TERMINAL)) + if stopped["status"] != "STOPPED" or stopped["trigger"] != "WORKER_FAILED": + raise PermissionError("The original stopped GPU2 attempt is required") + c["ordinal_recovery"] = dict(handoff="h-91e61871", previous_terminal_sha256=OLD_TERMINAL, + previous_gpu_wall_seconds=stopped["cumulative_gpu_wall_seconds"]) + c["code_sha256"].update({str(p): sha(p) for p in (ENTRY, WORKER)}) + return c + + +def contract(path): + c = previous.checked_entry.contract(path) + if c != expected_contract(): + raise PermissionError("Only the fixed ordinal recovery of this measurement is authorized") + return c + + +def query_nvidia(options): + r = subprocess.run(["nvidia-smi", options, "--format=csv,noheader"], + check=True, capture_output=True, text=True, timeout=3) + return [[v.strip() for v in row] for row in csv.reader(r.stdout.splitlines()) if row] + + +def device_ordinal(rows, uuid): + ordered = sorted(rows, key=lambda row: row[2].lower()) + if len({row[2].lower() for row in ordered}) != len(ordered): + raise ValueError("Nonunique GPU PCI identities") + if [row[0] for row in ordered] != [str(i) for i in range(len(ordered))]: + raise ValueError("NVIDIA indices do not match CUDA PCI bus order") + match = [row for row in ordered if row[1] == uuid and "V100" in row[3]] + if len(match) != 1: + raise ValueError("Expected exactly one reviewed physical V100") + return match[0][0] + + +def dispatch_environment(uuid, env): + if env["CUDA_VISIBLE_DEVICES"] != uuid: + raise PermissionError("Supervisor did not select the exact physical UUID") + rows = query_nvidia("--query-gpu=index,uuid,pci.bus_id,name") + ordinal = device_ordinal(rows, uuid) + return dict(env, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=ordinal), rows + + +def monitor_identity(root, uuid): + pending = root / "worker-device-witness.json" + verified = root / "worker-device-verified.json" + if not pending.is_file() or verified.exists(): + return + witness = read(pending) + spawn = read(root / "actual-worker-spawn.json") + if (witness["process"] != spawn["process"] + or witness["physical_uuid_expected"] != uuid + or witness["ordinal"] != spawn["ordinal"] + or witness["device_count"] != 1 or "V100" not in witness["device_name"] + or witness["first_allocation_bytes"] != 1): + raise RuntimeError("WORKER_DEVICE_WITNESS_IDENTITY_DIFFERS") + pid = witness["process"]["pid"] + observed = sorted({card for card, owner in query_nvidia("--query-compute-apps=gpu_uuid,pid") + if int(owner) == pid}) + if not observed: + return + if observed != [uuid]: + out = root / "worker-device-mismatch.json" + if not out.exists(): + save(out, dict(process=witness["process"], expected_uuid=uuid, observed_uuids=observed, + scientific_worker_released=False)) + raise RuntimeError("WORKER_GPU_UUID_MISMATCH") + save(verified, dict(process=witness["process"], physical_uuid=uuid, + ordinal=witness["ordinal"], observed_uuids=observed)) + + +def dispatch_guard(root, path): + root = Path(root) + c = contract(path) + old_charge = c["ordinal_recovery"]["previous_gpu_wall_seconds"] + total = c["limits"]["gpu_wall_seconds"] + spawned = None + started_dispatch = None + + def runtime_contract(checked_path): + runtime = deepcopy(contract(checked_path)) + runtime["limits"]["gpu_wall_seconds"] = total - old_charge + return runtime + + def snapshot(uuid): + state = original.snapshot(uuid) + if spawned is not None: + receipt = root / "actual-worker-spawn.json" + if not receipt.exists(): + try: + identity = process_identity(spawned.pid) + except psutil.NoSuchProcess: + return state + save(receipt, dict(process=identity, ordinal=started_dispatch["ordinal"], physical_uuid=uuid)) + monitor_identity(root, uuid) + return state + + def route_supervise(command, directory, mode, limits, *, poll, gpu_uuid): + if gpu_uuid != UUID or Path(command[1]).name != "track_b_gino_shapenet_worker.py": + raise PermissionError("Unexpected original guard worker route") + command[1] = str(WORKER) + + def spawn(args, **kwargs): + nonlocal spawned, started_dispatch + environment, rows = dispatch_environment(gpu_uuid, kwargs["env"]) + ordinal = environment["CUDA_VISIBLE_DEVICES"] + started_dispatch = dict(ordinal=ordinal, order="PCI_BUS_ID", + physical_uuid=gpu_uuid, gpu_inventory=rows) + save(root / "ordinal-dispatch.json", started_dispatch) + if time.monotonic() >= read(root / "dispatch.json")["start_by_monotonic"]: + raise RuntimeError("ORDINAL_DISPATCH_DEADLINE") + kwargs["env"] = environment + spawned = subprocess.Popen(args, **kwargs) + return spawned + + ns = dict(original.supervise.__globals__, + subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT)) + supervise = FunctionType(original.supervise.__code__, ns, "supervise") + return supervise(command, directory, mode, limits, poll=poll, gpu_uuid=gpu_uuid) + + namespace = dict(original.dispatch_guard.__globals__, contract=runtime_contract, + snapshot=snapshot, supervise=route_supervise) + code = FunctionType(original.dispatch_guard.__code__, namespace, "dispatch_guard")(root, path) + record = read(root / "complete.json") + new_charge = record["cumulative_gpu_wall_seconds"] + save(root / "recovery-settlement.json", dict(status=record["status"], trigger=record["trigger"], + previous_gpu_wall_seconds=old_charge, recovery_gpu_wall_seconds=new_charge, + cumulative_measurement_gpu_wall_seconds=old_charge + new_charge, total_cap_gpu_wall_seconds=total, + independent_attempts=2, no_retry=True, within_total_cap=old_charge + new_charge <= total)) + return code + + +def authorized_child(root): + namespace = dict(accepted.authorized_child.__globals__, ROOT=ROOT, contract=contract) + return FunctionType(accepted.authorized_child.__code__, namespace, "authorized_child")(root) + + +def run(root): + namespace = dict(accepted.run.__globals__, ROOT=ROOT, contract=contract, dispatch_guard=dispatch_guard) + return FunctionType(accepted.run.__code__, namespace, "run")(root) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + raise SystemExit(run(parser.parse_args().root)) diff --git a/tools/track_b_gino_ordinal_recovery_worker.py b/tools/track_b_gino_ordinal_recovery_worker.py new file mode 100644 index 000000000..b71289100 --- /dev/null +++ b/tools/track_b_gino_ordinal_recovery_worker.py @@ -0,0 +1,62 @@ +"""Prove the worker's first allocation is on the reviewed physical V100.""" + +import argparse +import os +from pathlib import Path +import sys +import time +from types import FunctionType + +from track_b_gino_ordinal_recovery import UUID, authorized_child +from track_b_gino_paper_keyword_patch import extract_patched_paper_code +from track_b_gino_shapenet_entry import process_identity, read, save +from track_b_gino_shapenet_worker import work + + +def wait_for_verified_device(root, c): + import torch + + dispatch = read(root / "ordinal-dispatch.json") + ordinal = dispatch["ordinal"] + if (not ordinal.isdecimal() or os.environ.get("CUDA_VISIBLE_DEVICES") != ordinal + or os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID" or dispatch["physical_uuid"] != UUID + or torch.cuda.device_count() != 1 or torch.version.cuda != c["environment"]["torch_cuda"]): + raise PermissionError("Exactly one device must be visible through the reviewed PCI ordinal") + torch.cuda.set_device(0) + name = torch.cuda.get_device_name(0) + if "V100" not in name: + raise PermissionError("Expected one V100 before creating the witness allocation") + probe = torch.empty(1, dtype=torch.uint8, device="cuda:0") + current = process_identity(os.getpid()) + save(root / "worker-device-witness.json", dict(process=current, ordinal=ordinal, device_count=1, + device_name=name, physical_uuid_expected=UUID, first_allocation_bytes=probe.numel())) + deadline = read(root / "dispatch.json")["start_by_monotonic"] + verified = root / "worker-device-verified.json" + while time.monotonic() < deadline: + if verified.exists(): + record = read(verified) + if (record["process"] != current or record["ordinal"] != ordinal or record["physical_uuid"] != UUID + or record["observed_uuids"] != [UUID]): + raise PermissionError("Verified physical GPU witness differs") + del probe + return ordinal + time.sleep(.1) + raise RuntimeError("Physical GPU identity was not verified in the bounded admission window") + + +def run(root): + c = authorized_child(root) + if str(Path(sys.executable).absolute()) != c["environment"]["python"]: + raise PermissionError("The worker did not start with the accepted interpreter") + ordinal = wait_for_verified_device(root, c) + # In the unchanged worker this field is read only to check the visible-device + # environment string. The coordinator and admission receipts keep the UUID. + c = dict(c, gpu_uuid=ordinal) + namespace = dict(work.__globals__, extract_paper_code=extract_patched_paper_code) + FunctionType(work.__code__, namespace, work.__name__)(root, c) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path) + run(parser.parse_args().root.resolve())