From 9a4f8563ee140b214021244e252a8df32e0c3f5f Mon Sep 17 00:00:00 2001 From: NewMeltAssist Date: Thu, 19 Nov 2026 16:06:50 +0800 Subject: [PATCH] Bind formal three-seed GINO with CPU guard and custody replay --- .../contract.json | 834 ++++++++++++++++++ .../cpu-preparation.json | 48 + .../cpu-replay.log | 14 + .../complete.json | 38 + .../recovery-settlement.json | 11 + .../worker-records/progress.json | 9 + ...k-b-gino-formal-reproduction-2026-11-19.md | 19 + tests/test_track_b_gino_formal.py | 204 +++++ tools/track_b_gino_formal.py | 314 +++++++ tools/track_b_gino_formal_guard.py | 175 ++++ tools/track_b_gino_formal_worker.py | 143 +++ 11 files changed, 1809 insertions(+) create mode 100644 artifacts/track-b-gino-shapenet611-formal-2026-11-19/contract.json create mode 100644 artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-preparation.json create mode 100644 artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-replay.log create mode 100644 artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json create mode 100644 artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json create mode 100644 artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json create mode 100644 docs/track-b-gino-formal-reproduction-2026-11-19.md create mode 100644 tests/test_track_b_gino_formal.py create mode 100644 tools/track_b_gino_formal.py create mode 100644 tools/track_b_gino_formal_guard.py create mode 100644 tools/track_b_gino_formal_worker.py diff --git a/artifacts/track-b-gino-shapenet611-formal-2026-11-19/contract.json b/artifacts/track-b-gino-shapenet611-formal-2026-11-19/contract.json new file mode 100644 index 000000000..853450ff5 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-formal-2026-11-19/contract.json @@ -0,0 +1,834 @@ +{ + "mode": "formal-gino-reproduction", + "ruling": "h-f84547d7", + "retry": false, + "evaluate_other_models": false, + "limits": { + "cleanup_seconds": 20, + "idle_wait_seconds": 60, + "rss_bytes": 12884901888, + "recorded_vram_bytes": 15032385536, + "device_vram_bytes": 16106127360, + "disk_bytes": 2147483648, + "torch_threads": 1, + "max_processes": 1, + "cpu_seconds_per_gpu_second": 8 + }, + "protocol": { + "model": "NeurIPS 2023 supplement SDFFNOGNO decoder", + "seeds": [ + 0, + 1, + 2 + ], + "grid": [ + 64, + 64, + 64 + ], + "modes": [ + 24, + 24, + 24 + ], + "hidden_channels": 32, + "radius": 0.05, + "epochs_per_seed": 100, + "batch_size": 1, + "learning_rate": 0.001, + "bounds_expansion": 0.01, + "pressure_epsilon": 1e-06, + "optimizer": "Adam", + "weight_decay": 0.0001, + "lr_step_epochs": 50, + "lr_gamma": 0.5, + "pressure_training_transform": "scalar Gaussian on the 500 learning cases", + "metric": "per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)", + "report": "equal mean over 111 cases for each seed, then equal mean over the three seeds", + "acceptance_percent": [ + 6.62, + 7.62 + ], + "paper_reference_percent": 7.12, + "checkpoint_epoch": 100, + "heldout_access_after_all_checkpoints_frozen": true + }, + "environment": { + "python": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "python_version": "3.10", + "torch_cuda": "11.6", + "neuralop_commit": "fbdc7f872792b75b84545c2f374e7a881c72a718", + "neuralop_file_sha256": { + "utils.py": "84554eed45ac415ab52f79e456eab70ba204e7ab7539c2f2841955a9145220b4", + "models/__init__.py": "c67679f1f8f90e1ab4eb4c2aa916aeaedf1b175d795fed03b8d4f7210883ebae", + "models/tfno.py": "d74b5805b69a64bf33e3ff5186709c75bd431a56c9548f06fb9635a859d849e4", + "models/spectral_convolution.py": "ccecf80806d93e8f35713ac6ab2c3b61c11739f5fcb8c0ba7397b7d4eb5a5d45" + }, + "package_versions": { + "torch": "1.13.1+cu116", + "open3d": "0.17.0", + "numpy": "1.24.3", + "torch-scatter": "2.1.1+pt113cu116", + "torch-sparse": "0.6.17+pt113cu116", + "torch-geometric": "2.3.1", + "tensorly": "0.8.1", + "tensorly-torch": "0.4.0" + } + }, + "comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-measurement-2026-11-19/comparison-contract.json", + "comparison_contract_sha256": "503f6c8ab1217d66b44e0d127555cfad5da4cd8a17b336299095c3cfd13041d7", + "code_sha256": { + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8", + "/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_shapenet.py": "557ad56277d7e85c6b5cc358eb151632d1757f5883f429988d36022d0d407550", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_entry.py": "e86d955c36e291e6bbef67947de663b24c8d66c5d72d9d28e4193fb8a3051ee2", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py": "7211be211c29472fd9be35a6bcb1f56025de4a63f36852bfafa10df97c003ff3", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_entry.py": "199d833f40339feb0433611ba27302416a280452f8918715e6ef95cb93c38721", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_worker.py": "3acf742dd4549e9f5491e10af5bb0b4acf00e2f47e91753100b5998cfcc61852", + "/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_import_checked.py": "fef485dd0c233c76a336239d5f29cbf44ac0fb62a6a297dff34293b23a1b75fe", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_paper_keyword_patch.py": "c61e2ed7d099eb125386857bf4d7ccdbfc3af522f80edb4f01b450cee91c0b95", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2.py": "7d3ab2412e5208892cffcfc0e1ec363eb5db2be3e10bf60595356f71e82d8552", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py": "ba1fd5ec837e884b000ba640fceafe30ea40ccf708847733df6a17a8f3599680", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery.py": "1e1827498afab62e2f28efb5d450376f2dca4e8bf6f74114f2914ec771872c86", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py": "7a68c77e9c913ba9161832f513dbf8754ebc5c270c1e742fa5ae97a7b14b9120", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_formal.py": "15ea113a0657715eff495ce17dc63a17a3ee984133907f8a4d5a83f2cf8a01a2", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_guard.py": "9b4020b4754b46325c9aab1d15448eb86b21ecf0e5dfb527a51907982db2c0fb", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_worker.py": "978f1c7cfb622300b8814e14e4a02e5cbb9e2ea7f6e5fee080b986c7409ade2c" + }, + "paper_data": { + "path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-car-pressure-data.zip", + "sha256": "b5f9e2b407d54e034596384253fb58196933c46f6b5b586383508da54475f89d" + }, + "paper_supplement": { + "path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-neurips-supplement.zip", + "sha256": "48cdae80ac2781e27ac00e10a37124711d6bb305593559002bda0527d11a533d" + }, + "paper_code_zip_sha256": "347a6f097e2b9958d1ab87930fa1d6227b793a9d213e12418ce9a157e83dce04", + "split_receipt": { + "path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-split.json", + "sha256": "5a96c25c2ec24cfc1e2264d4ebd0879c658c0f168e22711b576bf303151b8601" + }, + "mapping_audit": { + "path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-mapping-audit.json", + "sha256": "055d008a02494acd29df36c41c8e68db931a027f5635e8f5f765a96a54c62289" + }, + "split": { + "train": [ + 1, + 2, + 4, + 5, + 6, + 7, + 8, + 10, + 12, + 13, + 17, + 18, + 21, + 22, + 23, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 34, + 35, + 39, + 40, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 54, + 55, + 56, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 67, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 83, + 84, + 85, + 86, + 87, + 88, + 90, + 91, + 92, + 94, + 95, + 96, + 97, + 100, + 101, + 102, + 105, + 106, + 107, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 133, + 134, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 165, + 166, + 170, + 172, + 173, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 186, + 190, + 191, + 192, + 193, + 195, + 196, + 198, + 199, + 200, + 201, + 202, + 203, + 205, + 207, + 210, + 211, + 212, + 213, + 214, + 215, + 217, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 241, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 251, + 252, + 253, + 255, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 266, + 267, + 268, + 269, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 285, + 286, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 304, + 305, + 306, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 327, + 328, + 329, + 331, + 332, + 333, + 334, + 335, + 337, + 338, + 339, + 340, + 341, + 344, + 345, + 347, + 348, + 349, + 350, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 360, + 362, + 364, + 365, + 366, + 367, + 369, + 371, + 372, + 373, + 374, + 375, + 376, + 378, + 379, + 380, + 381, + 384, + 385, + 389, + 392, + 393, + 397, + 398, + 399, + 401, + 402, + 403, + 404, + 405, + 407, + 408, + 410, + 412, + 413, + 414, + 415, + 417, + 418, + 419, + 420, + 422, + 424, + 425, + 427, + 430, + 431, + 433, + 435, + 436, + 437, + 439, + 440, + 443, + 444, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 459, + 460, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 472, + 473, + 474, + 475, + 476, + 478, + 479, + 480, + 482, + 483, + 486, + 487, + 488, + 490, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 501, + 502, + 503, + 504, + 505, + 507, + 508, + 509, + 511, + 512, + 513, + 514, + 515, + 516, + 518, + 519, + 521, + 522, + 523, + 524, + 525, + 527, + 529, + 530, + 532, + 533, + 536, + 538, + 539, + 540, + 542, + 543, + 545, + 547, + 548, + 549, + 550, + 551, + 552, + 553, + 554, + 555, + 560, + 561, + 562, + 564, + 565, + 566, + 567, + 568, + 569, + 572, + 573, + 574, + 576, + 577, + 579, + 581, + 582, + 583, + 584, + 587, + 588, + 589, + 591, + 593, + 594, + 595, + 596, + 597, + 598, + 600, + 602, + 604, + 608, + 610, + 611, + 612, + 613, + 615, + 616, + 617, + 618, + 620, + 621, + 622, + 623, + 625, + 626, + 627, + 628, + 629, + 630, + 631, + 632, + 633, + 634, + 635, + 636, + 638, + 639, + 640, + 641, + 642, + 643, + 644, + 645, + 646, + 647, + 648, + 649, + 651, + 652, + 654, + 655, + 656, + 657 + ], + "test": [ + 658, + 659, + 660, + 662, + 663, + 664, + 665, + 666, + 667, + 668, + 672, + 673, + 674, + 675, + 676, + 677, + 678, + 679, + 681, + 683, + 684, + 686, + 687, + 688, + 689, + 690, + 691, + 692, + 693, + 695, + 696, + 697, + 700, + 701, + 702, + 703, + 704, + 705, + 708, + 709, + 710, + 711, + 712, + 713, + 715, + 717, + 718, + 719, + 721, + 722, + 723, + 724, + 725, + 726, + 727, + 729, + 730, + 731, + 733, + 734, + 735, + 736, + 737, + 738, + 739, + 740, + 741, + 743, + 744, + 745, + 746, + 747, + 748, + 749, + 750, + 751, + 752, + 753, + 757, + 758, + 760, + 762, + 763, + 764, + 765, + 766, + 767, + 768, + 769, + 770, + 771, + 772, + 773, + 775, + 776, + 777, + 778, + 779, + 781, + 782, + 783, + 784, + 785, + 787, + 789, + 790, + 791, + 792, + 793, + 795, + 797 + ], + "raw_global_bounds": [ + [ + -0.90252799, + 0.0066688, + -2.85272002 + ], + [ + 0.90253001, + 1.95825005, + 2.86385012 + ] + ] + }, + "environment_ruling": "h-0d21f3bd", + "environment_acceptance": { + "path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-import-checked-2026-11-19/cpu-environment-acceptance.json", + "sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3" + }, + "paper_compatibility": { + "handoff": "h-b8508140", + "identity": "supplement code + 2-keyword patch", + "patch_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/two-keyword-patch.json", + "patch_sha256": "7307ba49f83bfc46c5decf803a993a41ed602d987de65714958e738d2f52f077", + "search_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/keyword-search.json", + "search_sha256": "77a897bf534a9a0e9e6a7c072b4f914168648eadc2af3493b70d63d0b715b3d0" + }, + "formal": { + "handoff": "h-4e4e680b", + "ruling_slack_ts": "1795074651.222599", + "devices": { + "0": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "1": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "2": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c" + }, + "device_pci_ordinals": { + "0": 2, + "1": 3, + "2": 0 + }, + "reservation": { + "handoff": "h-a429935e", + "timezone": "Asia/Hong_Kong", + "start_inclusive": "2026-11-19T00:00:00+08:00", + "end_exclusive": "2026-11-20T00:00:00+08:00", + "gpu3_single_task": true, + "historical_inventory_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json", + "historical_inventory_sha256": "da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8", + "runtime_idle_check_required": true + }, + "budget_per_seed_stage_gpu_seconds": { + "0": { + "fit": 13034, + "score": 300 + }, + "1": { + "fit": 13033, + "score": 300 + }, + "2": { + "fit": 13033, + "score": 300 + } + }, + "total_gpu_wall_seconds": 40000, + "timing": { + "record_paths": { + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json": "db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json": "851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json": "8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf" + }, + "status": "CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED", + "sizing_only": true, + "epoch_seconds": 102.93777855299413, + "train_sizing_gpu_seconds": 38601.6669573728, + "train_sizing_multiplier": 1.25, + "score_reserved_gpu_seconds": 900, + "prior_measurement_gpu_seconds": 222.80500172509346 + }, + "heldout_policy": { + "preparation_reads": 0, + "training_reads": 0, + "score_reads_per_seed": 111, + "all_epoch100_checkpoints_required": true, + "retry": false + } + } +} diff --git a/artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-preparation.json b/artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-preparation.json new file mode 100644 index 000000000..f1ecd1b5b --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-preparation.json @@ -0,0 +1,48 @@ +{ + "status": "PASS", + "contract_sha256": "bcb25413f1bd29679116ba320fee1bcc6d5d66ec074665d61c6910a65fb97959", + "cuda_visible_devices": "", + "actual_nvidia_queries": 0, + "training_starts": 0, + "scoring_starts": 0, + "actual_heldout_reads": 0, + "independent_review": "PENDING", + "guard_replay": 9, + "cpu_replay_log_sha256": "c0d40741c701de9be3699fd66f82473c8ffdfe9df25c29c95f4fe2ac76689369", + "read_only_collection": { + "seeds": { + "0": { + "fit": { + "status": "NOT_STARTED" + }, + "score": { + "status": "NOT_STARTED" + } + }, + "1": { + "fit": { + "status": "NOT_STARTED" + }, + "score": { + "status": "NOT_STARTED" + } + }, + "2": { + "fit": { + "status": "NOT_STARTED" + }, + "score": { + "status": "NOT_STARTED" + } + } + }, + "mean_percent": null, + "within_registered_range": null, + "reproduction_identity": "supplement code + 2-keyword patch", + "prior_measurement_gpu_seconds": 222.80500172509346, + "settled_formal_gpu_seconds": 0, + "settled_combined_gpu_seconds": 222.80500172509346, + "unsettled_formal_bookings_gpu_seconds": 0, + "formal_gpu_cap_seconds": 40000 + } +} diff --git a/artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-replay.log b/artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-replay.log new file mode 100644 index 000000000..3e168744e --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-replay.log @@ -0,0 +1,14 @@ +test_card_reservation_expiry_and_total_budget (test_track_b_gino_formal.FormalBinding) ... ok +test_inclusive_gate_and_all_three_seeds_required (test_track_b_gino_formal.FormalBinding) ... ok +test_original_split_score_keeps_order_and_rejects_nan (test_track_b_gino_formal.FormalBinding) ... ok +test_score_opens_one_111_target_pass_and_rejects_a_second (test_track_b_gino_formal.FormalBinding) ... ok +test_scoring_needs_all_checkpoints_before_access (test_track_b_gino_formal.FormalBinding) ... ok +test_gpu_list_before_worker_exit (test_track_b_gino_formal.GuardReplay) ... ok +test_real_foreign_identity_has_exact_stop_receipt (test_track_b_gino_formal.GuardReplay) ... ok +test_reused_worker_pid_is_foreign (test_track_b_gino_formal.GuardReplay) ... ok +test_worker_already_exited_before_gpu_list (test_track_b_gino_formal.GuardReplay) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 0.656s + +OK diff --git a/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json new file mode 100644 index 000000000..1b4889a65 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json @@ -0,0 +1,38 @@ +{ + "status": "STOPPED", + "trigger": "FOREIGN_GPU_OCCUPANT", + "contract_sha256": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a", + "result_sha256": null, + "worker": { + "status": "PASS", + "command": [ + "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19" + ], + "returncode": 0, + "stop_reason": null, + "cpu_seconds": 1221.067037, + "wall_seconds": 178.46390844404232, + "peak_tree_rss_bytes": 4780380160, + "peak_process_count": 1, + "output_disk_bytes_before_receipt": 131072, + "max_poll_gap_seconds": 0.8362329140072688, + "poll_target_seconds": 0.2, + "reaped_orphan_cpu_seconds": 0.0, + "supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure", + "gpu_visible": true, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "retry": false, + "monitor_errors": 0, + "last_monitor_error": null + }, + "booked_idle_seconds": 20.519942376064137, + "cumulative_gpu_wall_seconds": 199.1106877680868, + "gpu_wall_cap_seconds": 3576.3056860429933, + "recorded_peaks": { + "peak_allocated_bytes": 2953057792, + "peak_reserved_bytes": 3980394496 + }, + "no_retry": true +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json new file mode 100644 index 000000000..a73608ee7 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json @@ -0,0 +1,11 @@ +{ + "status": "STOPPED", + "trigger": "FOREIGN_GPU_OCCUPANT", + "previous_gpu_wall_seconds": 23.69431395700667, + "recovery_gpu_wall_seconds": 199.1106877680868, + "cumulative_measurement_gpu_wall_seconds": 222.80500172509346, + "total_cap_gpu_wall_seconds": 3600, + "independent_attempts": 2, + "no_retry": true, + "within_total_cap": true +} diff --git a/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json new file mode 100644 index 000000000..6e25eaeb3 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json @@ -0,0 +1,9 @@ +{ + "stage": "EPOCH_COMPLETE", + "seed": 0, + "epoch": 1, + "examples": 500, + "seconds": 102.93777855299413, + "peak_allocated_bytes": 2953057792, + "peak_reserved_bytes": 3980394496 +} diff --git a/docs/track-b-gino-formal-reproduction-2026-11-19.md b/docs/track-b-gino-formal-reproduction-2026-11-19.md new file mode 100644 index 000000000..5e1c204a4 --- /dev/null +++ b/docs/track-b-gino-formal-reproduction-2026-11-19.md @@ -0,0 +1,19 @@ +# Track B GINO 三种子复现联合预审 + +本次交给 MeltAssist 一次联合预审,范围包括占卡守卫、训练入口和评分入口。当前只准备和回放。任何真实派发均需同一绑定的独立 PASS 和阶段授权。 [Claude 裁定](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795074651222599) + +## 卡和预算 + +依据 h-a429935e,种子 0、1、2 分别绑定物理 GPU2、GPU3、GPU0。GPU3 只允许单任务。香港时间 2026-11-19 当日预留,满额预约须能在当地午夜前结束。实际派发仍须通过三次空闲观察和临派复核。物理 UUID、PCI 顺序和真实 worker 首次一字节分配均须匹配,GPU1 不参与。 [组合合同](../artifacts/track-b-gino-shapenet611-formal-2026-11-19/contract.json) + +GPU2 恢复测量的 worker 完整跑了 500 样本、1 轮,用时 102.93777855299413 秒。该计时受抢占影响且未经守卫认证,只用于预算测算。原守卫仍是 `STOPPED / FOREIGN_GPU_OCCUPANT`,缺少占用者身份,历史根因未确定。两次旧测量累计 222.80500172509346 GPU 秒独立记账一次。正式预算为 40,000 GPU 秒,三种子 300 轮乘 1.25 得 38,601.667 GPU 秒。训练、入场和清理预留为 13,034 / 13,033 / 13,033 GPU 秒,每个评分另预留 300 GPU 秒。评分预留是预算假设,尚无实际评分计时。 [历史终止](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json) · [历史计时](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json) · [历史结算](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json) + +## 守卫、训练和评分 + +派发时记录真实 worker PID 和出生时间。GPU 占用采样前后都观察其进程树,保留已认领的进程身份,避免已退出 worker 被陈旧 GPU 记录误判。PID 复用时以不同出生时间区分。确认外部占用时记录采样时间、卡 UUID、占用 PID 和名称,只停止本次 worker 进程组。运行中的显存与结果中记录的显存峰值均纳入守卫。 [守卫](../tools/track_b_gino_formal_guard.py) + +沿用原始 500/111 划分、已验收的解释器、原加载器、模型、优化步骤和指标。复现身份为 `supplement code + 2-keyword patch`。每个种子从头训练 100 轮,冻结模型与只从学习集拟合的压力归一化参数。每次评分须先核对三份已通过守卫的第 100 轮检查点。然后该种子只开一次留出目标记录,按原序逐读 111 个目标并验证逐例哈希。三种子的结果全部列出。仅在三者均有效时,对各自的 111 件平均相对 L2 等权汇总,区间两端 6.62% 和 7.62% 均计入通过。 [训练与评分](../tools/track_b_gino_formal_worker.py) · [联合入口与结算](../tools/track_b_gino_formal.py) + +## CPU 回放 + +生产守卫配合真实 CPU 子进程回放了两种退出时序。GPU 列表先采到 worker 随后其退出、以及 worker 先退出而 GPU 列表仍返回旧 PID,均通过。另用真实活进程身份模拟外部占用,验证 STOP 回执的 PID、名称、采样时间和仅停止本方 worker。另验 PID 复用、日界和预算、汇总区间、留出顺序、三检查点屏障及只开一次 111 项的合成评分。首次回放的两项退出测试被测试夹具的 256 MiB RSS 帽拦截,夹具放宽到 1 GiB 后九项全部通过。正式 RSS 帽为 12 GiB。回放未访问目标归档,未执行真实训练或评分。 [CPU 回放日志](../artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-replay.log) · [回放用例](../tests/test_track_b_gino_formal.py) diff --git a/tests/test_track_b_gino_formal.py b/tests/test_track_b_gino_formal.py new file mode 100644 index 000000000..ebb6190f6 --- /dev/null +++ b/tests/test_track_b_gino_formal.py @@ -0,0 +1,204 @@ +"""CPU replay of the production GINO guard and formal custody gates.""" + +from pathlib import Path +import os +import sys +from tempfile import TemporaryDirectory +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import track_b_gino_formal as formal +import track_b_gino_formal_guard as gpu_guard +import track_b_gino_formal_worker as worker +from track_b_gino_shapenet_entry import read, save, sha + + +class GuardReplay(unittest.TestCase): + def replay(self, ordering): + uuid = 'GPU-fixture' + limit = dict(disk_bytes=2**20, idle_wait_seconds=10, cleanup_seconds=2, cpu_seconds=10, + rss_bytes=2**30, max_processes=1, device_vram_bytes=2**30, + recorded_vram_bytes=2**30) + with TemporaryDirectory() as folder: + root = Path(folder) + result = root / 'worker-records/result.json' + result.parent.mkdir() + # This subprocess has no GPU code. It waits until the replay lets + # it finish or until the production guard stops its process group. + command = [sys.executable, '-c', + "from pathlib import Path\nimport time,sys\n" + "root=Path(sys.argv[1])\n" + "while not (root/'exit').exists(): time.sleep(.01)\n" + "(root/'worker-records/result.json').write_text(" + "'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder] + observed = [] + stopped = [] + real_stop = gpu_guard.stop_group + real_tree = gpu_guard.process_tree + + def stop_owned(pid): + stopped.append(pid) + real_stop(pid) + + def exit_and_wait(): + (root / 'exit').touch() + pid = read(root / 'actual-worker-spawn.json')['process']['pid'] + by = time.monotonic() + 4 + while gpu_guard.current_identity(pid) is not None and time.monotonic() < by: + time.sleep(.01) + if gpu_guard.current_identity(pid) is not None: + raise RuntimeError('fixture worker did not exit and get reaped') + observed.append('worker-reaped') + + def tree(identity): + if (root / 'actual-worker-spawn.json').exists() and ( + ordering == 'tree-first' and 'worker-reaped' not in observed): + exit_and_wait() + observed.append('tree') + return real_tree(identity) + + def snapshot(card): + if card != uuid: + raise AssertionError(card) + spawn = root / 'actual-worker-spawn.json' + if not spawn.exists(): + return dict(idle=True, occupants=[], unix=10., memory_MiB=0) + observed.append('gpu-sample') + pid = os.getpid() if ordering == 'foreign' else read(spawn)['process']['pid'] + sample = dict(idle=False, unix=42.125, memory_MiB=4, + occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')]) + if ordering == 'gpu-first': + exit_and_wait() + return sample + + with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot), + patch.object(gpu_guard, 'process_tree', side_effect=tree), + patch.object(gpu_guard, 'stop_group', side_effect=stop_owned), + patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}), + patch.object(gpu_guard, 'dispatch_environment', + side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))): + began = time.monotonic() + outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15, + ticket=dict(ordinal='', guard_started_monotonic=began), validate_result=lambda: sha(result)) + spawn = read(root / 'actual-worker-spawn.json')['process'] + if ordering == 'foreign': + self.assertEqual(outcome['status'], 'STOPPED') + self.assertEqual(outcome['trigger'], 'FOREIGN_GPU_OCCUPANT') + self.assertIsNone(outcome['result_sha256']) + row = read(root / 'foreign-occupant.json') + self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid)) + self.assertEqual(row['occupants'][0]['pid'], os.getpid()) + self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python') + self.assertEqual(stopped, [spawn['pid']]) + else: + self.assertEqual(outcome['status'], 'COMPLETE', outcome) + self.assertFalse((root / 'foreign-occupant.json').exists()) + self.assertEqual(stopped, []) + self.assertEqual(outcome['result_sha256'], sha(result)) + return observed + + def test_gpu_list_before_worker_exit(self): + order = self.replay('gpu-first') + self.assertLess(order.index('gpu-sample'), order.index('worker-reaped')) + + def test_worker_already_exited_before_gpu_list(self): + order = self.replay('tree-first') + self.assertLess(order.index('worker-reaped'), order.index('gpu-sample')) + + def test_real_foreign_identity_has_exact_stop_receipt(self): + self.replay('foreign') + + def test_reused_worker_pid_is_foreign(self): + with TemporaryDirectory() as folder: + root = Path(folder) + original = dict(pid=12, created=1.) + current = dict(pid=12, created=2.) + sample = dict(unix=42., occupants=[dict(pid=12, process_name='new-process')], memory_MiB=4) + monitor = gpu_guard.OccupancyMonitor(root, 'fixture', original, 2**30) + with (patch.object(gpu_guard, 'snapshot', return_value=sample), + patch.object(gpu_guard, 'current_identity', return_value=current)): + with self.assertRaisesRegex(RuntimeError, 'FOREIGN_GPU_OCCUPANT'): + monitor.check() + row = read(root / 'foreign-occupant.json') + self.assertEqual(row['occupants'][0]['observed_process_identity'], current) + + +class FormalBinding(unittest.TestCase): + def test_inclusive_gate_and_all_three_seeds_required(self): + for endpoint in (6.62, 7.62): + self.assertEqual(formal.pool([endpoint] * 3, [6.62, 7.62]), (endpoint, True)) + self.assertEqual(formal.pool([7.12, 7.12, None], [6.62, 7.62]), (None, None)) + self.assertFalse(formal.pool([7.63] * 3, [6.62, 7.62])[1]) + + def test_original_split_score_keeps_order_and_rejects_nan(self): + ids = list(range(111)) + complete = [dict(case=i, relative_l2=.0712) for i in ids] + self.assertAlmostEqual(formal.score_mean(complete, ids), 7.12) + with self.assertRaises(ValueError): + formal.score_mean(complete[::-1], ids) + complete[0]['relative_l2'] = float('nan') + with self.assertRaises(ValueError): + formal.score_mean(complete, ids) + + def test_scoring_needs_all_checkpoints_before_access(self): + with TemporaryDirectory() as folder: + root = Path(folder) + (root / 'checkpoints-frozen.json').write_text('{}') + with patch.object(formal, 'all_checkpoints', side_effect=PermissionError('missing fit')): + with self.assertRaisesRegex(PermissionError, 'missing fit'): + worker.score({}, 0, None, root, None) + self.assertFalse((root / 'heldout-target-access.jsonl').exists()) + + def test_score_opens_one_111_target_pass_and_rejects_a_second(self): + import torch + + with TemporaryDirectory() as folder: + root = Path(folder) + path = root / 'checkpoint.pt' + torch.save(dict(state={}, pressure_mean=torch.tensor([2.]), pressure_std=torch.tensor([3.])), path) + roster = {'0': dict(checkpoint=dict(path=str(path), sha256=sha(path)))} + save(root / 'checkpoints-frozen.json', roster) + ids = list(range(111)) + reads = [] + + def load(ids_argument, role, log): + self.assertEqual((ids_argument, role), (ids, 'heldout')) + for case in ids_argument: + reads.append(case) + log.write(f'{case}\n') + return object() + + class Normalizer: + pass + + data = SimpleNamespace(test_ids=ids, normalizer_class=Normalizer, data=load) + torch_cpu_fixture = SimpleNamespace(load=torch.load, isfinite=torch.isfinite, + cuda=SimpleNamespace(synchronize=lambda: None)) + model = SimpleNamespace(to=lambda _: model, load_state_dict=lambda _state, **kw: None) + rows = [dict(case=i, relative_l2=.0712) for i in ids] + with (patch.object(formal, 'all_checkpoints', return_value=roster), + patch.object(worker, 'paper_model', return_value=model), + patch.object(worker, 'case_scores', return_value=rows)): + result = worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture) + with self.assertRaises(FileExistsError): + worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture) + self.assertEqual(reads, ids) + self.assertEqual(result['target_reads'], 111) + self.assertAlmostEqual(result['mean_percent'], 7.12) + + def test_card_reservation_expiry_and_total_budget(self): + binding = formal.expected_contract()['formal'] + budgets = binding['budget_per_seed_stage_gpu_seconds'] + self.assertEqual(sum(sum(b.values()) for b in budgets.values()), 40000) + self.assertEqual(binding['device_pci_ordinals'], {'0': 2, '1': 3, '2': 0}) + latest = formal.RESERVATION_END.timestamp() - 300 + formal.check_reservation(latest - 1, 300) + for when in (latest, formal.RESERVATION_END.timestamp(), formal.RESERVATION_START.timestamp() - 1): + with self.assertRaises(PermissionError): + formal.check_reservation(when, 300) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/track_b_gino_formal.py b/tools/track_b_gino_formal.py new file mode 100644 index 000000000..4bc33cf72 --- /dev/null +++ b/tools/track_b_gino_formal.py @@ -0,0 +1,314 @@ +"""Exact review, custody and resource binding for formal three-seed GINO.""" + +import argparse +from datetime import datetime +import fcntl +import json +import math +import os +from pathlib import Path +import signal +import time +from zoneinfo import ZoneInfo + +from phase2_gate_outcome_review import validate_review +from track_b_gino_formal_guard import guard +from track_b_gino_import_checked_entry import accepted_environment +from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT +from track_b_gino_shapenet_entry import BASE, checked, read, save, sha + + +ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-2026-11-19') +SOURCES = Path(__file__).resolve().parent +OLD_FILES = { + 'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a', + 'complete.json': 'db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d', + 'recovery-settlement.json': '851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec', + 'worker-records/progress.json': '8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf', +} +DEVICES = { + '0': 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', + '1': 'GPU-8b71b032-1b75-2891-935c-ce106a53034a', + '2': 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c', +} +ORDINALS = {'0': 2, '1': 3, '2': 0} +INVENTORY = BASE / 'artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json' +INVENTORY_SHA = 'da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8' +TZ = ZoneInfo('Asia/Hong_Kong') +RESERVATION_START = datetime(2026, 11, 19, tzinfo=TZ) +RESERVATION_END = datetime(2026, 11, 20, tzinfo=TZ) +NEW_SOURCES = ('track_b_gino_formal.py', 'track_b_gino_formal_guard.py', + 'track_b_gino_formal_worker.py') + + +def expected_contract(): + old = {name: read(checked(MEASUREMENT / name, digest)) for name, digest in OLD_FILES.items()} + c = old['contract.json'] + stop, epoch = old['complete.json'], old['worker-records/progress.json'] + account = old['recovery-settlement.json'] + if (stop['status'], stop['trigger'], stop['result_sha256']) != ('STOPPED', 'FOREIGN_GPU_OCCUPANT', None): + raise ValueError('Historical guard STOP differs') + if (epoch['stage'], epoch['seed'], epoch['epoch'], epoch['examples']) != ('EPOCH_COMPLETE', 0, 1, 500): + raise ValueError('Only the completed 500-example epoch can size this reproduction') + seconds = epoch['seconds'] + inventory = read(checked(INVENTORY, INVENTORY_SHA))['inventory'] + physical = {(int(index), uuid) for index, uuid, _, name in inventory if 'V100' in name} + if not {(ORDINALS[seed], uuid) for seed, uuid in DEVICES.items()} <= physical: + raise ValueError('Reviewed physical V100 inventory differs') + budgets = {seed: {'fit': 13033 + (seed == '0'), 'score': 300} for seed in DEVICES} + fit_reference = 100 * seconds * 1.25 + if (not math.isfinite(fit_reference) or fit_reference <= 0 + or min(b['fit'] for b in budgets.values()) <= fit_reference + or sum(b['fit'] + b['score'] for b in budgets.values()) != 40000): + raise ValueError('Formal booking must cover the sizing reference and score reserve') + c['mode'] = 'formal-gino-reproduction' + for key in ('gpu_uuid', 'measurement_allowed_heldout_labels', 'gpu_ruling', 'ordinal_recovery'): + del c[key] + c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES}) + c['formal'] = dict( + handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES, + device_pci_ordinals=ORDINALS, + reservation=dict(handoff='h-a429935e', timezone=TZ.key, + start_inclusive=RESERVATION_START.isoformat(), + end_exclusive=RESERVATION_END.isoformat(), gpu3_single_task=True, + historical_inventory_path=str(INVENTORY), historical_inventory_sha256=INVENTORY_SHA, + runtime_idle_check_required=True), + budget_per_seed_stage_gpu_seconds=budgets, total_gpu_wall_seconds=40000, + timing=dict(record_paths={str(MEASUREMENT / k): v for k, v in OLD_FILES.items()}, + status='CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED', sizing_only=True, + epoch_seconds=seconds, train_sizing_gpu_seconds=300 * seconds * 1.25, + train_sizing_multiplier=1.25, score_reserved_gpu_seconds=900, + prior_measurement_gpu_seconds=account['cumulative_measurement_gpu_wall_seconds']), + heldout_policy=dict(preparation_reads=0, training_reads=0, score_reads_per_seed=111, + all_epoch100_checkpoints_required=True, retry=False), + ) + del c['limits']['gpu_wall_seconds'] + c['limits']['cleanup_seconds'] = 20 + del c['limits']['cpu_seconds'] + c['limits']['cpu_seconds_per_gpu_second'] = 8 + return c + + +def contract(): + c = read(ROOT / 'contract.json') + if c != expected_contract(): + raise PermissionError('The sole formal contract or its implementation changed') + for path, digest in c['code_sha256'].items(): + checked(path, digest) + for name in ('paper_supplement', 'split_receipt', 'mapping_audit'): + pin = c[name] + checked(pin['path'], pin['sha256']) + # Archive members are opened only in the scoped workers, which verify each + # member before using it. Preparation does not hash the archive payload. + comparison = read(checked(c['comparison_contract'], c['comparison_contract_sha256'])) + split = read(c['split_receipt']['path']) + train, test = split['train'], split['test'] + if (c['protocol'] != comparison['protocol'] or len(train) != 500 or len(test) != 111 + or len(set(train) | set(test)) != 611 + or any(c['split'][k] != split[k] for k in ('train', 'test', 'raw_global_bounds')) + or c['protocol']['epochs_per_seed'] != 100 or c['protocol']['seeds'] != [0, 1, 2] + or c['protocol']['acceptance_percent'] != [6.62, 7.62] + or c['paper_compatibility']['identity'] != 'supplement code + 2-keyword patch' + or not Path(c['paper_data']['path']).is_file()): + raise ValueError('Original GINO comparison, split or pinned data path differs') + accepted_environment(c) + return c + + +def authorize(c): + validate_review(ROOT) + authorization = read(ROOT / 'stage-authorization.json') + required = dict(contract_sha256=sha(ROOT / 'contract.json'), + implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'), + decision='APPROVED', mode=c['mode'], reserved_devices=DEVICES, + reservation_end_exclusive=RESERVATION_END.isoformat()) + if any(authorization.get(k) != v for k, v in required.items()): + raise PermissionError('Exact execution and card-reservation authorization required') + + +def check_reservation(unix, duration): + if (unix < RESERVATION_START.timestamp() + or unix + duration >= RESERVATION_END.timestamp()): + raise PermissionError('The entire booking must fit the November 19 Hong Kong reservation') + + +def run_root(seed, phase): + return ROOT / f'seed-{seed}' / phase + + +def epoch100(root, seed, c): + progress = read(root / 'worker-records/progress.json') + if (progress['stage'], progress['seed'], progress['epoch'], progress['examples']) != ( + 'EPOCH_COMPLETE', seed, c['protocol']['epochs_per_seed'], 500): + raise ValueError('No completed 100th full training epoch for this seed') + return progress + + +def checkpoint(seed, result): + path = run_root(seed, 'fit') / 'epoch-100.pt' + if result['checkpoint']['path'] != str(path): + raise ValueError('Checkpoint path differs from this seed') + return checked(path, result['checkpoint']['sha256']) + + +def score_mean(rows, test_ids): + if [r['case'] for r in rows] != test_ids: + raise ValueError('Scoring requires the original 111-case order') + values = [r['relative_l2'] for r in rows] + if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values): + raise ValueError('Incomplete or invalid relative L2') + return 100 * (math.fsum(values) / len(values)) + + +def validate_result(c, seed, phase): + root = run_root(seed, phase) + path = root / 'worker-records/result.json' + result = read(path) + spawned = read(root / 'actual-worker-spawn.json') + device = DEVICES[str(seed)] + ordinal = str(ORDINALS[str(seed)]) + if (spawned['physical_uuid'], spawned['ordinal']) != (device, ordinal) or ( + read(root / 'worker-device-verified.json') != dict( + process=spawned['process'], physical_uuid=device, ordinal=ordinal, observed_uuids=[device])): + raise ValueError('Worker has no matching physical-device witness') + if (result['phase'], result['seed'], result['status']) != (phase, seed, 'COMPLETE'): + raise ValueError('Worker result identity differs') + if phase == 'fit': + epoch100(root, seed, c) + checkpoint(seed, result) + if (result['target_reads'], result['epochs'], result['training_examples']) != (0, 100, 500): + raise ValueError('Fit must complete all 100 epochs with zero held-out reads') + else: + fit_terminal = read(run_root(seed, 'fit') / 'complete.json') + fit_path = run_root(seed, 'fit') / 'worker-records/result.json' + original = read(checked(fit_path, fit_terminal['result_sha256'])) + access = root / 'heldout-target-access.jsonl' + mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']} + actual = [json.loads(line) for line in access.read_text().splitlines()] + expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']] + if (actual != expected or result['target_reads'] != 111 + or result['target_access_sha256'] != sha(access) + or result['checkpoints_manifest_sha256'] != sha(root / 'checkpoints-frozen.json') + or read(root / 'checkpoints-frozen.json') != all_checkpoints(c) + or result['checkpoint_sha256'] != original['checkpoint']['sha256'] + or score_mean(result['cases'], c['split']['test']) != result['mean_percent']): + raise ValueError('Score or held-out target audit differs') + return sha(path) + + +def accepted_terminal(c, seed, phase): + root = run_root(seed, phase) + record = read(root / 'complete.json') + if (record['status'], record['seed'], record['phase'], record['contract_sha256'], record['physical_uuid']) != ( + 'COMPLETE', seed, phase, sha(ROOT / 'contract.json'), DEVICES[str(seed)]): + raise PermissionError(f'Seed {seed} {phase} has no accepted terminal result') + booked = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase] + if (record['booked_gpu_seconds'] != booked or record['charged_gpu_seconds'] > booked + or record['result_sha256'] != validate_result(c, seed, phase)): + raise ValueError('Terminal result bytes or resource accounting differ') + return record + + +def all_checkpoints(c): + checkpoints = {} + for seed in c['protocol']['seeds']: + accepted_terminal(c, seed, 'fit') + root = run_root(seed, 'fit') + checkpoints[str(seed)] = dict( + result_sha256=sha(root / 'worker-records/result.json'), + terminal_sha256=sha(root / 'complete.json'), + checkpoint=read(root / 'worker-records/result.json')['checkpoint']) + return checkpoints + + +def dispatch(c, seed, phase): + began = time.monotonic() + unix = time.time() + authorize(c) + budget = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase] + check_reservation(unix, budget) + seed_root = run_root(seed, phase).parent + seed_root.mkdir(exist_ok=True) + with (seed_root / 'execution.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + roster = all_checkpoints(c) if phase == 'score' else None + root = run_root(seed, phase) + root.mkdir(exist_ok=False) + if roster is not None: + save(root / 'checkpoints-frozen.json', roster) + ticket = dict(seed=seed, phase=phase, ordinal=str(ORDINALS[str(seed)]), + booked_gpu_seconds=budget, unix=unix, + contract_sha256=sha(ROOT / 'contract.json'), + guard_started_monotonic=began, booked_deadline_monotonic=began + budget) + save(root / 'started.json', ticket) + command = [c['environment']['python'], str(SOURCES / 'track_b_gino_formal_worker.py'), + str(seed), phase] + + def request_stop(signum, _frame): + raise KeyboardInterrupt(signal.Signals(signum).name) + + previous = signal.signal(signal.SIGTERM, request_stop) + try: + limits = dict(c['limits'], cpu_seconds=c['limits']['cpu_seconds_per_gpu_second'] * budget) + result = guard(root, command, uuid=DEVICES[str(seed)], limits=limits, + deadline=began + budget, ticket=ticket, + validate_result=lambda: validate_result(c, seed, phase)) + finally: + signal.signal(signal.SIGTERM, previous) + return 0 if result['status'] == 'COMPLETE' else 1 + + +def pool(means, limits): + if len(means) != 3 or any(v is None for v in means): + return None, None + mean = math.fsum(means) / 3 + return mean, limits[0] <= mean <= limits[1] + + +def collect(c): + seeds = {} + accounted = unsettled = 0 + for seed in c['protocol']['seeds']: + rows = {} + for phase in ('fit', 'score'): + root = run_root(seed, phase) + terminal = root / 'complete.json' + if terminal.exists(): + record = read(terminal) + if record['status'] == 'COMPLETE': + accepted_terminal(c, seed, phase) + accounted += record['charged_gpu_seconds'] + rows[phase] = dict(status=record['status'], trigger=record['trigger'], + terminal_sha256=sha(terminal)) + if phase == 'score' and record['status'] == 'COMPLETE': + rows['mean_percent'] = read(root / 'worker-records/result.json')['mean_percent'] + elif (root / 'started.json').exists(): + rows[phase] = dict(status='NO_TERMINAL_RECEIPT') + unsettled += c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase] + else: + rows[phase] = dict(status='NOT_STARTED') + seeds[str(seed)] = rows + mean, verdict = pool([row.get('mean_percent') for row in seeds.values()], + c['protocol']['acceptance_percent']) + prior = c['formal']['timing']['prior_measurement_gpu_seconds'] + return dict(seeds=seeds, mean_percent=mean, within_registered_range=verdict, + reproduction_identity=c['paper_compatibility']['identity'], + prior_measurement_gpu_seconds=prior, settled_formal_gpu_seconds=accounted, + settled_combined_gpu_seconds=accounted + prior, + unsettled_formal_bookings_gpu_seconds=unsettled, formal_gpu_cap_seconds=40000) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('fit', 'score', 'collect')) + parser.add_argument('--seed', type=int, choices=(0, 1, 2)) + args = parser.parse_args() + c = contract() + if args.action == 'collect': + if args.seed is not None: + parser.error('collect reports all three seeds') + print(json.dumps(collect(c), indent=2, allow_nan=False)) + else: + if args.seed is None: + parser.error('--seed is required for fit or score') + raise SystemExit(dispatch(c, args.seed, args.action)) diff --git a/tools/track_b_gino_formal_guard.py b/tools/track_b_gino_formal_guard.py new file mode 100644 index 000000000..f9cda06e5 --- /dev/null +++ b/tools/track_b_gino_formal_guard.py @@ -0,0 +1,175 @@ +"""Supervise a formal GINO fit or score on one physical V100.""" + +import os +import shutil +import subprocess +import threading +import time +from types import FunctionType, SimpleNamespace + +import psutil + +from s2_gpu_idle_gate import snapshot, wait_idle +from s2_main_memory_guard import PeakRecords +from s2_recovery_gpu_supervisor import stop_group, supervise +from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity +from track_b_gino_shapenet_entry import process_identity, save + + +def current_identity(pid): + try: + return process_identity(pid) + except psutil.NoSuchProcess: + return None + + +def process_tree(worker): + if current_identity(worker['pid']) != worker: + return [] + try: + children = psutil.Process(worker['pid']).children(recursive=True) + except psutil.NoSuchProcess: + return [] + result = [] + for child in children: + try: + result.append(dict(pid=child.pid, created=child.create_time())) + except psutil.NoSuchProcess: + continue + return result + + +class OccupancyMonitor: + def __init__(self, root, uuid, worker, device_vram_bytes): + self.root = root + self.uuid = uuid + self.worker = worker + self.known = {worker['pid']: worker} + self.device_vram_bytes = device_vram_bytes + self.peak_device_bytes = 0 + + def check(self): + before = process_tree(self.worker) + state = snapshot(self.uuid) + after = process_tree(self.worker) + for identity in before + after: + if identity['pid'] != self.worker['pid']: + self.known[identity['pid']] = identity + foreign = [] + for row in state['occupants']: + pid = row['pid'] + current = current_identity(pid) + if pid not in self.known or (current is not None and current != self.known[pid]): + foreign.append(dict(row, observed_process_identity=current)) + if foreign: + save(self.root / 'foreign-occupant.json', dict( + trigger='FOREIGN_GPU_OCCUPANT', unix=state['unix'], physical_uuid=self.uuid, + occupants=foreign, own_spawn=self.worker, tree_before=before, tree_after=after)) + raise RuntimeError('FOREIGN_GPU_OCCUPANT') + self.peak_device_bytes = max(self.peak_device_bytes, state['memory_MiB'] * 2**20) + if self.peak_device_bytes > self.device_vram_bytes: + raise RuntimeError('DEVICE_VRAM_LIMIT') + + +def watch(root, uuid, worker, limits, done, tracker, state): + occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes']) + while not done.is_set(): + try: + occupancy.check() + state['peak_device_bytes'] = occupancy.peak_device_bytes + tracker.scan(root / 'worker-records') + if max(tracker.maxima.values()) > limits['recorded_vram_bytes']: + raise RuntimeError('RECORDED_VRAM_LIMIT') + monitor_identity(root, uuid) + except Exception as exc: + state['trigger'] = str(exc) or type(exc).__name__ + # Signal only the recorded worker group while its leader has the same identity. + if current_identity(worker['pid']) == worker: + stop_group(worker['pid']) + return + done.wait(.5) + + +def guard(root, command, *, uuid, limits, deadline, ticket, validate_result): + began = time.monotonic() + admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds']) + done = threading.Event() + tracker = PeakRecords() + state = dict(trigger=None, peak_device_bytes=0) + watcher = None + worker_receipt = None + status, trigger, result_hash = 'STOPPED', None, None + + def spawn(args, **kwargs): + nonlocal watcher + env, inventory = dispatch_environment(uuid, kwargs['env']) + ordinal = env['CUDA_VISIBLE_DEVICES'] + if ordinal != ticket['ordinal']: + raise RuntimeError('PINNED_PCI_ORDINAL_DIFFERS') + if not snapshot(uuid)['idle']: + raise RuntimeError('SELECTED_GPU_BECAME_BUSY') + save(root / 'ordinal-dispatch.json', dict(ordinal=ordinal, physical_uuid=uuid, inventory=inventory)) + if time.monotonic() >= admission: + raise RuntimeError('ADMISSION_DEADLINE') + proc = subprocess.Popen(args, **dict(kwargs, env=env)) + try: + worker = process_identity(proc.pid) + save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid)) + watcher = threading.Thread(target=watch, daemon=True, + args=(root, uuid, worker, limits, done, tracker, state)) + watcher.start() + except (Exception, KeyboardInterrupt): + # The shared supervisor has not yet received the process to reap. + stop_group(proc.pid) + proc.wait() + raise + return proc + + try: + if shutil.disk_usage(root).free < limits['disk_bytes']: + raise RuntimeError('DISK_HEADROOM') + idle = wait_idle(uuid, max(0., admission - time.monotonic()), lambda _: None, sample=snapshot) + save(root / 'idle-gate.json', idle) + if idle['status'] != 'IDLE_CONFIRMED' or time.monotonic() >= admission: + raise RuntimeError('IDLE_DEADLINE') + save(root / 'dispatch.json', dict(ticket, coordinator=process_identity(os.getpid()), + start_by_monotonic=admission)) + remaining = deadline - time.monotonic() - limits['cleanup_seconds'] + if remaining <= 0: + raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET') + ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT)) + run_worker = FunctionType(supervise.__code__, ns, supervise.__name__) + try: + worker_receipt = run_worker(command, root, 'gino', dict( + total_cpu_seconds=limits['cpu_seconds'], job_cpu_seconds=limits['cpu_seconds'], + job_wall_seconds=remaining, tree_rss_bytes=limits['rss_bytes'], + disk_bytes=limits['disk_bytes'], max_processes=limits['max_processes']), + poll=.2, gpu_uuid=uuid) + finally: + done.set() + if watcher is not None: + watcher.join(timeout=8) + if watcher is not None and watcher.is_alive(): + raise RuntimeError('MONITOR_DID_NOT_EXIT') + tracker.scan(root / 'worker-records') + if state['trigger']: + raise RuntimeError(state['trigger']) + if max(tracker.maxima.values()) > limits['recorded_vram_bytes']: + raise RuntimeError('RECORDED_VRAM_LIMIT') + if worker_receipt['status'] != 'PASS' or worker_receipt['monitor_errors']: + raise RuntimeError(worker_receipt.get('stop_reason') or 'WORKER_FAILED') + if set(tracker.witnesses) != set(tracker.maxima): + raise RuntimeError('MISSING_RECORDED_MEMORY_PEAKS') + result_hash = validate_result() + status = 'COMPLETE' + except (Exception, KeyboardInterrupt) as exc: + trigger = str(exc) or type(exc).__name__ + end = time.monotonic() + if end > deadline and status == 'COMPLETE': + status, trigger, result_hash = 'STOPPED', 'BOOKED_GPU_WALL_CAP', None + terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt, + physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state, + charged_gpu_seconds=end - ticket['guard_started_monotonic'], + booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True) + save(root / 'complete.json', terminal) + return terminal diff --git a/tools/track_b_gino_formal_worker.py b/tools/track_b_gino_formal_worker.py new file mode 100644 index 000000000..760781d53 --- /dev/null +++ b/tools/track_b_gino_formal_worker.py @@ -0,0 +1,143 @@ +"""Run one formally bound GINO fit or the corresponding one-pass score.""" + +import argparse +import ctypes +import os +import signal +import sys +from pathlib import Path +from tempfile import TemporaryDirectory +import time + +import track_b_gino_formal as formal +from track_b_gino_paper_keyword_patch import extract_patched_paper_code +from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha +from track_b_gino_shapenet_worker import ( + OriginalData, case_scores, cuda_peaks, paper_model, test_environment, train_seed, +) + + +def attach(root, seed, phase): + ticket = read(root / 'dispatch.json') + parent = os.getppid() + if (ticket['coordinator'] != process_identity(parent) + or ticket['contract_sha256'] != sha(formal.ROOT / 'contract.json') + or (ticket['seed'], ticket['phase']) != (seed, phase) + or time.monotonic() >= ticket['start_by_monotonic']): + raise PermissionError('Worker is not attached to the authorized guard') + if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent: + raise RuntimeError('Could not attach worker lifetime to its guard') + c = formal.contract() + formal.authorize(c) + if str(Path(sys.executable).absolute()) != c['environment']['python']: + raise PermissionError('Unreviewed interpreter') + return c, ticket + + +def bind_device(root, c, ticket, torch): + ordinal = ticket['ordinal'] + uuid = formal.DEVICES[str(ticket['seed'])] + dispatch = read(root / 'ordinal-dispatch.json') + if (dispatch['ordinal'] != ordinal or dispatch['physical_uuid'] != uuid + or os.environ.get('CUDA_VISIBLE_DEVICES') != ordinal + or os.environ.get('CUDA_DEVICE_ORDER') != 'PCI_BUS_ID' + or torch.cuda.device_count() != 1 or torch.version.cuda != c['environment']['torch_cuda']): + raise PermissionError('One reviewed CUDA PCI ordinal is required') + torch.cuda.set_device(0) + name = torch.cuda.get_device_name(0) + if 'V100' not in name: + raise PermissionError('The reviewed device must be a V100') + probe = torch.empty(1, dtype=torch.uint8, device='cuda:0') + current = process_identity(os.getpid()) + save(root / 'worker-device-witness.json', dict( + process=current, ordinal=ordinal, device_count=1, device_name=name, + physical_uuid_expected=uuid, first_allocation_bytes=probe.numel())) + verified = root / 'worker-device-verified.json' + while time.monotonic() < ticket['start_by_monotonic']: + if verified.exists(): + record = read(verified) + expected = dict(process=current, physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid]) + if record != expected: + raise PermissionError('Actual worker physical-device witness differs') + return + time.sleep(.1) + raise RuntimeError('Physical device was not verified in the admission window') + + +def train(c, seed, data, root, records, torch): + learning = data.data(data.train_ids, 'train') + epochs = c['protocol']['epochs_per_seed'] + model, times = train_seed(c, learning, seed, epochs, records / 'progress.json') + if len(times) != epochs: + raise ValueError('Full training did not finish') + formal.epoch100(root, seed, c) + path = root / 'epoch-100.pt' + temporary = root / 'epoch-100.pt.tmp' + with temporary.open('xb') as stream: + torch.save(dict(state=model.state_dict(), pressure_mean=data.normalizer.mean, + pressure_std=data.normalizer.std), stream) + temporary.rename(path) + return dict(checkpoint=dict(path=str(path), sha256=sha(path)), target_reads=0, + epochs=epochs, training_examples=500, epoch_seconds=times) + + +def score(c, seed, data, root, torch): + # Targets are still unopened while rechecking all three finished fits, + # initializing the exact model and loading its immutable checkpoint. + manifest = root / 'checkpoints-frozen.json' + manifest_sha = sha(manifest) + roster = read(manifest) + if roster != formal.all_checkpoints(c): + raise PermissionError('Three complete, pinned checkpoints must precede custody') + checkpoint = roster[str(seed)]['checkpoint'] + state = torch.load(checked(checkpoint['path'], checkpoint['sha256']), map_location='cpu') + model = paper_model(c).to('cuda') + model.load_state_dict(state['state'], strict=True) + # The pinned normalizer's encode/decode and device methods use these + # fitted tensors and epsilon. No learning or target dataset is refitted. + normalizer = object.__new__(data.normalizer_class) + normalizer.mean = state['pressure_mean'] + normalizer.std = state['pressure_std'] + normalizer.eps = c['protocol']['pressure_epsilon'] + if any(t.shape != (1,) or not bool(torch.isfinite(t).all()) + for t in (normalizer.mean, normalizer.std)) or bool(normalizer.std < 0): + raise ValueError('Invalid frozen learning-only pressure statistics') + data.normalizer = normalizer + access = root / 'heldout-target-access.jsonl' + with access.open('x') as log: + heldout = data.data(data.test_ids, 'heldout', log) + rows = case_scores(model, heldout, data.test_ids, data.normalizer) + torch.cuda.synchronize() + return dict(cases=rows, mean_percent=formal.score_mean(rows, data.test_ids), + target_reads=len(rows), target_access_sha256=sha(access), + checkpoints_manifest_sha256=manifest_sha, checkpoint_sha256=checkpoint['sha256']) + + +def run(seed, phase): + root = formal.run_root(seed, phase) + c, ticket = attach(root, seed, phase) + test_environment(c) + import torch + + torch.set_num_threads(c['limits']['torch_threads']) + torch.set_num_interop_threads(1) + bind_device(root, c, ticket, torch) + torch.cuda.reset_peak_memory_stats() + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + records = root / 'worker-records' + records.mkdir(exist_ok=False) + with TemporaryDirectory(prefix='formal-gino-', dir=root) as scratch: + extract_patched_paper_code(c, scratch) + data = OriginalData(c, scratch) + result = (train(c, seed, data, root, records, torch) if phase == 'fit' + else score(c, seed, data, root, torch)) + save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch))) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('seed', type=int, choices=(0, 1, 2)) + parser.add_argument('phase', choices=('fit', 'score')) + args = parser.parse_args() + run(args.seed, args.phase)