From ee8e01937e0c87d4992bee57bcf024f04d683044 Mon Sep 17 00:00:00 2001 From: NewMeltAssist Date: Thu, 19 Nov 2026 16:18:05 +0800 Subject: [PATCH] Bind GINO cleanup to unreaped worker group and replay actual signals --- .../reap-signal-replay.json | 24 + .../contract.json | 837 ++++++++++++++++++ .../cpu-preparation.json | 52 ++ .../cpu-replay.log | 18 + .../lifecycle-cpu-replay.json | 114 +++ ...rack-b-gino-formal-lifecycle-2026-11-19.md | 9 + tests/test_track_b_gino_formal.py | 61 +- tools/track_b_gino_formal.py | 3 +- tools/track_b_gino_formal_guard.py | 88 +- 9 files changed, 1185 insertions(+), 21 deletions(-) create mode 100644 artifacts/meltassist-track-b-gino-formal-joint-review-2026-11-19/reap-signal-replay.json create mode 100644 artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/contract.json create mode 100644 artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-preparation.json create mode 100644 artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-replay.log create mode 100644 artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/lifecycle-cpu-replay.json create mode 100644 docs/track-b-gino-formal-lifecycle-2026-11-19.md diff --git a/artifacts/meltassist-track-b-gino-formal-joint-review-2026-11-19/reap-signal-replay.json b/artifacts/meltassist-track-b-gino-formal-joint-review-2026-11-19/reap-signal-replay.json new file mode 100644 index 000000000..215cfed96 --- /dev/null +++ b/artifacts/meltassist-track-b-gino-formal-joint-review-2026-11-19/reap-signal-replay.json @@ -0,0 +1,24 @@ +{ + "production_guard_sha256": "2267318d0e99e8c465bbfdf9e5dc7cac46b3b3f5fe88ba5bad647c49222ae128", + "shared_supervisor_sha256": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "production_guard_status": "COMPLETE", + "spawned_cpu_fixture": { + "pid": 3, + "created": 1795075930.27 + }, + "observed_sequence": [ + { + "event": "wait4_reaped", + "pid": 3 + }, + { + "event": "killpg_intercepted", + "pgid": 3, + "signal": 9, + "leader_already_reaped": true + } + ], + "real_signals_sent_by_fixture": 0, + "real_gpu_queries": 0, + "real_pressure_members_opened": 0 +} diff --git a/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/contract.json b/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/contract.json new file mode 100644 index 000000000..dc78c9b46 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/contract.json @@ -0,0 +1,837 @@ +{ + "mode": "formal-gino-reproduction", + "ruling": "h-f84547d7", + "retry": false, + "evaluate_other_models": false, + "limits": { + "cleanup_seconds": 20, + "idle_wait_seconds": 60, + "rss_bytes": 12884901888, + "recorded_vram_bytes": 15032385536, + "device_vram_bytes": 16106127360, + "disk_bytes": 2147483648, + "torch_threads": 1, + "max_processes": 1, + "cpu_seconds_per_gpu_second": 8 + }, + "protocol": { + "model": "NeurIPS 2023 supplement SDFFNOGNO decoder", + "seeds": [ + 0, + 1, + 2 + ], + "grid": [ + 64, + 64, + 64 + ], + "modes": [ + 24, + 24, + 24 + ], + "hidden_channels": 32, + "radius": 0.05, + "epochs_per_seed": 100, + "batch_size": 1, + "learning_rate": 0.001, + "bounds_expansion": 0.01, + "pressure_epsilon": 1e-06, + "optimizer": "Adam", + "weight_decay": 0.0001, + "lr_step_epochs": 50, + "lr_gamma": 0.5, + "pressure_training_transform": "scalar Gaussian on the 500 learning cases", + "metric": "per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)", + "report": "equal mean over 111 cases for each seed, then equal mean over the three seeds", + "acceptance_percent": [ + 6.62, + 7.62 + ], + "paper_reference_percent": 7.12, + "checkpoint_epoch": 100, + "heldout_access_after_all_checkpoints_frozen": true + }, + "environment": { + "python": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python", + "python_version": "3.10", + "torch_cuda": "11.6", + "neuralop_commit": "fbdc7f872792b75b84545c2f374e7a881c72a718", + "neuralop_file_sha256": { + "utils.py": "84554eed45ac415ab52f79e456eab70ba204e7ab7539c2f2841955a9145220b4", + "models/__init__.py": "c67679f1f8f90e1ab4eb4c2aa916aeaedf1b175d795fed03b8d4f7210883ebae", + "models/tfno.py": "d74b5805b69a64bf33e3ff5186709c75bd431a56c9548f06fb9635a859d849e4", + "models/spectral_convolution.py": "ccecf80806d93e8f35713ac6ab2c3b61c11739f5fcb8c0ba7397b7d4eb5a5d45" + }, + "package_versions": { + "torch": "1.13.1+cu116", + "open3d": "0.17.0", + "numpy": "1.24.3", + "torch-scatter": "2.1.1+pt113cu116", + "torch-sparse": "0.6.17+pt113cu116", + "torch-geometric": "2.3.1", + "tensorly": "0.8.1", + "tensorly-torch": "0.4.0" + } + }, + "comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-measurement-2026-11-19/comparison-contract.json", + "comparison_contract_sha256": "503f6c8ab1217d66b44e0d127555cfad5da4cd8a17b336299095c3cfd13041d7", + "code_sha256": { + "/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8", + "/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20", + "/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff", + "/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd", + "/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62", + "/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32", + "/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d", + "/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8", + "/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_shapenet.py": "557ad56277d7e85c6b5cc358eb151632d1757f5883f429988d36022d0d407550", + "/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f", + "/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1", + "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65", + "/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_entry.py": "e86d955c36e291e6bbef67947de663b24c8d66c5d72d9d28e4193fb8a3051ee2", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py": "7211be211c29472fd9be35a6bcb1f56025de4a63f36852bfafa10df97c003ff3", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_entry.py": "199d833f40339feb0433611ba27302416a280452f8918715e6ef95cb93c38721", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_worker.py": "3acf742dd4549e9f5491e10af5bb0b4acf00e2f47e91753100b5998cfcc61852", + "/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_import_checked.py": "fef485dd0c233c76a336239d5f29cbf44ac0fb62a6a297dff34293b23a1b75fe", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_paper_keyword_patch.py": "c61e2ed7d099eb125386857bf4d7ccdbfc3af522f80edb4f01b450cee91c0b95", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2.py": "7d3ab2412e5208892cffcfc0e1ec363eb5db2be3e10bf60595356f71e82d8552", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py": "ba1fd5ec837e884b000ba640fceafe30ea40ccf708847733df6a17a8f3599680", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery.py": "1e1827498afab62e2f28efb5d450376f2dca4e8bf6f74114f2914ec771872c86", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py": "7a68c77e9c913ba9161832f513dbf8754ebc5c270c1e742fa5ae97a7b14b9120", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_formal.py": "aa7b1301203d433564491b72fc967a29898098ad8d0d3677cda88991fbc7b483", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_guard.py": "70820fb9208960485efe07654057977b676a3003563b6a54cf00c86dc838613d", + "/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_worker.py": "978f1c7cfb622300b8814e14e4a02e5cbb9e2ea7f6e5fee080b986c7409ade2c" + }, + "paper_data": { + "path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-car-pressure-data.zip", + "sha256": "b5f9e2b407d54e034596384253fb58196933c46f6b5b586383508da54475f89d" + }, + "paper_supplement": { + "path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-neurips-supplement.zip", + "sha256": "48cdae80ac2781e27ac00e10a37124711d6bb305593559002bda0527d11a533d" + }, + "paper_code_zip_sha256": "347a6f097e2b9958d1ab87930fa1d6227b793a9d213e12418ce9a157e83dce04", + "split_receipt": { + "path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-split.json", + "sha256": "5a96c25c2ec24cfc1e2264d4ebd0879c658c0f168e22711b576bf303151b8601" + }, + "mapping_audit": { + "path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-mapping-audit.json", + "sha256": "055d008a02494acd29df36c41c8e68db931a027f5635e8f5f765a96a54c62289" + }, + "split": { + "train": [ + 1, + 2, + 4, + 5, + 6, + 7, + 8, + 10, + 12, + 13, + 17, + 18, + 21, + 22, + 23, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 34, + 35, + 39, + 40, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 54, + 55, + 56, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 67, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 83, + 84, + 85, + 86, + 87, + 88, + 90, + 91, + 92, + 94, + 95, + 96, + 97, + 100, + 101, + 102, + 105, + 106, + 107, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 133, + 134, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 165, + 166, + 170, + 172, + 173, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 186, + 190, + 191, + 192, + 193, + 195, + 196, + 198, + 199, + 200, + 201, + 202, + 203, + 205, + 207, + 210, + 211, + 212, + 213, + 214, + 215, + 217, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 241, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 251, + 252, + 253, + 255, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 266, + 267, + 268, + 269, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 285, + 286, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 304, + 305, + 306, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 327, + 328, + 329, + 331, + 332, + 333, + 334, + 335, + 337, + 338, + 339, + 340, + 341, + 344, + 345, + 347, + 348, + 349, + 350, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 360, + 362, + 364, + 365, + 366, + 367, + 369, + 371, + 372, + 373, + 374, + 375, + 376, + 378, + 379, + 380, + 381, + 384, + 385, + 389, + 392, + 393, + 397, + 398, + 399, + 401, + 402, + 403, + 404, + 405, + 407, + 408, + 410, + 412, + 413, + 414, + 415, + 417, + 418, + 419, + 420, + 422, + 424, + 425, + 427, + 430, + 431, + 433, + 435, + 436, + 437, + 439, + 440, + 443, + 444, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 459, + 460, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 472, + 473, + 474, + 475, + 476, + 478, + 479, + 480, + 482, + 483, + 486, + 487, + 488, + 490, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 501, + 502, + 503, + 504, + 505, + 507, + 508, + 509, + 511, + 512, + 513, + 514, + 515, + 516, + 518, + 519, + 521, + 522, + 523, + 524, + 525, + 527, + 529, + 530, + 532, + 533, + 536, + 538, + 539, + 540, + 542, + 543, + 545, + 547, + 548, + 549, + 550, + 551, + 552, + 553, + 554, + 555, + 560, + 561, + 562, + 564, + 565, + 566, + 567, + 568, + 569, + 572, + 573, + 574, + 576, + 577, + 579, + 581, + 582, + 583, + 584, + 587, + 588, + 589, + 591, + 593, + 594, + 595, + 596, + 597, + 598, + 600, + 602, + 604, + 608, + 610, + 611, + 612, + 613, + 615, + 616, + 617, + 618, + 620, + 621, + 622, + 623, + 625, + 626, + 627, + 628, + 629, + 630, + 631, + 632, + 633, + 634, + 635, + 636, + 638, + 639, + 640, + 641, + 642, + 643, + 644, + 645, + 646, + 647, + 648, + 649, + 651, + 652, + 654, + 655, + 656, + 657 + ], + "test": [ + 658, + 659, + 660, + 662, + 663, + 664, + 665, + 666, + 667, + 668, + 672, + 673, + 674, + 675, + 676, + 677, + 678, + 679, + 681, + 683, + 684, + 686, + 687, + 688, + 689, + 690, + 691, + 692, + 693, + 695, + 696, + 697, + 700, + 701, + 702, + 703, + 704, + 705, + 708, + 709, + 710, + 711, + 712, + 713, + 715, + 717, + 718, + 719, + 721, + 722, + 723, + 724, + 725, + 726, + 727, + 729, + 730, + 731, + 733, + 734, + 735, + 736, + 737, + 738, + 739, + 740, + 741, + 743, + 744, + 745, + 746, + 747, + 748, + 749, + 750, + 751, + 752, + 753, + 757, + 758, + 760, + 762, + 763, + 764, + 765, + 766, + 767, + 768, + 769, + 770, + 771, + 772, + 773, + 775, + 776, + 777, + 778, + 779, + 781, + 782, + 783, + 784, + 785, + 787, + 789, + 790, + 791, + 792, + 793, + 795, + 797 + ], + "raw_global_bounds": [ + [ + -0.90252799, + 0.0066688, + -2.85272002 + ], + [ + 0.90253001, + 1.95825005, + 2.86385012 + ] + ] + }, + "environment_ruling": "h-0d21f3bd", + "environment_acceptance": { + "path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-import-checked-2026-11-19/cpu-environment-acceptance.json", + "sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3" + }, + "paper_compatibility": { + "handoff": "h-b8508140", + "identity": "supplement code + 2-keyword patch", + "patch_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/two-keyword-patch.json", + "patch_sha256": "7307ba49f83bfc46c5decf803a993a41ed602d987de65714958e738d2f52f077", + "search_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/keyword-search.json", + "search_sha256": "77a897bf534a9a0e9e6a7c072b4f914168648eadc2af3493b70d63d0b715b3d0" + }, + "formal": { + "handoff": "h-4e4e680b", + "ruling_slack_ts": "1795074651.222599", + "devices": { + "0": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "1": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "2": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c" + }, + "lifecycle_review_slack_ts": "1795075951.091959", + "device_pci_ordinals": { + "0": 2, + "1": 3, + "2": 0 + }, + "reservation": { + "handoff": "h-a429935e", + "timezone": "Asia/Hong_Kong", + "start_inclusive": "2026-11-19T00:00:00+08:00", + "end_exclusive": "2026-11-20T00:00:00+08:00", + "gpu3_single_task": true, + "historical_inventory_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json", + "historical_inventory_sha256": "da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8", + "runtime_idle_check_required": true, + "applies_to_new_dispatch_only": true, + "already_running_guards_unchanged": true + }, + "budget_per_seed_stage_gpu_seconds": { + "0": { + "fit": 13034, + "score": 300 + }, + "1": { + "fit": 13033, + "score": 300 + }, + "2": { + "fit": 13033, + "score": 300 + } + }, + "total_gpu_wall_seconds": 40000, + "timing": { + "record_paths": { + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json": "db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json": "851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec", + "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json": "8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf" + }, + "status": "CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED", + "sizing_only": true, + "epoch_seconds": 102.93777855299413, + "train_sizing_gpu_seconds": 38601.6669573728, + "train_sizing_multiplier": 1.25, + "score_reserved_gpu_seconds": 900, + "prior_measurement_gpu_seconds": 222.80500172509346 + }, + "heldout_policy": { + "preparation_reads": 0, + "training_reads": 0, + "score_reads_per_seed": 111, + "all_epoch100_checkpoints_required": true, + "retry": false + } + } +} diff --git a/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-preparation.json b/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-preparation.json new file mode 100644 index 000000000..dcd400f34 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-preparation.json @@ -0,0 +1,52 @@ +{ + "status": "PASS", + "contract_sha256": "c3c5c5f0fab049c757032c343673104eb0f99cbaead5e0f78247f2a3aa6173c3", + "cuda_visible_devices": "", + "actual_nvidia_queries": 0, + "training_starts": 0, + "scoring_starts": 0, + "actual_heldout_reads": 0, + "independent_review": "PENDING", + "guard_replay": 13, + "supersedes": "8af022e43410232d42d07c92e2f468342a385c7a", + "intermediary_blocker_evidence_sha256": "06f4b0c2a66d4ac8d285ee4e508388241798e60a4e500dc5ab5e1fde02a77720", + "unchanged_shared_supervisor_sha256": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "cpu_replay_log_sha256": "6bf84633d3dc09cbd5002a149a8d0ebe3ef205ccb9748096f50ac7ee906971a8", + "lifecycle_replay_sha256": "feb40e583810259537ad746ccfbbf2e56ff9bc969f002f1e7f74657e3b88cf8d", + "read_only_collection": { + "seeds": { + "0": { + "fit": { + "status": "NOT_STARTED" + }, + "score": { + "status": "NOT_STARTED" + } + }, + "1": { + "fit": { + "status": "NOT_STARTED" + }, + "score": { + "status": "NOT_STARTED" + } + }, + "2": { + "fit": { + "status": "NOT_STARTED" + }, + "score": { + "status": "NOT_STARTED" + } + } + }, + "mean_percent": null, + "within_registered_range": null, + "reproduction_identity": "supplement code + 2-keyword patch", + "prior_measurement_gpu_seconds": 222.80500172509346, + "settled_formal_gpu_seconds": 0, + "settled_combined_gpu_seconds": 222.80500172509346, + "unsettled_formal_bookings_gpu_seconds": 0, + "formal_gpu_cap_seconds": 40000 + } +} diff --git a/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-replay.log b/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-replay.log new file mode 100644 index 000000000..ca1ce21d8 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-replay.log @@ -0,0 +1,18 @@ +test_card_reservation_expiry_and_total_budget (test_track_b_gino_formal.FormalBinding) ... ok +test_inclusive_gate_and_all_three_seeds_required (test_track_b_gino_formal.FormalBinding) ... ok +test_original_split_score_keeps_order_and_rejects_nan (test_track_b_gino_formal.FormalBinding) ... ok +test_score_opens_one_111_target_pass_and_rejects_a_second (test_track_b_gino_formal.FormalBinding) ... ok +test_scoring_needs_all_checkpoints_before_access (test_track_b_gino_formal.FormalBinding) ... ok +test_failed_worker_exit_has_no_stale_group_signal (test_track_b_gino_formal.GuardReplay) ... ok +test_gpu_list_before_worker_exit (test_track_b_gino_formal.GuardReplay) ... ok +test_real_foreign_identity_has_exact_stop_receipt (test_track_b_gino_formal.GuardReplay) ... ok +test_reservation_expires_after_idle_but_before_spawn (test_track_b_gino_formal.GuardReplay) ... ok +test_reused_worker_pid_is_foreign (test_track_b_gino_formal.GuardReplay) ... ok +test_running_guard_continues_after_reservation_end (test_track_b_gino_formal.GuardReplay) ... ok +test_simulated_reused_pgid_cannot_be_signalled (test_track_b_gino_formal.GuardReplay) ... ok +test_worker_already_exited_before_gpu_list (test_track_b_gino_formal.GuardReplay) ... ok + +---------------------------------------------------------------------- +Ran 13 tests in 1.232s + +OK diff --git a/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/lifecycle-cpu-replay.json b/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/lifecycle-cpu-replay.json new file mode 100644 index 000000000..e541d7745 --- /dev/null +++ b/artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/lifecycle-cpu-replay.json @@ -0,0 +1,114 @@ +{ + "status": "PASS", + "tests": 13, + "real_gpu_queries": 0, + "pressure_members_opened": 0, + "cases": { + "failed-exit": { + "status": "STOPPED", + "trigger": "WORKER_FAILED", + "syscalls": [ + { + "event": "wait4_reaped", + "pid": 67 + } + ], + "lifecycle": [ + { + "event": "leader_reaped", + "pid": 67 + } + ] + }, + "gpu-first": { + "status": "COMPLETE", + "trigger": null, + "syscalls": [ + { + "event": "wait4_reaped", + "pid": 69 + } + ], + "lifecycle": [ + { + "event": "leader_reaped", + "pid": 69 + } + ] + }, + "foreign": { + "status": "STOPPED", + "trigger": "FOREIGN_GPU_OCCUPANT", + "syscalls": [ + { + "event": "killpg", + "pid": 71, + "signal": 9, + "leader_reaped": false + }, + { + "event": "wait4_reaped", + "pid": 71 + } + ], + "lifecycle": [ + { + "event": "group_signal", + "pid": 71, + "signal": 9, + "reason": "GUARD_STOP" + }, + { + "event": "leader_reaped", + "pid": 71 + } + ] + }, + "expired-before-spawn": { + "status": "STOPPED", + "trigger": "CARD_RESERVATION_EXPIRED_BEFORE_SPAWN", + "syscalls": [], + "lifecycle": [] + }, + "crossed-after-spawn": { + "status": "COMPLETE", + "trigger": null, + "syscalls": [ + { + "event": "wait4_reaped", + "pid": 73 + } + ], + "lifecycle": [ + { + "event": "leader_reaped", + "pid": 73 + } + ] + }, + "pgid-reuse": { + "reaped": true, + "simulated_new_identity": { + "pid": 12, + "created": 2.0 + }, + "signals": 0 + }, + "tree-first": { + "status": "COMPLETE", + "trigger": null, + "syscalls": [ + { + "event": "wait4_reaped", + "pid": 75 + } + ], + "lifecycle": [ + { + "event": "leader_reaped", + "pid": 75 + } + ] + } + } +} diff --git a/docs/track-b-gino-formal-lifecycle-2026-11-19.md b/docs/track-b-gino-formal-lifecycle-2026-11-19.md new file mode 100644 index 000000000..f1b743188 --- /dev/null +++ b/docs/track-b-gino-formal-lifecycle-2026-11-19.md @@ -0,0 +1,9 @@ +# Track B GINO 三种子复现联合预审定稿候选 + +本绑定替换中间提交 `8af022e4` 供同一次独立预审。MeltAssist 在旧生产守卫的 CPU 回放中捕获 `wait4` 已回收组长后的 `killpg(SIGKILL)`。旧回放与两次 GPU 测量终止均保留。 [MeltAssist 阻塞回执](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795075951091959) · [旧 CPU 调用次序](../artifacts/meltassist-track-b-gino-formal-joint-review-2026-11-19/reap-signal-replay.json) + +修复只接入本 GINO 守卫所用的监督器调用,原共享文件及别的运行固定版本保持原样。用 `waitid(WNOWAIT)` 核实固定 PID 仍是本方直接子进程。守卫在同一把锁下验证组长身份并发送信号,或交由 `wait4` 回收并标记结束。需要清理残留同组子进程时先清理再回收组长。组长回收后,旧清理调用不再发信号。 [生产守卫](../tools/track_b_gino_formal_guard.py) + +实际生产守卫 CPU 回放 13/13。拦截并记录所有组清理信号及 `wait4`。成功退出、非零退出、两类陈旧 GPU 列表和跨午夜运行都没有回收后信号。模拟真实外部占用时只向自己仍未回收的组发送一次 SIGKILL,随后实际回收,外部活进程未收信号。复用组 ID 负例到达清理入口时不触发信号。午夜到期且尚未派发的路径不创建 worker。 [调用及回收次序](../artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/lifecycle-cpu-replay.json) · [十三项 CPU 日志](../artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-replay.log) + +种子 0、1、2 分别在 GPU2、GPU3、GPU0,各 100 完整轮。香港时间 2026-11-19 当日预留只适用于新派发,已开始的守卫照常运作。三份检查点冻结后,每个种子各评分一次原 111 件,三种子等权平均门含两端 6.62% 和 7.62%。正式总帽 40,000 GPU 秒。102.9377786 秒为未经守卫认证的单轮计时,仅用于定预算,旧账 222.805002 GPU 秒独计一次。训练及评分身份为 `supplement code + 2-keyword patch`。本准备阶段没有真实 GPU 派发或留出目标读取。 [组合合同](../artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/contract.json) · [训练与评分](../tools/track_b_gino_formal_worker.py) diff --git a/tests/test_track_b_gino_formal.py b/tests/test_track_b_gino_formal.py index 57fd7bf70..4356b4291 100644 --- a/tests/test_track_b_gino_formal.py +++ b/tests/test_track_b_gino_formal.py @@ -15,6 +15,9 @@ import track_b_gino_formal_worker as worker from track_b_gino_shapenet_entry import read, save, sha +REPLAYS = {} + + class GuardReplay(unittest.TestCase): def replay(self, ordering): uuid = 'GPU-fixture' @@ -32,16 +35,28 @@ class GuardReplay(unittest.TestCase): "root=Path(sys.argv[1])\n" "while not (root/'exit').exists(): time.sleep(.01)\n" "(root/'worker-records/result.json').write_text(" - "'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder] + "'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n" + "if sys.argv[2] == 'failed-exit': raise SystemExit(7)\n", folder, ordering] observed = [] - stopped = [] - real_stop = gpu_guard.stop_group + syscalls = [] + real_kill = os.killpg + real_wait = os.wait4 real_tree = gpu_guard.process_tree ticket = dict(ordinal='', dispatch_before_unix=time.time() + 15) - def stop_owned(pid): - stopped.append(pid) - real_stop(pid) + def record_kill(pid, signal): + own = read(root / 'actual-worker-spawn.json')['process'] + is_reaped = any(e == dict(event='wait4_reaped', pid=pid) for e in syscalls) + syscalls.append(dict(event='killpg', pid=pid, signal=int(signal), leader_reaped=is_reaped)) + if is_reaped or pid != own['pid']: + raise AssertionError('Invalid group signal intercepted before delivery') + return real_kill(pid, signal) + + def record_wait(pid, options): + receipt = real_wait(pid, options) + if receipt[0]: + syscalls.append(dict(event='wait4_reaped', pid=receipt[0])) + return receipt def exit_and_wait(): (root / 'exit').touch() @@ -74,13 +89,14 @@ class GuardReplay(unittest.TestCase): occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')]) if ordering == 'crossed-after-spawn': ticket['dispatch_before_unix'] = time.time() - 1 - if ordering in ('gpu-first', 'crossed-after-spawn'): + if ordering in ('gpu-first', 'crossed-after-spawn', 'failed-exit'): exit_and_wait() return sample with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot), patch.object(gpu_guard, 'process_tree', side_effect=tree), - patch.object(gpu_guard, 'stop_group', side_effect=stop_owned), + patch.object(os, 'killpg', side_effect=record_kill), + patch.object(os, 'wait4', side_effect=record_wait), patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}), patch.object(gpu_guard, 'dispatch_environment', side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))): @@ -88,10 +104,13 @@ class GuardReplay(unittest.TestCase): ticket['guard_started_monotonic'] = began outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15, ticket=ticket, validate_result=lambda: sha(result)) + REPLAYS[ordering] = dict(status=outcome['status'], trigger=outcome['trigger'], + syscalls=syscalls, lifecycle=outcome['lifecycle']) if ordering == 'expired-before-spawn': self.assertEqual(outcome['status'], 'STOPPED') self.assertEqual(outcome['trigger'], 'CARD_RESERVATION_EXPIRED_BEFORE_SPAWN') self.assertFalse((root / 'actual-worker-spawn.json').exists()) + self.assertEqual(syscalls, []) return observed spawn = read(root / 'actual-worker-spawn.json')['process'] if ordering == 'foreign': @@ -102,11 +121,16 @@ class GuardReplay(unittest.TestCase): self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid)) self.assertEqual(row['occupants'][0]['pid'], os.getpid()) self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python') - self.assertEqual(stopped, [spawn['pid']]) + self.assertEqual(syscalls, [ + dict(event='killpg', pid=spawn['pid'], signal=9, leader_reaped=False), + dict(event='wait4_reaped', pid=spawn['pid'])]) + elif ordering == 'failed-exit': + self.assertEqual((outcome['status'], outcome['worker']['returncode']), ('STOPPED', 7)) + self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])]) else: self.assertEqual(outcome['status'], 'COMPLETE', outcome) self.assertFalse((root / 'foreign-occupant.json').exists()) - self.assertEqual(stopped, []) + self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])]) self.assertEqual(outcome['result_sha256'], sha(result)) return observed @@ -127,6 +151,23 @@ class GuardReplay(unittest.TestCase): def test_running_guard_continues_after_reservation_end(self): self.replay('crossed-after-spawn') + def test_failed_worker_exit_has_no_stale_group_signal(self): + self.replay('failed-exit') + + def test_simulated_reused_pgid_cannot_be_signalled(self): + group = gpu_guard.OwnedGroup() + group.pid = 12 + group.identity = dict(pid=12, created=1.) + group.reaped = True + reused = dict(pid=12, created=2.) + with (patch.object(gpu_guard, 'current_identity', return_value=reused), + patch.object(os, 'waitid') as observe, + patch.object(os, 'killpg') as intercept): + group.stop(12) + observe.assert_not_called() + intercept.assert_not_called() + REPLAYS['pgid-reuse'] = dict(reaped=group.reaped, simulated_new_identity=reused, signals=0) + def test_reused_worker_pid_is_foreign(self): with TemporaryDirectory() as folder: root = Path(folder) diff --git a/tools/track_b_gino_formal.py b/tools/track_b_gino_formal.py index 37b48db22..83bc5bb7b 100644 --- a/tools/track_b_gino_formal.py +++ b/tools/track_b_gino_formal.py @@ -18,7 +18,7 @@ from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT from track_b_gino_shapenet_entry import BASE, checked, read, save, sha -ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-v2-2026-11-19') +ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-v3-2026-11-19') SOURCES = Path(__file__).resolve().parent OLD_FILES = { 'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a', @@ -67,6 +67,7 @@ def expected_contract(): c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES}) c['formal'] = dict( handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES, + lifecycle_review_slack_ts='1795075951.091959', device_pci_ordinals=ORDINALS, reservation=dict(handoff='h-a429935e', timezone=TZ.key, start_inclusive=RESERVATION_START.isoformat(), diff --git a/tools/track_b_gino_formal_guard.py b/tools/track_b_gino_formal_guard.py index e4e0b6f8b..a6bf65ce3 100644 --- a/tools/track_b_gino_formal_guard.py +++ b/tools/track_b_gino_formal_guard.py @@ -2,6 +2,7 @@ import os import shutil +import signal import subprocess import threading import time @@ -11,7 +12,7 @@ import psutil from s2_gpu_idle_gate import snapshot, wait_idle from s2_main_memory_guard import PeakRecords -from s2_recovery_gpu_supervisor import stop_group, supervise +from s2_recovery_gpu_supervisor import supervise from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity from track_b_gino_shapenet_entry import process_identity, save @@ -71,7 +72,70 @@ class OccupancyMonitor: raise RuntimeError('DEVICE_VRAM_LIMIT') -def watch(root, uuid, worker, limits, done, tracker, state): +class OwnedGroup: + """Tie group signals to an unreaped session leader we directly spawned.""" + + def __init__(self): + self.pid = None + self.identity = None + self.reaped = False + self.signalled = False + self.lock = threading.Lock() + self.events = [] + + def observe_exit(self, pid, nonblocking): + flags = os.WEXITED | os.WNOWAIT | (os.WNOHANG if nonblocking else 0) + return os.waitid(os.P_PID, pid, flags) + + def stop_locked(self, pid, reason): + if pid != self.pid or self.reaped or self.signalled: + return + # WNOWAIT verifies this PID is still our child without permitting PID + # reuse. The same lock prevents our wait4 from reaping before killpg. + try: + self.observe_exit(pid, nonblocking=True) + if os.getpgid(pid) != pid: + raise RuntimeError('WORKER_IS_NOT_ITS_OWN_SESSION_LEADER') + if self.identity is not None and current_identity(pid) != self.identity: + raise RuntimeError('WORKER_BIRTH_IDENTITY_DIFFERS') + os.killpg(pid, signal.SIGKILL) + except (ChildProcessError, ProcessLookupError): + return + self.signalled = True + self.events.append(dict(event='group_signal', pid=pid, signal=int(signal.SIGKILL), reason=reason)) + + def stop(self, pid): + with self.lock: + self.stop_locked(pid, 'GUARD_STOP') + + def wait4(self, pid, options): + if pid != self.pid: + # The shared supervisor separately collects adopted descendants + # with a negative group selector. That path never sends signals. + return os.wait4(pid, options) + # Never hold the lock on a blocking wait. The watcher may still need + # it to stop the worker. + if self.observe_exit(pid, nonblocking=bool(options & os.WNOHANG)) is None: + return 0, 0, None + with self.lock: + # With subreaping enabled, children the leader left in its group + # are now ours. Clean them up while the zombie still pins PGID. + for child in psutil.Process().children(): + try: + in_group = child.pid != pid and os.getpgid(child.pid) == pid + except ProcessLookupError: + continue + if in_group: + self.stop_locked(pid, 'REMAINING_GROUP_MEMBER') + break + result = os.wait4(pid, options) + if result[0] == pid: + self.reaped = True + self.events.append(dict(event='leader_reaped', pid=pid)) + return result + + +def watch(root, uuid, worker, group, limits, done, tracker, state): occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes']) while not done.is_set(): try: @@ -83,9 +147,7 @@ def watch(root, uuid, worker, limits, done, tracker, state): monitor_identity(root, uuid) except Exception as exc: state['trigger'] = str(exc) or type(exc).__name__ - # Signal only the recorded worker group while its leader has the same identity. - if current_identity(worker['pid']) == worker: - stop_group(worker['pid']) + group.stop(worker['pid']) return done.wait(.5) @@ -95,6 +157,7 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result): admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds']) done = threading.Event() tracker = PeakRecords() + group = OwnedGroup() state = dict(trigger=None, peak_device_bytes=0) watcher = None worker_receipt = None @@ -114,16 +177,19 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result): if time.time() >= ticket['dispatch_before_unix']: raise PermissionError('CARD_RESERVATION_EXPIRED_BEFORE_SPAWN') proc = subprocess.Popen(args, **dict(kwargs, env=env)) + group.pid = proc.pid try: worker = process_identity(proc.pid) + group.identity = worker save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid)) watcher = threading.Thread(target=watch, daemon=True, - args=(root, uuid, worker, limits, done, tracker, state)) + args=(root, uuid, worker, group, limits, done, tracker, state)) watcher.start() except (Exception, KeyboardInterrupt): # The shared supervisor has not yet received the process to reap. - stop_group(proc.pid) - proc.wait() + group.stop(proc.pid) + _, status, _ = group.wait4(proc.pid, 0) + proc.returncode = os.waitstatus_to_exitcode(status) raise return proc @@ -139,7 +205,8 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result): remaining = deadline - time.monotonic() - limits['cleanup_seconds'] if remaining <= 0: raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET') - ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT)) + ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT), + os=SimpleNamespace(**dict(vars(os), wait4=group.wait4)), stop_group=group.stop) run_worker = FunctionType(supervise.__code__, ns, supervise.__name__) try: worker_receipt = run_worker(command, root, 'gino', dict( @@ -172,6 +239,7 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result): terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt, physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state, charged_gpu_seconds=end - ticket['guard_started_monotonic'], - booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True) + booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, + lifecycle=group.events, no_retry=True) save(root / 'complete.json', terminal) return terminal