Bind GINO cleanup to unreaped worker group and replay actual signals

This commit is contained in:
NewMeltAssist committed 2026-11-19 16:18:05 +08:00
1 parent 8af022e434
commit ee8e01937e
9 files changed
+1185 -21

No files matched your search

@@ -0,0 +1,24 @@
{
"production_guard_sha256": "2267318d0e99e8c465bbfdf9e5dc7cac46b3b3f5fe88ba5bad647c49222ae128",
"shared_supervisor_sha256": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"production_guard_status": "COMPLETE",
"spawned_cpu_fixture": {
"pid": 3,
"created": 1795075930.27
},
"observed_sequence": [
{
"event": "wait4_reaped",
"pid": 3
},
{
"event": "killpg_intercepted",
"pgid": 3,
"signal": 9,
"leader_already_reaped": true
}
],
"real_signals_sent_by_fixture": 0,
"real_gpu_queries": 0,
"real_pressure_members_opened": 0
}
@@ -0,0 +1,837 @@
{
"mode": "formal-gino-reproduction",
"ruling": "h-f84547d7",
"retry": false,
"evaluate_other_models": false,
"limits": {
"cleanup_seconds": 20,
"idle_wait_seconds": 60,
"rss_bytes": 12884901888,
"recorded_vram_bytes": 15032385536,
"device_vram_bytes": 16106127360,
"disk_bytes": 2147483648,
"torch_threads": 1,
"max_processes": 1,
"cpu_seconds_per_gpu_second": 8
},
"protocol": {
"model": "NeurIPS 2023 supplement SDFFNOGNO decoder",
"seeds": [
0,
1,
2
],
"grid": [
64,
64,
64
],
"modes": [
24,
24,
24
],
"hidden_channels": 32,
"radius": 0.05,
"epochs_per_seed": 100,
"batch_size": 1,
"learning_rate": 0.001,
"bounds_expansion": 0.01,
"pressure_epsilon": 1e-06,
"optimizer": "Adam",
"weight_decay": 0.0001,
"lr_step_epochs": 50,
"lr_gamma": 0.5,
"pressure_training_transform": "scalar Gaussian on the 500 learning cases",
"metric": "per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)",
"report": "equal mean over 111 cases for each seed, then equal mean over the three seeds",
"acceptance_percent": [
6.62,
7.62
],
"paper_reference_percent": 7.12,
"checkpoint_epoch": 100,
"heldout_access_after_all_checkpoints_frozen": true
},
"environment": {
"python": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
"python_version": "3.10",
"torch_cuda": "11.6",
"neuralop_commit": "fbdc7f872792b75b84545c2f374e7a881c72a718",
"neuralop_file_sha256": {
"utils.py": "84554eed45ac415ab52f79e456eab70ba204e7ab7539c2f2841955a9145220b4",
"models/__init__.py": "c67679f1f8f90e1ab4eb4c2aa916aeaedf1b175d795fed03b8d4f7210883ebae",
"models/tfno.py": "d74b5805b69a64bf33e3ff5186709c75bd431a56c9548f06fb9635a859d849e4",
"models/spectral_convolution.py": "ccecf80806d93e8f35713ac6ab2c3b61c11739f5fcb8c0ba7397b7d4eb5a5d45"
},
"package_versions": {
"torch": "1.13.1+cu116",
"open3d": "0.17.0",
"numpy": "1.24.3",
"torch-scatter": "2.1.1+pt113cu116",
"torch-sparse": "0.6.17+pt113cu116",
"torch-geometric": "2.3.1",
"tensorly": "0.8.1",
"tensorly-torch": "0.4.0"
}
},
"comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-measurement-2026-11-19/comparison-contract.json",
"comparison_contract_sha256": "503f6c8ab1217d66b44e0d127555cfad5da4cd8a17b336299095c3cfd13041d7",
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_shapenet.py": "557ad56277d7e85c6b5cc358eb151632d1757f5883f429988d36022d0d407550",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_entry.py": "e86d955c36e291e6bbef67947de663b24c8d66c5d72d9d28e4193fb8a3051ee2",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py": "7211be211c29472fd9be35a6bcb1f56025de4a63f36852bfafa10df97c003ff3",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_entry.py": "199d833f40339feb0433611ba27302416a280452f8918715e6ef95cb93c38721",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_worker.py": "3acf742dd4549e9f5491e10af5bb0b4acf00e2f47e91753100b5998cfcc61852",
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_import_checked.py": "fef485dd0c233c76a336239d5f29cbf44ac0fb62a6a297dff34293b23a1b75fe",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_paper_keyword_patch.py": "c61e2ed7d099eb125386857bf4d7ccdbfc3af522f80edb4f01b450cee91c0b95",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2.py": "7d3ab2412e5208892cffcfc0e1ec363eb5db2be3e10bf60595356f71e82d8552",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py": "ba1fd5ec837e884b000ba640fceafe30ea40ccf708847733df6a17a8f3599680",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery.py": "1e1827498afab62e2f28efb5d450376f2dca4e8bf6f74114f2914ec771872c86",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py": "7a68c77e9c913ba9161832f513dbf8754ebc5c270c1e742fa5ae97a7b14b9120",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal.py": "aa7b1301203d433564491b72fc967a29898098ad8d0d3677cda88991fbc7b483",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_guard.py": "70820fb9208960485efe07654057977b676a3003563b6a54cf00c86dc838613d",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_worker.py": "978f1c7cfb622300b8814e14e4a02e5cbb9e2ea7f6e5fee080b986c7409ade2c"
},
"paper_data": {
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-car-pressure-data.zip",
"sha256": "b5f9e2b407d54e034596384253fb58196933c46f6b5b586383508da54475f89d"
},
"paper_supplement": {
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-neurips-supplement.zip",
"sha256": "48cdae80ac2781e27ac00e10a37124711d6bb305593559002bda0527d11a533d"
},
"paper_code_zip_sha256": "347a6f097e2b9958d1ab87930fa1d6227b793a9d213e12418ce9a157e83dce04",
"split_receipt": {
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-split.json",
"sha256": "5a96c25c2ec24cfc1e2264d4ebd0879c658c0f168e22711b576bf303151b8601"
},
"mapping_audit": {
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-mapping-audit.json",
"sha256": "055d008a02494acd29df36c41c8e68db931a027f5635e8f5f765a96a54c62289"
},
"split": {
"train": [
1,
2,
4,
5,
6,
7,
8,
10,
12,
13,
17,
18,
21,
22,
23,
25,
26,
27,
28,
29,
30,
31,
32,
34,
35,
39,
40,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
54,
55,
56,
58,
59,
60,
61,
62,
63,
64,
65,
67,
69,
70,
71,
72,
73,
74,
75,
76,
77,
78,
79,
80,
81,
83,
84,
85,
86,
87,
88,
90,
91,
92,
94,
95,
96,
97,
100,
101,
102,
105,
106,
107,
109,
110,
111,
112,
113,
114,
115,
116,
117,
118,
119,
120,
121,
123,
124,
125,
126,
127,
128,
129,
130,
131,
133,
134,
136,
137,
138,
139,
140,
141,
142,
143,
144,
145,
146,
147,
148,
149,
150,
151,
152,
153,
155,
156,
157,
158,
159,
160,
161,
162,
163,
165,
166,
170,
172,
173,
175,
176,
177,
178,
179,
180,
181,
182,
183,
184,
186,
190,
191,
192,
193,
195,
196,
198,
199,
200,
201,
202,
203,
205,
207,
210,
211,
212,
213,
214,
215,
217,
219,
220,
221,
222,
223,
224,
225,
227,
228,
229,
230,
231,
232,
233,
234,
235,
236,
237,
241,
243,
244,
245,
246,
247,
248,
249,
251,
252,
253,
255,
257,
258,
259,
260,
261,
262,
263,
264,
266,
267,
268,
269,
271,
272,
273,
274,
275,
276,
277,
278,
279,
280,
281,
282,
283,
285,
286,
289,
290,
291,
292,
293,
294,
295,
296,
297,
298,
299,
300,
301,
302,
304,
305,
306,
308,
309,
310,
311,
312,
313,
314,
315,
319,
320,
321,
322,
323,
324,
325,
327,
328,
329,
331,
332,
333,
334,
335,
337,
338,
339,
340,
341,
344,
345,
347,
348,
349,
350,
352,
353,
354,
355,
356,
357,
358,
360,
362,
364,
365,
366,
367,
369,
371,
372,
373,
374,
375,
376,
378,
379,
380,
381,
384,
385,
389,
392,
393,
397,
398,
399,
401,
402,
403,
404,
405,
407,
408,
410,
412,
413,
414,
415,
417,
418,
419,
420,
422,
424,
425,
427,
430,
431,
433,
435,
436,
437,
439,
440,
443,
444,
446,
447,
448,
449,
450,
451,
452,
453,
454,
455,
456,
457,
459,
460,
462,
463,
464,
465,
466,
467,
468,
469,
470,
472,
473,
474,
475,
476,
478,
479,
480,
482,
483,
486,
487,
488,
490,
493,
494,
495,
496,
497,
498,
499,
501,
502,
503,
504,
505,
507,
508,
509,
511,
512,
513,
514,
515,
516,
518,
519,
521,
522,
523,
524,
525,
527,
529,
530,
532,
533,
536,
538,
539,
540,
542,
543,
545,
547,
548,
549,
550,
551,
552,
553,
554,
555,
560,
561,
562,
564,
565,
566,
567,
568,
569,
572,
573,
574,
576,
577,
579,
581,
582,
583,
584,
587,
588,
589,
591,
593,
594,
595,
596,
597,
598,
600,
602,
604,
608,
610,
611,
612,
613,
615,
616,
617,
618,
620,
621,
622,
623,
625,
626,
627,
628,
629,
630,
631,
632,
633,
634,
635,
636,
638,
639,
640,
641,
642,
643,
644,
645,
646,
647,
648,
649,
651,
652,
654,
655,
656,
657
],
"test": [
658,
659,
660,
662,
663,
664,
665,
666,
667,
668,
672,
673,
674,
675,
676,
677,
678,
679,
681,
683,
684,
686,
687,
688,
689,
690,
691,
692,
693,
695,
696,
697,
700,
701,
702,
703,
704,
705,
708,
709,
710,
711,
712,
713,
715,
717,
718,
719,
721,
722,
723,
724,
725,
726,
727,
729,
730,
731,
733,
734,
735,
736,
737,
738,
739,
740,
741,
743,
744,
745,
746,
747,
748,
749,
750,
751,
752,
753,
757,
758,
760,
762,
763,
764,
765,
766,
767,
768,
769,
770,
771,
772,
773,
775,
776,
777,
778,
779,
781,
782,
783,
784,
785,
787,
789,
790,
791,
792,
793,
795,
797
],
"raw_global_bounds": [
[
-0.90252799,
0.0066688,
-2.85272002
],
[
0.90253001,
1.95825005,
2.86385012
]
]
},
"environment_ruling": "h-0d21f3bd",
"environment_acceptance": {
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-import-checked-2026-11-19/cpu-environment-acceptance.json",
"sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3"
},
"paper_compatibility": {
"handoff": "h-b8508140",
"identity": "supplement code + 2-keyword patch",
"patch_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/two-keyword-patch.json",
"patch_sha256": "7307ba49f83bfc46c5decf803a993a41ed602d987de65714958e738d2f52f077",
"search_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/keyword-search.json",
"search_sha256": "77a897bf534a9a0e9e6a7c072b4f914168648eadc2af3493b70d63d0b715b3d0"
},
"formal": {
"handoff": "h-4e4e680b",
"ruling_slack_ts": "1795074651.222599",
"devices": {
"0": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"1": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"2": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c"
},
"lifecycle_review_slack_ts": "1795075951.091959",
"device_pci_ordinals": {
"0": 2,
"1": 3,
"2": 0
},
"reservation": {
"handoff": "h-a429935e",
"timezone": "Asia/Hong_Kong",
"start_inclusive": "2026-11-19T00:00:00+08:00",
"end_exclusive": "2026-11-20T00:00:00+08:00",
"gpu3_single_task": true,
"historical_inventory_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json",
"historical_inventory_sha256": "da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8",
"runtime_idle_check_required": true,
"applies_to_new_dispatch_only": true,
"already_running_guards_unchanged": true
},
"budget_per_seed_stage_gpu_seconds": {
"0": {
"fit": 13034,
"score": 300
},
"1": {
"fit": 13033,
"score": 300
},
"2": {
"fit": 13033,
"score": 300
}
},
"total_gpu_wall_seconds": 40000,
"timing": {
"record_paths": {
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a",
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json": "db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d",
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json": "851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec",
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json": "8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf"
},
"status": "CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED",
"sizing_only": true,
"epoch_seconds": 102.93777855299413,
"train_sizing_gpu_seconds": 38601.6669573728,
"train_sizing_multiplier": 1.25,
"score_reserved_gpu_seconds": 900,
"prior_measurement_gpu_seconds": 222.80500172509346
},
"heldout_policy": {
"preparation_reads": 0,
"training_reads": 0,
"score_reads_per_seed": 111,
"all_epoch100_checkpoints_required": true,
"retry": false
}
}
}
@@ -0,0 +1,52 @@
{
"status": "PASS",
"contract_sha256": "c3c5c5f0fab049c757032c343673104eb0f99cbaead5e0f78247f2a3aa6173c3",
"cuda_visible_devices": "",
"actual_nvidia_queries": 0,
"training_starts": 0,
"scoring_starts": 0,
"actual_heldout_reads": 0,
"independent_review": "PENDING",
"guard_replay": 13,
"supersedes": "8af022e43410232d42d07c92e2f468342a385c7a",
"intermediary_blocker_evidence_sha256": "06f4b0c2a66d4ac8d285ee4e508388241798e60a4e500dc5ab5e1fde02a77720",
"unchanged_shared_supervisor_sha256": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"cpu_replay_log_sha256": "6bf84633d3dc09cbd5002a149a8d0ebe3ef205ccb9748096f50ac7ee906971a8",
"lifecycle_replay_sha256": "feb40e583810259537ad746ccfbbf2e56ff9bc969f002f1e7f74657e3b88cf8d",
"read_only_collection": {
"seeds": {
"0": {
"fit": {
"status": "NOT_STARTED"
},
"score": {
"status": "NOT_STARTED"
}
},
"1": {
"fit": {
"status": "NOT_STARTED"
},
"score": {
"status": "NOT_STARTED"
}
},
"2": {
"fit": {
"status": "NOT_STARTED"
},
"score": {
"status": "NOT_STARTED"
}
}
},
"mean_percent": null,
"within_registered_range": null,
"reproduction_identity": "supplement code + 2-keyword patch",
"prior_measurement_gpu_seconds": 222.80500172509346,
"settled_formal_gpu_seconds": 0,
"settled_combined_gpu_seconds": 222.80500172509346,
"unsettled_formal_bookings_gpu_seconds": 0,
"formal_gpu_cap_seconds": 40000
}
}
@@ -0,0 +1,18 @@
test_card_reservation_expiry_and_total_budget (test_track_b_gino_formal.FormalBinding) ... ok
test_inclusive_gate_and_all_three_seeds_required (test_track_b_gino_formal.FormalBinding) ... ok
test_original_split_score_keeps_order_and_rejects_nan (test_track_b_gino_formal.FormalBinding) ... ok
test_score_opens_one_111_target_pass_and_rejects_a_second (test_track_b_gino_formal.FormalBinding) ... ok
test_scoring_needs_all_checkpoints_before_access (test_track_b_gino_formal.FormalBinding) ... ok
test_failed_worker_exit_has_no_stale_group_signal (test_track_b_gino_formal.GuardReplay) ... ok
test_gpu_list_before_worker_exit (test_track_b_gino_formal.GuardReplay) ... ok
test_real_foreign_identity_has_exact_stop_receipt (test_track_b_gino_formal.GuardReplay) ... ok
test_reservation_expires_after_idle_but_before_spawn (test_track_b_gino_formal.GuardReplay) ... ok
test_reused_worker_pid_is_foreign (test_track_b_gino_formal.GuardReplay) ... ok
test_running_guard_continues_after_reservation_end (test_track_b_gino_formal.GuardReplay) ... ok
test_simulated_reused_pgid_cannot_be_signalled (test_track_b_gino_formal.GuardReplay) ... ok
test_worker_already_exited_before_gpu_list (test_track_b_gino_formal.GuardReplay) ... ok
----------------------------------------------------------------------
Ran 13 tests in 1.232s
OK
@@ -0,0 +1,114 @@
{
"status": "PASS",
"tests": 13,
"real_gpu_queries": 0,
"pressure_members_opened": 0,
"cases": {
"failed-exit": {
"status": "STOPPED",
"trigger": "WORKER_FAILED",
"syscalls": [
{
"event": "wait4_reaped",
"pid": 67
}
],
"lifecycle": [
{
"event": "leader_reaped",
"pid": 67
}
]
},
"gpu-first": {
"status": "COMPLETE",
"trigger": null,
"syscalls": [
{
"event": "wait4_reaped",
"pid": 69
}
],
"lifecycle": [
{
"event": "leader_reaped",
"pid": 69
}
]
},
"foreign": {
"status": "STOPPED",
"trigger": "FOREIGN_GPU_OCCUPANT",
"syscalls": [
{
"event": "killpg",
"pid": 71,
"signal": 9,
"leader_reaped": false
},
{
"event": "wait4_reaped",
"pid": 71
}
],
"lifecycle": [
{
"event": "group_signal",
"pid": 71,
"signal": 9,
"reason": "GUARD_STOP"
},
{
"event": "leader_reaped",
"pid": 71
}
]
},
"expired-before-spawn": {
"status": "STOPPED",
"trigger": "CARD_RESERVATION_EXPIRED_BEFORE_SPAWN",
"syscalls": [],
"lifecycle": []
},
"crossed-after-spawn": {
"status": "COMPLETE",
"trigger": null,
"syscalls": [
{
"event": "wait4_reaped",
"pid": 73
}
],
"lifecycle": [
{
"event": "leader_reaped",
"pid": 73
}
]
},
"pgid-reuse": {
"reaped": true,
"simulated_new_identity": {
"pid": 12,
"created": 2.0
},
"signals": 0
},
"tree-first": {
"status": "COMPLETE",
"trigger": null,
"syscalls": [
{
"event": "wait4_reaped",
"pid": 75
}
],
"lifecycle": [
{
"event": "leader_reaped",
"pid": 75
}
]
}
}
}
@@ -0,0 +1,9 @@
# Track B GINO 三种子复现联合预审定稿候选
本绑定替换中间提交 `8af022e4` 供同一次独立预审。MeltAssist 在旧生产守卫的 CPU 回放中捕获 `wait4` 已回收组长后的 `killpg(SIGKILL)`。旧回放与两次 GPU 测量终止均保留。 [MeltAssist 阻塞回执](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795075951091959) · [旧 CPU 调用次序](../artifacts/meltassist-track-b-gino-formal-joint-review-2026-11-19/reap-signal-replay.json)
修复只接入本 GINO 守卫所用的监督器调用,原共享文件及别的运行固定版本保持原样。用 `waitid(WNOWAIT)` 核实固定 PID 仍是本方直接子进程。守卫在同一把锁下验证组长身份并发送信号,或交由 `wait4` 回收并标记结束。需要清理残留同组子进程时先清理再回收组长。组长回收后,旧清理调用不再发信号。 [生产守卫](../tools/track_b_gino_formal_guard.py)
实际生产守卫 CPU 回放 13/13。拦截并记录所有组清理信号及 `wait4`。成功退出、非零退出、两类陈旧 GPU 列表和跨午夜运行都没有回收后信号。模拟真实外部占用时只向自己仍未回收的组发送一次 SIGKILL,随后实际回收,外部活进程未收信号。复用组 ID 负例到达清理入口时不触发信号。午夜到期且尚未派发的路径不创建 worker。 [调用及回收次序](../artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/lifecycle-cpu-replay.json) · [十三项 CPU 日志](../artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-replay.log)
种子 0、1、2 分别在 GPU2、GPU3、GPU0,各 100 完整轮。香港时间 2026-11-19 当日预留只适用于新派发,已开始的守卫照常运作。三份检查点冻结后,每个种子各评分一次原 111 件,三种子等权平均门含两端 6.62% 和 7.62%。正式总帽 40,000 GPU 秒。102.9377786 秒为未经守卫认证的单轮计时,仅用于定预算,旧账 222.805002 GPU 秒独计一次。训练及评分身份为 `supplement code + 2-keyword patch`。本准备阶段没有真实 GPU 派发或留出目标读取。 [组合合同](../artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/contract.json) · [训练与评分](../tools/track_b_gino_formal_worker.py)
+51 -10
View File
@@ -15,6 +15,9 @@ import track_b_gino_formal_worker as worker
from track_b_gino_shapenet_entry import read, save, sha
REPLAYS = {}
class GuardReplay(unittest.TestCase):
def replay(self, ordering):
uuid = 'GPU-fixture'
@@ -32,16 +35,28 @@ class GuardReplay(unittest.TestCase):
"root=Path(sys.argv[1])\n"
"while not (root/'exit').exists(): time.sleep(.01)\n"
"(root/'worker-records/result.json').write_text("
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder]
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n"
"if sys.argv[2] == 'failed-exit': raise SystemExit(7)\n", folder, ordering]
observed = []
stopped = []
real_stop = gpu_guard.stop_group
syscalls = []
real_kill = os.killpg
real_wait = os.wait4
real_tree = gpu_guard.process_tree
ticket = dict(ordinal='', dispatch_before_unix=time.time() + 15)
def stop_owned(pid):
stopped.append(pid)
real_stop(pid)
def record_kill(pid, signal):
own = read(root / 'actual-worker-spawn.json')['process']
is_reaped = any(e == dict(event='wait4_reaped', pid=pid) for e in syscalls)
syscalls.append(dict(event='killpg', pid=pid, signal=int(signal), leader_reaped=is_reaped))
if is_reaped or pid != own['pid']:
raise AssertionError('Invalid group signal intercepted before delivery')
return real_kill(pid, signal)
def record_wait(pid, options):
receipt = real_wait(pid, options)
if receipt[0]:
syscalls.append(dict(event='wait4_reaped', pid=receipt[0]))
return receipt
def exit_and_wait():
(root / 'exit').touch()
@@ -74,13 +89,14 @@ class GuardReplay(unittest.TestCase):
occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')])
if ordering == 'crossed-after-spawn':
ticket['dispatch_before_unix'] = time.time() - 1
if ordering in ('gpu-first', 'crossed-after-spawn'):
if ordering in ('gpu-first', 'crossed-after-spawn', 'failed-exit'):
exit_and_wait()
return sample
with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot),
patch.object(gpu_guard, 'process_tree', side_effect=tree),
patch.object(gpu_guard, 'stop_group', side_effect=stop_owned),
patch.object(os, 'killpg', side_effect=record_kill),
patch.object(os, 'wait4', side_effect=record_wait),
patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}),
patch.object(gpu_guard, 'dispatch_environment',
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))):
@@ -88,10 +104,13 @@ class GuardReplay(unittest.TestCase):
ticket['guard_started_monotonic'] = began
outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15,
ticket=ticket, validate_result=lambda: sha(result))
REPLAYS[ordering] = dict(status=outcome['status'], trigger=outcome['trigger'],
syscalls=syscalls, lifecycle=outcome['lifecycle'])
if ordering == 'expired-before-spawn':
self.assertEqual(outcome['status'], 'STOPPED')
self.assertEqual(outcome['trigger'], 'CARD_RESERVATION_EXPIRED_BEFORE_SPAWN')
self.assertFalse((root / 'actual-worker-spawn.json').exists())
self.assertEqual(syscalls, [])
return observed
spawn = read(root / 'actual-worker-spawn.json')['process']
if ordering == 'foreign':
@@ -102,11 +121,16 @@ class GuardReplay(unittest.TestCase):
self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid))
self.assertEqual(row['occupants'][0]['pid'], os.getpid())
self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python')
self.assertEqual(stopped, [spawn['pid']])
self.assertEqual(syscalls, [
dict(event='killpg', pid=spawn['pid'], signal=9, leader_reaped=False),
dict(event='wait4_reaped', pid=spawn['pid'])])
elif ordering == 'failed-exit':
self.assertEqual((outcome['status'], outcome['worker']['returncode']), ('STOPPED', 7))
self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])])
else:
self.assertEqual(outcome['status'], 'COMPLETE', outcome)
self.assertFalse((root / 'foreign-occupant.json').exists())
self.assertEqual(stopped, [])
self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])])
self.assertEqual(outcome['result_sha256'], sha(result))
return observed
@@ -127,6 +151,23 @@ class GuardReplay(unittest.TestCase):
def test_running_guard_continues_after_reservation_end(self):
self.replay('crossed-after-spawn')
def test_failed_worker_exit_has_no_stale_group_signal(self):
self.replay('failed-exit')
def test_simulated_reused_pgid_cannot_be_signalled(self):
group = gpu_guard.OwnedGroup()
group.pid = 12
group.identity = dict(pid=12, created=1.)
group.reaped = True
reused = dict(pid=12, created=2.)
with (patch.object(gpu_guard, 'current_identity', return_value=reused),
patch.object(os, 'waitid') as observe,
patch.object(os, 'killpg') as intercept):
group.stop(12)
observe.assert_not_called()
intercept.assert_not_called()
REPLAYS['pgid-reuse'] = dict(reaped=group.reaped, simulated_new_identity=reused, signals=0)
def test_reused_worker_pid_is_foreign(self):
with TemporaryDirectory() as folder:
root = Path(folder)
+2 -1
View File
@@ -18,7 +18,7 @@ from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-v2-2026-11-19')
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-v3-2026-11-19')
SOURCES = Path(__file__).resolve().parent
OLD_FILES = {
'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a',
@@ -67,6 +67,7 @@ def expected_contract():
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES})
c['formal'] = dict(
handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES,
lifecycle_review_slack_ts='1795075951.091959',
device_pci_ordinals=ORDINALS,
reservation=dict(handoff='h-a429935e', timezone=TZ.key,
start_inclusive=RESERVATION_START.isoformat(),
+78 -10
View File
@@ -2,6 +2,7 @@
import os
import shutil
import signal
import subprocess
import threading
import time
@@ -11,7 +12,7 @@ import psutil
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords
from s2_recovery_gpu_supervisor import stop_group, supervise
from s2_recovery_gpu_supervisor import supervise
from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity
from track_b_gino_shapenet_entry import process_identity, save
@@ -71,7 +72,70 @@ class OccupancyMonitor:
raise RuntimeError('DEVICE_VRAM_LIMIT')
def watch(root, uuid, worker, limits, done, tracker, state):
class OwnedGroup:
"""Tie group signals to an unreaped session leader we directly spawned."""
def __init__(self):
self.pid = None
self.identity = None
self.reaped = False
self.signalled = False
self.lock = threading.Lock()
self.events = []
def observe_exit(self, pid, nonblocking):
flags = os.WEXITED | os.WNOWAIT | (os.WNOHANG if nonblocking else 0)
return os.waitid(os.P_PID, pid, flags)
def stop_locked(self, pid, reason):
if pid != self.pid or self.reaped or self.signalled:
return
# WNOWAIT verifies this PID is still our child without permitting PID
# reuse. The same lock prevents our wait4 from reaping before killpg.
try:
self.observe_exit(pid, nonblocking=True)
if os.getpgid(pid) != pid:
raise RuntimeError('WORKER_IS_NOT_ITS_OWN_SESSION_LEADER')
if self.identity is not None and current_identity(pid) != self.identity:
raise RuntimeError('WORKER_BIRTH_IDENTITY_DIFFERS')
os.killpg(pid, signal.SIGKILL)
except (ChildProcessError, ProcessLookupError):
return
self.signalled = True
self.events.append(dict(event='group_signal', pid=pid, signal=int(signal.SIGKILL), reason=reason))
def stop(self, pid):
with self.lock:
self.stop_locked(pid, 'GUARD_STOP')
def wait4(self, pid, options):
if pid != self.pid:
# The shared supervisor separately collects adopted descendants
# with a negative group selector. That path never sends signals.
return os.wait4(pid, options)
# Never hold the lock on a blocking wait. The watcher may still need
# it to stop the worker.
if self.observe_exit(pid, nonblocking=bool(options & os.WNOHANG)) is None:
return 0, 0, None
with self.lock:
# With subreaping enabled, children the leader left in its group
# are now ours. Clean them up while the zombie still pins PGID.
for child in psutil.Process().children():
try:
in_group = child.pid != pid and os.getpgid(child.pid) == pid
except ProcessLookupError:
continue
if in_group:
self.stop_locked(pid, 'REMAINING_GROUP_MEMBER')
break
result = os.wait4(pid, options)
if result[0] == pid:
self.reaped = True
self.events.append(dict(event='leader_reaped', pid=pid))
return result
def watch(root, uuid, worker, group, limits, done, tracker, state):
occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes'])
while not done.is_set():
try:
@@ -83,9 +147,7 @@ def watch(root, uuid, worker, limits, done, tracker, state):
monitor_identity(root, uuid)
except Exception as exc:
state['trigger'] = str(exc) or type(exc).__name__
# Signal only the recorded worker group while its leader has the same identity.
if current_identity(worker['pid']) == worker:
stop_group(worker['pid'])
group.stop(worker['pid'])
return
done.wait(.5)
@@ -95,6 +157,7 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds'])
done = threading.Event()
tracker = PeakRecords()
group = OwnedGroup()
state = dict(trigger=None, peak_device_bytes=0)
watcher = None
worker_receipt = None
@@ -114,16 +177,19 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
if time.time() >= ticket['dispatch_before_unix']:
raise PermissionError('CARD_RESERVATION_EXPIRED_BEFORE_SPAWN')
proc = subprocess.Popen(args, **dict(kwargs, env=env))
group.pid = proc.pid
try:
worker = process_identity(proc.pid)
group.identity = worker
save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid))
watcher = threading.Thread(target=watch, daemon=True,
args=(root, uuid, worker, limits, done, tracker, state))
args=(root, uuid, worker, group, limits, done, tracker, state))
watcher.start()
except (Exception, KeyboardInterrupt):
# The shared supervisor has not yet received the process to reap.
stop_group(proc.pid)
proc.wait()
group.stop(proc.pid)
_, status, _ = group.wait4(proc.pid, 0)
proc.returncode = os.waitstatus_to_exitcode(status)
raise
return proc
@@ -139,7 +205,8 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
remaining = deadline - time.monotonic() - limits['cleanup_seconds']
if remaining <= 0:
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT),
os=SimpleNamespace(**dict(vars(os), wait4=group.wait4)), stop_group=group.stop)
run_worker = FunctionType(supervise.__code__, ns, supervise.__name__)
try:
worker_receipt = run_worker(command, root, 'gino', dict(
@@ -172,6 +239,7 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt,
physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state,
charged_gpu_seconds=end - ticket['guard_started_monotonic'],
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True)
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima,
lifecycle=group.events, no_retry=True)
save(root / 'complete.json', terminal)
return terminal