Bind GINO cleanup to unreaped worker group and replay actual signals
This commit is contained in:
1 parent
8af022e434
commit
ee8e01937e
9 files changed
+1185
-21
No files matched your search
+24
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"production_guard_sha256": "2267318d0e99e8c465bbfdf9e5dc7cac46b3b3f5fe88ba5bad647c49222ae128",
|
||||
"shared_supervisor_sha256": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"production_guard_status": "COMPLETE",
|
||||
"spawned_cpu_fixture": {
|
||||
"pid": 3,
|
||||
"created": 1795075930.27
|
||||
},
|
||||
"observed_sequence": [
|
||||
{
|
||||
"event": "wait4_reaped",
|
||||
"pid": 3
|
||||
},
|
||||
{
|
||||
"event": "killpg_intercepted",
|
||||
"pgid": 3,
|
||||
"signal": 9,
|
||||
"leader_already_reaped": true
|
||||
}
|
||||
],
|
||||
"real_signals_sent_by_fixture": 0,
|
||||
"real_gpu_queries": 0,
|
||||
"real_pressure_members_opened": 0
|
||||
}
|
||||
@@ -0,0 +1,837 @@
|
||||
{
|
||||
"mode": "formal-gino-reproduction",
|
||||
"ruling": "h-f84547d7",
|
||||
"retry": false,
|
||||
"evaluate_other_models": false,
|
||||
"limits": {
|
||||
"cleanup_seconds": 20,
|
||||
"idle_wait_seconds": 60,
|
||||
"rss_bytes": 12884901888,
|
||||
"recorded_vram_bytes": 15032385536,
|
||||
"device_vram_bytes": 16106127360,
|
||||
"disk_bytes": 2147483648,
|
||||
"torch_threads": 1,
|
||||
"max_processes": 1,
|
||||
"cpu_seconds_per_gpu_second": 8
|
||||
},
|
||||
"protocol": {
|
||||
"model": "NeurIPS 2023 supplement SDFFNOGNO decoder",
|
||||
"seeds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"grid": [
|
||||
64,
|
||||
64,
|
||||
64
|
||||
],
|
||||
"modes": [
|
||||
24,
|
||||
24,
|
||||
24
|
||||
],
|
||||
"hidden_channels": 32,
|
||||
"radius": 0.05,
|
||||
"epochs_per_seed": 100,
|
||||
"batch_size": 1,
|
||||
"learning_rate": 0.001,
|
||||
"bounds_expansion": 0.01,
|
||||
"pressure_epsilon": 1e-06,
|
||||
"optimizer": "Adam",
|
||||
"weight_decay": 0.0001,
|
||||
"lr_step_epochs": 50,
|
||||
"lr_gamma": 0.5,
|
||||
"pressure_training_transform": "scalar Gaussian on the 500 learning cases",
|
||||
"metric": "per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)",
|
||||
"report": "equal mean over 111 cases for each seed, then equal mean over the three seeds",
|
||||
"acceptance_percent": [
|
||||
6.62,
|
||||
7.62
|
||||
],
|
||||
"paper_reference_percent": 7.12,
|
||||
"checkpoint_epoch": 100,
|
||||
"heldout_access_after_all_checkpoints_frozen": true
|
||||
},
|
||||
"environment": {
|
||||
"python": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"python_version": "3.10",
|
||||
"torch_cuda": "11.6",
|
||||
"neuralop_commit": "fbdc7f872792b75b84545c2f374e7a881c72a718",
|
||||
"neuralop_file_sha256": {
|
||||
"utils.py": "84554eed45ac415ab52f79e456eab70ba204e7ab7539c2f2841955a9145220b4",
|
||||
"models/__init__.py": "c67679f1f8f90e1ab4eb4c2aa916aeaedf1b175d795fed03b8d4f7210883ebae",
|
||||
"models/tfno.py": "d74b5805b69a64bf33e3ff5186709c75bd431a56c9548f06fb9635a859d849e4",
|
||||
"models/spectral_convolution.py": "ccecf80806d93e8f35713ac6ab2c3b61c11739f5fcb8c0ba7397b7d4eb5a5d45"
|
||||
},
|
||||
"package_versions": {
|
||||
"torch": "1.13.1+cu116",
|
||||
"open3d": "0.17.0",
|
||||
"numpy": "1.24.3",
|
||||
"torch-scatter": "2.1.1+pt113cu116",
|
||||
"torch-sparse": "0.6.17+pt113cu116",
|
||||
"torch-geometric": "2.3.1",
|
||||
"tensorly": "0.8.1",
|
||||
"tensorly-torch": "0.4.0"
|
||||
}
|
||||
},
|
||||
"comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-measurement-2026-11-19/comparison-contract.json",
|
||||
"comparison_contract_sha256": "503f6c8ab1217d66b44e0d127555cfad5da4cd8a17b336299095c3cfd13041d7",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_shapenet.py": "557ad56277d7e85c6b5cc358eb151632d1757f5883f429988d36022d0d407550",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_entry.py": "e86d955c36e291e6bbef67947de663b24c8d66c5d72d9d28e4193fb8a3051ee2",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py": "7211be211c29472fd9be35a6bcb1f56025de4a63f36852bfafa10df97c003ff3",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_entry.py": "199d833f40339feb0433611ba27302416a280452f8918715e6ef95cb93c38721",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_worker.py": "3acf742dd4549e9f5491e10af5bb0b4acf00e2f47e91753100b5998cfcc61852",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_import_checked.py": "fef485dd0c233c76a336239d5f29cbf44ac0fb62a6a297dff34293b23a1b75fe",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_paper_keyword_patch.py": "c61e2ed7d099eb125386857bf4d7ccdbfc3af522f80edb4f01b450cee91c0b95",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2.py": "7d3ab2412e5208892cffcfc0e1ec363eb5db2be3e10bf60595356f71e82d8552",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py": "ba1fd5ec837e884b000ba640fceafe30ea40ccf708847733df6a17a8f3599680",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery.py": "1e1827498afab62e2f28efb5d450376f2dca4e8bf6f74114f2914ec771872c86",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py": "7a68c77e9c913ba9161832f513dbf8754ebc5c270c1e742fa5ae97a7b14b9120",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal.py": "aa7b1301203d433564491b72fc967a29898098ad8d0d3677cda88991fbc7b483",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_guard.py": "70820fb9208960485efe07654057977b676a3003563b6a54cf00c86dc838613d",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_worker.py": "978f1c7cfb622300b8814e14e4a02e5cbb9e2ea7f6e5fee080b986c7409ade2c"
|
||||
},
|
||||
"paper_data": {
|
||||
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-car-pressure-data.zip",
|
||||
"sha256": "b5f9e2b407d54e034596384253fb58196933c46f6b5b586383508da54475f89d"
|
||||
},
|
||||
"paper_supplement": {
|
||||
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-neurips-supplement.zip",
|
||||
"sha256": "48cdae80ac2781e27ac00e10a37124711d6bb305593559002bda0527d11a533d"
|
||||
},
|
||||
"paper_code_zip_sha256": "347a6f097e2b9958d1ab87930fa1d6227b793a9d213e12418ce9a157e83dce04",
|
||||
"split_receipt": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-split.json",
|
||||
"sha256": "5a96c25c2ec24cfc1e2264d4ebd0879c658c0f168e22711b576bf303151b8601"
|
||||
},
|
||||
"mapping_audit": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-mapping-audit.json",
|
||||
"sha256": "055d008a02494acd29df36c41c8e68db931a027f5635e8f5f765a96a54c62289"
|
||||
},
|
||||
"split": {
|
||||
"train": [
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
10,
|
||||
12,
|
||||
13,
|
||||
17,
|
||||
18,
|
||||
21,
|
||||
22,
|
||||
23,
|
||||
25,
|
||||
26,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31,
|
||||
32,
|
||||
34,
|
||||
35,
|
||||
39,
|
||||
40,
|
||||
43,
|
||||
44,
|
||||
45,
|
||||
46,
|
||||
47,
|
||||
48,
|
||||
49,
|
||||
50,
|
||||
51,
|
||||
52,
|
||||
54,
|
||||
55,
|
||||
56,
|
||||
58,
|
||||
59,
|
||||
60,
|
||||
61,
|
||||
62,
|
||||
63,
|
||||
64,
|
||||
65,
|
||||
67,
|
||||
69,
|
||||
70,
|
||||
71,
|
||||
72,
|
||||
73,
|
||||
74,
|
||||
75,
|
||||
76,
|
||||
77,
|
||||
78,
|
||||
79,
|
||||
80,
|
||||
81,
|
||||
83,
|
||||
84,
|
||||
85,
|
||||
86,
|
||||
87,
|
||||
88,
|
||||
90,
|
||||
91,
|
||||
92,
|
||||
94,
|
||||
95,
|
||||
96,
|
||||
97,
|
||||
100,
|
||||
101,
|
||||
102,
|
||||
105,
|
||||
106,
|
||||
107,
|
||||
109,
|
||||
110,
|
||||
111,
|
||||
112,
|
||||
113,
|
||||
114,
|
||||
115,
|
||||
116,
|
||||
117,
|
||||
118,
|
||||
119,
|
||||
120,
|
||||
121,
|
||||
123,
|
||||
124,
|
||||
125,
|
||||
126,
|
||||
127,
|
||||
128,
|
||||
129,
|
||||
130,
|
||||
131,
|
||||
133,
|
||||
134,
|
||||
136,
|
||||
137,
|
||||
138,
|
||||
139,
|
||||
140,
|
||||
141,
|
||||
142,
|
||||
143,
|
||||
144,
|
||||
145,
|
||||
146,
|
||||
147,
|
||||
148,
|
||||
149,
|
||||
150,
|
||||
151,
|
||||
152,
|
||||
153,
|
||||
155,
|
||||
156,
|
||||
157,
|
||||
158,
|
||||
159,
|
||||
160,
|
||||
161,
|
||||
162,
|
||||
163,
|
||||
165,
|
||||
166,
|
||||
170,
|
||||
172,
|
||||
173,
|
||||
175,
|
||||
176,
|
||||
177,
|
||||
178,
|
||||
179,
|
||||
180,
|
||||
181,
|
||||
182,
|
||||
183,
|
||||
184,
|
||||
186,
|
||||
190,
|
||||
191,
|
||||
192,
|
||||
193,
|
||||
195,
|
||||
196,
|
||||
198,
|
||||
199,
|
||||
200,
|
||||
201,
|
||||
202,
|
||||
203,
|
||||
205,
|
||||
207,
|
||||
210,
|
||||
211,
|
||||
212,
|
||||
213,
|
||||
214,
|
||||
215,
|
||||
217,
|
||||
219,
|
||||
220,
|
||||
221,
|
||||
222,
|
||||
223,
|
||||
224,
|
||||
225,
|
||||
227,
|
||||
228,
|
||||
229,
|
||||
230,
|
||||
231,
|
||||
232,
|
||||
233,
|
||||
234,
|
||||
235,
|
||||
236,
|
||||
237,
|
||||
241,
|
||||
243,
|
||||
244,
|
||||
245,
|
||||
246,
|
||||
247,
|
||||
248,
|
||||
249,
|
||||
251,
|
||||
252,
|
||||
253,
|
||||
255,
|
||||
257,
|
||||
258,
|
||||
259,
|
||||
260,
|
||||
261,
|
||||
262,
|
||||
263,
|
||||
264,
|
||||
266,
|
||||
267,
|
||||
268,
|
||||
269,
|
||||
271,
|
||||
272,
|
||||
273,
|
||||
274,
|
||||
275,
|
||||
276,
|
||||
277,
|
||||
278,
|
||||
279,
|
||||
280,
|
||||
281,
|
||||
282,
|
||||
283,
|
||||
285,
|
||||
286,
|
||||
289,
|
||||
290,
|
||||
291,
|
||||
292,
|
||||
293,
|
||||
294,
|
||||
295,
|
||||
296,
|
||||
297,
|
||||
298,
|
||||
299,
|
||||
300,
|
||||
301,
|
||||
302,
|
||||
304,
|
||||
305,
|
||||
306,
|
||||
308,
|
||||
309,
|
||||
310,
|
||||
311,
|
||||
312,
|
||||
313,
|
||||
314,
|
||||
315,
|
||||
319,
|
||||
320,
|
||||
321,
|
||||
322,
|
||||
323,
|
||||
324,
|
||||
325,
|
||||
327,
|
||||
328,
|
||||
329,
|
||||
331,
|
||||
332,
|
||||
333,
|
||||
334,
|
||||
335,
|
||||
337,
|
||||
338,
|
||||
339,
|
||||
340,
|
||||
341,
|
||||
344,
|
||||
345,
|
||||
347,
|
||||
348,
|
||||
349,
|
||||
350,
|
||||
352,
|
||||
353,
|
||||
354,
|
||||
355,
|
||||
356,
|
||||
357,
|
||||
358,
|
||||
360,
|
||||
362,
|
||||
364,
|
||||
365,
|
||||
366,
|
||||
367,
|
||||
369,
|
||||
371,
|
||||
372,
|
||||
373,
|
||||
374,
|
||||
375,
|
||||
376,
|
||||
378,
|
||||
379,
|
||||
380,
|
||||
381,
|
||||
384,
|
||||
385,
|
||||
389,
|
||||
392,
|
||||
393,
|
||||
397,
|
||||
398,
|
||||
399,
|
||||
401,
|
||||
402,
|
||||
403,
|
||||
404,
|
||||
405,
|
||||
407,
|
||||
408,
|
||||
410,
|
||||
412,
|
||||
413,
|
||||
414,
|
||||
415,
|
||||
417,
|
||||
418,
|
||||
419,
|
||||
420,
|
||||
422,
|
||||
424,
|
||||
425,
|
||||
427,
|
||||
430,
|
||||
431,
|
||||
433,
|
||||
435,
|
||||
436,
|
||||
437,
|
||||
439,
|
||||
440,
|
||||
443,
|
||||
444,
|
||||
446,
|
||||
447,
|
||||
448,
|
||||
449,
|
||||
450,
|
||||
451,
|
||||
452,
|
||||
453,
|
||||
454,
|
||||
455,
|
||||
456,
|
||||
457,
|
||||
459,
|
||||
460,
|
||||
462,
|
||||
463,
|
||||
464,
|
||||
465,
|
||||
466,
|
||||
467,
|
||||
468,
|
||||
469,
|
||||
470,
|
||||
472,
|
||||
473,
|
||||
474,
|
||||
475,
|
||||
476,
|
||||
478,
|
||||
479,
|
||||
480,
|
||||
482,
|
||||
483,
|
||||
486,
|
||||
487,
|
||||
488,
|
||||
490,
|
||||
493,
|
||||
494,
|
||||
495,
|
||||
496,
|
||||
497,
|
||||
498,
|
||||
499,
|
||||
501,
|
||||
502,
|
||||
503,
|
||||
504,
|
||||
505,
|
||||
507,
|
||||
508,
|
||||
509,
|
||||
511,
|
||||
512,
|
||||
513,
|
||||
514,
|
||||
515,
|
||||
516,
|
||||
518,
|
||||
519,
|
||||
521,
|
||||
522,
|
||||
523,
|
||||
524,
|
||||
525,
|
||||
527,
|
||||
529,
|
||||
530,
|
||||
532,
|
||||
533,
|
||||
536,
|
||||
538,
|
||||
539,
|
||||
540,
|
||||
542,
|
||||
543,
|
||||
545,
|
||||
547,
|
||||
548,
|
||||
549,
|
||||
550,
|
||||
551,
|
||||
552,
|
||||
553,
|
||||
554,
|
||||
555,
|
||||
560,
|
||||
561,
|
||||
562,
|
||||
564,
|
||||
565,
|
||||
566,
|
||||
567,
|
||||
568,
|
||||
569,
|
||||
572,
|
||||
573,
|
||||
574,
|
||||
576,
|
||||
577,
|
||||
579,
|
||||
581,
|
||||
582,
|
||||
583,
|
||||
584,
|
||||
587,
|
||||
588,
|
||||
589,
|
||||
591,
|
||||
593,
|
||||
594,
|
||||
595,
|
||||
596,
|
||||
597,
|
||||
598,
|
||||
600,
|
||||
602,
|
||||
604,
|
||||
608,
|
||||
610,
|
||||
611,
|
||||
612,
|
||||
613,
|
||||
615,
|
||||
616,
|
||||
617,
|
||||
618,
|
||||
620,
|
||||
621,
|
||||
622,
|
||||
623,
|
||||
625,
|
||||
626,
|
||||
627,
|
||||
628,
|
||||
629,
|
||||
630,
|
||||
631,
|
||||
632,
|
||||
633,
|
||||
634,
|
||||
635,
|
||||
636,
|
||||
638,
|
||||
639,
|
||||
640,
|
||||
641,
|
||||
642,
|
||||
643,
|
||||
644,
|
||||
645,
|
||||
646,
|
||||
647,
|
||||
648,
|
||||
649,
|
||||
651,
|
||||
652,
|
||||
654,
|
||||
655,
|
||||
656,
|
||||
657
|
||||
],
|
||||
"test": [
|
||||
658,
|
||||
659,
|
||||
660,
|
||||
662,
|
||||
663,
|
||||
664,
|
||||
665,
|
||||
666,
|
||||
667,
|
||||
668,
|
||||
672,
|
||||
673,
|
||||
674,
|
||||
675,
|
||||
676,
|
||||
677,
|
||||
678,
|
||||
679,
|
||||
681,
|
||||
683,
|
||||
684,
|
||||
686,
|
||||
687,
|
||||
688,
|
||||
689,
|
||||
690,
|
||||
691,
|
||||
692,
|
||||
693,
|
||||
695,
|
||||
696,
|
||||
697,
|
||||
700,
|
||||
701,
|
||||
702,
|
||||
703,
|
||||
704,
|
||||
705,
|
||||
708,
|
||||
709,
|
||||
710,
|
||||
711,
|
||||
712,
|
||||
713,
|
||||
715,
|
||||
717,
|
||||
718,
|
||||
719,
|
||||
721,
|
||||
722,
|
||||
723,
|
||||
724,
|
||||
725,
|
||||
726,
|
||||
727,
|
||||
729,
|
||||
730,
|
||||
731,
|
||||
733,
|
||||
734,
|
||||
735,
|
||||
736,
|
||||
737,
|
||||
738,
|
||||
739,
|
||||
740,
|
||||
741,
|
||||
743,
|
||||
744,
|
||||
745,
|
||||
746,
|
||||
747,
|
||||
748,
|
||||
749,
|
||||
750,
|
||||
751,
|
||||
752,
|
||||
753,
|
||||
757,
|
||||
758,
|
||||
760,
|
||||
762,
|
||||
763,
|
||||
764,
|
||||
765,
|
||||
766,
|
||||
767,
|
||||
768,
|
||||
769,
|
||||
770,
|
||||
771,
|
||||
772,
|
||||
773,
|
||||
775,
|
||||
776,
|
||||
777,
|
||||
778,
|
||||
779,
|
||||
781,
|
||||
782,
|
||||
783,
|
||||
784,
|
||||
785,
|
||||
787,
|
||||
789,
|
||||
790,
|
||||
791,
|
||||
792,
|
||||
793,
|
||||
795,
|
||||
797
|
||||
],
|
||||
"raw_global_bounds": [
|
||||
[
|
||||
-0.90252799,
|
||||
0.0066688,
|
||||
-2.85272002
|
||||
],
|
||||
[
|
||||
0.90253001,
|
||||
1.95825005,
|
||||
2.86385012
|
||||
]
|
||||
]
|
||||
},
|
||||
"environment_ruling": "h-0d21f3bd",
|
||||
"environment_acceptance": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-import-checked-2026-11-19/cpu-environment-acceptance.json",
|
||||
"sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3"
|
||||
},
|
||||
"paper_compatibility": {
|
||||
"handoff": "h-b8508140",
|
||||
"identity": "supplement code + 2-keyword patch",
|
||||
"patch_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/two-keyword-patch.json",
|
||||
"patch_sha256": "7307ba49f83bfc46c5decf803a993a41ed602d987de65714958e738d2f52f077",
|
||||
"search_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/keyword-search.json",
|
||||
"search_sha256": "77a897bf534a9a0e9e6a7c072b4f914168648eadc2af3493b70d63d0b715b3d0"
|
||||
},
|
||||
"formal": {
|
||||
"handoff": "h-4e4e680b",
|
||||
"ruling_slack_ts": "1795074651.222599",
|
||||
"devices": {
|
||||
"0": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"1": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"2": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c"
|
||||
},
|
||||
"lifecycle_review_slack_ts": "1795075951.091959",
|
||||
"device_pci_ordinals": {
|
||||
"0": 2,
|
||||
"1": 3,
|
||||
"2": 0
|
||||
},
|
||||
"reservation": {
|
||||
"handoff": "h-a429935e",
|
||||
"timezone": "Asia/Hong_Kong",
|
||||
"start_inclusive": "2026-11-19T00:00:00+08:00",
|
||||
"end_exclusive": "2026-11-20T00:00:00+08:00",
|
||||
"gpu3_single_task": true,
|
||||
"historical_inventory_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json",
|
||||
"historical_inventory_sha256": "da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8",
|
||||
"runtime_idle_check_required": true,
|
||||
"applies_to_new_dispatch_only": true,
|
||||
"already_running_guards_unchanged": true
|
||||
},
|
||||
"budget_per_seed_stage_gpu_seconds": {
|
||||
"0": {
|
||||
"fit": 13034,
|
||||
"score": 300
|
||||
},
|
||||
"1": {
|
||||
"fit": 13033,
|
||||
"score": 300
|
||||
},
|
||||
"2": {
|
||||
"fit": 13033,
|
||||
"score": 300
|
||||
}
|
||||
},
|
||||
"total_gpu_wall_seconds": 40000,
|
||||
"timing": {
|
||||
"record_paths": {
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json": "db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json": "851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json": "8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf"
|
||||
},
|
||||
"status": "CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED",
|
||||
"sizing_only": true,
|
||||
"epoch_seconds": 102.93777855299413,
|
||||
"train_sizing_gpu_seconds": 38601.6669573728,
|
||||
"train_sizing_multiplier": 1.25,
|
||||
"score_reserved_gpu_seconds": 900,
|
||||
"prior_measurement_gpu_seconds": 222.80500172509346
|
||||
},
|
||||
"heldout_policy": {
|
||||
"preparation_reads": 0,
|
||||
"training_reads": 0,
|
||||
"score_reads_per_seed": 111,
|
||||
"all_epoch100_checkpoints_required": true,
|
||||
"retry": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"contract_sha256": "c3c5c5f0fab049c757032c343673104eb0f99cbaead5e0f78247f2a3aa6173c3",
|
||||
"cuda_visible_devices": "",
|
||||
"actual_nvidia_queries": 0,
|
||||
"training_starts": 0,
|
||||
"scoring_starts": 0,
|
||||
"actual_heldout_reads": 0,
|
||||
"independent_review": "PENDING",
|
||||
"guard_replay": 13,
|
||||
"supersedes": "8af022e43410232d42d07c92e2f468342a385c7a",
|
||||
"intermediary_blocker_evidence_sha256": "06f4b0c2a66d4ac8d285ee4e508388241798e60a4e500dc5ab5e1fde02a77720",
|
||||
"unchanged_shared_supervisor_sha256": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"cpu_replay_log_sha256": "6bf84633d3dc09cbd5002a149a8d0ebe3ef205ccb9748096f50ac7ee906971a8",
|
||||
"lifecycle_replay_sha256": "feb40e583810259537ad746ccfbbf2e56ff9bc969f002f1e7f74657e3b88cf8d",
|
||||
"read_only_collection": {
|
||||
"seeds": {
|
||||
"0": {
|
||||
"fit": {
|
||||
"status": "NOT_STARTED"
|
||||
},
|
||||
"score": {
|
||||
"status": "NOT_STARTED"
|
||||
}
|
||||
},
|
||||
"1": {
|
||||
"fit": {
|
||||
"status": "NOT_STARTED"
|
||||
},
|
||||
"score": {
|
||||
"status": "NOT_STARTED"
|
||||
}
|
||||
},
|
||||
"2": {
|
||||
"fit": {
|
||||
"status": "NOT_STARTED"
|
||||
},
|
||||
"score": {
|
||||
"status": "NOT_STARTED"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mean_percent": null,
|
||||
"within_registered_range": null,
|
||||
"reproduction_identity": "supplement code + 2-keyword patch",
|
||||
"prior_measurement_gpu_seconds": 222.80500172509346,
|
||||
"settled_formal_gpu_seconds": 0,
|
||||
"settled_combined_gpu_seconds": 222.80500172509346,
|
||||
"unsettled_formal_bookings_gpu_seconds": 0,
|
||||
"formal_gpu_cap_seconds": 40000
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
test_card_reservation_expiry_and_total_budget (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_inclusive_gate_and_all_three_seeds_required (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_original_split_score_keeps_order_and_rejects_nan (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_score_opens_one_111_target_pass_and_rejects_a_second (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_scoring_needs_all_checkpoints_before_access (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_failed_worker_exit_has_no_stale_group_signal (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_gpu_list_before_worker_exit (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_real_foreign_identity_has_exact_stop_receipt (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_reservation_expires_after_idle_but_before_spawn (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_reused_worker_pid_is_foreign (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_running_guard_continues_after_reservation_end (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_simulated_reused_pgid_cannot_be_signalled (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_worker_already_exited_before_gpu_list (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 13 tests in 1.232s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,114 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 13,
|
||||
"real_gpu_queries": 0,
|
||||
"pressure_members_opened": 0,
|
||||
"cases": {
|
||||
"failed-exit": {
|
||||
"status": "STOPPED",
|
||||
"trigger": "WORKER_FAILED",
|
||||
"syscalls": [
|
||||
{
|
||||
"event": "wait4_reaped",
|
||||
"pid": 67
|
||||
}
|
||||
],
|
||||
"lifecycle": [
|
||||
{
|
||||
"event": "leader_reaped",
|
||||
"pid": 67
|
||||
}
|
||||
]
|
||||
},
|
||||
"gpu-first": {
|
||||
"status": "COMPLETE",
|
||||
"trigger": null,
|
||||
"syscalls": [
|
||||
{
|
||||
"event": "wait4_reaped",
|
||||
"pid": 69
|
||||
}
|
||||
],
|
||||
"lifecycle": [
|
||||
{
|
||||
"event": "leader_reaped",
|
||||
"pid": 69
|
||||
}
|
||||
]
|
||||
},
|
||||
"foreign": {
|
||||
"status": "STOPPED",
|
||||
"trigger": "FOREIGN_GPU_OCCUPANT",
|
||||
"syscalls": [
|
||||
{
|
||||
"event": "killpg",
|
||||
"pid": 71,
|
||||
"signal": 9,
|
||||
"leader_reaped": false
|
||||
},
|
||||
{
|
||||
"event": "wait4_reaped",
|
||||
"pid": 71
|
||||
}
|
||||
],
|
||||
"lifecycle": [
|
||||
{
|
||||
"event": "group_signal",
|
||||
"pid": 71,
|
||||
"signal": 9,
|
||||
"reason": "GUARD_STOP"
|
||||
},
|
||||
{
|
||||
"event": "leader_reaped",
|
||||
"pid": 71
|
||||
}
|
||||
]
|
||||
},
|
||||
"expired-before-spawn": {
|
||||
"status": "STOPPED",
|
||||
"trigger": "CARD_RESERVATION_EXPIRED_BEFORE_SPAWN",
|
||||
"syscalls": [],
|
||||
"lifecycle": []
|
||||
},
|
||||
"crossed-after-spawn": {
|
||||
"status": "COMPLETE",
|
||||
"trigger": null,
|
||||
"syscalls": [
|
||||
{
|
||||
"event": "wait4_reaped",
|
||||
"pid": 73
|
||||
}
|
||||
],
|
||||
"lifecycle": [
|
||||
{
|
||||
"event": "leader_reaped",
|
||||
"pid": 73
|
||||
}
|
||||
]
|
||||
},
|
||||
"pgid-reuse": {
|
||||
"reaped": true,
|
||||
"simulated_new_identity": {
|
||||
"pid": 12,
|
||||
"created": 2.0
|
||||
},
|
||||
"signals": 0
|
||||
},
|
||||
"tree-first": {
|
||||
"status": "COMPLETE",
|
||||
"trigger": null,
|
||||
"syscalls": [
|
||||
{
|
||||
"event": "wait4_reaped",
|
||||
"pid": 75
|
||||
}
|
||||
],
|
||||
"lifecycle": [
|
||||
{
|
||||
"event": "leader_reaped",
|
||||
"pid": 75
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# Track B GINO 三种子复现联合预审定稿候选
|
||||
|
||||
本绑定替换中间提交 `8af022e4` 供同一次独立预审。MeltAssist 在旧生产守卫的 CPU 回放中捕获 `wait4` 已回收组长后的 `killpg(SIGKILL)`。旧回放与两次 GPU 测量终止均保留。 [MeltAssist 阻塞回执](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795075951091959) · [旧 CPU 调用次序](../artifacts/meltassist-track-b-gino-formal-joint-review-2026-11-19/reap-signal-replay.json)
|
||||
|
||||
修复只接入本 GINO 守卫所用的监督器调用,原共享文件及别的运行固定版本保持原样。用 `waitid(WNOWAIT)` 核实固定 PID 仍是本方直接子进程。守卫在同一把锁下验证组长身份并发送信号,或交由 `wait4` 回收并标记结束。需要清理残留同组子进程时先清理再回收组长。组长回收后,旧清理调用不再发信号。 [生产守卫](../tools/track_b_gino_formal_guard.py)
|
||||
|
||||
实际生产守卫 CPU 回放 13/13。拦截并记录所有组清理信号及 `wait4`。成功退出、非零退出、两类陈旧 GPU 列表和跨午夜运行都没有回收后信号。模拟真实外部占用时只向自己仍未回收的组发送一次 SIGKILL,随后实际回收,外部活进程未收信号。复用组 ID 负例到达清理入口时不触发信号。午夜到期且尚未派发的路径不创建 worker。 [调用及回收次序](../artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/lifecycle-cpu-replay.json) · [十三项 CPU 日志](../artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/cpu-replay.log)
|
||||
|
||||
种子 0、1、2 分别在 GPU2、GPU3、GPU0,各 100 完整轮。香港时间 2026-11-19 当日预留只适用于新派发,已开始的守卫照常运作。三份检查点冻结后,每个种子各评分一次原 111 件,三种子等权平均门含两端 6.62% 和 7.62%。正式总帽 40,000 GPU 秒。102.9377786 秒为未经守卫认证的单轮计时,仅用于定预算,旧账 222.805002 GPU 秒独计一次。训练及评分身份为 `supplement code + 2-keyword patch`。本准备阶段没有真实 GPU 派发或留出目标读取。 [组合合同](../artifacts/track-b-gino-shapenet611-formal-v3-2026-11-19/contract.json) · [训练与评分](../tools/track_b_gino_formal_worker.py)
|
||||
@@ -15,6 +15,9 @@ import track_b_gino_formal_worker as worker
|
||||
from track_b_gino_shapenet_entry import read, save, sha
|
||||
|
||||
|
||||
REPLAYS = {}
|
||||
|
||||
|
||||
class GuardReplay(unittest.TestCase):
|
||||
def replay(self, ordering):
|
||||
uuid = 'GPU-fixture'
|
||||
@@ -32,16 +35,28 @@ class GuardReplay(unittest.TestCase):
|
||||
"root=Path(sys.argv[1])\n"
|
||||
"while not (root/'exit').exists(): time.sleep(.01)\n"
|
||||
"(root/'worker-records/result.json').write_text("
|
||||
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder]
|
||||
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n"
|
||||
"if sys.argv[2] == 'failed-exit': raise SystemExit(7)\n", folder, ordering]
|
||||
observed = []
|
||||
stopped = []
|
||||
real_stop = gpu_guard.stop_group
|
||||
syscalls = []
|
||||
real_kill = os.killpg
|
||||
real_wait = os.wait4
|
||||
real_tree = gpu_guard.process_tree
|
||||
ticket = dict(ordinal='', dispatch_before_unix=time.time() + 15)
|
||||
|
||||
def stop_owned(pid):
|
||||
stopped.append(pid)
|
||||
real_stop(pid)
|
||||
def record_kill(pid, signal):
|
||||
own = read(root / 'actual-worker-spawn.json')['process']
|
||||
is_reaped = any(e == dict(event='wait4_reaped', pid=pid) for e in syscalls)
|
||||
syscalls.append(dict(event='killpg', pid=pid, signal=int(signal), leader_reaped=is_reaped))
|
||||
if is_reaped or pid != own['pid']:
|
||||
raise AssertionError('Invalid group signal intercepted before delivery')
|
||||
return real_kill(pid, signal)
|
||||
|
||||
def record_wait(pid, options):
|
||||
receipt = real_wait(pid, options)
|
||||
if receipt[0]:
|
||||
syscalls.append(dict(event='wait4_reaped', pid=receipt[0]))
|
||||
return receipt
|
||||
|
||||
def exit_and_wait():
|
||||
(root / 'exit').touch()
|
||||
@@ -74,13 +89,14 @@ class GuardReplay(unittest.TestCase):
|
||||
occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')])
|
||||
if ordering == 'crossed-after-spawn':
|
||||
ticket['dispatch_before_unix'] = time.time() - 1
|
||||
if ordering in ('gpu-first', 'crossed-after-spawn'):
|
||||
if ordering in ('gpu-first', 'crossed-after-spawn', 'failed-exit'):
|
||||
exit_and_wait()
|
||||
return sample
|
||||
|
||||
with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot),
|
||||
patch.object(gpu_guard, 'process_tree', side_effect=tree),
|
||||
patch.object(gpu_guard, 'stop_group', side_effect=stop_owned),
|
||||
patch.object(os, 'killpg', side_effect=record_kill),
|
||||
patch.object(os, 'wait4', side_effect=record_wait),
|
||||
patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}),
|
||||
patch.object(gpu_guard, 'dispatch_environment',
|
||||
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))):
|
||||
@@ -88,10 +104,13 @@ class GuardReplay(unittest.TestCase):
|
||||
ticket['guard_started_monotonic'] = began
|
||||
outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15,
|
||||
ticket=ticket, validate_result=lambda: sha(result))
|
||||
REPLAYS[ordering] = dict(status=outcome['status'], trigger=outcome['trigger'],
|
||||
syscalls=syscalls, lifecycle=outcome['lifecycle'])
|
||||
if ordering == 'expired-before-spawn':
|
||||
self.assertEqual(outcome['status'], 'STOPPED')
|
||||
self.assertEqual(outcome['trigger'], 'CARD_RESERVATION_EXPIRED_BEFORE_SPAWN')
|
||||
self.assertFalse((root / 'actual-worker-spawn.json').exists())
|
||||
self.assertEqual(syscalls, [])
|
||||
return observed
|
||||
spawn = read(root / 'actual-worker-spawn.json')['process']
|
||||
if ordering == 'foreign':
|
||||
@@ -102,11 +121,16 @@ class GuardReplay(unittest.TestCase):
|
||||
self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid))
|
||||
self.assertEqual(row['occupants'][0]['pid'], os.getpid())
|
||||
self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python')
|
||||
self.assertEqual(stopped, [spawn['pid']])
|
||||
self.assertEqual(syscalls, [
|
||||
dict(event='killpg', pid=spawn['pid'], signal=9, leader_reaped=False),
|
||||
dict(event='wait4_reaped', pid=spawn['pid'])])
|
||||
elif ordering == 'failed-exit':
|
||||
self.assertEqual((outcome['status'], outcome['worker']['returncode']), ('STOPPED', 7))
|
||||
self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])])
|
||||
else:
|
||||
self.assertEqual(outcome['status'], 'COMPLETE', outcome)
|
||||
self.assertFalse((root / 'foreign-occupant.json').exists())
|
||||
self.assertEqual(stopped, [])
|
||||
self.assertEqual(syscalls, [dict(event='wait4_reaped', pid=spawn['pid'])])
|
||||
self.assertEqual(outcome['result_sha256'], sha(result))
|
||||
return observed
|
||||
|
||||
@@ -127,6 +151,23 @@ class GuardReplay(unittest.TestCase):
|
||||
def test_running_guard_continues_after_reservation_end(self):
|
||||
self.replay('crossed-after-spawn')
|
||||
|
||||
def test_failed_worker_exit_has_no_stale_group_signal(self):
|
||||
self.replay('failed-exit')
|
||||
|
||||
def test_simulated_reused_pgid_cannot_be_signalled(self):
|
||||
group = gpu_guard.OwnedGroup()
|
||||
group.pid = 12
|
||||
group.identity = dict(pid=12, created=1.)
|
||||
group.reaped = True
|
||||
reused = dict(pid=12, created=2.)
|
||||
with (patch.object(gpu_guard, 'current_identity', return_value=reused),
|
||||
patch.object(os, 'waitid') as observe,
|
||||
patch.object(os, 'killpg') as intercept):
|
||||
group.stop(12)
|
||||
observe.assert_not_called()
|
||||
intercept.assert_not_called()
|
||||
REPLAYS['pgid-reuse'] = dict(reaped=group.reaped, simulated_new_identity=reused, signals=0)
|
||||
|
||||
def test_reused_worker_pid_is_foreign(self):
|
||||
with TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
|
||||
@@ -18,7 +18,7 @@ from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT
|
||||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||||
|
||||
|
||||
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-v2-2026-11-19')
|
||||
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-v3-2026-11-19')
|
||||
SOURCES = Path(__file__).resolve().parent
|
||||
OLD_FILES = {
|
||||
'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a',
|
||||
@@ -67,6 +67,7 @@ def expected_contract():
|
||||
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES})
|
||||
c['formal'] = dict(
|
||||
handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES,
|
||||
lifecycle_review_slack_ts='1795075951.091959',
|
||||
device_pci_ordinals=ORDINALS,
|
||||
reservation=dict(handoff='h-a429935e', timezone=TZ.key,
|
||||
start_inclusive=RESERVATION_START.isoformat(),
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
@@ -11,7 +12,7 @@ import psutil
|
||||
|
||||
from s2_gpu_idle_gate import snapshot, wait_idle
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_recovery_gpu_supervisor import stop_group, supervise
|
||||
from s2_recovery_gpu_supervisor import supervise
|
||||
from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity
|
||||
from track_b_gino_shapenet_entry import process_identity, save
|
||||
|
||||
@@ -71,7 +72,70 @@ class OccupancyMonitor:
|
||||
raise RuntimeError('DEVICE_VRAM_LIMIT')
|
||||
|
||||
|
||||
def watch(root, uuid, worker, limits, done, tracker, state):
|
||||
class OwnedGroup:
|
||||
"""Tie group signals to an unreaped session leader we directly spawned."""
|
||||
|
||||
def __init__(self):
|
||||
self.pid = None
|
||||
self.identity = None
|
||||
self.reaped = False
|
||||
self.signalled = False
|
||||
self.lock = threading.Lock()
|
||||
self.events = []
|
||||
|
||||
def observe_exit(self, pid, nonblocking):
|
||||
flags = os.WEXITED | os.WNOWAIT | (os.WNOHANG if nonblocking else 0)
|
||||
return os.waitid(os.P_PID, pid, flags)
|
||||
|
||||
def stop_locked(self, pid, reason):
|
||||
if pid != self.pid or self.reaped or self.signalled:
|
||||
return
|
||||
# WNOWAIT verifies this PID is still our child without permitting PID
|
||||
# reuse. The same lock prevents our wait4 from reaping before killpg.
|
||||
try:
|
||||
self.observe_exit(pid, nonblocking=True)
|
||||
if os.getpgid(pid) != pid:
|
||||
raise RuntimeError('WORKER_IS_NOT_ITS_OWN_SESSION_LEADER')
|
||||
if self.identity is not None and current_identity(pid) != self.identity:
|
||||
raise RuntimeError('WORKER_BIRTH_IDENTITY_DIFFERS')
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except (ChildProcessError, ProcessLookupError):
|
||||
return
|
||||
self.signalled = True
|
||||
self.events.append(dict(event='group_signal', pid=pid, signal=int(signal.SIGKILL), reason=reason))
|
||||
|
||||
def stop(self, pid):
|
||||
with self.lock:
|
||||
self.stop_locked(pid, 'GUARD_STOP')
|
||||
|
||||
def wait4(self, pid, options):
|
||||
if pid != self.pid:
|
||||
# The shared supervisor separately collects adopted descendants
|
||||
# with a negative group selector. That path never sends signals.
|
||||
return os.wait4(pid, options)
|
||||
# Never hold the lock on a blocking wait. The watcher may still need
|
||||
# it to stop the worker.
|
||||
if self.observe_exit(pid, nonblocking=bool(options & os.WNOHANG)) is None:
|
||||
return 0, 0, None
|
||||
with self.lock:
|
||||
# With subreaping enabled, children the leader left in its group
|
||||
# are now ours. Clean them up while the zombie still pins PGID.
|
||||
for child in psutil.Process().children():
|
||||
try:
|
||||
in_group = child.pid != pid and os.getpgid(child.pid) == pid
|
||||
except ProcessLookupError:
|
||||
continue
|
||||
if in_group:
|
||||
self.stop_locked(pid, 'REMAINING_GROUP_MEMBER')
|
||||
break
|
||||
result = os.wait4(pid, options)
|
||||
if result[0] == pid:
|
||||
self.reaped = True
|
||||
self.events.append(dict(event='leader_reaped', pid=pid))
|
||||
return result
|
||||
|
||||
|
||||
def watch(root, uuid, worker, group, limits, done, tracker, state):
|
||||
occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes'])
|
||||
while not done.is_set():
|
||||
try:
|
||||
@@ -83,9 +147,7 @@ def watch(root, uuid, worker, limits, done, tracker, state):
|
||||
monitor_identity(root, uuid)
|
||||
except Exception as exc:
|
||||
state['trigger'] = str(exc) or type(exc).__name__
|
||||
# Signal only the recorded worker group while its leader has the same identity.
|
||||
if current_identity(worker['pid']) == worker:
|
||||
stop_group(worker['pid'])
|
||||
group.stop(worker['pid'])
|
||||
return
|
||||
done.wait(.5)
|
||||
|
||||
@@ -95,6 +157,7 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds'])
|
||||
done = threading.Event()
|
||||
tracker = PeakRecords()
|
||||
group = OwnedGroup()
|
||||
state = dict(trigger=None, peak_device_bytes=0)
|
||||
watcher = None
|
||||
worker_receipt = None
|
||||
@@ -114,16 +177,19 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
if time.time() >= ticket['dispatch_before_unix']:
|
||||
raise PermissionError('CARD_RESERVATION_EXPIRED_BEFORE_SPAWN')
|
||||
proc = subprocess.Popen(args, **dict(kwargs, env=env))
|
||||
group.pid = proc.pid
|
||||
try:
|
||||
worker = process_identity(proc.pid)
|
||||
group.identity = worker
|
||||
save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid))
|
||||
watcher = threading.Thread(target=watch, daemon=True,
|
||||
args=(root, uuid, worker, limits, done, tracker, state))
|
||||
args=(root, uuid, worker, group, limits, done, tracker, state))
|
||||
watcher.start()
|
||||
except (Exception, KeyboardInterrupt):
|
||||
# The shared supervisor has not yet received the process to reap.
|
||||
stop_group(proc.pid)
|
||||
proc.wait()
|
||||
group.stop(proc.pid)
|
||||
_, status, _ = group.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(status)
|
||||
raise
|
||||
return proc
|
||||
|
||||
@@ -139,7 +205,8 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
remaining = deadline - time.monotonic() - limits['cleanup_seconds']
|
||||
if remaining <= 0:
|
||||
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
|
||||
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
||||
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT),
|
||||
os=SimpleNamespace(**dict(vars(os), wait4=group.wait4)), stop_group=group.stop)
|
||||
run_worker = FunctionType(supervise.__code__, ns, supervise.__name__)
|
||||
try:
|
||||
worker_receipt = run_worker(command, root, 'gino', dict(
|
||||
@@ -172,6 +239,7 @@ def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt,
|
||||
physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state,
|
||||
charged_gpu_seconds=end - ticket['guard_started_monotonic'],
|
||||
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True)
|
||||
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima,
|
||||
lifecycle=group.events, no_retry=True)
|
||||
save(root / 'complete.json', terminal)
|
||||
return terminal
|
||||
Reference in new issue
Block a user