Freeze the one GINO ordinal recovery, physical witness and cumulative budget
This commit is contained in:
1 parent
17a40a6c1b
commit
057843ae2a
7 files changed
+1303
No files matched your search
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"inventory": [
|
||||
[
|
||||
"0",
|
||||
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"00000000:84:00.0",
|
||||
"Tesla V100-SXM2-16GB"
|
||||
],
|
||||
[
|
||||
"1",
|
||||
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"00000000:85:00.0",
|
||||
"Tesla V100-SXM2-16GB"
|
||||
],
|
||||
[
|
||||
"2",
|
||||
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"00000000:A4:00.0",
|
||||
"Tesla V100-SXM2-16GB"
|
||||
],
|
||||
[
|
||||
"3",
|
||||
"GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"00000000:A5:00.0",
|
||||
"Tesla V100-SXM2-16GB"
|
||||
]
|
||||
],
|
||||
"occupants": [
|
||||
[
|
||||
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"2512405",
|
||||
"820 MiB"
|
||||
],
|
||||
[
|
||||
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"2517067",
|
||||
"1708 MiB"
|
||||
]
|
||||
],
|
||||
"source": "nvidia-smi actual after the original UUID stop"
|
||||
}
|
||||
@@ -0,0 +1,779 @@
|
||||
{
|
||||
"mode": "measurement",
|
||||
"ruling": "h-f84547d7",
|
||||
"retry": false,
|
||||
"evaluate_other_models": false,
|
||||
"measurement_allowed_heldout_labels": false,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"limits": {
|
||||
"gpu_wall_seconds": 3600,
|
||||
"cleanup_seconds": 60,
|
||||
"idle_wait_seconds": 60,
|
||||
"cpu_seconds": 3500,
|
||||
"rss_bytes": 12884901888,
|
||||
"recorded_vram_bytes": 15032385536,
|
||||
"device_vram_bytes": 16106127360,
|
||||
"disk_bytes": 2147483648,
|
||||
"torch_threads": 1,
|
||||
"max_processes": 1
|
||||
},
|
||||
"protocol": {
|
||||
"model": "NeurIPS 2023 supplement SDFFNOGNO decoder",
|
||||
"seeds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"grid": [
|
||||
64,
|
||||
64,
|
||||
64
|
||||
],
|
||||
"modes": [
|
||||
24,
|
||||
24,
|
||||
24
|
||||
],
|
||||
"hidden_channels": 32,
|
||||
"radius": 0.05,
|
||||
"epochs_per_seed": 100,
|
||||
"batch_size": 1,
|
||||
"learning_rate": 0.001,
|
||||
"bounds_expansion": 0.01,
|
||||
"pressure_epsilon": 1e-06,
|
||||
"optimizer": "Adam",
|
||||
"weight_decay": 0.0001,
|
||||
"lr_step_epochs": 50,
|
||||
"lr_gamma": 0.5,
|
||||
"pressure_training_transform": "scalar Gaussian on the 500 learning cases",
|
||||
"metric": "per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)",
|
||||
"report": "equal mean over 111 cases for each seed, then equal mean over the three seeds",
|
||||
"acceptance_percent": [
|
||||
6.62,
|
||||
7.62
|
||||
],
|
||||
"paper_reference_percent": 7.12,
|
||||
"checkpoint_epoch": 100,
|
||||
"heldout_access_after_all_checkpoints_frozen": true
|
||||
},
|
||||
"environment": {
|
||||
"python": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"python_version": "3.10",
|
||||
"torch_cuda": "11.6",
|
||||
"neuralop_commit": "fbdc7f872792b75b84545c2f374e7a881c72a718",
|
||||
"neuralop_file_sha256": {
|
||||
"utils.py": "84554eed45ac415ab52f79e456eab70ba204e7ab7539c2f2841955a9145220b4",
|
||||
"models/__init__.py": "c67679f1f8f90e1ab4eb4c2aa916aeaedf1b175d795fed03b8d4f7210883ebae",
|
||||
"models/tfno.py": "d74b5805b69a64bf33e3ff5186709c75bd431a56c9548f06fb9635a859d849e4",
|
||||
"models/spectral_convolution.py": "ccecf80806d93e8f35713ac6ab2c3b61c11739f5fcb8c0ba7397b7d4eb5a5d45"
|
||||
},
|
||||
"package_versions": {
|
||||
"torch": "1.13.1+cu116",
|
||||
"open3d": "0.17.0",
|
||||
"numpy": "1.24.3",
|
||||
"torch-scatter": "2.1.1+pt113cu116",
|
||||
"torch-sparse": "0.6.17+pt113cu116",
|
||||
"torch-geometric": "2.3.1",
|
||||
"tensorly": "0.8.1",
|
||||
"tensorly-torch": "0.4.0"
|
||||
}
|
||||
},
|
||||
"comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-measurement-2026-11-19/comparison-contract.json",
|
||||
"comparison_contract_sha256": "503f6c8ab1217d66b44e0d127555cfad5da4cd8a17b336299095c3cfd13041d7",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_shapenet.py": "557ad56277d7e85c6b5cc358eb151632d1757f5883f429988d36022d0d407550",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_entry.py": "e86d955c36e291e6bbef67947de663b24c8d66c5d72d9d28e4193fb8a3051ee2",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py": "7211be211c29472fd9be35a6bcb1f56025de4a63f36852bfafa10df97c003ff3",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_entry.py": "199d833f40339feb0433611ba27302416a280452f8918715e6ef95cb93c38721",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_worker.py": "3acf742dd4549e9f5491e10af5bb0b4acf00e2f47e91753100b5998cfcc61852",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_import_checked.py": "fef485dd0c233c76a336239d5f29cbf44ac0fb62a6a297dff34293b23a1b75fe",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_paper_keyword_patch.py": "c61e2ed7d099eb125386857bf4d7ccdbfc3af522f80edb4f01b450cee91c0b95",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2.py": "7d3ab2412e5208892cffcfc0e1ec363eb5db2be3e10bf60595356f71e82d8552",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py": "ba1fd5ec837e884b000ba640fceafe30ea40ccf708847733df6a17a8f3599680",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery.py": "1e1827498afab62e2f28efb5d450376f2dca4e8bf6f74114f2914ec771872c86",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py": "7a68c77e9c913ba9161832f513dbf8754ebc5c270c1e742fa5ae97a7b14b9120"
|
||||
},
|
||||
"paper_data": {
|
||||
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-car-pressure-data.zip",
|
||||
"sha256": "b5f9e2b407d54e034596384253fb58196933c46f6b5b586383508da54475f89d"
|
||||
},
|
||||
"paper_supplement": {
|
||||
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-neurips-supplement.zip",
|
||||
"sha256": "48cdae80ac2781e27ac00e10a37124711d6bb305593559002bda0527d11a533d"
|
||||
},
|
||||
"paper_code_zip_sha256": "347a6f097e2b9958d1ab87930fa1d6227b793a9d213e12418ce9a157e83dce04",
|
||||
"split_receipt": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-split.json",
|
||||
"sha256": "5a96c25c2ec24cfc1e2264d4ebd0879c658c0f168e22711b576bf303151b8601"
|
||||
},
|
||||
"mapping_audit": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-mapping-audit.json",
|
||||
"sha256": "055d008a02494acd29df36c41c8e68db931a027f5635e8f5f765a96a54c62289"
|
||||
},
|
||||
"split": {
|
||||
"train": [
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
10,
|
||||
12,
|
||||
13,
|
||||
17,
|
||||
18,
|
||||
21,
|
||||
22,
|
||||
23,
|
||||
25,
|
||||
26,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31,
|
||||
32,
|
||||
34,
|
||||
35,
|
||||
39,
|
||||
40,
|
||||
43,
|
||||
44,
|
||||
45,
|
||||
46,
|
||||
47,
|
||||
48,
|
||||
49,
|
||||
50,
|
||||
51,
|
||||
52,
|
||||
54,
|
||||
55,
|
||||
56,
|
||||
58,
|
||||
59,
|
||||
60,
|
||||
61,
|
||||
62,
|
||||
63,
|
||||
64,
|
||||
65,
|
||||
67,
|
||||
69,
|
||||
70,
|
||||
71,
|
||||
72,
|
||||
73,
|
||||
74,
|
||||
75,
|
||||
76,
|
||||
77,
|
||||
78,
|
||||
79,
|
||||
80,
|
||||
81,
|
||||
83,
|
||||
84,
|
||||
85,
|
||||
86,
|
||||
87,
|
||||
88,
|
||||
90,
|
||||
91,
|
||||
92,
|
||||
94,
|
||||
95,
|
||||
96,
|
||||
97,
|
||||
100,
|
||||
101,
|
||||
102,
|
||||
105,
|
||||
106,
|
||||
107,
|
||||
109,
|
||||
110,
|
||||
111,
|
||||
112,
|
||||
113,
|
||||
114,
|
||||
115,
|
||||
116,
|
||||
117,
|
||||
118,
|
||||
119,
|
||||
120,
|
||||
121,
|
||||
123,
|
||||
124,
|
||||
125,
|
||||
126,
|
||||
127,
|
||||
128,
|
||||
129,
|
||||
130,
|
||||
131,
|
||||
133,
|
||||
134,
|
||||
136,
|
||||
137,
|
||||
138,
|
||||
139,
|
||||
140,
|
||||
141,
|
||||
142,
|
||||
143,
|
||||
144,
|
||||
145,
|
||||
146,
|
||||
147,
|
||||
148,
|
||||
149,
|
||||
150,
|
||||
151,
|
||||
152,
|
||||
153,
|
||||
155,
|
||||
156,
|
||||
157,
|
||||
158,
|
||||
159,
|
||||
160,
|
||||
161,
|
||||
162,
|
||||
163,
|
||||
165,
|
||||
166,
|
||||
170,
|
||||
172,
|
||||
173,
|
||||
175,
|
||||
176,
|
||||
177,
|
||||
178,
|
||||
179,
|
||||
180,
|
||||
181,
|
||||
182,
|
||||
183,
|
||||
184,
|
||||
186,
|
||||
190,
|
||||
191,
|
||||
192,
|
||||
193,
|
||||
195,
|
||||
196,
|
||||
198,
|
||||
199,
|
||||
200,
|
||||
201,
|
||||
202,
|
||||
203,
|
||||
205,
|
||||
207,
|
||||
210,
|
||||
211,
|
||||
212,
|
||||
213,
|
||||
214,
|
||||
215,
|
||||
217,
|
||||
219,
|
||||
220,
|
||||
221,
|
||||
222,
|
||||
223,
|
||||
224,
|
||||
225,
|
||||
227,
|
||||
228,
|
||||
229,
|
||||
230,
|
||||
231,
|
||||
232,
|
||||
233,
|
||||
234,
|
||||
235,
|
||||
236,
|
||||
237,
|
||||
241,
|
||||
243,
|
||||
244,
|
||||
245,
|
||||
246,
|
||||
247,
|
||||
248,
|
||||
249,
|
||||
251,
|
||||
252,
|
||||
253,
|
||||
255,
|
||||
257,
|
||||
258,
|
||||
259,
|
||||
260,
|
||||
261,
|
||||
262,
|
||||
263,
|
||||
264,
|
||||
266,
|
||||
267,
|
||||
268,
|
||||
269,
|
||||
271,
|
||||
272,
|
||||
273,
|
||||
274,
|
||||
275,
|
||||
276,
|
||||
277,
|
||||
278,
|
||||
279,
|
||||
280,
|
||||
281,
|
||||
282,
|
||||
283,
|
||||
285,
|
||||
286,
|
||||
289,
|
||||
290,
|
||||
291,
|
||||
292,
|
||||
293,
|
||||
294,
|
||||
295,
|
||||
296,
|
||||
297,
|
||||
298,
|
||||
299,
|
||||
300,
|
||||
301,
|
||||
302,
|
||||
304,
|
||||
305,
|
||||
306,
|
||||
308,
|
||||
309,
|
||||
310,
|
||||
311,
|
||||
312,
|
||||
313,
|
||||
314,
|
||||
315,
|
||||
319,
|
||||
320,
|
||||
321,
|
||||
322,
|
||||
323,
|
||||
324,
|
||||
325,
|
||||
327,
|
||||
328,
|
||||
329,
|
||||
331,
|
||||
332,
|
||||
333,
|
||||
334,
|
||||
335,
|
||||
337,
|
||||
338,
|
||||
339,
|
||||
340,
|
||||
341,
|
||||
344,
|
||||
345,
|
||||
347,
|
||||
348,
|
||||
349,
|
||||
350,
|
||||
352,
|
||||
353,
|
||||
354,
|
||||
355,
|
||||
356,
|
||||
357,
|
||||
358,
|
||||
360,
|
||||
362,
|
||||
364,
|
||||
365,
|
||||
366,
|
||||
367,
|
||||
369,
|
||||
371,
|
||||
372,
|
||||
373,
|
||||
374,
|
||||
375,
|
||||
376,
|
||||
378,
|
||||
379,
|
||||
380,
|
||||
381,
|
||||
384,
|
||||
385,
|
||||
389,
|
||||
392,
|
||||
393,
|
||||
397,
|
||||
398,
|
||||
399,
|
||||
401,
|
||||
402,
|
||||
403,
|
||||
404,
|
||||
405,
|
||||
407,
|
||||
408,
|
||||
410,
|
||||
412,
|
||||
413,
|
||||
414,
|
||||
415,
|
||||
417,
|
||||
418,
|
||||
419,
|
||||
420,
|
||||
422,
|
||||
424,
|
||||
425,
|
||||
427,
|
||||
430,
|
||||
431,
|
||||
433,
|
||||
435,
|
||||
436,
|
||||
437,
|
||||
439,
|
||||
440,
|
||||
443,
|
||||
444,
|
||||
446,
|
||||
447,
|
||||
448,
|
||||
449,
|
||||
450,
|
||||
451,
|
||||
452,
|
||||
453,
|
||||
454,
|
||||
455,
|
||||
456,
|
||||
457,
|
||||
459,
|
||||
460,
|
||||
462,
|
||||
463,
|
||||
464,
|
||||
465,
|
||||
466,
|
||||
467,
|
||||
468,
|
||||
469,
|
||||
470,
|
||||
472,
|
||||
473,
|
||||
474,
|
||||
475,
|
||||
476,
|
||||
478,
|
||||
479,
|
||||
480,
|
||||
482,
|
||||
483,
|
||||
486,
|
||||
487,
|
||||
488,
|
||||
490,
|
||||
493,
|
||||
494,
|
||||
495,
|
||||
496,
|
||||
497,
|
||||
498,
|
||||
499,
|
||||
501,
|
||||
502,
|
||||
503,
|
||||
504,
|
||||
505,
|
||||
507,
|
||||
508,
|
||||
509,
|
||||
511,
|
||||
512,
|
||||
513,
|
||||
514,
|
||||
515,
|
||||
516,
|
||||
518,
|
||||
519,
|
||||
521,
|
||||
522,
|
||||
523,
|
||||
524,
|
||||
525,
|
||||
527,
|
||||
529,
|
||||
530,
|
||||
532,
|
||||
533,
|
||||
536,
|
||||
538,
|
||||
539,
|
||||
540,
|
||||
542,
|
||||
543,
|
||||
545,
|
||||
547,
|
||||
548,
|
||||
549,
|
||||
550,
|
||||
551,
|
||||
552,
|
||||
553,
|
||||
554,
|
||||
555,
|
||||
560,
|
||||
561,
|
||||
562,
|
||||
564,
|
||||
565,
|
||||
566,
|
||||
567,
|
||||
568,
|
||||
569,
|
||||
572,
|
||||
573,
|
||||
574,
|
||||
576,
|
||||
577,
|
||||
579,
|
||||
581,
|
||||
582,
|
||||
583,
|
||||
584,
|
||||
587,
|
||||
588,
|
||||
589,
|
||||
591,
|
||||
593,
|
||||
594,
|
||||
595,
|
||||
596,
|
||||
597,
|
||||
598,
|
||||
600,
|
||||
602,
|
||||
604,
|
||||
608,
|
||||
610,
|
||||
611,
|
||||
612,
|
||||
613,
|
||||
615,
|
||||
616,
|
||||
617,
|
||||
618,
|
||||
620,
|
||||
621,
|
||||
622,
|
||||
623,
|
||||
625,
|
||||
626,
|
||||
627,
|
||||
628,
|
||||
629,
|
||||
630,
|
||||
631,
|
||||
632,
|
||||
633,
|
||||
634,
|
||||
635,
|
||||
636,
|
||||
638,
|
||||
639,
|
||||
640,
|
||||
641,
|
||||
642,
|
||||
643,
|
||||
644,
|
||||
645,
|
||||
646,
|
||||
647,
|
||||
648,
|
||||
649,
|
||||
651,
|
||||
652,
|
||||
654,
|
||||
655,
|
||||
656,
|
||||
657
|
||||
],
|
||||
"test": [
|
||||
658,
|
||||
659,
|
||||
660,
|
||||
662,
|
||||
663,
|
||||
664,
|
||||
665,
|
||||
666,
|
||||
667,
|
||||
668,
|
||||
672,
|
||||
673,
|
||||
674,
|
||||
675,
|
||||
676,
|
||||
677,
|
||||
678,
|
||||
679,
|
||||
681,
|
||||
683,
|
||||
684,
|
||||
686,
|
||||
687,
|
||||
688,
|
||||
689,
|
||||
690,
|
||||
691,
|
||||
692,
|
||||
693,
|
||||
695,
|
||||
696,
|
||||
697,
|
||||
700,
|
||||
701,
|
||||
702,
|
||||
703,
|
||||
704,
|
||||
705,
|
||||
708,
|
||||
709,
|
||||
710,
|
||||
711,
|
||||
712,
|
||||
713,
|
||||
715,
|
||||
717,
|
||||
718,
|
||||
719,
|
||||
721,
|
||||
722,
|
||||
723,
|
||||
724,
|
||||
725,
|
||||
726,
|
||||
727,
|
||||
729,
|
||||
730,
|
||||
731,
|
||||
733,
|
||||
734,
|
||||
735,
|
||||
736,
|
||||
737,
|
||||
738,
|
||||
739,
|
||||
740,
|
||||
741,
|
||||
743,
|
||||
744,
|
||||
745,
|
||||
746,
|
||||
747,
|
||||
748,
|
||||
749,
|
||||
750,
|
||||
751,
|
||||
752,
|
||||
753,
|
||||
757,
|
||||
758,
|
||||
760,
|
||||
762,
|
||||
763,
|
||||
764,
|
||||
765,
|
||||
766,
|
||||
767,
|
||||
768,
|
||||
769,
|
||||
770,
|
||||
771,
|
||||
772,
|
||||
773,
|
||||
775,
|
||||
776,
|
||||
777,
|
||||
778,
|
||||
779,
|
||||
781,
|
||||
782,
|
||||
783,
|
||||
784,
|
||||
785,
|
||||
787,
|
||||
789,
|
||||
790,
|
||||
791,
|
||||
792,
|
||||
793,
|
||||
795,
|
||||
797
|
||||
],
|
||||
"raw_global_bounds": [
|
||||
[
|
||||
-0.90252799,
|
||||
0.0066688,
|
||||
-2.85272002
|
||||
],
|
||||
[
|
||||
0.90253001,
|
||||
1.95825005,
|
||||
2.86385012
|
||||
]
|
||||
]
|
||||
},
|
||||
"environment_ruling": "h-0d21f3bd",
|
||||
"environment_acceptance": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-import-checked-2026-11-19/cpu-environment-acceptance.json",
|
||||
"sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3"
|
||||
},
|
||||
"paper_compatibility": {
|
||||
"handoff": "h-b8508140",
|
||||
"identity": "supplement code + 2-keyword patch",
|
||||
"patch_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/two-keyword-patch.json",
|
||||
"patch_sha256": "7307ba49f83bfc46c5decf803a993a41ed602d987de65714958e738d2f52f077",
|
||||
"search_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/keyword-search.json",
|
||||
"search_sha256": "77a897bf534a9a0e9e6a7c072b4f914168648eadc2af3493b70d63d0b715b3d0"
|
||||
},
|
||||
"gpu_ruling": "h-bc42181f",
|
||||
"ordinal_recovery": {
|
||||
"handoff": "h-91e61871",
|
||||
"previous_terminal_sha256": "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e",
|
||||
"previous_gpu_wall_seconds": 23.69431395700667
|
||||
}
|
||||
}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"ruling": "h-91e61871",
|
||||
"old_terminal_sha256": "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e",
|
||||
"contract_sha256": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a",
|
||||
"same_scientific_protocol_and_caps": true,
|
||||
"hardware_inventory": {
|
||||
"inventory": [
|
||||
[
|
||||
"0",
|
||||
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"00000000:84:00.0",
|
||||
"Tesla V100-SXM2-16GB"
|
||||
],
|
||||
[
|
||||
"1",
|
||||
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"00000000:85:00.0",
|
||||
"Tesla V100-SXM2-16GB"
|
||||
],
|
||||
[
|
||||
"2",
|
||||
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"00000000:A4:00.0",
|
||||
"Tesla V100-SXM2-16GB"
|
||||
],
|
||||
[
|
||||
"3",
|
||||
"GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"00000000:A5:00.0",
|
||||
"Tesla V100-SXM2-16GB"
|
||||
]
|
||||
],
|
||||
"occupants": [
|
||||
[
|
||||
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"2512405",
|
||||
"820 MiB"
|
||||
],
|
||||
[
|
||||
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"2517067",
|
||||
"1708 MiB"
|
||||
]
|
||||
],
|
||||
"source": "nvidia-smi actual after the original UUID stop"
|
||||
},
|
||||
"inconsistent_pci_order_rejected": true,
|
||||
"advance_checks": {
|
||||
"matching": {
|
||||
"captured": {
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py",
|
||||
"/tmp/track-b-cuda-identity-replay-8m_kv8v_"
|
||||
],
|
||||
"device_order": "PCI_BUS_ID",
|
||||
"visible_devices": "2",
|
||||
"start_new_session": true
|
||||
},
|
||||
"verified": true,
|
||||
"mismatch": false,
|
||||
"stop_called": false,
|
||||
"terminal_trigger": "MOCK_WORK_REPLACED",
|
||||
"budget": {
|
||||
"status": "STOPPED",
|
||||
"trigger": "MOCK_WORK_REPLACED",
|
||||
"previous_gpu_wall_seconds": 23.69431395700667,
|
||||
"recovery_gpu_wall_seconds": 0.6289700350025669,
|
||||
"cumulative_measurement_gpu_wall_seconds": 24.323283992009237,
|
||||
"total_cap_gpu_wall_seconds": 3600,
|
||||
"independent_attempts": 2,
|
||||
"no_retry": true,
|
||||
"within_total_cap": true
|
||||
}
|
||||
},
|
||||
"mismatch": {
|
||||
"captured": {
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py",
|
||||
"/tmp/track-b-cuda-identity-replay-mh5jfyj7"
|
||||
],
|
||||
"device_order": "PCI_BUS_ID",
|
||||
"visible_devices": "2",
|
||||
"start_new_session": true
|
||||
},
|
||||
"verified": false,
|
||||
"mismatch": true,
|
||||
"stop_called": true,
|
||||
"terminal_trigger": "MONITOR_FAILURE: WORKER_GPU_UUID_MISMATCH",
|
||||
"budget": {
|
||||
"status": "STOPPED",
|
||||
"trigger": "MONITOR_FAILURE: WORKER_GPU_UUID_MISMATCH",
|
||||
"previous_gpu_wall_seconds": 23.69431395700667,
|
||||
"recovery_gpu_wall_seconds": 0.757885464001447,
|
||||
"cumulative_measurement_gpu_wall_seconds": 24.452199421008117,
|
||||
"total_cap_gpu_wall_seconds": 3600,
|
||||
"independent_attempts": 2,
|
||||
"no_retry": true,
|
||||
"within_total_cap": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"real_spawns": 0,
|
||||
"real_cuda_calls": 0,
|
||||
"heldout_targets_read": 0,
|
||||
"independent_review": "PENDING"
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# Track B|GPU2 单轮测量的设备绑定恢复
|
||||
|
||||
原 GPU2 唯一尝试通过空闲门并派发。Torch 1.13.1 在以物理 UUID 作为 `CUDA_VISIBLE_DEVICES` 时,Python NVML 枚举将该字符串解析为负序号并返回可见卡数 0,`get_device_name(0)` 抛出 `Invalid device id`。原 worker 未进入样本准备,无优化步或计时产物。原申请、工作者退出、终止日志和 CPU Mock 复现已固定。原终止实付 23.69431395700667 GPU-s。 [原终止及 CPU 复现](https://gitea.frank14f.online/Frank14f/MeltMind/src/commit/17a40a6c1b3f765827b2d7c4950b6e6c5a8d1ad6/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/terminal-uuid-enumeration-audit.json)
|
||||
|
||||
Claude h-91e61871 批准唯一一次环境恢复。在原 UUID 守卫中临派时要求 `nvidia-smi` 的序号与 PCI 总线排序严格一致。物理 GPU2 在实核中为序号 2,PCI 地址 A4:00.0,V100,未见计算进程。只在单个 worker 进程设 `CUDA_DEVICE_ORDER=PCI_BUS_ID`、`CUDA_VISIBLE_DEVICES=2`。在读取任何样本之前,worker 核验只见一张 V100,然后作一字节标记分配。守卫须从实际 worker PID 核实该分配确在固定 UUID,随后才放行原工作体。卡不匹配时由原守卫停止该工作者。 [Claude 裁决](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795072229918399)
|
||||
|
||||
CPU 回放验证了正确卡的环境与放行、错误卡的终止、PCI 序号冲突的拒绝,以及旧账计一次。原单轮训练、学习代理、补丁、环境及选择 UUID 均沿用。总帽为 3,600 GPU-s,独立新根的原守卫实际余量为 3,576.305686 GPU-s,支出含等待与设置。独立审查和阶段门在真实开始之前执行。原 111 个留出标签保持封存。 [恢复合同](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json) · [CPU 回放](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/ordinal-cpu-replay.json)
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Bind the one approved recovery and replay ordinal dispatch and mismatch."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from tempfile import TemporaryDirectory
|
||||
import threading
|
||||
import time
|
||||
from types import FunctionType, SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import track_b_gino_import_checked_entry as accepted
|
||||
import track_b_gino_ordinal_recovery as recovery
|
||||
import track_b_gino_shapenet_entry as original
|
||||
|
||||
|
||||
halt = threading.Event()
|
||||
|
||||
|
||||
def fake_supervisor(command, root, mode, limits, *, poll, gpu_uuid):
|
||||
"""Executed in a fixture namespace with the real recovery spawn adapter."""
|
||||
process = subprocess.Popen(command, env={"CUDA_VISIBLE_DEVICES": gpu_uuid}, start_new_session=True)
|
||||
deadline = time.monotonic() + 4
|
||||
matched = Path(root) / "worker-device-verified.json"
|
||||
while not matched.exists() and not halt.is_set() and time.monotonic() < deadline:
|
||||
time.sleep(.01)
|
||||
if not (matched.exists() or halt.is_set()):
|
||||
raise AssertionError("Fixture monitor never verified or stopped the worker")
|
||||
return dict(status="STOPPED_OR_FAILED", returncode=-9, monitor_errors=0,
|
||||
stop_reason="MOCK_WORK_REPLACED", pid=process.pid)
|
||||
|
||||
|
||||
def replay(inventory, slot):
|
||||
halt = threading.Event()
|
||||
pid = 12345
|
||||
identity = {"pid": pid, "created": 123.}
|
||||
captured = {}
|
||||
|
||||
def query(option):
|
||||
if option == "--query-gpu=index,uuid,pci.bus_id,name":
|
||||
return inventory
|
||||
if option != "--query-compute-apps=gpu_uuid,pid":
|
||||
raise AssertionError("Unexpected inventory query")
|
||||
return [[slot, str(pid)]]
|
||||
|
||||
def spawn(command, *, env, start_new_session):
|
||||
captured.update(command=list(command), device_order=env["CUDA_DEVICE_ORDER"],
|
||||
visible_devices=env["CUDA_VISIBLE_DEVICES"], start_new_session=start_new_session)
|
||||
original.save(root / "worker-device-witness.json", dict(
|
||||
process=identity, ordinal="2", device_count=1, device_name="Tesla V100-SXM2-16GB",
|
||||
physical_uuid_expected=recovery.UUID, first_allocation_bytes=1))
|
||||
return SimpleNamespace(pid=pid)
|
||||
|
||||
def stop(pid_to_stop):
|
||||
if pid_to_stop != pid:
|
||||
raise AssertionError("Attempted to stop the wrong process")
|
||||
halt.set()
|
||||
|
||||
with TemporaryDirectory(prefix="track-b-cuda-identity-replay-") as temp:
|
||||
root = Path(temp)
|
||||
path = root / "contract.json"
|
||||
original.save(path, recovery.expected_contract())
|
||||
isolated = dict(fake_supervisor.__globals__, halt=halt)
|
||||
fake = FunctionType(fake_supervisor.__code__, isolated, fake_supervisor.__name__)
|
||||
with (patch.object(recovery, "query_nvidia", side_effect=query),
|
||||
patch.object(recovery, "process_identity", return_value=identity),
|
||||
patch.object(recovery, "subprocess", SimpleNamespace(Popen=spawn, STDOUT=-2)),
|
||||
patch.object(original, "supervise", fake),
|
||||
patch.object(original, "snapshot", return_value=dict(idle=True, memory_MiB=5, occupants=[])),
|
||||
patch.object(original, "wait_idle", return_value=dict(status="IDLE_CONFIRMED")),
|
||||
patch.object(original, "active_command_pids", return_value=[pid]),
|
||||
patch.object(original, "stop_group", side_effect=stop)):
|
||||
code = recovery.dispatch_guard(root, path)
|
||||
witness, mismatch = root / "worker-device-verified.json", root / "worker-device-mismatch.json"
|
||||
terminal = original.read(root / "complete.json")
|
||||
budget = original.read(root / "recovery-settlement.json")
|
||||
assert code == 1 and bool(captured)
|
||||
assert captured["device_order"] == "PCI_BUS_ID" and captured["visible_devices"] == "2"
|
||||
assert captured["command"][1] == str(recovery.WORKER)
|
||||
assert budget["previous_gpu_wall_seconds"] == 23.69431395700667
|
||||
assert budget["cumulative_measurement_gpu_wall_seconds"] == (
|
||||
budget["previous_gpu_wall_seconds"] + budget["recovery_gpu_wall_seconds"])
|
||||
assert terminal["gpu_wall_cap_seconds"] == 3600 - budget["previous_gpu_wall_seconds"]
|
||||
if slot == recovery.UUID:
|
||||
assert witness.exists() and not mismatch.exists() and not halt.is_set()
|
||||
else:
|
||||
assert mismatch.exists() and not witness.exists() and halt.is_set()
|
||||
assert "WORKER_GPU_UUID_MISMATCH" in terminal["trigger"]
|
||||
return dict(captured=captured, verified=witness.exists(), mismatch=mismatch.exists(),
|
||||
stop_called=halt.is_set(), terminal_trigger=terminal["trigger"], budget=budget)
|
||||
|
||||
|
||||
def prepare(inventory_path):
|
||||
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
|
||||
raise PermissionError("The replay requires an empty device mask")
|
||||
source = json.loads(inventory_path.read_text())
|
||||
rows = source["inventory"]
|
||||
assert recovery.device_ordinal(rows, recovery.UUID) == "2"
|
||||
assert not any(card == recovery.UUID for card, *_ in source["occupants"])
|
||||
mismatched = [row.copy() for row in rows]
|
||||
mismatched[0][0], mismatched[2][0] = mismatched[2][0], mismatched[0][0]
|
||||
try:
|
||||
recovery.device_ordinal(mismatched, recovery.UUID)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("A conflicting PCI order was accepted")
|
||||
c = recovery.expected_contract()
|
||||
old = original.read(recovery.previous.ROOT / "contract.json")
|
||||
assert c["gpu_uuid"] == old["gpu_uuid"] and c["limits"] == old["limits"]
|
||||
assert set(k for k in c if c[k] != old.get(k)) == {"ordinal_recovery", "code_sha256"}
|
||||
assert accepted.accepted_environment(c)["heldout_target_reads"] == 0
|
||||
checks = {"matching": replay(rows, recovery.UUID),
|
||||
"mismatch": replay(rows, rows[0][1])}
|
||||
assert not (recovery.ROOT / "started.json").exists()
|
||||
recovery.ROOT.mkdir(exist_ok=False)
|
||||
original.save(recovery.ROOT / "contract.json", c)
|
||||
original.save(recovery.ROOT / "ordinal-cpu-replay.json", dict(
|
||||
status="PASS", ruling="h-91e61871", old_terminal_sha256=recovery.OLD_TERMINAL,
|
||||
contract_sha256=original.sha(recovery.ROOT / "contract.json"),
|
||||
same_scientific_protocol_and_caps=True, hardware_inventory=source,
|
||||
inconsistent_pci_order_rejected=True, advance_checks=checks, real_spawns=0, real_cuda_calls=0,
|
||||
heldout_targets_read=0, independent_review="PENDING"))
|
||||
print("PASS", original.sha(recovery.ROOT / "contract.json"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("inventory", type=Path)
|
||||
prepare(parser.parse_args().inventory)
|
||||
@@ -0,0 +1,173 @@
|
||||
"""One labelled recovery for the historical Torch UUID-counting failure."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
from types import FunctionType, SimpleNamespace
|
||||
|
||||
import psutil
|
||||
|
||||
import track_b_gino_import_checked_entry as accepted
|
||||
import track_b_gino_measurement_gpu2 as previous
|
||||
import track_b_gino_shapenet_entry as original
|
||||
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
|
||||
|
||||
|
||||
ROOT = previous.ROOT.with_name("track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19")
|
||||
ENTRY = Path(__file__).resolve()
|
||||
WORKER = ENTRY.with_name("track_b_gino_ordinal_recovery_worker.py")
|
||||
UUID = previous.UUID
|
||||
OLD_CONTRACT = "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f"
|
||||
OLD_TERMINAL = "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e"
|
||||
|
||||
|
||||
def expected_contract():
|
||||
c = read(checked(previous.ROOT / "contract.json", OLD_CONTRACT))
|
||||
stopped = read(checked(previous.ROOT / "complete.json", OLD_TERMINAL))
|
||||
if stopped["status"] != "STOPPED" or stopped["trigger"] != "WORKER_FAILED":
|
||||
raise PermissionError("The original stopped GPU2 attempt is required")
|
||||
c["ordinal_recovery"] = dict(handoff="h-91e61871", previous_terminal_sha256=OLD_TERMINAL,
|
||||
previous_gpu_wall_seconds=stopped["cumulative_gpu_wall_seconds"])
|
||||
c["code_sha256"].update({str(p): sha(p) for p in (ENTRY, WORKER)})
|
||||
return c
|
||||
|
||||
|
||||
def contract(path):
|
||||
c = previous.checked_entry.contract(path)
|
||||
if c != expected_contract():
|
||||
raise PermissionError("Only the fixed ordinal recovery of this measurement is authorized")
|
||||
return c
|
||||
|
||||
|
||||
def query_nvidia(options):
|
||||
r = subprocess.run(["nvidia-smi", options, "--format=csv,noheader"],
|
||||
check=True, capture_output=True, text=True, timeout=3)
|
||||
return [[v.strip() for v in row] for row in csv.reader(r.stdout.splitlines()) if row]
|
||||
|
||||
|
||||
def device_ordinal(rows, uuid):
|
||||
ordered = sorted(rows, key=lambda row: row[2].lower())
|
||||
if len({row[2].lower() for row in ordered}) != len(ordered):
|
||||
raise ValueError("Nonunique GPU PCI identities")
|
||||
if [row[0] for row in ordered] != [str(i) for i in range(len(ordered))]:
|
||||
raise ValueError("NVIDIA indices do not match CUDA PCI bus order")
|
||||
match = [row for row in ordered if row[1] == uuid and "V100" in row[3]]
|
||||
if len(match) != 1:
|
||||
raise ValueError("Expected exactly one reviewed physical V100")
|
||||
return match[0][0]
|
||||
|
||||
|
||||
def dispatch_environment(uuid, env):
|
||||
if env["CUDA_VISIBLE_DEVICES"] != uuid:
|
||||
raise PermissionError("Supervisor did not select the exact physical UUID")
|
||||
rows = query_nvidia("--query-gpu=index,uuid,pci.bus_id,name")
|
||||
ordinal = device_ordinal(rows, uuid)
|
||||
return dict(env, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=ordinal), rows
|
||||
|
||||
|
||||
def monitor_identity(root, uuid):
|
||||
pending = root / "worker-device-witness.json"
|
||||
verified = root / "worker-device-verified.json"
|
||||
if not pending.is_file() or verified.exists():
|
||||
return
|
||||
witness = read(pending)
|
||||
spawn = read(root / "actual-worker-spawn.json")
|
||||
if (witness["process"] != spawn["process"]
|
||||
or witness["physical_uuid_expected"] != uuid
|
||||
or witness["ordinal"] != spawn["ordinal"]
|
||||
or witness["device_count"] != 1 or "V100" not in witness["device_name"]
|
||||
or witness["first_allocation_bytes"] != 1):
|
||||
raise RuntimeError("WORKER_DEVICE_WITNESS_IDENTITY_DIFFERS")
|
||||
pid = witness["process"]["pid"]
|
||||
observed = sorted({card for card, owner in query_nvidia("--query-compute-apps=gpu_uuid,pid")
|
||||
if int(owner) == pid})
|
||||
if not observed:
|
||||
return
|
||||
if observed != [uuid]:
|
||||
out = root / "worker-device-mismatch.json"
|
||||
if not out.exists():
|
||||
save(out, dict(process=witness["process"], expected_uuid=uuid, observed_uuids=observed,
|
||||
scientific_worker_released=False))
|
||||
raise RuntimeError("WORKER_GPU_UUID_MISMATCH")
|
||||
save(verified, dict(process=witness["process"], physical_uuid=uuid,
|
||||
ordinal=witness["ordinal"], observed_uuids=observed))
|
||||
|
||||
|
||||
def dispatch_guard(root, path):
|
||||
root = Path(root)
|
||||
c = contract(path)
|
||||
old_charge = c["ordinal_recovery"]["previous_gpu_wall_seconds"]
|
||||
total = c["limits"]["gpu_wall_seconds"]
|
||||
spawned = None
|
||||
started_dispatch = None
|
||||
|
||||
def runtime_contract(checked_path):
|
||||
runtime = deepcopy(contract(checked_path))
|
||||
runtime["limits"]["gpu_wall_seconds"] = total - old_charge
|
||||
return runtime
|
||||
|
||||
def snapshot(uuid):
|
||||
state = original.snapshot(uuid)
|
||||
if spawned is not None:
|
||||
receipt = root / "actual-worker-spawn.json"
|
||||
if not receipt.exists():
|
||||
try:
|
||||
identity = process_identity(spawned.pid)
|
||||
except psutil.NoSuchProcess:
|
||||
return state
|
||||
save(receipt, dict(process=identity, ordinal=started_dispatch["ordinal"], physical_uuid=uuid))
|
||||
monitor_identity(root, uuid)
|
||||
return state
|
||||
|
||||
def route_supervise(command, directory, mode, limits, *, poll, gpu_uuid):
|
||||
if gpu_uuid != UUID or Path(command[1]).name != "track_b_gino_shapenet_worker.py":
|
||||
raise PermissionError("Unexpected original guard worker route")
|
||||
command[1] = str(WORKER)
|
||||
|
||||
def spawn(args, **kwargs):
|
||||
nonlocal spawned, started_dispatch
|
||||
environment, rows = dispatch_environment(gpu_uuid, kwargs["env"])
|
||||
ordinal = environment["CUDA_VISIBLE_DEVICES"]
|
||||
started_dispatch = dict(ordinal=ordinal, order="PCI_BUS_ID",
|
||||
physical_uuid=gpu_uuid, gpu_inventory=rows)
|
||||
save(root / "ordinal-dispatch.json", started_dispatch)
|
||||
if time.monotonic() >= read(root / "dispatch.json")["start_by_monotonic"]:
|
||||
raise RuntimeError("ORDINAL_DISPATCH_DEADLINE")
|
||||
kwargs["env"] = environment
|
||||
spawned = subprocess.Popen(args, **kwargs)
|
||||
return spawned
|
||||
|
||||
ns = dict(original.supervise.__globals__,
|
||||
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
||||
supervise = FunctionType(original.supervise.__code__, ns, "supervise")
|
||||
return supervise(command, directory, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
|
||||
|
||||
namespace = dict(original.dispatch_guard.__globals__, contract=runtime_contract,
|
||||
snapshot=snapshot, supervise=route_supervise)
|
||||
code = FunctionType(original.dispatch_guard.__code__, namespace, "dispatch_guard")(root, path)
|
||||
record = read(root / "complete.json")
|
||||
new_charge = record["cumulative_gpu_wall_seconds"]
|
||||
save(root / "recovery-settlement.json", dict(status=record["status"], trigger=record["trigger"],
|
||||
previous_gpu_wall_seconds=old_charge, recovery_gpu_wall_seconds=new_charge,
|
||||
cumulative_measurement_gpu_wall_seconds=old_charge + new_charge, total_cap_gpu_wall_seconds=total,
|
||||
independent_attempts=2, no_retry=True, within_total_cap=old_charge + new_charge <= total))
|
||||
return code
|
||||
|
||||
|
||||
def authorized_child(root):
|
||||
namespace = dict(accepted.authorized_child.__globals__, ROOT=ROOT, contract=contract)
|
||||
return FunctionType(accepted.authorized_child.__code__, namespace, "authorized_child")(root)
|
||||
|
||||
|
||||
def run(root):
|
||||
namespace = dict(accepted.run.__globals__, ROOT=ROOT, contract=contract, dispatch_guard=dispatch_guard)
|
||||
return FunctionType(accepted.run.__code__, namespace, "run")(root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("root", type=Path)
|
||||
raise SystemExit(run(parser.parse_args().root))
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Prove the worker's first allocation is on the reviewed physical V100."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
from types import FunctionType
|
||||
|
||||
from track_b_gino_ordinal_recovery import UUID, authorized_child
|
||||
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
|
||||
from track_b_gino_shapenet_entry import process_identity, read, save
|
||||
from track_b_gino_shapenet_worker import work
|
||||
|
||||
|
||||
def wait_for_verified_device(root, c):
|
||||
import torch
|
||||
|
||||
dispatch = read(root / "ordinal-dispatch.json")
|
||||
ordinal = dispatch["ordinal"]
|
||||
if (not ordinal.isdecimal() or os.environ.get("CUDA_VISIBLE_DEVICES") != ordinal
|
||||
or os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID" or dispatch["physical_uuid"] != UUID
|
||||
or torch.cuda.device_count() != 1 or torch.version.cuda != c["environment"]["torch_cuda"]):
|
||||
raise PermissionError("Exactly one device must be visible through the reviewed PCI ordinal")
|
||||
torch.cuda.set_device(0)
|
||||
name = torch.cuda.get_device_name(0)
|
||||
if "V100" not in name:
|
||||
raise PermissionError("Expected one V100 before creating the witness allocation")
|
||||
probe = torch.empty(1, dtype=torch.uint8, device="cuda:0")
|
||||
current = process_identity(os.getpid())
|
||||
save(root / "worker-device-witness.json", dict(process=current, ordinal=ordinal, device_count=1,
|
||||
device_name=name, physical_uuid_expected=UUID, first_allocation_bytes=probe.numel()))
|
||||
deadline = read(root / "dispatch.json")["start_by_monotonic"]
|
||||
verified = root / "worker-device-verified.json"
|
||||
while time.monotonic() < deadline:
|
||||
if verified.exists():
|
||||
record = read(verified)
|
||||
if (record["process"] != current or record["ordinal"] != ordinal or record["physical_uuid"] != UUID
|
||||
or record["observed_uuids"] != [UUID]):
|
||||
raise PermissionError("Verified physical GPU witness differs")
|
||||
del probe
|
||||
return ordinal
|
||||
time.sleep(.1)
|
||||
raise RuntimeError("Physical GPU identity was not verified in the bounded admission window")
|
||||
|
||||
|
||||
def run(root):
|
||||
c = authorized_child(root)
|
||||
if str(Path(sys.executable).absolute()) != c["environment"]["python"]:
|
||||
raise PermissionError("The worker did not start with the accepted interpreter")
|
||||
ordinal = wait_for_verified_device(root, c)
|
||||
# In the unchanged worker this field is read only to check the visible-device
|
||||
# environment string. The coordinator and admission receipts keep the UUID.
|
||||
c = dict(c, gpu_uuid=ordinal)
|
||||
namespace = dict(work.__globals__, extract_paper_code=extract_patched_paper_code)
|
||||
FunctionType(work.__code__, namespace, work.__name__)(root, c)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("root", type=Path)
|
||||
run(parser.parse_args().root.resolve())
|
||||
Reference in new issue
Block a user