Freeze the one GINO ordinal recovery, physical witness and cumulative budget

This commit is contained in:
NewMeltAssist committed 2026-11-19 15:18:33 +08:00
1 parent 17a40a6c1b
commit 057843ae2a
7 files changed
+1303

No files matched your search

@@ -0,0 +1,41 @@
{
"inventory": [
[
"0",
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"00000000:84:00.0",
"Tesla V100-SXM2-16GB"
],
[
"1",
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"00000000:85:00.0",
"Tesla V100-SXM2-16GB"
],
[
"2",
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"00000000:A4:00.0",
"Tesla V100-SXM2-16GB"
],
[
"3",
"GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"00000000:A5:00.0",
"Tesla V100-SXM2-16GB"
]
],
"occupants": [
[
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"2512405",
"820 MiB"
],
[
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"2517067",
"1708 MiB"
]
],
"source": "nvidia-smi actual after the original UUID stop"
}
@@ -0,0 +1,779 @@
{
"mode": "measurement",
"ruling": "h-f84547d7",
"retry": false,
"evaluate_other_models": false,
"measurement_allowed_heldout_labels": false,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"limits": {
"gpu_wall_seconds": 3600,
"cleanup_seconds": 60,
"idle_wait_seconds": 60,
"cpu_seconds": 3500,
"rss_bytes": 12884901888,
"recorded_vram_bytes": 15032385536,
"device_vram_bytes": 16106127360,
"disk_bytes": 2147483648,
"torch_threads": 1,
"max_processes": 1
},
"protocol": {
"model": "NeurIPS 2023 supplement SDFFNOGNO decoder",
"seeds": [
0,
1,
2
],
"grid": [
64,
64,
64
],
"modes": [
24,
24,
24
],
"hidden_channels": 32,
"radius": 0.05,
"epochs_per_seed": 100,
"batch_size": 1,
"learning_rate": 0.001,
"bounds_expansion": 0.01,
"pressure_epsilon": 1e-06,
"optimizer": "Adam",
"weight_decay": 0.0001,
"lr_step_epochs": 50,
"lr_gamma": 0.5,
"pressure_training_transform": "scalar Gaussian on the 500 learning cases",
"metric": "per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)",
"report": "equal mean over 111 cases for each seed, then equal mean over the three seeds",
"acceptance_percent": [
6.62,
7.62
],
"paper_reference_percent": 7.12,
"checkpoint_epoch": 100,
"heldout_access_after_all_checkpoints_frozen": true
},
"environment": {
"python": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
"python_version": "3.10",
"torch_cuda": "11.6",
"neuralop_commit": "fbdc7f872792b75b84545c2f374e7a881c72a718",
"neuralop_file_sha256": {
"utils.py": "84554eed45ac415ab52f79e456eab70ba204e7ab7539c2f2841955a9145220b4",
"models/__init__.py": "c67679f1f8f90e1ab4eb4c2aa916aeaedf1b175d795fed03b8d4f7210883ebae",
"models/tfno.py": "d74b5805b69a64bf33e3ff5186709c75bd431a56c9548f06fb9635a859d849e4",
"models/spectral_convolution.py": "ccecf80806d93e8f35713ac6ab2c3b61c11739f5fcb8c0ba7397b7d4eb5a5d45"
},
"package_versions": {
"torch": "1.13.1+cu116",
"open3d": "0.17.0",
"numpy": "1.24.3",
"torch-scatter": "2.1.1+pt113cu116",
"torch-sparse": "0.6.17+pt113cu116",
"torch-geometric": "2.3.1",
"tensorly": "0.8.1",
"tensorly-torch": "0.4.0"
}
},
"comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-measurement-2026-11-19/comparison-contract.json",
"comparison_contract_sha256": "503f6c8ab1217d66b44e0d127555cfad5da4cd8a17b336299095c3cfd13041d7",
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_shapenet.py": "557ad56277d7e85c6b5cc358eb151632d1757f5883f429988d36022d0d407550",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_entry.py": "e86d955c36e291e6bbef67947de663b24c8d66c5d72d9d28e4193fb8a3051ee2",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py": "7211be211c29472fd9be35a6bcb1f56025de4a63f36852bfafa10df97c003ff3",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_entry.py": "199d833f40339feb0433611ba27302416a280452f8918715e6ef95cb93c38721",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_worker.py": "3acf742dd4549e9f5491e10af5bb0b4acf00e2f47e91753100b5998cfcc61852",
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_import_checked.py": "fef485dd0c233c76a336239d5f29cbf44ac0fb62a6a297dff34293b23a1b75fe",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_paper_keyword_patch.py": "c61e2ed7d099eb125386857bf4d7ccdbfc3af522f80edb4f01b450cee91c0b95",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2.py": "7d3ab2412e5208892cffcfc0e1ec363eb5db2be3e10bf60595356f71e82d8552",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py": "ba1fd5ec837e884b000ba640fceafe30ea40ccf708847733df6a17a8f3599680",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery.py": "1e1827498afab62e2f28efb5d450376f2dca4e8bf6f74114f2914ec771872c86",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py": "7a68c77e9c913ba9161832f513dbf8754ebc5c270c1e742fa5ae97a7b14b9120"
},
"paper_data": {
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-car-pressure-data.zip",
"sha256": "b5f9e2b407d54e034596384253fb58196933c46f6b5b586383508da54475f89d"
},
"paper_supplement": {
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-neurips-supplement.zip",
"sha256": "48cdae80ac2781e27ac00e10a37124711d6bb305593559002bda0527d11a533d"
},
"paper_code_zip_sha256": "347a6f097e2b9958d1ab87930fa1d6227b793a9d213e12418ce9a157e83dce04",
"split_receipt": {
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-split.json",
"sha256": "5a96c25c2ec24cfc1e2264d4ebd0879c658c0f168e22711b576bf303151b8601"
},
"mapping_audit": {
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-mapping-audit.json",
"sha256": "055d008a02494acd29df36c41c8e68db931a027f5635e8f5f765a96a54c62289"
},
"split": {
"train": [
1,
2,
4,
5,
6,
7,
8,
10,
12,
13,
17,
18,
21,
22,
23,
25,
26,
27,
28,
29,
30,
31,
32,
34,
35,
39,
40,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
54,
55,
56,
58,
59,
60,
61,
62,
63,
64,
65,
67,
69,
70,
71,
72,
73,
74,
75,
76,
77,
78,
79,
80,
81,
83,
84,
85,
86,
87,
88,
90,
91,
92,
94,
95,
96,
97,
100,
101,
102,
105,
106,
107,
109,
110,
111,
112,
113,
114,
115,
116,
117,
118,
119,
120,
121,
123,
124,
125,
126,
127,
128,
129,
130,
131,
133,
134,
136,
137,
138,
139,
140,
141,
142,
143,
144,
145,
146,
147,
148,
149,
150,
151,
152,
153,
155,
156,
157,
158,
159,
160,
161,
162,
163,
165,
166,
170,
172,
173,
175,
176,
177,
178,
179,
180,
181,
182,
183,
184,
186,
190,
191,
192,
193,
195,
196,
198,
199,
200,
201,
202,
203,
205,
207,
210,
211,
212,
213,
214,
215,
217,
219,
220,
221,
222,
223,
224,
225,
227,
228,
229,
230,
231,
232,
233,
234,
235,
236,
237,
241,
243,
244,
245,
246,
247,
248,
249,
251,
252,
253,
255,
257,
258,
259,
260,
261,
262,
263,
264,
266,
267,
268,
269,
271,
272,
273,
274,
275,
276,
277,
278,
279,
280,
281,
282,
283,
285,
286,
289,
290,
291,
292,
293,
294,
295,
296,
297,
298,
299,
300,
301,
302,
304,
305,
306,
308,
309,
310,
311,
312,
313,
314,
315,
319,
320,
321,
322,
323,
324,
325,
327,
328,
329,
331,
332,
333,
334,
335,
337,
338,
339,
340,
341,
344,
345,
347,
348,
349,
350,
352,
353,
354,
355,
356,
357,
358,
360,
362,
364,
365,
366,
367,
369,
371,
372,
373,
374,
375,
376,
378,
379,
380,
381,
384,
385,
389,
392,
393,
397,
398,
399,
401,
402,
403,
404,
405,
407,
408,
410,
412,
413,
414,
415,
417,
418,
419,
420,
422,
424,
425,
427,
430,
431,
433,
435,
436,
437,
439,
440,
443,
444,
446,
447,
448,
449,
450,
451,
452,
453,
454,
455,
456,
457,
459,
460,
462,
463,
464,
465,
466,
467,
468,
469,
470,
472,
473,
474,
475,
476,
478,
479,
480,
482,
483,
486,
487,
488,
490,
493,
494,
495,
496,
497,
498,
499,
501,
502,
503,
504,
505,
507,
508,
509,
511,
512,
513,
514,
515,
516,
518,
519,
521,
522,
523,
524,
525,
527,
529,
530,
532,
533,
536,
538,
539,
540,
542,
543,
545,
547,
548,
549,
550,
551,
552,
553,
554,
555,
560,
561,
562,
564,
565,
566,
567,
568,
569,
572,
573,
574,
576,
577,
579,
581,
582,
583,
584,
587,
588,
589,
591,
593,
594,
595,
596,
597,
598,
600,
602,
604,
608,
610,
611,
612,
613,
615,
616,
617,
618,
620,
621,
622,
623,
625,
626,
627,
628,
629,
630,
631,
632,
633,
634,
635,
636,
638,
639,
640,
641,
642,
643,
644,
645,
646,
647,
648,
649,
651,
652,
654,
655,
656,
657
],
"test": [
658,
659,
660,
662,
663,
664,
665,
666,
667,
668,
672,
673,
674,
675,
676,
677,
678,
679,
681,
683,
684,
686,
687,
688,
689,
690,
691,
692,
693,
695,
696,
697,
700,
701,
702,
703,
704,
705,
708,
709,
710,
711,
712,
713,
715,
717,
718,
719,
721,
722,
723,
724,
725,
726,
727,
729,
730,
731,
733,
734,
735,
736,
737,
738,
739,
740,
741,
743,
744,
745,
746,
747,
748,
749,
750,
751,
752,
753,
757,
758,
760,
762,
763,
764,
765,
766,
767,
768,
769,
770,
771,
772,
773,
775,
776,
777,
778,
779,
781,
782,
783,
784,
785,
787,
789,
790,
791,
792,
793,
795,
797
],
"raw_global_bounds": [
[
-0.90252799,
0.0066688,
-2.85272002
],
[
0.90253001,
1.95825005,
2.86385012
]
]
},
"environment_ruling": "h-0d21f3bd",
"environment_acceptance": {
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-import-checked-2026-11-19/cpu-environment-acceptance.json",
"sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3"
},
"paper_compatibility": {
"handoff": "h-b8508140",
"identity": "supplement code + 2-keyword patch",
"patch_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/two-keyword-patch.json",
"patch_sha256": "7307ba49f83bfc46c5decf803a993a41ed602d987de65714958e738d2f52f077",
"search_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/keyword-search.json",
"search_sha256": "77a897bf534a9a0e9e6a7c072b4f914168648eadc2af3493b70d63d0b715b3d0"
},
"gpu_ruling": "h-bc42181f",
"ordinal_recovery": {
"handoff": "h-91e61871",
"previous_terminal_sha256": "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e",
"previous_gpu_wall_seconds": 23.69431395700667
}
}
@@ -0,0 +1,109 @@
{
"status": "PASS",
"ruling": "h-91e61871",
"old_terminal_sha256": "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e",
"contract_sha256": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a",
"same_scientific_protocol_and_caps": true,
"hardware_inventory": {
"inventory": [
[
"0",
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"00000000:84:00.0",
"Tesla V100-SXM2-16GB"
],
[
"1",
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"00000000:85:00.0",
"Tesla V100-SXM2-16GB"
],
[
"2",
"GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"00000000:A4:00.0",
"Tesla V100-SXM2-16GB"
],
[
"3",
"GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"00000000:A5:00.0",
"Tesla V100-SXM2-16GB"
]
],
"occupants": [
[
"GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"2512405",
"820 MiB"
],
[
"GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"2517067",
"1708 MiB"
]
],
"source": "nvidia-smi actual after the original UUID stop"
},
"inconsistent_pci_order_rejected": true,
"advance_checks": {
"matching": {
"captured": {
"command": [
"/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py",
"/tmp/track-b-cuda-identity-replay-8m_kv8v_"
],
"device_order": "PCI_BUS_ID",
"visible_devices": "2",
"start_new_session": true
},
"verified": true,
"mismatch": false,
"stop_called": false,
"terminal_trigger": "MOCK_WORK_REPLACED",
"budget": {
"status": "STOPPED",
"trigger": "MOCK_WORK_REPLACED",
"previous_gpu_wall_seconds": 23.69431395700667,
"recovery_gpu_wall_seconds": 0.6289700350025669,
"cumulative_measurement_gpu_wall_seconds": 24.323283992009237,
"total_cap_gpu_wall_seconds": 3600,
"independent_attempts": 2,
"no_retry": true,
"within_total_cap": true
}
},
"mismatch": {
"captured": {
"command": [
"/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py",
"/tmp/track-b-cuda-identity-replay-mh5jfyj7"
],
"device_order": "PCI_BUS_ID",
"visible_devices": "2",
"start_new_session": true
},
"verified": false,
"mismatch": true,
"stop_called": true,
"terminal_trigger": "MONITOR_FAILURE: WORKER_GPU_UUID_MISMATCH",
"budget": {
"status": "STOPPED",
"trigger": "MONITOR_FAILURE: WORKER_GPU_UUID_MISMATCH",
"previous_gpu_wall_seconds": 23.69431395700667,
"recovery_gpu_wall_seconds": 0.757885464001447,
"cumulative_measurement_gpu_wall_seconds": 24.452199421008117,
"total_cap_gpu_wall_seconds": 3600,
"independent_attempts": 2,
"no_retry": true,
"within_total_cap": true
}
}
},
"real_spawns": 0,
"real_cuda_calls": 0,
"heldout_targets_read": 0,
"independent_review": "PENDING"
}
@@ -0,0 +1,7 @@
# Track B|GPU2 单轮测量的设备绑定恢复
原 GPU2 唯一尝试通过空闲门并派发。Torch 1.13.1 在以物理 UUID 作为 `CUDA_VISIBLE_DEVICES` 时,Python NVML 枚举将该字符串解析为负序号并返回可见卡数 0,`get_device_name(0)` 抛出 `Invalid device id`。原 worker 未进入样本准备,无优化步或计时产物。原申请、工作者退出、终止日志和 CPU Mock 复现已固定。原终止实付 23.69431395700667 GPU-s。 [原终止及 CPU 复现](https://gitea.frank14f.online/Frank14f/MeltMind/src/commit/17a40a6c1b3f765827b2d7c4950b6e6c5a8d1ad6/artifacts/track-b-gino-shapenet611-gpu2-measurement-2026-11-19/terminal-uuid-enumeration-audit.json)
Claude h-91e61871 批准唯一一次环境恢复。在原 UUID 守卫中临派时要求 `nvidia-smi` 的序号与 PCI 总线排序严格一致。物理 GPU2 在实核中为序号 2,PCI 地址 A4:00.0,V100,未见计算进程。只在单个 worker 进程设 `CUDA_DEVICE_ORDER=PCI_BUS_ID`、`CUDA_VISIBLE_DEVICES=2`。在读取任何样本之前,worker 核验只见一张 V100,然后作一字节标记分配。守卫须从实际 worker PID 核实该分配确在固定 UUID,随后才放行原工作体。卡不匹配时由原守卫停止该工作者。 [Claude 裁决](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795072229918399)
CPU 回放验证了正确卡的环境与放行、错误卡的终止、PCI 序号冲突的拒绝,以及旧账计一次。原单轮训练、学习代理、补丁、环境及选择 UUID 均沿用。总帽为 3,600 GPU-s,独立新根的原守卫实际余量为 3,576.305686 GPU-s,支出含等待与设置。独立审查和阶段门在真实开始之前执行。原 111 个留出标签保持封存。 [恢复合同](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json) · [CPU 回放](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/ordinal-cpu-replay.json)
@@ -0,0 +1,132 @@
"""Bind the one approved recovery and replay ordinal dispatch and mismatch."""
import argparse
import json
import os
from pathlib import Path
import subprocess
from tempfile import TemporaryDirectory
import threading
import time
from types import FunctionType, SimpleNamespace
from unittest.mock import patch
import track_b_gino_import_checked_entry as accepted
import track_b_gino_ordinal_recovery as recovery
import track_b_gino_shapenet_entry as original
halt = threading.Event()
def fake_supervisor(command, root, mode, limits, *, poll, gpu_uuid):
"""Executed in a fixture namespace with the real recovery spawn adapter."""
process = subprocess.Popen(command, env={"CUDA_VISIBLE_DEVICES": gpu_uuid}, start_new_session=True)
deadline = time.monotonic() + 4
matched = Path(root) / "worker-device-verified.json"
while not matched.exists() and not halt.is_set() and time.monotonic() < deadline:
time.sleep(.01)
if not (matched.exists() or halt.is_set()):
raise AssertionError("Fixture monitor never verified or stopped the worker")
return dict(status="STOPPED_OR_FAILED", returncode=-9, monitor_errors=0,
stop_reason="MOCK_WORK_REPLACED", pid=process.pid)
def replay(inventory, slot):
halt = threading.Event()
pid = 12345
identity = {"pid": pid, "created": 123.}
captured = {}
def query(option):
if option == "--query-gpu=index,uuid,pci.bus_id,name":
return inventory
if option != "--query-compute-apps=gpu_uuid,pid":
raise AssertionError("Unexpected inventory query")
return [[slot, str(pid)]]
def spawn(command, *, env, start_new_session):
captured.update(command=list(command), device_order=env["CUDA_DEVICE_ORDER"],
visible_devices=env["CUDA_VISIBLE_DEVICES"], start_new_session=start_new_session)
original.save(root / "worker-device-witness.json", dict(
process=identity, ordinal="2", device_count=1, device_name="Tesla V100-SXM2-16GB",
physical_uuid_expected=recovery.UUID, first_allocation_bytes=1))
return SimpleNamespace(pid=pid)
def stop(pid_to_stop):
if pid_to_stop != pid:
raise AssertionError("Attempted to stop the wrong process")
halt.set()
with TemporaryDirectory(prefix="track-b-cuda-identity-replay-") as temp:
root = Path(temp)
path = root / "contract.json"
original.save(path, recovery.expected_contract())
isolated = dict(fake_supervisor.__globals__, halt=halt)
fake = FunctionType(fake_supervisor.__code__, isolated, fake_supervisor.__name__)
with (patch.object(recovery, "query_nvidia", side_effect=query),
patch.object(recovery, "process_identity", return_value=identity),
patch.object(recovery, "subprocess", SimpleNamespace(Popen=spawn, STDOUT=-2)),
patch.object(original, "supervise", fake),
patch.object(original, "snapshot", return_value=dict(idle=True, memory_MiB=5, occupants=[])),
patch.object(original, "wait_idle", return_value=dict(status="IDLE_CONFIRMED")),
patch.object(original, "active_command_pids", return_value=[pid]),
patch.object(original, "stop_group", side_effect=stop)):
code = recovery.dispatch_guard(root, path)
witness, mismatch = root / "worker-device-verified.json", root / "worker-device-mismatch.json"
terminal = original.read(root / "complete.json")
budget = original.read(root / "recovery-settlement.json")
assert code == 1 and bool(captured)
assert captured["device_order"] == "PCI_BUS_ID" and captured["visible_devices"] == "2"
assert captured["command"][1] == str(recovery.WORKER)
assert budget["previous_gpu_wall_seconds"] == 23.69431395700667
assert budget["cumulative_measurement_gpu_wall_seconds"] == (
budget["previous_gpu_wall_seconds"] + budget["recovery_gpu_wall_seconds"])
assert terminal["gpu_wall_cap_seconds"] == 3600 - budget["previous_gpu_wall_seconds"]
if slot == recovery.UUID:
assert witness.exists() and not mismatch.exists() and not halt.is_set()
else:
assert mismatch.exists() and not witness.exists() and halt.is_set()
assert "WORKER_GPU_UUID_MISMATCH" in terminal["trigger"]
return dict(captured=captured, verified=witness.exists(), mismatch=mismatch.exists(),
stop_called=halt.is_set(), terminal_trigger=terminal["trigger"], budget=budget)
def prepare(inventory_path):
if os.environ.get("CUDA_VISIBLE_DEVICES") != "":
raise PermissionError("The replay requires an empty device mask")
source = json.loads(inventory_path.read_text())
rows = source["inventory"]
assert recovery.device_ordinal(rows, recovery.UUID) == "2"
assert not any(card == recovery.UUID for card, *_ in source["occupants"])
mismatched = [row.copy() for row in rows]
mismatched[0][0], mismatched[2][0] = mismatched[2][0], mismatched[0][0]
try:
recovery.device_ordinal(mismatched, recovery.UUID)
except ValueError:
pass
else:
raise AssertionError("A conflicting PCI order was accepted")
c = recovery.expected_contract()
old = original.read(recovery.previous.ROOT / "contract.json")
assert c["gpu_uuid"] == old["gpu_uuid"] and c["limits"] == old["limits"]
assert set(k for k in c if c[k] != old.get(k)) == {"ordinal_recovery", "code_sha256"}
assert accepted.accepted_environment(c)["heldout_target_reads"] == 0
checks = {"matching": replay(rows, recovery.UUID),
"mismatch": replay(rows, rows[0][1])}
assert not (recovery.ROOT / "started.json").exists()
recovery.ROOT.mkdir(exist_ok=False)
original.save(recovery.ROOT / "contract.json", c)
original.save(recovery.ROOT / "ordinal-cpu-replay.json", dict(
status="PASS", ruling="h-91e61871", old_terminal_sha256=recovery.OLD_TERMINAL,
contract_sha256=original.sha(recovery.ROOT / "contract.json"),
same_scientific_protocol_and_caps=True, hardware_inventory=source,
inconsistent_pci_order_rejected=True, advance_checks=checks, real_spawns=0, real_cuda_calls=0,
heldout_targets_read=0, independent_review="PENDING"))
print("PASS", original.sha(recovery.ROOT / "contract.json"))
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("inventory", type=Path)
prepare(parser.parse_args().inventory)
+173
View File
@@ -0,0 +1,173 @@
"""One labelled recovery for the historical Torch UUID-counting failure."""
import argparse
import csv
from copy import deepcopy
from pathlib import Path
import subprocess
import time
from types import FunctionType, SimpleNamespace
import psutil
import track_b_gino_import_checked_entry as accepted
import track_b_gino_measurement_gpu2 as previous
import track_b_gino_shapenet_entry as original
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
ROOT = previous.ROOT.with_name("track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19")
ENTRY = Path(__file__).resolve()
WORKER = ENTRY.with_name("track_b_gino_ordinal_recovery_worker.py")
UUID = previous.UUID
OLD_CONTRACT = "e4545bb5e9eea0a5a501cbfd538ee67468224da36ea151951c7affc5bfc4719f"
OLD_TERMINAL = "c983c05f9531119dad905d71c73399d788837299c44e4736b50a42262b4e9e6e"
def expected_contract():
c = read(checked(previous.ROOT / "contract.json", OLD_CONTRACT))
stopped = read(checked(previous.ROOT / "complete.json", OLD_TERMINAL))
if stopped["status"] != "STOPPED" or stopped["trigger"] != "WORKER_FAILED":
raise PermissionError("The original stopped GPU2 attempt is required")
c["ordinal_recovery"] = dict(handoff="h-91e61871", previous_terminal_sha256=OLD_TERMINAL,
previous_gpu_wall_seconds=stopped["cumulative_gpu_wall_seconds"])
c["code_sha256"].update({str(p): sha(p) for p in (ENTRY, WORKER)})
return c
def contract(path):
c = previous.checked_entry.contract(path)
if c != expected_contract():
raise PermissionError("Only the fixed ordinal recovery of this measurement is authorized")
return c
def query_nvidia(options):
r = subprocess.run(["nvidia-smi", options, "--format=csv,noheader"],
check=True, capture_output=True, text=True, timeout=3)
return [[v.strip() for v in row] for row in csv.reader(r.stdout.splitlines()) if row]
def device_ordinal(rows, uuid):
ordered = sorted(rows, key=lambda row: row[2].lower())
if len({row[2].lower() for row in ordered}) != len(ordered):
raise ValueError("Nonunique GPU PCI identities")
if [row[0] for row in ordered] != [str(i) for i in range(len(ordered))]:
raise ValueError("NVIDIA indices do not match CUDA PCI bus order")
match = [row for row in ordered if row[1] == uuid and "V100" in row[3]]
if len(match) != 1:
raise ValueError("Expected exactly one reviewed physical V100")
return match[0][0]
def dispatch_environment(uuid, env):
if env["CUDA_VISIBLE_DEVICES"] != uuid:
raise PermissionError("Supervisor did not select the exact physical UUID")
rows = query_nvidia("--query-gpu=index,uuid,pci.bus_id,name")
ordinal = device_ordinal(rows, uuid)
return dict(env, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=ordinal), rows
def monitor_identity(root, uuid):
pending = root / "worker-device-witness.json"
verified = root / "worker-device-verified.json"
if not pending.is_file() or verified.exists():
return
witness = read(pending)
spawn = read(root / "actual-worker-spawn.json")
if (witness["process"] != spawn["process"]
or witness["physical_uuid_expected"] != uuid
or witness["ordinal"] != spawn["ordinal"]
or witness["device_count"] != 1 or "V100" not in witness["device_name"]
or witness["first_allocation_bytes"] != 1):
raise RuntimeError("WORKER_DEVICE_WITNESS_IDENTITY_DIFFERS")
pid = witness["process"]["pid"]
observed = sorted({card for card, owner in query_nvidia("--query-compute-apps=gpu_uuid,pid")
if int(owner) == pid})
if not observed:
return
if observed != [uuid]:
out = root / "worker-device-mismatch.json"
if not out.exists():
save(out, dict(process=witness["process"], expected_uuid=uuid, observed_uuids=observed,
scientific_worker_released=False))
raise RuntimeError("WORKER_GPU_UUID_MISMATCH")
save(verified, dict(process=witness["process"], physical_uuid=uuid,
ordinal=witness["ordinal"], observed_uuids=observed))
def dispatch_guard(root, path):
root = Path(root)
c = contract(path)
old_charge = c["ordinal_recovery"]["previous_gpu_wall_seconds"]
total = c["limits"]["gpu_wall_seconds"]
spawned = None
started_dispatch = None
def runtime_contract(checked_path):
runtime = deepcopy(contract(checked_path))
runtime["limits"]["gpu_wall_seconds"] = total - old_charge
return runtime
def snapshot(uuid):
state = original.snapshot(uuid)
if spawned is not None:
receipt = root / "actual-worker-spawn.json"
if not receipt.exists():
try:
identity = process_identity(spawned.pid)
except psutil.NoSuchProcess:
return state
save(receipt, dict(process=identity, ordinal=started_dispatch["ordinal"], physical_uuid=uuid))
monitor_identity(root, uuid)
return state
def route_supervise(command, directory, mode, limits, *, poll, gpu_uuid):
if gpu_uuid != UUID or Path(command[1]).name != "track_b_gino_shapenet_worker.py":
raise PermissionError("Unexpected original guard worker route")
command[1] = str(WORKER)
def spawn(args, **kwargs):
nonlocal spawned, started_dispatch
environment, rows = dispatch_environment(gpu_uuid, kwargs["env"])
ordinal = environment["CUDA_VISIBLE_DEVICES"]
started_dispatch = dict(ordinal=ordinal, order="PCI_BUS_ID",
physical_uuid=gpu_uuid, gpu_inventory=rows)
save(root / "ordinal-dispatch.json", started_dispatch)
if time.monotonic() >= read(root / "dispatch.json")["start_by_monotonic"]:
raise RuntimeError("ORDINAL_DISPATCH_DEADLINE")
kwargs["env"] = environment
spawned = subprocess.Popen(args, **kwargs)
return spawned
ns = dict(original.supervise.__globals__,
subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
supervise = FunctionType(original.supervise.__code__, ns, "supervise")
return supervise(command, directory, mode, limits, poll=poll, gpu_uuid=gpu_uuid)
namespace = dict(original.dispatch_guard.__globals__, contract=runtime_contract,
snapshot=snapshot, supervise=route_supervise)
code = FunctionType(original.dispatch_guard.__code__, namespace, "dispatch_guard")(root, path)
record = read(root / "complete.json")
new_charge = record["cumulative_gpu_wall_seconds"]
save(root / "recovery-settlement.json", dict(status=record["status"], trigger=record["trigger"],
previous_gpu_wall_seconds=old_charge, recovery_gpu_wall_seconds=new_charge,
cumulative_measurement_gpu_wall_seconds=old_charge + new_charge, total_cap_gpu_wall_seconds=total,
independent_attempts=2, no_retry=True, within_total_cap=old_charge + new_charge <= total))
return code
def authorized_child(root):
namespace = dict(accepted.authorized_child.__globals__, ROOT=ROOT, contract=contract)
return FunctionType(accepted.authorized_child.__code__, namespace, "authorized_child")(root)
def run(root):
namespace = dict(accepted.run.__globals__, ROOT=ROOT, contract=contract, dispatch_guard=dispatch_guard)
return FunctionType(accepted.run.__code__, namespace, "run")(root)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", type=Path)
raise SystemExit(run(parser.parse_args().root))
@@ -0,0 +1,62 @@
"""Prove the worker's first allocation is on the reviewed physical V100."""
import argparse
import os
from pathlib import Path
import sys
import time
from types import FunctionType
from track_b_gino_ordinal_recovery import UUID, authorized_child
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
from track_b_gino_shapenet_entry import process_identity, read, save
from track_b_gino_shapenet_worker import work
def wait_for_verified_device(root, c):
import torch
dispatch = read(root / "ordinal-dispatch.json")
ordinal = dispatch["ordinal"]
if (not ordinal.isdecimal() or os.environ.get("CUDA_VISIBLE_DEVICES") != ordinal
or os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID" or dispatch["physical_uuid"] != UUID
or torch.cuda.device_count() != 1 or torch.version.cuda != c["environment"]["torch_cuda"]):
raise PermissionError("Exactly one device must be visible through the reviewed PCI ordinal")
torch.cuda.set_device(0)
name = torch.cuda.get_device_name(0)
if "V100" not in name:
raise PermissionError("Expected one V100 before creating the witness allocation")
probe = torch.empty(1, dtype=torch.uint8, device="cuda:0")
current = process_identity(os.getpid())
save(root / "worker-device-witness.json", dict(process=current, ordinal=ordinal, device_count=1,
device_name=name, physical_uuid_expected=UUID, first_allocation_bytes=probe.numel()))
deadline = read(root / "dispatch.json")["start_by_monotonic"]
verified = root / "worker-device-verified.json"
while time.monotonic() < deadline:
if verified.exists():
record = read(verified)
if (record["process"] != current or record["ordinal"] != ordinal or record["physical_uuid"] != UUID
or record["observed_uuids"] != [UUID]):
raise PermissionError("Verified physical GPU witness differs")
del probe
return ordinal
time.sleep(.1)
raise RuntimeError("Physical GPU identity was not verified in the bounded admission window")
def run(root):
c = authorized_child(root)
if str(Path(sys.executable).absolute()) != c["environment"]["python"]:
raise PermissionError("The worker did not start with the accepted interpreter")
ordinal = wait_for_verified_device(root, c)
# In the unchanged worker this field is read only to check the visible-device
# environment string. The coordinator and admission receipts keep the UUID.
c = dict(c, gpu_uuid=ordinal)
namespace = dict(work.__globals__, extract_paper_code=extract_patched_paper_code)
FunctionType(work.__code__, namespace, work.__name__)(root, c)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", type=Path)
run(parser.parse_args().root.resolve())