Bind formal three-seed GINO with CPU guard and custody replay
This commit is contained in:
1 parent
93d7262e83
commit
9a4f8563ee
11 files changed
+1809
No files matched your search
@@ -0,0 +1,834 @@
|
||||
{
|
||||
"mode": "formal-gino-reproduction",
|
||||
"ruling": "h-f84547d7",
|
||||
"retry": false,
|
||||
"evaluate_other_models": false,
|
||||
"limits": {
|
||||
"cleanup_seconds": 20,
|
||||
"idle_wait_seconds": 60,
|
||||
"rss_bytes": 12884901888,
|
||||
"recorded_vram_bytes": 15032385536,
|
||||
"device_vram_bytes": 16106127360,
|
||||
"disk_bytes": 2147483648,
|
||||
"torch_threads": 1,
|
||||
"max_processes": 1,
|
||||
"cpu_seconds_per_gpu_second": 8
|
||||
},
|
||||
"protocol": {
|
||||
"model": "NeurIPS 2023 supplement SDFFNOGNO decoder",
|
||||
"seeds": [
|
||||
0,
|
||||
1,
|
||||
2
|
||||
],
|
||||
"grid": [
|
||||
64,
|
||||
64,
|
||||
64
|
||||
],
|
||||
"modes": [
|
||||
24,
|
||||
24,
|
||||
24
|
||||
],
|
||||
"hidden_channels": 32,
|
||||
"radius": 0.05,
|
||||
"epochs_per_seed": 100,
|
||||
"batch_size": 1,
|
||||
"learning_rate": 0.001,
|
||||
"bounds_expansion": 0.01,
|
||||
"pressure_epsilon": 1e-06,
|
||||
"optimizer": "Adam",
|
||||
"weight_decay": 0.0001,
|
||||
"lr_step_epochs": 50,
|
||||
"lr_gamma": 0.5,
|
||||
"pressure_training_transform": "scalar Gaussian on the 500 learning cases",
|
||||
"metric": "per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)",
|
||||
"report": "equal mean over 111 cases for each seed, then equal mean over the three seeds",
|
||||
"acceptance_percent": [
|
||||
6.62,
|
||||
7.62
|
||||
],
|
||||
"paper_reference_percent": 7.12,
|
||||
"checkpoint_epoch": 100,
|
||||
"heldout_access_after_all_checkpoints_frozen": true
|
||||
},
|
||||
"environment": {
|
||||
"python": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"python_version": "3.10",
|
||||
"torch_cuda": "11.6",
|
||||
"neuralop_commit": "fbdc7f872792b75b84545c2f374e7a881c72a718",
|
||||
"neuralop_file_sha256": {
|
||||
"utils.py": "84554eed45ac415ab52f79e456eab70ba204e7ab7539c2f2841955a9145220b4",
|
||||
"models/__init__.py": "c67679f1f8f90e1ab4eb4c2aa916aeaedf1b175d795fed03b8d4f7210883ebae",
|
||||
"models/tfno.py": "d74b5805b69a64bf33e3ff5186709c75bd431a56c9548f06fb9635a859d849e4",
|
||||
"models/spectral_convolution.py": "ccecf80806d93e8f35713ac6ab2c3b61c11739f5fcb8c0ba7397b7d4eb5a5d45"
|
||||
},
|
||||
"package_versions": {
|
||||
"torch": "1.13.1+cu116",
|
||||
"open3d": "0.17.0",
|
||||
"numpy": "1.24.3",
|
||||
"torch-scatter": "2.1.1+pt113cu116",
|
||||
"torch-sparse": "0.6.17+pt113cu116",
|
||||
"torch-geometric": "2.3.1",
|
||||
"tensorly": "0.8.1",
|
||||
"tensorly-torch": "0.4.0"
|
||||
}
|
||||
},
|
||||
"comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-measurement-2026-11-19/comparison-contract.json",
|
||||
"comparison_contract_sha256": "503f6c8ab1217d66b44e0d127555cfad5da4cd8a17b336299095c3cfd13041d7",
|
||||
"code_sha256": {
|
||||
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
|
||||
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_shapenet.py": "557ad56277d7e85c6b5cc358eb151632d1757f5883f429988d36022d0d407550",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_entry.py": "e86d955c36e291e6bbef67947de663b24c8d66c5d72d9d28e4193fb8a3051ee2",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py": "7211be211c29472fd9be35a6bcb1f56025de4a63f36852bfafa10df97c003ff3",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_entry.py": "199d833f40339feb0433611ba27302416a280452f8918715e6ef95cb93c38721",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_worker.py": "3acf742dd4549e9f5491e10af5bb0b4acf00e2f47e91753100b5998cfcc61852",
|
||||
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_import_checked.py": "fef485dd0c233c76a336239d5f29cbf44ac0fb62a6a297dff34293b23a1b75fe",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_paper_keyword_patch.py": "c61e2ed7d099eb125386857bf4d7ccdbfc3af522f80edb4f01b450cee91c0b95",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2.py": "7d3ab2412e5208892cffcfc0e1ec363eb5db2be3e10bf60595356f71e82d8552",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py": "ba1fd5ec837e884b000ba640fceafe30ea40ccf708847733df6a17a8f3599680",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery.py": "1e1827498afab62e2f28efb5d450376f2dca4e8bf6f74114f2914ec771872c86",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py": "7a68c77e9c913ba9161832f513dbf8754ebc5c270c1e742fa5ae97a7b14b9120",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal.py": "15ea113a0657715eff495ce17dc63a17a3ee984133907f8a4d5a83f2cf8a01a2",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_guard.py": "9b4020b4754b46325c9aab1d15448eb86b21ecf0e5dfb527a51907982db2c0fb",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_worker.py": "978f1c7cfb622300b8814e14e4a02e5cbb9e2ea7f6e5fee080b986c7409ade2c"
|
||||
},
|
||||
"paper_data": {
|
||||
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-car-pressure-data.zip",
|
||||
"sha256": "b5f9e2b407d54e034596384253fb58196933c46f6b5b586383508da54475f89d"
|
||||
},
|
||||
"paper_supplement": {
|
||||
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-neurips-supplement.zip",
|
||||
"sha256": "48cdae80ac2781e27ac00e10a37124711d6bb305593559002bda0527d11a533d"
|
||||
},
|
||||
"paper_code_zip_sha256": "347a6f097e2b9958d1ab87930fa1d6227b793a9d213e12418ce9a157e83dce04",
|
||||
"split_receipt": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-split.json",
|
||||
"sha256": "5a96c25c2ec24cfc1e2264d4ebd0879c658c0f168e22711b576bf303151b8601"
|
||||
},
|
||||
"mapping_audit": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-mapping-audit.json",
|
||||
"sha256": "055d008a02494acd29df36c41c8e68db931a027f5635e8f5f765a96a54c62289"
|
||||
},
|
||||
"split": {
|
||||
"train": [
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
10,
|
||||
12,
|
||||
13,
|
||||
17,
|
||||
18,
|
||||
21,
|
||||
22,
|
||||
23,
|
||||
25,
|
||||
26,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31,
|
||||
32,
|
||||
34,
|
||||
35,
|
||||
39,
|
||||
40,
|
||||
43,
|
||||
44,
|
||||
45,
|
||||
46,
|
||||
47,
|
||||
48,
|
||||
49,
|
||||
50,
|
||||
51,
|
||||
52,
|
||||
54,
|
||||
55,
|
||||
56,
|
||||
58,
|
||||
59,
|
||||
60,
|
||||
61,
|
||||
62,
|
||||
63,
|
||||
64,
|
||||
65,
|
||||
67,
|
||||
69,
|
||||
70,
|
||||
71,
|
||||
72,
|
||||
73,
|
||||
74,
|
||||
75,
|
||||
76,
|
||||
77,
|
||||
78,
|
||||
79,
|
||||
80,
|
||||
81,
|
||||
83,
|
||||
84,
|
||||
85,
|
||||
86,
|
||||
87,
|
||||
88,
|
||||
90,
|
||||
91,
|
||||
92,
|
||||
94,
|
||||
95,
|
||||
96,
|
||||
97,
|
||||
100,
|
||||
101,
|
||||
102,
|
||||
105,
|
||||
106,
|
||||
107,
|
||||
109,
|
||||
110,
|
||||
111,
|
||||
112,
|
||||
113,
|
||||
114,
|
||||
115,
|
||||
116,
|
||||
117,
|
||||
118,
|
||||
119,
|
||||
120,
|
||||
121,
|
||||
123,
|
||||
124,
|
||||
125,
|
||||
126,
|
||||
127,
|
||||
128,
|
||||
129,
|
||||
130,
|
||||
131,
|
||||
133,
|
||||
134,
|
||||
136,
|
||||
137,
|
||||
138,
|
||||
139,
|
||||
140,
|
||||
141,
|
||||
142,
|
||||
143,
|
||||
144,
|
||||
145,
|
||||
146,
|
||||
147,
|
||||
148,
|
||||
149,
|
||||
150,
|
||||
151,
|
||||
152,
|
||||
153,
|
||||
155,
|
||||
156,
|
||||
157,
|
||||
158,
|
||||
159,
|
||||
160,
|
||||
161,
|
||||
162,
|
||||
163,
|
||||
165,
|
||||
166,
|
||||
170,
|
||||
172,
|
||||
173,
|
||||
175,
|
||||
176,
|
||||
177,
|
||||
178,
|
||||
179,
|
||||
180,
|
||||
181,
|
||||
182,
|
||||
183,
|
||||
184,
|
||||
186,
|
||||
190,
|
||||
191,
|
||||
192,
|
||||
193,
|
||||
195,
|
||||
196,
|
||||
198,
|
||||
199,
|
||||
200,
|
||||
201,
|
||||
202,
|
||||
203,
|
||||
205,
|
||||
207,
|
||||
210,
|
||||
211,
|
||||
212,
|
||||
213,
|
||||
214,
|
||||
215,
|
||||
217,
|
||||
219,
|
||||
220,
|
||||
221,
|
||||
222,
|
||||
223,
|
||||
224,
|
||||
225,
|
||||
227,
|
||||
228,
|
||||
229,
|
||||
230,
|
||||
231,
|
||||
232,
|
||||
233,
|
||||
234,
|
||||
235,
|
||||
236,
|
||||
237,
|
||||
241,
|
||||
243,
|
||||
244,
|
||||
245,
|
||||
246,
|
||||
247,
|
||||
248,
|
||||
249,
|
||||
251,
|
||||
252,
|
||||
253,
|
||||
255,
|
||||
257,
|
||||
258,
|
||||
259,
|
||||
260,
|
||||
261,
|
||||
262,
|
||||
263,
|
||||
264,
|
||||
266,
|
||||
267,
|
||||
268,
|
||||
269,
|
||||
271,
|
||||
272,
|
||||
273,
|
||||
274,
|
||||
275,
|
||||
276,
|
||||
277,
|
||||
278,
|
||||
279,
|
||||
280,
|
||||
281,
|
||||
282,
|
||||
283,
|
||||
285,
|
||||
286,
|
||||
289,
|
||||
290,
|
||||
291,
|
||||
292,
|
||||
293,
|
||||
294,
|
||||
295,
|
||||
296,
|
||||
297,
|
||||
298,
|
||||
299,
|
||||
300,
|
||||
301,
|
||||
302,
|
||||
304,
|
||||
305,
|
||||
306,
|
||||
308,
|
||||
309,
|
||||
310,
|
||||
311,
|
||||
312,
|
||||
313,
|
||||
314,
|
||||
315,
|
||||
319,
|
||||
320,
|
||||
321,
|
||||
322,
|
||||
323,
|
||||
324,
|
||||
325,
|
||||
327,
|
||||
328,
|
||||
329,
|
||||
331,
|
||||
332,
|
||||
333,
|
||||
334,
|
||||
335,
|
||||
337,
|
||||
338,
|
||||
339,
|
||||
340,
|
||||
341,
|
||||
344,
|
||||
345,
|
||||
347,
|
||||
348,
|
||||
349,
|
||||
350,
|
||||
352,
|
||||
353,
|
||||
354,
|
||||
355,
|
||||
356,
|
||||
357,
|
||||
358,
|
||||
360,
|
||||
362,
|
||||
364,
|
||||
365,
|
||||
366,
|
||||
367,
|
||||
369,
|
||||
371,
|
||||
372,
|
||||
373,
|
||||
374,
|
||||
375,
|
||||
376,
|
||||
378,
|
||||
379,
|
||||
380,
|
||||
381,
|
||||
384,
|
||||
385,
|
||||
389,
|
||||
392,
|
||||
393,
|
||||
397,
|
||||
398,
|
||||
399,
|
||||
401,
|
||||
402,
|
||||
403,
|
||||
404,
|
||||
405,
|
||||
407,
|
||||
408,
|
||||
410,
|
||||
412,
|
||||
413,
|
||||
414,
|
||||
415,
|
||||
417,
|
||||
418,
|
||||
419,
|
||||
420,
|
||||
422,
|
||||
424,
|
||||
425,
|
||||
427,
|
||||
430,
|
||||
431,
|
||||
433,
|
||||
435,
|
||||
436,
|
||||
437,
|
||||
439,
|
||||
440,
|
||||
443,
|
||||
444,
|
||||
446,
|
||||
447,
|
||||
448,
|
||||
449,
|
||||
450,
|
||||
451,
|
||||
452,
|
||||
453,
|
||||
454,
|
||||
455,
|
||||
456,
|
||||
457,
|
||||
459,
|
||||
460,
|
||||
462,
|
||||
463,
|
||||
464,
|
||||
465,
|
||||
466,
|
||||
467,
|
||||
468,
|
||||
469,
|
||||
470,
|
||||
472,
|
||||
473,
|
||||
474,
|
||||
475,
|
||||
476,
|
||||
478,
|
||||
479,
|
||||
480,
|
||||
482,
|
||||
483,
|
||||
486,
|
||||
487,
|
||||
488,
|
||||
490,
|
||||
493,
|
||||
494,
|
||||
495,
|
||||
496,
|
||||
497,
|
||||
498,
|
||||
499,
|
||||
501,
|
||||
502,
|
||||
503,
|
||||
504,
|
||||
505,
|
||||
507,
|
||||
508,
|
||||
509,
|
||||
511,
|
||||
512,
|
||||
513,
|
||||
514,
|
||||
515,
|
||||
516,
|
||||
518,
|
||||
519,
|
||||
521,
|
||||
522,
|
||||
523,
|
||||
524,
|
||||
525,
|
||||
527,
|
||||
529,
|
||||
530,
|
||||
532,
|
||||
533,
|
||||
536,
|
||||
538,
|
||||
539,
|
||||
540,
|
||||
542,
|
||||
543,
|
||||
545,
|
||||
547,
|
||||
548,
|
||||
549,
|
||||
550,
|
||||
551,
|
||||
552,
|
||||
553,
|
||||
554,
|
||||
555,
|
||||
560,
|
||||
561,
|
||||
562,
|
||||
564,
|
||||
565,
|
||||
566,
|
||||
567,
|
||||
568,
|
||||
569,
|
||||
572,
|
||||
573,
|
||||
574,
|
||||
576,
|
||||
577,
|
||||
579,
|
||||
581,
|
||||
582,
|
||||
583,
|
||||
584,
|
||||
587,
|
||||
588,
|
||||
589,
|
||||
591,
|
||||
593,
|
||||
594,
|
||||
595,
|
||||
596,
|
||||
597,
|
||||
598,
|
||||
600,
|
||||
602,
|
||||
604,
|
||||
608,
|
||||
610,
|
||||
611,
|
||||
612,
|
||||
613,
|
||||
615,
|
||||
616,
|
||||
617,
|
||||
618,
|
||||
620,
|
||||
621,
|
||||
622,
|
||||
623,
|
||||
625,
|
||||
626,
|
||||
627,
|
||||
628,
|
||||
629,
|
||||
630,
|
||||
631,
|
||||
632,
|
||||
633,
|
||||
634,
|
||||
635,
|
||||
636,
|
||||
638,
|
||||
639,
|
||||
640,
|
||||
641,
|
||||
642,
|
||||
643,
|
||||
644,
|
||||
645,
|
||||
646,
|
||||
647,
|
||||
648,
|
||||
649,
|
||||
651,
|
||||
652,
|
||||
654,
|
||||
655,
|
||||
656,
|
||||
657
|
||||
],
|
||||
"test": [
|
||||
658,
|
||||
659,
|
||||
660,
|
||||
662,
|
||||
663,
|
||||
664,
|
||||
665,
|
||||
666,
|
||||
667,
|
||||
668,
|
||||
672,
|
||||
673,
|
||||
674,
|
||||
675,
|
||||
676,
|
||||
677,
|
||||
678,
|
||||
679,
|
||||
681,
|
||||
683,
|
||||
684,
|
||||
686,
|
||||
687,
|
||||
688,
|
||||
689,
|
||||
690,
|
||||
691,
|
||||
692,
|
||||
693,
|
||||
695,
|
||||
696,
|
||||
697,
|
||||
700,
|
||||
701,
|
||||
702,
|
||||
703,
|
||||
704,
|
||||
705,
|
||||
708,
|
||||
709,
|
||||
710,
|
||||
711,
|
||||
712,
|
||||
713,
|
||||
715,
|
||||
717,
|
||||
718,
|
||||
719,
|
||||
721,
|
||||
722,
|
||||
723,
|
||||
724,
|
||||
725,
|
||||
726,
|
||||
727,
|
||||
729,
|
||||
730,
|
||||
731,
|
||||
733,
|
||||
734,
|
||||
735,
|
||||
736,
|
||||
737,
|
||||
738,
|
||||
739,
|
||||
740,
|
||||
741,
|
||||
743,
|
||||
744,
|
||||
745,
|
||||
746,
|
||||
747,
|
||||
748,
|
||||
749,
|
||||
750,
|
||||
751,
|
||||
752,
|
||||
753,
|
||||
757,
|
||||
758,
|
||||
760,
|
||||
762,
|
||||
763,
|
||||
764,
|
||||
765,
|
||||
766,
|
||||
767,
|
||||
768,
|
||||
769,
|
||||
770,
|
||||
771,
|
||||
772,
|
||||
773,
|
||||
775,
|
||||
776,
|
||||
777,
|
||||
778,
|
||||
779,
|
||||
781,
|
||||
782,
|
||||
783,
|
||||
784,
|
||||
785,
|
||||
787,
|
||||
789,
|
||||
790,
|
||||
791,
|
||||
792,
|
||||
793,
|
||||
795,
|
||||
797
|
||||
],
|
||||
"raw_global_bounds": [
|
||||
[
|
||||
-0.90252799,
|
||||
0.0066688,
|
||||
-2.85272002
|
||||
],
|
||||
[
|
||||
0.90253001,
|
||||
1.95825005,
|
||||
2.86385012
|
||||
]
|
||||
]
|
||||
},
|
||||
"environment_ruling": "h-0d21f3bd",
|
||||
"environment_acceptance": {
|
||||
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-import-checked-2026-11-19/cpu-environment-acceptance.json",
|
||||
"sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3"
|
||||
},
|
||||
"paper_compatibility": {
|
||||
"handoff": "h-b8508140",
|
||||
"identity": "supplement code + 2-keyword patch",
|
||||
"patch_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/two-keyword-patch.json",
|
||||
"patch_sha256": "7307ba49f83bfc46c5decf803a993a41ed602d987de65714958e738d2f52f077",
|
||||
"search_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/keyword-search.json",
|
||||
"search_sha256": "77a897bf534a9a0e9e6a7c072b4f914168648eadc2af3493b70d63d0b715b3d0"
|
||||
},
|
||||
"formal": {
|
||||
"handoff": "h-4e4e680b",
|
||||
"ruling_slack_ts": "1795074651.222599",
|
||||
"devices": {
|
||||
"0": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"1": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"2": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c"
|
||||
},
|
||||
"device_pci_ordinals": {
|
||||
"0": 2,
|
||||
"1": 3,
|
||||
"2": 0
|
||||
},
|
||||
"reservation": {
|
||||
"handoff": "h-a429935e",
|
||||
"timezone": "Asia/Hong_Kong",
|
||||
"start_inclusive": "2026-11-19T00:00:00+08:00",
|
||||
"end_exclusive": "2026-11-20T00:00:00+08:00",
|
||||
"gpu3_single_task": true,
|
||||
"historical_inventory_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json",
|
||||
"historical_inventory_sha256": "da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8",
|
||||
"runtime_idle_check_required": true
|
||||
},
|
||||
"budget_per_seed_stage_gpu_seconds": {
|
||||
"0": {
|
||||
"fit": 13034,
|
||||
"score": 300
|
||||
},
|
||||
"1": {
|
||||
"fit": 13033,
|
||||
"score": 300
|
||||
},
|
||||
"2": {
|
||||
"fit": 13033,
|
||||
"score": 300
|
||||
}
|
||||
},
|
||||
"total_gpu_wall_seconds": 40000,
|
||||
"timing": {
|
||||
"record_paths": {
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json": "db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json": "851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json": "8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf"
|
||||
},
|
||||
"status": "CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED",
|
||||
"sizing_only": true,
|
||||
"epoch_seconds": 102.93777855299413,
|
||||
"train_sizing_gpu_seconds": 38601.6669573728,
|
||||
"train_sizing_multiplier": 1.25,
|
||||
"score_reserved_gpu_seconds": 900,
|
||||
"prior_measurement_gpu_seconds": 222.80500172509346
|
||||
},
|
||||
"heldout_policy": {
|
||||
"preparation_reads": 0,
|
||||
"training_reads": 0,
|
||||
"score_reads_per_seed": 111,
|
||||
"all_epoch100_checkpoints_required": true,
|
||||
"retry": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"contract_sha256": "bcb25413f1bd29679116ba320fee1bcc6d5d66ec074665d61c6910a65fb97959",
|
||||
"cuda_visible_devices": "",
|
||||
"actual_nvidia_queries": 0,
|
||||
"training_starts": 0,
|
||||
"scoring_starts": 0,
|
||||
"actual_heldout_reads": 0,
|
||||
"independent_review": "PENDING",
|
||||
"guard_replay": 9,
|
||||
"cpu_replay_log_sha256": "c0d40741c701de9be3699fd66f82473c8ffdfe9df25c29c95f4fe2ac76689369",
|
||||
"read_only_collection": {
|
||||
"seeds": {
|
||||
"0": {
|
||||
"fit": {
|
||||
"status": "NOT_STARTED"
|
||||
},
|
||||
"score": {
|
||||
"status": "NOT_STARTED"
|
||||
}
|
||||
},
|
||||
"1": {
|
||||
"fit": {
|
||||
"status": "NOT_STARTED"
|
||||
},
|
||||
"score": {
|
||||
"status": "NOT_STARTED"
|
||||
}
|
||||
},
|
||||
"2": {
|
||||
"fit": {
|
||||
"status": "NOT_STARTED"
|
||||
},
|
||||
"score": {
|
||||
"status": "NOT_STARTED"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mean_percent": null,
|
||||
"within_registered_range": null,
|
||||
"reproduction_identity": "supplement code + 2-keyword patch",
|
||||
"prior_measurement_gpu_seconds": 222.80500172509346,
|
||||
"settled_formal_gpu_seconds": 0,
|
||||
"settled_combined_gpu_seconds": 222.80500172509346,
|
||||
"unsettled_formal_bookings_gpu_seconds": 0,
|
||||
"formal_gpu_cap_seconds": 40000
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
test_card_reservation_expiry_and_total_budget (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_inclusive_gate_and_all_three_seeds_required (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_original_split_score_keeps_order_and_rejects_nan (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_score_opens_one_111_target_pass_and_rejects_a_second (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_scoring_needs_all_checkpoints_before_access (test_track_b_gino_formal.FormalBinding) ... ok
|
||||
test_gpu_list_before_worker_exit (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_real_foreign_identity_has_exact_stop_receipt (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_reused_worker_pid_is_foreign (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
test_worker_already_exited_before_gpu_list (test_track_b_gino_formal.GuardReplay) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 9 tests in 0.656s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"status": "STOPPED",
|
||||
"trigger": "FOREIGN_GPU_OCCUPANT",
|
||||
"contract_sha256": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a",
|
||||
"result_sha256": null,
|
||||
"worker": {
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
|
||||
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py",
|
||||
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 1221.067037,
|
||||
"wall_seconds": 178.46390844404232,
|
||||
"peak_tree_rss_bytes": 4780380160,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 131072,
|
||||
"max_poll_gap_seconds": 0.8362329140072688,
|
||||
"poll_target_seconds": 0.2,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
},
|
||||
"booked_idle_seconds": 20.519942376064137,
|
||||
"cumulative_gpu_wall_seconds": 199.1106877680868,
|
||||
"gpu_wall_cap_seconds": 3576.3056860429933,
|
||||
"recorded_peaks": {
|
||||
"peak_allocated_bytes": 2953057792,
|
||||
"peak_reserved_bytes": 3980394496
|
||||
},
|
||||
"no_retry": true
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"status": "STOPPED",
|
||||
"trigger": "FOREIGN_GPU_OCCUPANT",
|
||||
"previous_gpu_wall_seconds": 23.69431395700667,
|
||||
"recovery_gpu_wall_seconds": 199.1106877680868,
|
||||
"cumulative_measurement_gpu_wall_seconds": 222.80500172509346,
|
||||
"total_cap_gpu_wall_seconds": 3600,
|
||||
"independent_attempts": 2,
|
||||
"no_retry": true,
|
||||
"within_total_cap": true
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"stage": "EPOCH_COMPLETE",
|
||||
"seed": 0,
|
||||
"epoch": 1,
|
||||
"examples": 500,
|
||||
"seconds": 102.93777855299413,
|
||||
"peak_allocated_bytes": 2953057792,
|
||||
"peak_reserved_bytes": 3980394496
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
# Track B GINO 三种子复现联合预审
|
||||
|
||||
本次交给 MeltAssist 一次联合预审,范围包括占卡守卫、训练入口和评分入口。当前只准备和回放。任何真实派发均需同一绑定的独立 PASS 和阶段授权。 [Claude 裁定](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795074651222599)
|
||||
|
||||
## 卡和预算
|
||||
|
||||
依据 h-a429935e,种子 0、1、2 分别绑定物理 GPU2、GPU3、GPU0。GPU3 只允许单任务。香港时间 2026-11-19 当日预留,满额预约须能在当地午夜前结束。实际派发仍须通过三次空闲观察和临派复核。物理 UUID、PCI 顺序和真实 worker 首次一字节分配均须匹配,GPU1 不参与。 [组合合同](../artifacts/track-b-gino-shapenet611-formal-2026-11-19/contract.json)
|
||||
|
||||
GPU2 恢复测量的 worker 完整跑了 500 样本、1 轮,用时 102.93777855299413 秒。该计时受抢占影响且未经守卫认证,只用于预算测算。原守卫仍是 `STOPPED / FOREIGN_GPU_OCCUPANT`,缺少占用者身份,历史根因未确定。两次旧测量累计 222.80500172509346 GPU 秒独立记账一次。正式预算为 40,000 GPU 秒,三种子 300 轮乘 1.25 得 38,601.667 GPU 秒。训练、入场和清理预留为 13,034 / 13,033 / 13,033 GPU 秒,每个评分另预留 300 GPU 秒。评分预留是预算假设,尚无实际评分计时。 [历史终止](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json) · [历史计时](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json) · [历史结算](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json)
|
||||
|
||||
## 守卫、训练和评分
|
||||
|
||||
派发时记录真实 worker PID 和出生时间。GPU 占用采样前后都观察其进程树,保留已认领的进程身份,避免已退出 worker 被陈旧 GPU 记录误判。PID 复用时以不同出生时间区分。确认外部占用时记录采样时间、卡 UUID、占用 PID 和名称,只停止本次 worker 进程组。运行中的显存与结果中记录的显存峰值均纳入守卫。 [守卫](../tools/track_b_gino_formal_guard.py)
|
||||
|
||||
沿用原始 500/111 划分、已验收的解释器、原加载器、模型、优化步骤和指标。复现身份为 `supplement code + 2-keyword patch`。每个种子从头训练 100 轮,冻结模型与只从学习集拟合的压力归一化参数。每次评分须先核对三份已通过守卫的第 100 轮检查点。然后该种子只开一次留出目标记录,按原序逐读 111 个目标并验证逐例哈希。三种子的结果全部列出。仅在三者均有效时,对各自的 111 件平均相对 L2 等权汇总,区间两端 6.62% 和 7.62% 均计入通过。 [训练与评分](../tools/track_b_gino_formal_worker.py) · [联合入口与结算](../tools/track_b_gino_formal.py)
|
||||
|
||||
## CPU 回放
|
||||
|
||||
生产守卫配合真实 CPU 子进程回放了两种退出时序。GPU 列表先采到 worker 随后其退出、以及 worker 先退出而 GPU 列表仍返回旧 PID,均通过。另用真实活进程身份模拟外部占用,验证 STOP 回执的 PID、名称、采样时间和仅停止本方 worker。另验 PID 复用、日界和预算、汇总区间、留出顺序、三检查点屏障及只开一次 111 项的合成评分。首次回放的两项退出测试被测试夹具的 256 MiB RSS 帽拦截,夹具放宽到 1 GiB 后九项全部通过。正式 RSS 帽为 12 GiB。回放未访问目标归档,未执行真实训练或评分。 [CPU 回放日志](../artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-replay.log) · [回放用例](../tests/test_track_b_gino_formal.py)
|
||||
@@ -0,0 +1,204 @@
|
||||
"""CPU replay of the production GINO guard and formal custody gates."""
|
||||
|
||||
from pathlib import Path
|
||||
import os
|
||||
import sys
|
||||
from tempfile import TemporaryDirectory
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import track_b_gino_formal as formal
|
||||
import track_b_gino_formal_guard as gpu_guard
|
||||
import track_b_gino_formal_worker as worker
|
||||
from track_b_gino_shapenet_entry import read, save, sha
|
||||
|
||||
|
||||
class GuardReplay(unittest.TestCase):
|
||||
def replay(self, ordering):
|
||||
uuid = 'GPU-fixture'
|
||||
limit = dict(disk_bytes=2**20, idle_wait_seconds=10, cleanup_seconds=2, cpu_seconds=10,
|
||||
rss_bytes=2**30, max_processes=1, device_vram_bytes=2**30,
|
||||
recorded_vram_bytes=2**30)
|
||||
with TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
result = root / 'worker-records/result.json'
|
||||
result.parent.mkdir()
|
||||
# This subprocess has no GPU code. It waits until the replay lets
|
||||
# it finish or until the production guard stops its process group.
|
||||
command = [sys.executable, '-c',
|
||||
"from pathlib import Path\nimport time,sys\n"
|
||||
"root=Path(sys.argv[1])\n"
|
||||
"while not (root/'exit').exists(): time.sleep(.01)\n"
|
||||
"(root/'worker-records/result.json').write_text("
|
||||
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder]
|
||||
observed = []
|
||||
stopped = []
|
||||
real_stop = gpu_guard.stop_group
|
||||
real_tree = gpu_guard.process_tree
|
||||
|
||||
def stop_owned(pid):
|
||||
stopped.append(pid)
|
||||
real_stop(pid)
|
||||
|
||||
def exit_and_wait():
|
||||
(root / 'exit').touch()
|
||||
pid = read(root / 'actual-worker-spawn.json')['process']['pid']
|
||||
by = time.monotonic() + 4
|
||||
while gpu_guard.current_identity(pid) is not None and time.monotonic() < by:
|
||||
time.sleep(.01)
|
||||
if gpu_guard.current_identity(pid) is not None:
|
||||
raise RuntimeError('fixture worker did not exit and get reaped')
|
||||
observed.append('worker-reaped')
|
||||
|
||||
def tree(identity):
|
||||
if (root / 'actual-worker-spawn.json').exists() and (
|
||||
ordering == 'tree-first' and 'worker-reaped' not in observed):
|
||||
exit_and_wait()
|
||||
observed.append('tree')
|
||||
return real_tree(identity)
|
||||
|
||||
def snapshot(card):
|
||||
if card != uuid:
|
||||
raise AssertionError(card)
|
||||
spawn = root / 'actual-worker-spawn.json'
|
||||
if not spawn.exists():
|
||||
return dict(idle=True, occupants=[], unix=10., memory_MiB=0)
|
||||
observed.append('gpu-sample')
|
||||
pid = os.getpid() if ordering == 'foreign' else read(spawn)['process']['pid']
|
||||
sample = dict(idle=False, unix=42.125, memory_MiB=4,
|
||||
occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')])
|
||||
if ordering == 'gpu-first':
|
||||
exit_and_wait()
|
||||
return sample
|
||||
|
||||
with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot),
|
||||
patch.object(gpu_guard, 'process_tree', side_effect=tree),
|
||||
patch.object(gpu_guard, 'stop_group', side_effect=stop_owned),
|
||||
patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}),
|
||||
patch.object(gpu_guard, 'dispatch_environment',
|
||||
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))):
|
||||
began = time.monotonic()
|
||||
outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15,
|
||||
ticket=dict(ordinal='', guard_started_monotonic=began), validate_result=lambda: sha(result))
|
||||
spawn = read(root / 'actual-worker-spawn.json')['process']
|
||||
if ordering == 'foreign':
|
||||
self.assertEqual(outcome['status'], 'STOPPED')
|
||||
self.assertEqual(outcome['trigger'], 'FOREIGN_GPU_OCCUPANT')
|
||||
self.assertIsNone(outcome['result_sha256'])
|
||||
row = read(root / 'foreign-occupant.json')
|
||||
self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid))
|
||||
self.assertEqual(row['occupants'][0]['pid'], os.getpid())
|
||||
self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python')
|
||||
self.assertEqual(stopped, [spawn['pid']])
|
||||
else:
|
||||
self.assertEqual(outcome['status'], 'COMPLETE', outcome)
|
||||
self.assertFalse((root / 'foreign-occupant.json').exists())
|
||||
self.assertEqual(stopped, [])
|
||||
self.assertEqual(outcome['result_sha256'], sha(result))
|
||||
return observed
|
||||
|
||||
def test_gpu_list_before_worker_exit(self):
|
||||
order = self.replay('gpu-first')
|
||||
self.assertLess(order.index('gpu-sample'), order.index('worker-reaped'))
|
||||
|
||||
def test_worker_already_exited_before_gpu_list(self):
|
||||
order = self.replay('tree-first')
|
||||
self.assertLess(order.index('worker-reaped'), order.index('gpu-sample'))
|
||||
|
||||
def test_real_foreign_identity_has_exact_stop_receipt(self):
|
||||
self.replay('foreign')
|
||||
|
||||
def test_reused_worker_pid_is_foreign(self):
|
||||
with TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
original = dict(pid=12, created=1.)
|
||||
current = dict(pid=12, created=2.)
|
||||
sample = dict(unix=42., occupants=[dict(pid=12, process_name='new-process')], memory_MiB=4)
|
||||
monitor = gpu_guard.OccupancyMonitor(root, 'fixture', original, 2**30)
|
||||
with (patch.object(gpu_guard, 'snapshot', return_value=sample),
|
||||
patch.object(gpu_guard, 'current_identity', return_value=current)):
|
||||
with self.assertRaisesRegex(RuntimeError, 'FOREIGN_GPU_OCCUPANT'):
|
||||
monitor.check()
|
||||
row = read(root / 'foreign-occupant.json')
|
||||
self.assertEqual(row['occupants'][0]['observed_process_identity'], current)
|
||||
|
||||
|
||||
class FormalBinding(unittest.TestCase):
|
||||
def test_inclusive_gate_and_all_three_seeds_required(self):
|
||||
for endpoint in (6.62, 7.62):
|
||||
self.assertEqual(formal.pool([endpoint] * 3, [6.62, 7.62]), (endpoint, True))
|
||||
self.assertEqual(formal.pool([7.12, 7.12, None], [6.62, 7.62]), (None, None))
|
||||
self.assertFalse(formal.pool([7.63] * 3, [6.62, 7.62])[1])
|
||||
|
||||
def test_original_split_score_keeps_order_and_rejects_nan(self):
|
||||
ids = list(range(111))
|
||||
complete = [dict(case=i, relative_l2=.0712) for i in ids]
|
||||
self.assertAlmostEqual(formal.score_mean(complete, ids), 7.12)
|
||||
with self.assertRaises(ValueError):
|
||||
formal.score_mean(complete[::-1], ids)
|
||||
complete[0]['relative_l2'] = float('nan')
|
||||
with self.assertRaises(ValueError):
|
||||
formal.score_mean(complete, ids)
|
||||
|
||||
def test_scoring_needs_all_checkpoints_before_access(self):
|
||||
with TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
(root / 'checkpoints-frozen.json').write_text('{}')
|
||||
with patch.object(formal, 'all_checkpoints', side_effect=PermissionError('missing fit')):
|
||||
with self.assertRaisesRegex(PermissionError, 'missing fit'):
|
||||
worker.score({}, 0, None, root, None)
|
||||
self.assertFalse((root / 'heldout-target-access.jsonl').exists())
|
||||
|
||||
def test_score_opens_one_111_target_pass_and_rejects_a_second(self):
|
||||
import torch
|
||||
|
||||
with TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
path = root / 'checkpoint.pt'
|
||||
torch.save(dict(state={}, pressure_mean=torch.tensor([2.]), pressure_std=torch.tensor([3.])), path)
|
||||
roster = {'0': dict(checkpoint=dict(path=str(path), sha256=sha(path)))}
|
||||
save(root / 'checkpoints-frozen.json', roster)
|
||||
ids = list(range(111))
|
||||
reads = []
|
||||
|
||||
def load(ids_argument, role, log):
|
||||
self.assertEqual((ids_argument, role), (ids, 'heldout'))
|
||||
for case in ids_argument:
|
||||
reads.append(case)
|
||||
log.write(f'{case}\n')
|
||||
return object()
|
||||
|
||||
class Normalizer:
|
||||
pass
|
||||
|
||||
data = SimpleNamespace(test_ids=ids, normalizer_class=Normalizer, data=load)
|
||||
torch_cpu_fixture = SimpleNamespace(load=torch.load, isfinite=torch.isfinite,
|
||||
cuda=SimpleNamespace(synchronize=lambda: None))
|
||||
model = SimpleNamespace(to=lambda _: model, load_state_dict=lambda _state, **kw: None)
|
||||
rows = [dict(case=i, relative_l2=.0712) for i in ids]
|
||||
with (patch.object(formal, 'all_checkpoints', return_value=roster),
|
||||
patch.object(worker, 'paper_model', return_value=model),
|
||||
patch.object(worker, 'case_scores', return_value=rows)):
|
||||
result = worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture)
|
||||
with self.assertRaises(FileExistsError):
|
||||
worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture)
|
||||
self.assertEqual(reads, ids)
|
||||
self.assertEqual(result['target_reads'], 111)
|
||||
self.assertAlmostEqual(result['mean_percent'], 7.12)
|
||||
|
||||
def test_card_reservation_expiry_and_total_budget(self):
|
||||
binding = formal.expected_contract()['formal']
|
||||
budgets = binding['budget_per_seed_stage_gpu_seconds']
|
||||
self.assertEqual(sum(sum(b.values()) for b in budgets.values()), 40000)
|
||||
self.assertEqual(binding['device_pci_ordinals'], {'0': 2, '1': 3, '2': 0})
|
||||
latest = formal.RESERVATION_END.timestamp() - 300
|
||||
formal.check_reservation(latest - 1, 300)
|
||||
for when in (latest, formal.RESERVATION_END.timestamp(), formal.RESERVATION_START.timestamp() - 1):
|
||||
with self.assertRaises(PermissionError):
|
||||
formal.check_reservation(when, 300)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,314 @@
|
||||
"""Exact review, custody and resource binding for formal three-seed GINO."""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import time
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from track_b_gino_formal_guard import guard
|
||||
from track_b_gino_import_checked_entry import accepted_environment
|
||||
from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT
|
||||
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
||||
|
||||
|
||||
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-2026-11-19')
|
||||
SOURCES = Path(__file__).resolve().parent
|
||||
OLD_FILES = {
|
||||
'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a',
|
||||
'complete.json': 'db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d',
|
||||
'recovery-settlement.json': '851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec',
|
||||
'worker-records/progress.json': '8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf',
|
||||
}
|
||||
DEVICES = {
|
||||
'0': 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76',
|
||||
'1': 'GPU-8b71b032-1b75-2891-935c-ce106a53034a',
|
||||
'2': 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
|
||||
}
|
||||
ORDINALS = {'0': 2, '1': 3, '2': 0}
|
||||
INVENTORY = BASE / 'artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json'
|
||||
INVENTORY_SHA = 'da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8'
|
||||
TZ = ZoneInfo('Asia/Hong_Kong')
|
||||
RESERVATION_START = datetime(2026, 11, 19, tzinfo=TZ)
|
||||
RESERVATION_END = datetime(2026, 11, 20, tzinfo=TZ)
|
||||
NEW_SOURCES = ('track_b_gino_formal.py', 'track_b_gino_formal_guard.py',
|
||||
'track_b_gino_formal_worker.py')
|
||||
|
||||
|
||||
def expected_contract():
|
||||
old = {name: read(checked(MEASUREMENT / name, digest)) for name, digest in OLD_FILES.items()}
|
||||
c = old['contract.json']
|
||||
stop, epoch = old['complete.json'], old['worker-records/progress.json']
|
||||
account = old['recovery-settlement.json']
|
||||
if (stop['status'], stop['trigger'], stop['result_sha256']) != ('STOPPED', 'FOREIGN_GPU_OCCUPANT', None):
|
||||
raise ValueError('Historical guard STOP differs')
|
||||
if (epoch['stage'], epoch['seed'], epoch['epoch'], epoch['examples']) != ('EPOCH_COMPLETE', 0, 1, 500):
|
||||
raise ValueError('Only the completed 500-example epoch can size this reproduction')
|
||||
seconds = epoch['seconds']
|
||||
inventory = read(checked(INVENTORY, INVENTORY_SHA))['inventory']
|
||||
physical = {(int(index), uuid) for index, uuid, _, name in inventory if 'V100' in name}
|
||||
if not {(ORDINALS[seed], uuid) for seed, uuid in DEVICES.items()} <= physical:
|
||||
raise ValueError('Reviewed physical V100 inventory differs')
|
||||
budgets = {seed: {'fit': 13033 + (seed == '0'), 'score': 300} for seed in DEVICES}
|
||||
fit_reference = 100 * seconds * 1.25
|
||||
if (not math.isfinite(fit_reference) or fit_reference <= 0
|
||||
or min(b['fit'] for b in budgets.values()) <= fit_reference
|
||||
or sum(b['fit'] + b['score'] for b in budgets.values()) != 40000):
|
||||
raise ValueError('Formal booking must cover the sizing reference and score reserve')
|
||||
c['mode'] = 'formal-gino-reproduction'
|
||||
for key in ('gpu_uuid', 'measurement_allowed_heldout_labels', 'gpu_ruling', 'ordinal_recovery'):
|
||||
del c[key]
|
||||
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES})
|
||||
c['formal'] = dict(
|
||||
handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES,
|
||||
device_pci_ordinals=ORDINALS,
|
||||
reservation=dict(handoff='h-a429935e', timezone=TZ.key,
|
||||
start_inclusive=RESERVATION_START.isoformat(),
|
||||
end_exclusive=RESERVATION_END.isoformat(), gpu3_single_task=True,
|
||||
historical_inventory_path=str(INVENTORY), historical_inventory_sha256=INVENTORY_SHA,
|
||||
runtime_idle_check_required=True),
|
||||
budget_per_seed_stage_gpu_seconds=budgets, total_gpu_wall_seconds=40000,
|
||||
timing=dict(record_paths={str(MEASUREMENT / k): v for k, v in OLD_FILES.items()},
|
||||
status='CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED', sizing_only=True,
|
||||
epoch_seconds=seconds, train_sizing_gpu_seconds=300 * seconds * 1.25,
|
||||
train_sizing_multiplier=1.25, score_reserved_gpu_seconds=900,
|
||||
prior_measurement_gpu_seconds=account['cumulative_measurement_gpu_wall_seconds']),
|
||||
heldout_policy=dict(preparation_reads=0, training_reads=0, score_reads_per_seed=111,
|
||||
all_epoch100_checkpoints_required=True, retry=False),
|
||||
)
|
||||
del c['limits']['gpu_wall_seconds']
|
||||
c['limits']['cleanup_seconds'] = 20
|
||||
del c['limits']['cpu_seconds']
|
||||
c['limits']['cpu_seconds_per_gpu_second'] = 8
|
||||
return c
|
||||
|
||||
|
||||
def contract():
|
||||
c = read(ROOT / 'contract.json')
|
||||
if c != expected_contract():
|
||||
raise PermissionError('The sole formal contract or its implementation changed')
|
||||
for path, digest in c['code_sha256'].items():
|
||||
checked(path, digest)
|
||||
for name in ('paper_supplement', 'split_receipt', 'mapping_audit'):
|
||||
pin = c[name]
|
||||
checked(pin['path'], pin['sha256'])
|
||||
# Archive members are opened only in the scoped workers, which verify each
|
||||
# member before using it. Preparation does not hash the archive payload.
|
||||
comparison = read(checked(c['comparison_contract'], c['comparison_contract_sha256']))
|
||||
split = read(c['split_receipt']['path'])
|
||||
train, test = split['train'], split['test']
|
||||
if (c['protocol'] != comparison['protocol'] or len(train) != 500 or len(test) != 111
|
||||
or len(set(train) | set(test)) != 611
|
||||
or any(c['split'][k] != split[k] for k in ('train', 'test', 'raw_global_bounds'))
|
||||
or c['protocol']['epochs_per_seed'] != 100 or c['protocol']['seeds'] != [0, 1, 2]
|
||||
or c['protocol']['acceptance_percent'] != [6.62, 7.62]
|
||||
or c['paper_compatibility']['identity'] != 'supplement code + 2-keyword patch'
|
||||
or not Path(c['paper_data']['path']).is_file()):
|
||||
raise ValueError('Original GINO comparison, split or pinned data path differs')
|
||||
accepted_environment(c)
|
||||
return c
|
||||
|
||||
|
||||
def authorize(c):
|
||||
validate_review(ROOT)
|
||||
authorization = read(ROOT / 'stage-authorization.json')
|
||||
required = dict(contract_sha256=sha(ROOT / 'contract.json'),
|
||||
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
|
||||
decision='APPROVED', mode=c['mode'], reserved_devices=DEVICES,
|
||||
reservation_end_exclusive=RESERVATION_END.isoformat())
|
||||
if any(authorization.get(k) != v for k, v in required.items()):
|
||||
raise PermissionError('Exact execution and card-reservation authorization required')
|
||||
|
||||
|
||||
def check_reservation(unix, duration):
|
||||
if (unix < RESERVATION_START.timestamp()
|
||||
or unix + duration >= RESERVATION_END.timestamp()):
|
||||
raise PermissionError('The entire booking must fit the November 19 Hong Kong reservation')
|
||||
|
||||
|
||||
def run_root(seed, phase):
|
||||
return ROOT / f'seed-{seed}' / phase
|
||||
|
||||
|
||||
def epoch100(root, seed, c):
|
||||
progress = read(root / 'worker-records/progress.json')
|
||||
if (progress['stage'], progress['seed'], progress['epoch'], progress['examples']) != (
|
||||
'EPOCH_COMPLETE', seed, c['protocol']['epochs_per_seed'], 500):
|
||||
raise ValueError('No completed 100th full training epoch for this seed')
|
||||
return progress
|
||||
|
||||
|
||||
def checkpoint(seed, result):
|
||||
path = run_root(seed, 'fit') / 'epoch-100.pt'
|
||||
if result['checkpoint']['path'] != str(path):
|
||||
raise ValueError('Checkpoint path differs from this seed')
|
||||
return checked(path, result['checkpoint']['sha256'])
|
||||
|
||||
|
||||
def score_mean(rows, test_ids):
|
||||
if [r['case'] for r in rows] != test_ids:
|
||||
raise ValueError('Scoring requires the original 111-case order')
|
||||
values = [r['relative_l2'] for r in rows]
|
||||
if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values):
|
||||
raise ValueError('Incomplete or invalid relative L2')
|
||||
return 100 * (math.fsum(values) / len(values))
|
||||
|
||||
|
||||
def validate_result(c, seed, phase):
|
||||
root = run_root(seed, phase)
|
||||
path = root / 'worker-records/result.json'
|
||||
result = read(path)
|
||||
spawned = read(root / 'actual-worker-spawn.json')
|
||||
device = DEVICES[str(seed)]
|
||||
ordinal = str(ORDINALS[str(seed)])
|
||||
if (spawned['physical_uuid'], spawned['ordinal']) != (device, ordinal) or (
|
||||
read(root / 'worker-device-verified.json') != dict(
|
||||
process=spawned['process'], physical_uuid=device, ordinal=ordinal, observed_uuids=[device])):
|
||||
raise ValueError('Worker has no matching physical-device witness')
|
||||
if (result['phase'], result['seed'], result['status']) != (phase, seed, 'COMPLETE'):
|
||||
raise ValueError('Worker result identity differs')
|
||||
if phase == 'fit':
|
||||
epoch100(root, seed, c)
|
||||
checkpoint(seed, result)
|
||||
if (result['target_reads'], result['epochs'], result['training_examples']) != (0, 100, 500):
|
||||
raise ValueError('Fit must complete all 100 epochs with zero held-out reads')
|
||||
else:
|
||||
fit_terminal = read(run_root(seed, 'fit') / 'complete.json')
|
||||
fit_path = run_root(seed, 'fit') / 'worker-records/result.json'
|
||||
original = read(checked(fit_path, fit_terminal['result_sha256']))
|
||||
access = root / 'heldout-target-access.jsonl'
|
||||
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
|
||||
actual = [json.loads(line) for line in access.read_text().splitlines()]
|
||||
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
|
||||
if (actual != expected or result['target_reads'] != 111
|
||||
or result['target_access_sha256'] != sha(access)
|
||||
or result['checkpoints_manifest_sha256'] != sha(root / 'checkpoints-frozen.json')
|
||||
or read(root / 'checkpoints-frozen.json') != all_checkpoints(c)
|
||||
or result['checkpoint_sha256'] != original['checkpoint']['sha256']
|
||||
or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
|
||||
raise ValueError('Score or held-out target audit differs')
|
||||
return sha(path)
|
||||
|
||||
|
||||
def accepted_terminal(c, seed, phase):
|
||||
root = run_root(seed, phase)
|
||||
record = read(root / 'complete.json')
|
||||
if (record['status'], record['seed'], record['phase'], record['contract_sha256'], record['physical_uuid']) != (
|
||||
'COMPLETE', seed, phase, sha(ROOT / 'contract.json'), DEVICES[str(seed)]):
|
||||
raise PermissionError(f'Seed {seed} {phase} has no accepted terminal result')
|
||||
booked = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
|
||||
if (record['booked_gpu_seconds'] != booked or record['charged_gpu_seconds'] > booked
|
||||
or record['result_sha256'] != validate_result(c, seed, phase)):
|
||||
raise ValueError('Terminal result bytes or resource accounting differ')
|
||||
return record
|
||||
|
||||
|
||||
def all_checkpoints(c):
|
||||
checkpoints = {}
|
||||
for seed in c['protocol']['seeds']:
|
||||
accepted_terminal(c, seed, 'fit')
|
||||
root = run_root(seed, 'fit')
|
||||
checkpoints[str(seed)] = dict(
|
||||
result_sha256=sha(root / 'worker-records/result.json'),
|
||||
terminal_sha256=sha(root / 'complete.json'),
|
||||
checkpoint=read(root / 'worker-records/result.json')['checkpoint'])
|
||||
return checkpoints
|
||||
|
||||
|
||||
def dispatch(c, seed, phase):
|
||||
began = time.monotonic()
|
||||
unix = time.time()
|
||||
authorize(c)
|
||||
budget = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
|
||||
check_reservation(unix, budget)
|
||||
seed_root = run_root(seed, phase).parent
|
||||
seed_root.mkdir(exist_ok=True)
|
||||
with (seed_root / 'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
roster = all_checkpoints(c) if phase == 'score' else None
|
||||
root = run_root(seed, phase)
|
||||
root.mkdir(exist_ok=False)
|
||||
if roster is not None:
|
||||
save(root / 'checkpoints-frozen.json', roster)
|
||||
ticket = dict(seed=seed, phase=phase, ordinal=str(ORDINALS[str(seed)]),
|
||||
booked_gpu_seconds=budget, unix=unix,
|
||||
contract_sha256=sha(ROOT / 'contract.json'),
|
||||
guard_started_monotonic=began, booked_deadline_monotonic=began + budget)
|
||||
save(root / 'started.json', ticket)
|
||||
command = [c['environment']['python'], str(SOURCES / 'track_b_gino_formal_worker.py'),
|
||||
str(seed), phase]
|
||||
|
||||
def request_stop(signum, _frame):
|
||||
raise KeyboardInterrupt(signal.Signals(signum).name)
|
||||
|
||||
previous = signal.signal(signal.SIGTERM, request_stop)
|
||||
try:
|
||||
limits = dict(c['limits'], cpu_seconds=c['limits']['cpu_seconds_per_gpu_second'] * budget)
|
||||
result = guard(root, command, uuid=DEVICES[str(seed)], limits=limits,
|
||||
deadline=began + budget, ticket=ticket,
|
||||
validate_result=lambda: validate_result(c, seed, phase))
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, previous)
|
||||
return 0 if result['status'] == 'COMPLETE' else 1
|
||||
|
||||
|
||||
def pool(means, limits):
|
||||
if len(means) != 3 or any(v is None for v in means):
|
||||
return None, None
|
||||
mean = math.fsum(means) / 3
|
||||
return mean, limits[0] <= mean <= limits[1]
|
||||
|
||||
|
||||
def collect(c):
|
||||
seeds = {}
|
||||
accounted = unsettled = 0
|
||||
for seed in c['protocol']['seeds']:
|
||||
rows = {}
|
||||
for phase in ('fit', 'score'):
|
||||
root = run_root(seed, phase)
|
||||
terminal = root / 'complete.json'
|
||||
if terminal.exists():
|
||||
record = read(terminal)
|
||||
if record['status'] == 'COMPLETE':
|
||||
accepted_terminal(c, seed, phase)
|
||||
accounted += record['charged_gpu_seconds']
|
||||
rows[phase] = dict(status=record['status'], trigger=record['trigger'],
|
||||
terminal_sha256=sha(terminal))
|
||||
if phase == 'score' and record['status'] == 'COMPLETE':
|
||||
rows['mean_percent'] = read(root / 'worker-records/result.json')['mean_percent']
|
||||
elif (root / 'started.json').exists():
|
||||
rows[phase] = dict(status='NO_TERMINAL_RECEIPT')
|
||||
unsettled += c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
|
||||
else:
|
||||
rows[phase] = dict(status='NOT_STARTED')
|
||||
seeds[str(seed)] = rows
|
||||
mean, verdict = pool([row.get('mean_percent') for row in seeds.values()],
|
||||
c['protocol']['acceptance_percent'])
|
||||
prior = c['formal']['timing']['prior_measurement_gpu_seconds']
|
||||
return dict(seeds=seeds, mean_percent=mean, within_registered_range=verdict,
|
||||
reproduction_identity=c['paper_compatibility']['identity'],
|
||||
prior_measurement_gpu_seconds=prior, settled_formal_gpu_seconds=accounted,
|
||||
settled_combined_gpu_seconds=accounted + prior,
|
||||
unsettled_formal_bookings_gpu_seconds=unsettled, formal_gpu_cap_seconds=40000)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=('fit', 'score', 'collect'))
|
||||
parser.add_argument('--seed', type=int, choices=(0, 1, 2))
|
||||
args = parser.parse_args()
|
||||
c = contract()
|
||||
if args.action == 'collect':
|
||||
if args.seed is not None:
|
||||
parser.error('collect reports all three seeds')
|
||||
print(json.dumps(collect(c), indent=2, allow_nan=False))
|
||||
else:
|
||||
if args.seed is None:
|
||||
parser.error('--seed is required for fit or score')
|
||||
raise SystemExit(dispatch(c, args.seed, args.action))
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Supervise a formal GINO fit or score on one physical V100."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from types import FunctionType, SimpleNamespace
|
||||
|
||||
import psutil
|
||||
|
||||
from s2_gpu_idle_gate import snapshot, wait_idle
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_recovery_gpu_supervisor import stop_group, supervise
|
||||
from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity
|
||||
from track_b_gino_shapenet_entry import process_identity, save
|
||||
|
||||
|
||||
def current_identity(pid):
|
||||
try:
|
||||
return process_identity(pid)
|
||||
except psutil.NoSuchProcess:
|
||||
return None
|
||||
|
||||
|
||||
def process_tree(worker):
|
||||
if current_identity(worker['pid']) != worker:
|
||||
return []
|
||||
try:
|
||||
children = psutil.Process(worker['pid']).children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return []
|
||||
result = []
|
||||
for child in children:
|
||||
try:
|
||||
result.append(dict(pid=child.pid, created=child.create_time()))
|
||||
except psutil.NoSuchProcess:
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
class OccupancyMonitor:
|
||||
def __init__(self, root, uuid, worker, device_vram_bytes):
|
||||
self.root = root
|
||||
self.uuid = uuid
|
||||
self.worker = worker
|
||||
self.known = {worker['pid']: worker}
|
||||
self.device_vram_bytes = device_vram_bytes
|
||||
self.peak_device_bytes = 0
|
||||
|
||||
def check(self):
|
||||
before = process_tree(self.worker)
|
||||
state = snapshot(self.uuid)
|
||||
after = process_tree(self.worker)
|
||||
for identity in before + after:
|
||||
if identity['pid'] != self.worker['pid']:
|
||||
self.known[identity['pid']] = identity
|
||||
foreign = []
|
||||
for row in state['occupants']:
|
||||
pid = row['pid']
|
||||
current = current_identity(pid)
|
||||
if pid not in self.known or (current is not None and current != self.known[pid]):
|
||||
foreign.append(dict(row, observed_process_identity=current))
|
||||
if foreign:
|
||||
save(self.root / 'foreign-occupant.json', dict(
|
||||
trigger='FOREIGN_GPU_OCCUPANT', unix=state['unix'], physical_uuid=self.uuid,
|
||||
occupants=foreign, own_spawn=self.worker, tree_before=before, tree_after=after))
|
||||
raise RuntimeError('FOREIGN_GPU_OCCUPANT')
|
||||
self.peak_device_bytes = max(self.peak_device_bytes, state['memory_MiB'] * 2**20)
|
||||
if self.peak_device_bytes > self.device_vram_bytes:
|
||||
raise RuntimeError('DEVICE_VRAM_LIMIT')
|
||||
|
||||
|
||||
def watch(root, uuid, worker, limits, done, tracker, state):
|
||||
occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes'])
|
||||
while not done.is_set():
|
||||
try:
|
||||
occupancy.check()
|
||||
state['peak_device_bytes'] = occupancy.peak_device_bytes
|
||||
tracker.scan(root / 'worker-records')
|
||||
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
|
||||
raise RuntimeError('RECORDED_VRAM_LIMIT')
|
||||
monitor_identity(root, uuid)
|
||||
except Exception as exc:
|
||||
state['trigger'] = str(exc) or type(exc).__name__
|
||||
# Signal only the recorded worker group while its leader has the same identity.
|
||||
if current_identity(worker['pid']) == worker:
|
||||
stop_group(worker['pid'])
|
||||
return
|
||||
done.wait(.5)
|
||||
|
||||
|
||||
def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
|
||||
began = time.monotonic()
|
||||
admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds'])
|
||||
done = threading.Event()
|
||||
tracker = PeakRecords()
|
||||
state = dict(trigger=None, peak_device_bytes=0)
|
||||
watcher = None
|
||||
worker_receipt = None
|
||||
status, trigger, result_hash = 'STOPPED', None, None
|
||||
|
||||
def spawn(args, **kwargs):
|
||||
nonlocal watcher
|
||||
env, inventory = dispatch_environment(uuid, kwargs['env'])
|
||||
ordinal = env['CUDA_VISIBLE_DEVICES']
|
||||
if ordinal != ticket['ordinal']:
|
||||
raise RuntimeError('PINNED_PCI_ORDINAL_DIFFERS')
|
||||
if not snapshot(uuid)['idle']:
|
||||
raise RuntimeError('SELECTED_GPU_BECAME_BUSY')
|
||||
save(root / 'ordinal-dispatch.json', dict(ordinal=ordinal, physical_uuid=uuid, inventory=inventory))
|
||||
if time.monotonic() >= admission:
|
||||
raise RuntimeError('ADMISSION_DEADLINE')
|
||||
proc = subprocess.Popen(args, **dict(kwargs, env=env))
|
||||
try:
|
||||
worker = process_identity(proc.pid)
|
||||
save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid))
|
||||
watcher = threading.Thread(target=watch, daemon=True,
|
||||
args=(root, uuid, worker, limits, done, tracker, state))
|
||||
watcher.start()
|
||||
except (Exception, KeyboardInterrupt):
|
||||
# The shared supervisor has not yet received the process to reap.
|
||||
stop_group(proc.pid)
|
||||
proc.wait()
|
||||
raise
|
||||
return proc
|
||||
|
||||
try:
|
||||
if shutil.disk_usage(root).free < limits['disk_bytes']:
|
||||
raise RuntimeError('DISK_HEADROOM')
|
||||
idle = wait_idle(uuid, max(0., admission - time.monotonic()), lambda _: None, sample=snapshot)
|
||||
save(root / 'idle-gate.json', idle)
|
||||
if idle['status'] != 'IDLE_CONFIRMED' or time.monotonic() >= admission:
|
||||
raise RuntimeError('IDLE_DEADLINE')
|
||||
save(root / 'dispatch.json', dict(ticket, coordinator=process_identity(os.getpid()),
|
||||
start_by_monotonic=admission))
|
||||
remaining = deadline - time.monotonic() - limits['cleanup_seconds']
|
||||
if remaining <= 0:
|
||||
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
|
||||
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
|
||||
run_worker = FunctionType(supervise.__code__, ns, supervise.__name__)
|
||||
try:
|
||||
worker_receipt = run_worker(command, root, 'gino', dict(
|
||||
total_cpu_seconds=limits['cpu_seconds'], job_cpu_seconds=limits['cpu_seconds'],
|
||||
job_wall_seconds=remaining, tree_rss_bytes=limits['rss_bytes'],
|
||||
disk_bytes=limits['disk_bytes'], max_processes=limits['max_processes']),
|
||||
poll=.2, gpu_uuid=uuid)
|
||||
finally:
|
||||
done.set()
|
||||
if watcher is not None:
|
||||
watcher.join(timeout=8)
|
||||
if watcher is not None and watcher.is_alive():
|
||||
raise RuntimeError('MONITOR_DID_NOT_EXIT')
|
||||
tracker.scan(root / 'worker-records')
|
||||
if state['trigger']:
|
||||
raise RuntimeError(state['trigger'])
|
||||
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
|
||||
raise RuntimeError('RECORDED_VRAM_LIMIT')
|
||||
if worker_receipt['status'] != 'PASS' or worker_receipt['monitor_errors']:
|
||||
raise RuntimeError(worker_receipt.get('stop_reason') or 'WORKER_FAILED')
|
||||
if set(tracker.witnesses) != set(tracker.maxima):
|
||||
raise RuntimeError('MISSING_RECORDED_MEMORY_PEAKS')
|
||||
result_hash = validate_result()
|
||||
status = 'COMPLETE'
|
||||
except (Exception, KeyboardInterrupt) as exc:
|
||||
trigger = str(exc) or type(exc).__name__
|
||||
end = time.monotonic()
|
||||
if end > deadline and status == 'COMPLETE':
|
||||
status, trigger, result_hash = 'STOPPED', 'BOOKED_GPU_WALL_CAP', None
|
||||
terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt,
|
||||
physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state,
|
||||
charged_gpu_seconds=end - ticket['guard_started_monotonic'],
|
||||
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True)
|
||||
save(root / 'complete.json', terminal)
|
||||
return terminal
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Run one formally bound GINO fit or the corresponding one-pass score."""
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import time
|
||||
|
||||
import track_b_gino_formal as formal
|
||||
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
|
||||
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
|
||||
from track_b_gino_shapenet_worker import (
|
||||
OriginalData, case_scores, cuda_peaks, paper_model, test_environment, train_seed,
|
||||
)
|
||||
|
||||
|
||||
def attach(root, seed, phase):
|
||||
ticket = read(root / 'dispatch.json')
|
||||
parent = os.getppid()
|
||||
if (ticket['coordinator'] != process_identity(parent)
|
||||
or ticket['contract_sha256'] != sha(formal.ROOT / 'contract.json')
|
||||
or (ticket['seed'], ticket['phase']) != (seed, phase)
|
||||
or time.monotonic() >= ticket['start_by_monotonic']):
|
||||
raise PermissionError('Worker is not attached to the authorized guard')
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent:
|
||||
raise RuntimeError('Could not attach worker lifetime to its guard')
|
||||
c = formal.contract()
|
||||
formal.authorize(c)
|
||||
if str(Path(sys.executable).absolute()) != c['environment']['python']:
|
||||
raise PermissionError('Unreviewed interpreter')
|
||||
return c, ticket
|
||||
|
||||
|
||||
def bind_device(root, c, ticket, torch):
|
||||
ordinal = ticket['ordinal']
|
||||
uuid = formal.DEVICES[str(ticket['seed'])]
|
||||
dispatch = read(root / 'ordinal-dispatch.json')
|
||||
if (dispatch['ordinal'] != ordinal or dispatch['physical_uuid'] != uuid
|
||||
or os.environ.get('CUDA_VISIBLE_DEVICES') != ordinal
|
||||
or os.environ.get('CUDA_DEVICE_ORDER') != 'PCI_BUS_ID'
|
||||
or torch.cuda.device_count() != 1 or torch.version.cuda != c['environment']['torch_cuda']):
|
||||
raise PermissionError('One reviewed CUDA PCI ordinal is required')
|
||||
torch.cuda.set_device(0)
|
||||
name = torch.cuda.get_device_name(0)
|
||||
if 'V100' not in name:
|
||||
raise PermissionError('The reviewed device must be a V100')
|
||||
probe = torch.empty(1, dtype=torch.uint8, device='cuda:0')
|
||||
current = process_identity(os.getpid())
|
||||
save(root / 'worker-device-witness.json', dict(
|
||||
process=current, ordinal=ordinal, device_count=1, device_name=name,
|
||||
physical_uuid_expected=uuid, first_allocation_bytes=probe.numel()))
|
||||
verified = root / 'worker-device-verified.json'
|
||||
while time.monotonic() < ticket['start_by_monotonic']:
|
||||
if verified.exists():
|
||||
record = read(verified)
|
||||
expected = dict(process=current, physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])
|
||||
if record != expected:
|
||||
raise PermissionError('Actual worker physical-device witness differs')
|
||||
return
|
||||
time.sleep(.1)
|
||||
raise RuntimeError('Physical device was not verified in the admission window')
|
||||
|
||||
|
||||
def train(c, seed, data, root, records, torch):
|
||||
learning = data.data(data.train_ids, 'train')
|
||||
epochs = c['protocol']['epochs_per_seed']
|
||||
model, times = train_seed(c, learning, seed, epochs, records / 'progress.json')
|
||||
if len(times) != epochs:
|
||||
raise ValueError('Full training did not finish')
|
||||
formal.epoch100(root, seed, c)
|
||||
path = root / 'epoch-100.pt'
|
||||
temporary = root / 'epoch-100.pt.tmp'
|
||||
with temporary.open('xb') as stream:
|
||||
torch.save(dict(state=model.state_dict(), pressure_mean=data.normalizer.mean,
|
||||
pressure_std=data.normalizer.std), stream)
|
||||
temporary.rename(path)
|
||||
return dict(checkpoint=dict(path=str(path), sha256=sha(path)), target_reads=0,
|
||||
epochs=epochs, training_examples=500, epoch_seconds=times)
|
||||
|
||||
|
||||
def score(c, seed, data, root, torch):
|
||||
# Targets are still unopened while rechecking all three finished fits,
|
||||
# initializing the exact model and loading its immutable checkpoint.
|
||||
manifest = root / 'checkpoints-frozen.json'
|
||||
manifest_sha = sha(manifest)
|
||||
roster = read(manifest)
|
||||
if roster != formal.all_checkpoints(c):
|
||||
raise PermissionError('Three complete, pinned checkpoints must precede custody')
|
||||
checkpoint = roster[str(seed)]['checkpoint']
|
||||
state = torch.load(checked(checkpoint['path'], checkpoint['sha256']), map_location='cpu')
|
||||
model = paper_model(c).to('cuda')
|
||||
model.load_state_dict(state['state'], strict=True)
|
||||
# The pinned normalizer's encode/decode and device methods use these
|
||||
# fitted tensors and epsilon. No learning or target dataset is refitted.
|
||||
normalizer = object.__new__(data.normalizer_class)
|
||||
normalizer.mean = state['pressure_mean']
|
||||
normalizer.std = state['pressure_std']
|
||||
normalizer.eps = c['protocol']['pressure_epsilon']
|
||||
if any(t.shape != (1,) or not bool(torch.isfinite(t).all())
|
||||
for t in (normalizer.mean, normalizer.std)) or bool(normalizer.std < 0):
|
||||
raise ValueError('Invalid frozen learning-only pressure statistics')
|
||||
data.normalizer = normalizer
|
||||
access = root / 'heldout-target-access.jsonl'
|
||||
with access.open('x') as log:
|
||||
heldout = data.data(data.test_ids, 'heldout', log)
|
||||
rows = case_scores(model, heldout, data.test_ids, data.normalizer)
|
||||
torch.cuda.synchronize()
|
||||
return dict(cases=rows, mean_percent=formal.score_mean(rows, data.test_ids),
|
||||
target_reads=len(rows), target_access_sha256=sha(access),
|
||||
checkpoints_manifest_sha256=manifest_sha, checkpoint_sha256=checkpoint['sha256'])
|
||||
|
||||
|
||||
def run(seed, phase):
|
||||
root = formal.run_root(seed, phase)
|
||||
c, ticket = attach(root, seed, phase)
|
||||
test_environment(c)
|
||||
import torch
|
||||
|
||||
torch.set_num_threads(c['limits']['torch_threads'])
|
||||
torch.set_num_interop_threads(1)
|
||||
bind_device(root, c, ticket, torch)
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.backends.cudnn.allow_tf32 = False
|
||||
records = root / 'worker-records'
|
||||
records.mkdir(exist_ok=False)
|
||||
with TemporaryDirectory(prefix='formal-gino-', dir=root) as scratch:
|
||||
extract_patched_paper_code(c, scratch)
|
||||
data = OriginalData(c, scratch)
|
||||
result = (train(c, seed, data, root, records, torch) if phase == 'fit'
|
||||
else score(c, seed, data, root, torch))
|
||||
save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('seed', type=int, choices=(0, 1, 2))
|
||||
parser.add_argument('phase', choices=('fit', 'score'))
|
||||
args = parser.parse_args()
|
||||
run(args.seed, args.phase)
|
||||
Reference in new issue
Block a user