Bind formal three-seed GINO with CPU guard and custody replay

This commit is contained in:
NewMeltAssist committed 2026-11-19 16:06:50 +08:00
1 parent 93d7262e83
commit 9a4f8563ee
11 files changed
+1809

No files matched your search

@@ -0,0 +1,834 @@
{
"mode": "formal-gino-reproduction",
"ruling": "h-f84547d7",
"retry": false,
"evaluate_other_models": false,
"limits": {
"cleanup_seconds": 20,
"idle_wait_seconds": 60,
"rss_bytes": 12884901888,
"recorded_vram_bytes": 15032385536,
"device_vram_bytes": 16106127360,
"disk_bytes": 2147483648,
"torch_threads": 1,
"max_processes": 1,
"cpu_seconds_per_gpu_second": 8
},
"protocol": {
"model": "NeurIPS 2023 supplement SDFFNOGNO decoder",
"seeds": [
0,
1,
2
],
"grid": [
64,
64,
64
],
"modes": [
24,
24,
24
],
"hidden_channels": 32,
"radius": 0.05,
"epochs_per_seed": 100,
"batch_size": 1,
"learning_rate": 0.001,
"bounds_expansion": 0.01,
"pressure_epsilon": 1e-06,
"optimizer": "Adam",
"weight_decay": 0.0001,
"lr_step_epochs": 50,
"lr_gamma": 0.5,
"pressure_training_transform": "scalar Gaussian on the 500 learning cases",
"metric": "per-case L2(predicted minus true decoded pressure) / L2(true decoded pressure)",
"report": "equal mean over 111 cases for each seed, then equal mean over the three seeds",
"acceptance_percent": [
6.62,
7.62
],
"paper_reference_percent": 7.12,
"checkpoint_epoch": 100,
"heldout_access_after_all_checkpoints_frozen": true
},
"environment": {
"python": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
"python_version": "3.10",
"torch_cuda": "11.6",
"neuralop_commit": "fbdc7f872792b75b84545c2f374e7a881c72a718",
"neuralop_file_sha256": {
"utils.py": "84554eed45ac415ab52f79e456eab70ba204e7ab7539c2f2841955a9145220b4",
"models/__init__.py": "c67679f1f8f90e1ab4eb4c2aa916aeaedf1b175d795fed03b8d4f7210883ebae",
"models/tfno.py": "d74b5805b69a64bf33e3ff5186709c75bd431a56c9548f06fb9635a859d849e4",
"models/spectral_convolution.py": "ccecf80806d93e8f35713ac6ab2c3b61c11739f5fcb8c0ba7397b7d4eb5a5d45"
},
"package_versions": {
"torch": "1.13.1+cu116",
"open3d": "0.17.0",
"numpy": "1.24.3",
"torch-scatter": "2.1.1+pt113cu116",
"torch-sparse": "0.6.17+pt113cu116",
"torch-geometric": "2.3.1",
"tensorly": "0.8.1",
"tensorly-torch": "0.4.0"
}
},
"comparison_contract": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-measurement-2026-11-19/comparison-contract.json",
"comparison_contract_sha256": "503f6c8ab1217d66b44e0d127555cfad5da4cd8a17b336299095c3cfd13041d7",
"code_sha256": {
"/home/frank14f/optane/MeltMind/tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"/home/frank14f/optane/MeltMind/tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
"/home/frank14f/optane/MeltMind/tools/phase2_gate_outcome_review.py": "915617aeb8292d03c654e637bc34a014c254fd178a773f135d1806c4b41372a8",
"/home/frank14f/optane/MeltMind/tools/phase2_h1_worker_v2.py": "1543df85b615bb69b79bacd572074a7c77ae19dab110197601573a7e5805ed20",
"/home/frank14f/optane/MeltMind/tools/phase2_home_freeze_io.py": "bf09204b85ed2a7038acdd202257f618474d1110a9d6d772f105a3ffb334d3ff",
"/home/frank14f/optane/MeltMind/tools/phase2_local_freeze_io.py": "60adf3382577613c5234f5cf30779336b9951b66e90c075cb7cd0adab9cc42dd",
"/home/frank14f/optane/MeltMind/tools/phase2_residual_math.py": "ffbd9624dec79bdaabf95244fe87209055908f31a977e73ea4377045d6f8fe62",
"/home/frank14f/optane/MeltMind/tools/phase3_item1b_common.py": "4e25effe8326ff8c7c3f751590315040209af054b32b0505dc3c6b001589db32",
"/home/frank14f/optane/MeltMind/tools/phase3_seed1_v2_common.py": "25ca6be14027369393bd3616209b08f30e25892cccca83300483c5a116e8d75d",
"/home/frank14f/optane/MeltMind/tools/phase3_shared_capacity.py": "7c07b0aaf1bc4e6a980d016defcac326c3bd871134d2583bb4dcd88bb34728f8",
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_shapenet.py": "557ad56277d7e85c6b5cc358eb151632d1757f5883f429988d36022d0d407550",
"/home/frank14f/optane/MeltMind/tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"/home/frank14f/optane/MeltMind/tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"/home/frank14f/optane/MeltMind/tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"/home/frank14f/optane/MeltMind/tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_common.py": "79918e7de887d1f0a2de5523156b27d00dc2d85ac930a07efc34d817d904fe65",
"/home/frank14f/optane/MeltMind/tools/track_a_h2_prepare.py": "f657be036dc48fc5936f56c4b8510a4b295d4787af6877cc5ccaad663107be17",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_entry.py": "e86d955c36e291e6bbef67947de663b24c8d66c5d72d9d28e4193fb8a3051ee2",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_shapenet_worker.py": "7211be211c29472fd9be35a6bcb1f56025de4a63f36852bfafa10df97c003ff3",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_entry.py": "199d833f40339feb0433611ba27302416a280452f8918715e6ef95cb93c38721",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_import_checked_worker.py": "3acf742dd4549e9f5491e10af5bb0b4acf00e2f47e91753100b5998cfcc61852",
"/home/frank14f/optane/MeltMind/tools/prepare_track_b_gino_import_checked.py": "fef485dd0c233c76a336239d5f29cbf44ac0fb62a6a297dff34293b23a1b75fe",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_paper_keyword_patch.py": "c61e2ed7d099eb125386857bf4d7ccdbfc3af522f80edb4f01b450cee91c0b95",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2.py": "7d3ab2412e5208892cffcfc0e1ec363eb5db2be3e10bf60595356f71e82d8552",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_measurement_gpu2_worker.py": "ba1fd5ec837e884b000ba640fceafe30ea40ccf708847733df6a17a8f3599680",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery.py": "1e1827498afab62e2f28efb5d450376f2dca4e8bf6f74114f2914ec771872c86",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py": "7a68c77e9c913ba9161832f513dbf8754ebc5c270c1e742fa5ae97a7b14b9120",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal.py": "15ea113a0657715eff495ce17dc63a17a3ee984133907f8a4d5a83f2cf8a01a2",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_guard.py": "9b4020b4754b46325c9aab1d15448eb86b21ecf0e5dfb527a51907982db2c0fb",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_formal_worker.py": "978f1c7cfb622300b8814e14e4a02e5cbb9e2ea7f6e5fee080b986c7409ade2c"
},
"paper_data": {
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-car-pressure-data.zip",
"sha256": "b5f9e2b407d54e034596384253fb58196933c46f6b5b586383508da54475f89d"
},
"paper_supplement": {
"path": "/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/paper-neurips-supplement.zip",
"sha256": "48cdae80ac2781e27ac00e10a37124711d6bb305593559002bda0527d11a533d"
},
"paper_code_zip_sha256": "347a6f097e2b9958d1ab87930fa1d6227b793a9d213e12418ce9a157e83dce04",
"split_receipt": {
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-split.json",
"sha256": "5a96c25c2ec24cfc1e2264d4ebd0879c658c0f168e22711b576bf303151b8601"
},
"mapping_audit": {
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-2026-11-19/paper-mapping-audit.json",
"sha256": "055d008a02494acd29df36c41c8e68db931a027f5635e8f5f765a96a54c62289"
},
"split": {
"train": [
1,
2,
4,
5,
6,
7,
8,
10,
12,
13,
17,
18,
21,
22,
23,
25,
26,
27,
28,
29,
30,
31,
32,
34,
35,
39,
40,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
54,
55,
56,
58,
59,
60,
61,
62,
63,
64,
65,
67,
69,
70,
71,
72,
73,
74,
75,
76,
77,
78,
79,
80,
81,
83,
84,
85,
86,
87,
88,
90,
91,
92,
94,
95,
96,
97,
100,
101,
102,
105,
106,
107,
109,
110,
111,
112,
113,
114,
115,
116,
117,
118,
119,
120,
121,
123,
124,
125,
126,
127,
128,
129,
130,
131,
133,
134,
136,
137,
138,
139,
140,
141,
142,
143,
144,
145,
146,
147,
148,
149,
150,
151,
152,
153,
155,
156,
157,
158,
159,
160,
161,
162,
163,
165,
166,
170,
172,
173,
175,
176,
177,
178,
179,
180,
181,
182,
183,
184,
186,
190,
191,
192,
193,
195,
196,
198,
199,
200,
201,
202,
203,
205,
207,
210,
211,
212,
213,
214,
215,
217,
219,
220,
221,
222,
223,
224,
225,
227,
228,
229,
230,
231,
232,
233,
234,
235,
236,
237,
241,
243,
244,
245,
246,
247,
248,
249,
251,
252,
253,
255,
257,
258,
259,
260,
261,
262,
263,
264,
266,
267,
268,
269,
271,
272,
273,
274,
275,
276,
277,
278,
279,
280,
281,
282,
283,
285,
286,
289,
290,
291,
292,
293,
294,
295,
296,
297,
298,
299,
300,
301,
302,
304,
305,
306,
308,
309,
310,
311,
312,
313,
314,
315,
319,
320,
321,
322,
323,
324,
325,
327,
328,
329,
331,
332,
333,
334,
335,
337,
338,
339,
340,
341,
344,
345,
347,
348,
349,
350,
352,
353,
354,
355,
356,
357,
358,
360,
362,
364,
365,
366,
367,
369,
371,
372,
373,
374,
375,
376,
378,
379,
380,
381,
384,
385,
389,
392,
393,
397,
398,
399,
401,
402,
403,
404,
405,
407,
408,
410,
412,
413,
414,
415,
417,
418,
419,
420,
422,
424,
425,
427,
430,
431,
433,
435,
436,
437,
439,
440,
443,
444,
446,
447,
448,
449,
450,
451,
452,
453,
454,
455,
456,
457,
459,
460,
462,
463,
464,
465,
466,
467,
468,
469,
470,
472,
473,
474,
475,
476,
478,
479,
480,
482,
483,
486,
487,
488,
490,
493,
494,
495,
496,
497,
498,
499,
501,
502,
503,
504,
505,
507,
508,
509,
511,
512,
513,
514,
515,
516,
518,
519,
521,
522,
523,
524,
525,
527,
529,
530,
532,
533,
536,
538,
539,
540,
542,
543,
545,
547,
548,
549,
550,
551,
552,
553,
554,
555,
560,
561,
562,
564,
565,
566,
567,
568,
569,
572,
573,
574,
576,
577,
579,
581,
582,
583,
584,
587,
588,
589,
591,
593,
594,
595,
596,
597,
598,
600,
602,
604,
608,
610,
611,
612,
613,
615,
616,
617,
618,
620,
621,
622,
623,
625,
626,
627,
628,
629,
630,
631,
632,
633,
634,
635,
636,
638,
639,
640,
641,
642,
643,
644,
645,
646,
647,
648,
649,
651,
652,
654,
655,
656,
657
],
"test": [
658,
659,
660,
662,
663,
664,
665,
666,
667,
668,
672,
673,
674,
675,
676,
677,
678,
679,
681,
683,
684,
686,
687,
688,
689,
690,
691,
692,
693,
695,
696,
697,
700,
701,
702,
703,
704,
705,
708,
709,
710,
711,
712,
713,
715,
717,
718,
719,
721,
722,
723,
724,
725,
726,
727,
729,
730,
731,
733,
734,
735,
736,
737,
738,
739,
740,
741,
743,
744,
745,
746,
747,
748,
749,
750,
751,
752,
753,
757,
758,
760,
762,
763,
764,
765,
766,
767,
768,
769,
770,
771,
772,
773,
775,
776,
777,
778,
779,
781,
782,
783,
784,
785,
787,
789,
790,
791,
792,
793,
795,
797
],
"raw_global_bounds": [
[
-0.90252799,
0.0066688,
-2.85272002
],
[
0.90253001,
1.95825005,
2.86385012
]
]
},
"environment_ruling": "h-0d21f3bd",
"environment_acceptance": {
"path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-import-checked-2026-11-19/cpu-environment-acceptance.json",
"sha256": "093e646890929af6f520eb5eecf8c751025c0a24b8b5689094b1a0ae4da08dc3"
},
"paper_compatibility": {
"handoff": "h-b8508140",
"identity": "supplement code + 2-keyword patch",
"patch_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/two-keyword-patch.json",
"patch_sha256": "7307ba49f83bfc46c5decf803a993a41ed602d987de65714958e738d2f52f077",
"search_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-paper-keyword-compatibility-2026-11-19/keyword-search.json",
"search_sha256": "77a897bf534a9a0e9e6a7c072b4f914168648eadc2af3493b70d63d0b715b3d0"
},
"formal": {
"handoff": "h-4e4e680b",
"ruling_slack_ts": "1795074651.222599",
"devices": {
"0": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"1": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"2": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c"
},
"device_pci_ordinals": {
"0": 2,
"1": 3,
"2": 0
},
"reservation": {
"handoff": "h-a429935e",
"timezone": "Asia/Hong_Kong",
"start_inclusive": "2026-11-19T00:00:00+08:00",
"end_exclusive": "2026-11-20T00:00:00+08:00",
"gpu3_single_task": true,
"historical_inventory_path": "/home/frank14f/optane/MeltMind/artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json",
"historical_inventory_sha256": "da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8",
"runtime_idle_check_required": true
},
"budget_per_seed_stage_gpu_seconds": {
"0": {
"fit": 13034,
"score": 300
},
"1": {
"fit": 13033,
"score": 300
},
"2": {
"fit": 13033,
"score": 300
}
},
"total_gpu_wall_seconds": 40000,
"timing": {
"record_paths": {
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/contract.json": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a",
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json": "db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d",
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json": "851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec",
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json": "8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf"
},
"status": "CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED",
"sizing_only": true,
"epoch_seconds": 102.93777855299413,
"train_sizing_gpu_seconds": 38601.6669573728,
"train_sizing_multiplier": 1.25,
"score_reserved_gpu_seconds": 900,
"prior_measurement_gpu_seconds": 222.80500172509346
},
"heldout_policy": {
"preparation_reads": 0,
"training_reads": 0,
"score_reads_per_seed": 111,
"all_epoch100_checkpoints_required": true,
"retry": false
}
}
}
@@ -0,0 +1,48 @@
{
"status": "PASS",
"contract_sha256": "bcb25413f1bd29679116ba320fee1bcc6d5d66ec074665d61c6910a65fb97959",
"cuda_visible_devices": "",
"actual_nvidia_queries": 0,
"training_starts": 0,
"scoring_starts": 0,
"actual_heldout_reads": 0,
"independent_review": "PENDING",
"guard_replay": 9,
"cpu_replay_log_sha256": "c0d40741c701de9be3699fd66f82473c8ffdfe9df25c29c95f4fe2ac76689369",
"read_only_collection": {
"seeds": {
"0": {
"fit": {
"status": "NOT_STARTED"
},
"score": {
"status": "NOT_STARTED"
}
},
"1": {
"fit": {
"status": "NOT_STARTED"
},
"score": {
"status": "NOT_STARTED"
}
},
"2": {
"fit": {
"status": "NOT_STARTED"
},
"score": {
"status": "NOT_STARTED"
}
}
},
"mean_percent": null,
"within_registered_range": null,
"reproduction_identity": "supplement code + 2-keyword patch",
"prior_measurement_gpu_seconds": 222.80500172509346,
"settled_formal_gpu_seconds": 0,
"settled_combined_gpu_seconds": 222.80500172509346,
"unsettled_formal_bookings_gpu_seconds": 0,
"formal_gpu_cap_seconds": 40000
}
}
@@ -0,0 +1,14 @@
test_card_reservation_expiry_and_total_budget (test_track_b_gino_formal.FormalBinding) ... ok
test_inclusive_gate_and_all_three_seeds_required (test_track_b_gino_formal.FormalBinding) ... ok
test_original_split_score_keeps_order_and_rejects_nan (test_track_b_gino_formal.FormalBinding) ... ok
test_score_opens_one_111_target_pass_and_rejects_a_second (test_track_b_gino_formal.FormalBinding) ... ok
test_scoring_needs_all_checkpoints_before_access (test_track_b_gino_formal.FormalBinding) ... ok
test_gpu_list_before_worker_exit (test_track_b_gino_formal.GuardReplay) ... ok
test_real_foreign_identity_has_exact_stop_receipt (test_track_b_gino_formal.GuardReplay) ... ok
test_reused_worker_pid_is_foreign (test_track_b_gino_formal.GuardReplay) ... ok
test_worker_already_exited_before_gpu_list (test_track_b_gino_formal.GuardReplay) ... ok
----------------------------------------------------------------------
Ran 9 tests in 0.656s
OK
@@ -0,0 +1,38 @@
{
"status": "STOPPED",
"trigger": "FOREIGN_GPU_OCCUPANT",
"contract_sha256": "ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a",
"result_sha256": null,
"worker": {
"status": "PASS",
"command": [
"/home/frank14f/optane/MeltMind/data/track-b-gino-shapenet611-2026-11-19/runtime/bin/python",
"/home/frank14f/optane/MeltMind/tools/track_b_gino_ordinal_recovery_worker.py",
"/home/frank14f/optane/MeltMind/artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 1221.067037,
"wall_seconds": 178.46390844404232,
"peak_tree_rss_bytes": 4780380160,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 131072,
"max_poll_gap_seconds": 0.8362329140072688,
"poll_target_seconds": 0.2,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"booked_idle_seconds": 20.519942376064137,
"cumulative_gpu_wall_seconds": 199.1106877680868,
"gpu_wall_cap_seconds": 3576.3056860429933,
"recorded_peaks": {
"peak_allocated_bytes": 2953057792,
"peak_reserved_bytes": 3980394496
},
"no_retry": true
}
@@ -0,0 +1,11 @@
{
"status": "STOPPED",
"trigger": "FOREIGN_GPU_OCCUPANT",
"previous_gpu_wall_seconds": 23.69431395700667,
"recovery_gpu_wall_seconds": 199.1106877680868,
"cumulative_measurement_gpu_wall_seconds": 222.80500172509346,
"total_cap_gpu_wall_seconds": 3600,
"independent_attempts": 2,
"no_retry": true,
"within_total_cap": true
}
@@ -0,0 +1,9 @@
{
"stage": "EPOCH_COMPLETE",
"seed": 0,
"epoch": 1,
"examples": 500,
"seconds": 102.93777855299413,
"peak_allocated_bytes": 2953057792,
"peak_reserved_bytes": 3980394496
}
@@ -0,0 +1,19 @@
# Track B GINO 三种子复现联合预审
本次交给 MeltAssist 一次联合预审,范围包括占卡守卫、训练入口和评分入口。当前只准备和回放。任何真实派发均需同一绑定的独立 PASS 和阶段授权。 [Claude 裁定](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1795074651222599)
## 卡和预算
依据 h-a429935e,种子 0、1、2 分别绑定物理 GPU2、GPU3、GPU0。GPU3 只允许单任务。香港时间 2026-11-19 当日预留,满额预约须能在当地午夜前结束。实际派发仍须通过三次空闲观察和临派复核。物理 UUID、PCI 顺序和真实 worker 首次一字节分配均须匹配,GPU1 不参与。 [组合合同](../artifacts/track-b-gino-shapenet611-formal-2026-11-19/contract.json)
GPU2 恢复测量的 worker 完整跑了 500 样本、1 轮,用时 102.93777855299413 秒。该计时受抢占影响且未经守卫认证,只用于预算测算。原守卫仍是 `STOPPED / FOREIGN_GPU_OCCUPANT`,缺少占用者身份,历史根因未确定。两次旧测量累计 222.80500172509346 GPU 秒独立记账一次。正式预算为 40,000 GPU 秒,三种子 300 轮乘 1.25 得 38,601.667 GPU 秒。训练、入场和清理预留为 13,034 / 13,033 / 13,033 GPU 秒,每个评分另预留 300 GPU 秒。评分预留是预算假设,尚无实际评分计时。 [历史终止](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/complete.json) · [历史计时](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/worker-records/progress.json) · [历史结算](../artifacts/track-b-gino-shapenet611-gpu2-ordinal-recovery-2026-11-19/recovery-settlement.json)
## 守卫、训练和评分
派发时记录真实 worker PID 和出生时间。GPU 占用采样前后都观察其进程树,保留已认领的进程身份,避免已退出 worker 被陈旧 GPU 记录误判。PID 复用时以不同出生时间区分。确认外部占用时记录采样时间、卡 UUID、占用 PID 和名称,只停止本次 worker 进程组。运行中的显存与结果中记录的显存峰值均纳入守卫。 [守卫](../tools/track_b_gino_formal_guard.py)
沿用原始 500/111 划分、已验收的解释器、原加载器、模型、优化步骤和指标。复现身份为 `supplement code + 2-keyword patch`。每个种子从头训练 100 轮,冻结模型与只从学习集拟合的压力归一化参数。每次评分须先核对三份已通过守卫的第 100 轮检查点。然后该种子只开一次留出目标记录,按原序逐读 111 个目标并验证逐例哈希。三种子的结果全部列出。仅在三者均有效时,对各自的 111 件平均相对 L2 等权汇总,区间两端 6.62% 和 7.62% 均计入通过。 [训练与评分](../tools/track_b_gino_formal_worker.py) · [联合入口与结算](../tools/track_b_gino_formal.py)
## CPU 回放
生产守卫配合真实 CPU 子进程回放了两种退出时序。GPU 列表先采到 worker 随后其退出、以及 worker 先退出而 GPU 列表仍返回旧 PID,均通过。另用真实活进程身份模拟外部占用,验证 STOP 回执的 PID、名称、采样时间和仅停止本方 worker。另验 PID 复用、日界和预算、汇总区间、留出顺序、三检查点屏障及只开一次 111 项的合成评分。首次回放的两项退出测试被测试夹具的 256 MiB RSS 帽拦截,夹具放宽到 1 GiB 后九项全部通过。正式 RSS 帽为 12 GiB。回放未访问目标归档,未执行真实训练或评分。 [CPU 回放日志](../artifacts/track-b-gino-shapenet611-formal-2026-11-19/cpu-replay.log) · [回放用例](../tests/test_track_b_gino_formal.py)
+204
View File
@@ -0,0 +1,204 @@
"""CPU replay of the production GINO guard and formal custody gates."""
from pathlib import Path
import os
import sys
from tempfile import TemporaryDirectory
import time
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import track_b_gino_formal as formal
import track_b_gino_formal_guard as gpu_guard
import track_b_gino_formal_worker as worker
from track_b_gino_shapenet_entry import read, save, sha
class GuardReplay(unittest.TestCase):
def replay(self, ordering):
uuid = 'GPU-fixture'
limit = dict(disk_bytes=2**20, idle_wait_seconds=10, cleanup_seconds=2, cpu_seconds=10,
rss_bytes=2**30, max_processes=1, device_vram_bytes=2**30,
recorded_vram_bytes=2**30)
with TemporaryDirectory() as folder:
root = Path(folder)
result = root / 'worker-records/result.json'
result.parent.mkdir()
# This subprocess has no GPU code. It waits until the replay lets
# it finish or until the production guard stops its process group.
command = [sys.executable, '-c',
"from pathlib import Path\nimport time,sys\n"
"root=Path(sys.argv[1])\n"
"while not (root/'exit').exists(): time.sleep(.01)\n"
"(root/'worker-records/result.json').write_text("
"'{\"peak_allocated_bytes\":1,\"peak_reserved_bytes\":2}')\n", folder]
observed = []
stopped = []
real_stop = gpu_guard.stop_group
real_tree = gpu_guard.process_tree
def stop_owned(pid):
stopped.append(pid)
real_stop(pid)
def exit_and_wait():
(root / 'exit').touch()
pid = read(root / 'actual-worker-spawn.json')['process']['pid']
by = time.monotonic() + 4
while gpu_guard.current_identity(pid) is not None and time.monotonic() < by:
time.sleep(.01)
if gpu_guard.current_identity(pid) is not None:
raise RuntimeError('fixture worker did not exit and get reaped')
observed.append('worker-reaped')
def tree(identity):
if (root / 'actual-worker-spawn.json').exists() and (
ordering == 'tree-first' and 'worker-reaped' not in observed):
exit_and_wait()
observed.append('tree')
return real_tree(identity)
def snapshot(card):
if card != uuid:
raise AssertionError(card)
spawn = root / 'actual-worker-spawn.json'
if not spawn.exists():
return dict(idle=True, occupants=[], unix=10., memory_MiB=0)
observed.append('gpu-sample')
pid = os.getpid() if ordering == 'foreign' else read(spawn)['process']['pid']
sample = dict(idle=False, unix=42.125, memory_MiB=4,
occupants=[dict(pid=pid, process_name='fixture-python', memory_MiB='4')])
if ordering == 'gpu-first':
exit_and_wait()
return sample
with (patch.object(gpu_guard, 'snapshot', side_effect=snapshot),
patch.object(gpu_guard, 'process_tree', side_effect=tree),
patch.object(gpu_guard, 'stop_group', side_effect=stop_owned),
patch.object(gpu_guard, 'wait_idle', return_value={'status': 'IDLE_CONFIRMED'}),
patch.object(gpu_guard, 'dispatch_environment',
side_effect=lambda _, env: (dict(env, CUDA_VISIBLE_DEVICES=''), []))):
began = time.monotonic()
outcome = gpu_guard.guard(root, command, uuid=uuid, limits=limit, deadline=began + 15,
ticket=dict(ordinal='', guard_started_monotonic=began), validate_result=lambda: sha(result))
spawn = read(root / 'actual-worker-spawn.json')['process']
if ordering == 'foreign':
self.assertEqual(outcome['status'], 'STOPPED')
self.assertEqual(outcome['trigger'], 'FOREIGN_GPU_OCCUPANT')
self.assertIsNone(outcome['result_sha256'])
row = read(root / 'foreign-occupant.json')
self.assertEqual((row['unix'], row['physical_uuid']), (42.125, uuid))
self.assertEqual(row['occupants'][0]['pid'], os.getpid())
self.assertEqual(row['occupants'][0]['process_name'], 'fixture-python')
self.assertEqual(stopped, [spawn['pid']])
else:
self.assertEqual(outcome['status'], 'COMPLETE', outcome)
self.assertFalse((root / 'foreign-occupant.json').exists())
self.assertEqual(stopped, [])
self.assertEqual(outcome['result_sha256'], sha(result))
return observed
def test_gpu_list_before_worker_exit(self):
order = self.replay('gpu-first')
self.assertLess(order.index('gpu-sample'), order.index('worker-reaped'))
def test_worker_already_exited_before_gpu_list(self):
order = self.replay('tree-first')
self.assertLess(order.index('worker-reaped'), order.index('gpu-sample'))
def test_real_foreign_identity_has_exact_stop_receipt(self):
self.replay('foreign')
def test_reused_worker_pid_is_foreign(self):
with TemporaryDirectory() as folder:
root = Path(folder)
original = dict(pid=12, created=1.)
current = dict(pid=12, created=2.)
sample = dict(unix=42., occupants=[dict(pid=12, process_name='new-process')], memory_MiB=4)
monitor = gpu_guard.OccupancyMonitor(root, 'fixture', original, 2**30)
with (patch.object(gpu_guard, 'snapshot', return_value=sample),
patch.object(gpu_guard, 'current_identity', return_value=current)):
with self.assertRaisesRegex(RuntimeError, 'FOREIGN_GPU_OCCUPANT'):
monitor.check()
row = read(root / 'foreign-occupant.json')
self.assertEqual(row['occupants'][0]['observed_process_identity'], current)
class FormalBinding(unittest.TestCase):
def test_inclusive_gate_and_all_three_seeds_required(self):
for endpoint in (6.62, 7.62):
self.assertEqual(formal.pool([endpoint] * 3, [6.62, 7.62]), (endpoint, True))
self.assertEqual(formal.pool([7.12, 7.12, None], [6.62, 7.62]), (None, None))
self.assertFalse(formal.pool([7.63] * 3, [6.62, 7.62])[1])
def test_original_split_score_keeps_order_and_rejects_nan(self):
ids = list(range(111))
complete = [dict(case=i, relative_l2=.0712) for i in ids]
self.assertAlmostEqual(formal.score_mean(complete, ids), 7.12)
with self.assertRaises(ValueError):
formal.score_mean(complete[::-1], ids)
complete[0]['relative_l2'] = float('nan')
with self.assertRaises(ValueError):
formal.score_mean(complete, ids)
def test_scoring_needs_all_checkpoints_before_access(self):
with TemporaryDirectory() as folder:
root = Path(folder)
(root / 'checkpoints-frozen.json').write_text('{}')
with patch.object(formal, 'all_checkpoints', side_effect=PermissionError('missing fit')):
with self.assertRaisesRegex(PermissionError, 'missing fit'):
worker.score({}, 0, None, root, None)
self.assertFalse((root / 'heldout-target-access.jsonl').exists())
def test_score_opens_one_111_target_pass_and_rejects_a_second(self):
import torch
with TemporaryDirectory() as folder:
root = Path(folder)
path = root / 'checkpoint.pt'
torch.save(dict(state={}, pressure_mean=torch.tensor([2.]), pressure_std=torch.tensor([3.])), path)
roster = {'0': dict(checkpoint=dict(path=str(path), sha256=sha(path)))}
save(root / 'checkpoints-frozen.json', roster)
ids = list(range(111))
reads = []
def load(ids_argument, role, log):
self.assertEqual((ids_argument, role), (ids, 'heldout'))
for case in ids_argument:
reads.append(case)
log.write(f'{case}\n')
return object()
class Normalizer:
pass
data = SimpleNamespace(test_ids=ids, normalizer_class=Normalizer, data=load)
torch_cpu_fixture = SimpleNamespace(load=torch.load, isfinite=torch.isfinite,
cuda=SimpleNamespace(synchronize=lambda: None))
model = SimpleNamespace(to=lambda _: model, load_state_dict=lambda _state, **kw: None)
rows = [dict(case=i, relative_l2=.0712) for i in ids]
with (patch.object(formal, 'all_checkpoints', return_value=roster),
patch.object(worker, 'paper_model', return_value=model),
patch.object(worker, 'case_scores', return_value=rows)):
result = worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture)
with self.assertRaises(FileExistsError):
worker.score(dict(protocol={'pressure_epsilon': 1e-6}), 0, data, root, torch_cpu_fixture)
self.assertEqual(reads, ids)
self.assertEqual(result['target_reads'], 111)
self.assertAlmostEqual(result['mean_percent'], 7.12)
def test_card_reservation_expiry_and_total_budget(self):
binding = formal.expected_contract()['formal']
budgets = binding['budget_per_seed_stage_gpu_seconds']
self.assertEqual(sum(sum(b.values()) for b in budgets.values()), 40000)
self.assertEqual(binding['device_pci_ordinals'], {'0': 2, '1': 3, '2': 0})
latest = formal.RESERVATION_END.timestamp() - 300
formal.check_reservation(latest - 1, 300)
for when in (latest, formal.RESERVATION_END.timestamp(), formal.RESERVATION_START.timestamp() - 1):
with self.assertRaises(PermissionError):
formal.check_reservation(when, 300)
if __name__ == '__main__':
unittest.main()
+314
View File
@@ -0,0 +1,314 @@
"""Exact review, custody and resource binding for formal three-seed GINO."""
import argparse
from datetime import datetime
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import time
from zoneinfo import ZoneInfo
from phase2_gate_outcome_review import validate_review
from track_b_gino_formal_guard import guard
from track_b_gino_import_checked_entry import accepted_environment
from track_b_gino_ordinal_recovery import ROOT as MEASUREMENT
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
ROOT = MEASUREMENT.with_name('track-b-gino-shapenet611-formal-2026-11-19')
SOURCES = Path(__file__).resolve().parent
OLD_FILES = {
'contract.json': 'ad9daa2cfb7e2a92bd57507221eba4b94d642bd25e072a6cabbbdd27984ff68a',
'complete.json': 'db3605159c5ade3deda5313ab409e2f1c8bf630511a617f527177da06d6c9a2d',
'recovery-settlement.json': '851eaedf716f01c1d80ce637f7de3edd0664c734bcbfed5622623d99f572fcec',
'worker-records/progress.json': '8f8fb1902c44e2f9c3198608cbecb3a74967ed00fdbcaf1e990d146c894b0daf',
}
DEVICES = {
'0': 'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76',
'1': 'GPU-8b71b032-1b75-2891-935c-ce106a53034a',
'2': 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
}
ORDINALS = {'0': 2, '1': 3, '2': 0}
INVENTORY = BASE / 'artifacts/track-b-gino-gpu2-ordinal-preparation-2026-11-19/physical-inventory.json'
INVENTORY_SHA = 'da2a7b1bf168c7cc4d45b6eaa2e007ae90cb3640eeadde94c10fd1d32fa30dd8'
TZ = ZoneInfo('Asia/Hong_Kong')
RESERVATION_START = datetime(2026, 11, 19, tzinfo=TZ)
RESERVATION_END = datetime(2026, 11, 20, tzinfo=TZ)
NEW_SOURCES = ('track_b_gino_formal.py', 'track_b_gino_formal_guard.py',
'track_b_gino_formal_worker.py')
def expected_contract():
old = {name: read(checked(MEASUREMENT / name, digest)) for name, digest in OLD_FILES.items()}
c = old['contract.json']
stop, epoch = old['complete.json'], old['worker-records/progress.json']
account = old['recovery-settlement.json']
if (stop['status'], stop['trigger'], stop['result_sha256']) != ('STOPPED', 'FOREIGN_GPU_OCCUPANT', None):
raise ValueError('Historical guard STOP differs')
if (epoch['stage'], epoch['seed'], epoch['epoch'], epoch['examples']) != ('EPOCH_COMPLETE', 0, 1, 500):
raise ValueError('Only the completed 500-example epoch can size this reproduction')
seconds = epoch['seconds']
inventory = read(checked(INVENTORY, INVENTORY_SHA))['inventory']
physical = {(int(index), uuid) for index, uuid, _, name in inventory if 'V100' in name}
if not {(ORDINALS[seed], uuid) for seed, uuid in DEVICES.items()} <= physical:
raise ValueError('Reviewed physical V100 inventory differs')
budgets = {seed: {'fit': 13033 + (seed == '0'), 'score': 300} for seed in DEVICES}
fit_reference = 100 * seconds * 1.25
if (not math.isfinite(fit_reference) or fit_reference <= 0
or min(b['fit'] for b in budgets.values()) <= fit_reference
or sum(b['fit'] + b['score'] for b in budgets.values()) != 40000):
raise ValueError('Formal booking must cover the sizing reference and score reserve')
c['mode'] = 'formal-gino-reproduction'
for key in ('gpu_uuid', 'measurement_allowed_heldout_labels', 'gpu_ruling', 'ordinal_recovery'):
del c[key]
c['code_sha256'].update({str(SOURCES / name): sha(SOURCES / name) for name in NEW_SOURCES})
c['formal'] = dict(
handoff='h-4e4e680b', ruling_slack_ts='1795074651.222599', devices=DEVICES,
device_pci_ordinals=ORDINALS,
reservation=dict(handoff='h-a429935e', timezone=TZ.key,
start_inclusive=RESERVATION_START.isoformat(),
end_exclusive=RESERVATION_END.isoformat(), gpu3_single_task=True,
historical_inventory_path=str(INVENTORY), historical_inventory_sha256=INVENTORY_SHA,
runtime_idle_check_required=True),
budget_per_seed_stage_gpu_seconds=budgets, total_gpu_wall_seconds=40000,
timing=dict(record_paths={str(MEASUREMENT / k): v for k, v in OLD_FILES.items()},
status='CONTENTION_AFFECTED_NOT_GUARD_CERTIFIED', sizing_only=True,
epoch_seconds=seconds, train_sizing_gpu_seconds=300 * seconds * 1.25,
train_sizing_multiplier=1.25, score_reserved_gpu_seconds=900,
prior_measurement_gpu_seconds=account['cumulative_measurement_gpu_wall_seconds']),
heldout_policy=dict(preparation_reads=0, training_reads=0, score_reads_per_seed=111,
all_epoch100_checkpoints_required=True, retry=False),
)
del c['limits']['gpu_wall_seconds']
c['limits']['cleanup_seconds'] = 20
del c['limits']['cpu_seconds']
c['limits']['cpu_seconds_per_gpu_second'] = 8
return c
def contract():
c = read(ROOT / 'contract.json')
if c != expected_contract():
raise PermissionError('The sole formal contract or its implementation changed')
for path, digest in c['code_sha256'].items():
checked(path, digest)
for name in ('paper_supplement', 'split_receipt', 'mapping_audit'):
pin = c[name]
checked(pin['path'], pin['sha256'])
# Archive members are opened only in the scoped workers, which verify each
# member before using it. Preparation does not hash the archive payload.
comparison = read(checked(c['comparison_contract'], c['comparison_contract_sha256']))
split = read(c['split_receipt']['path'])
train, test = split['train'], split['test']
if (c['protocol'] != comparison['protocol'] or len(train) != 500 or len(test) != 111
or len(set(train) | set(test)) != 611
or any(c['split'][k] != split[k] for k in ('train', 'test', 'raw_global_bounds'))
or c['protocol']['epochs_per_seed'] != 100 or c['protocol']['seeds'] != [0, 1, 2]
or c['protocol']['acceptance_percent'] != [6.62, 7.62]
or c['paper_compatibility']['identity'] != 'supplement code + 2-keyword patch'
or not Path(c['paper_data']['path']).is_file()):
raise ValueError('Original GINO comparison, split or pinned data path differs')
accepted_environment(c)
return c
def authorize(c):
validate_review(ROOT)
authorization = read(ROOT / 'stage-authorization.json')
required = dict(contract_sha256=sha(ROOT / 'contract.json'),
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
decision='APPROVED', mode=c['mode'], reserved_devices=DEVICES,
reservation_end_exclusive=RESERVATION_END.isoformat())
if any(authorization.get(k) != v for k, v in required.items()):
raise PermissionError('Exact execution and card-reservation authorization required')
def check_reservation(unix, duration):
if (unix < RESERVATION_START.timestamp()
or unix + duration >= RESERVATION_END.timestamp()):
raise PermissionError('The entire booking must fit the November 19 Hong Kong reservation')
def run_root(seed, phase):
return ROOT / f'seed-{seed}' / phase
def epoch100(root, seed, c):
progress = read(root / 'worker-records/progress.json')
if (progress['stage'], progress['seed'], progress['epoch'], progress['examples']) != (
'EPOCH_COMPLETE', seed, c['protocol']['epochs_per_seed'], 500):
raise ValueError('No completed 100th full training epoch for this seed')
return progress
def checkpoint(seed, result):
path = run_root(seed, 'fit') / 'epoch-100.pt'
if result['checkpoint']['path'] != str(path):
raise ValueError('Checkpoint path differs from this seed')
return checked(path, result['checkpoint']['sha256'])
def score_mean(rows, test_ids):
if [r['case'] for r in rows] != test_ids:
raise ValueError('Scoring requires the original 111-case order')
values = [r['relative_l2'] for r in rows]
if any(type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in values):
raise ValueError('Incomplete or invalid relative L2')
return 100 * (math.fsum(values) / len(values))
def validate_result(c, seed, phase):
root = run_root(seed, phase)
path = root / 'worker-records/result.json'
result = read(path)
spawned = read(root / 'actual-worker-spawn.json')
device = DEVICES[str(seed)]
ordinal = str(ORDINALS[str(seed)])
if (spawned['physical_uuid'], spawned['ordinal']) != (device, ordinal) or (
read(root / 'worker-device-verified.json') != dict(
process=spawned['process'], physical_uuid=device, ordinal=ordinal, observed_uuids=[device])):
raise ValueError('Worker has no matching physical-device witness')
if (result['phase'], result['seed'], result['status']) != (phase, seed, 'COMPLETE'):
raise ValueError('Worker result identity differs')
if phase == 'fit':
epoch100(root, seed, c)
checkpoint(seed, result)
if (result['target_reads'], result['epochs'], result['training_examples']) != (0, 100, 500):
raise ValueError('Fit must complete all 100 epochs with zero held-out reads')
else:
fit_terminal = read(run_root(seed, 'fit') / 'complete.json')
fit_path = run_root(seed, 'fit') / 'worker-records/result.json'
original = read(checked(fit_path, fit_terminal['result_sha256']))
access = root / 'heldout-target-access.jsonl'
mapping = {row['case']: row['pressure_sha256'] for row in read(c['mapping_audit']['path'])['rows']}
actual = [json.loads(line) for line in access.read_text().splitlines()]
expected = [dict(case=i, role='heldout', sha256=mapping[i]) for i in c['split']['test']]
if (actual != expected or result['target_reads'] != 111
or result['target_access_sha256'] != sha(access)
or result['checkpoints_manifest_sha256'] != sha(root / 'checkpoints-frozen.json')
or read(root / 'checkpoints-frozen.json') != all_checkpoints(c)
or result['checkpoint_sha256'] != original['checkpoint']['sha256']
or score_mean(result['cases'], c['split']['test']) != result['mean_percent']):
raise ValueError('Score or held-out target audit differs')
return sha(path)
def accepted_terminal(c, seed, phase):
root = run_root(seed, phase)
record = read(root / 'complete.json')
if (record['status'], record['seed'], record['phase'], record['contract_sha256'], record['physical_uuid']) != (
'COMPLETE', seed, phase, sha(ROOT / 'contract.json'), DEVICES[str(seed)]):
raise PermissionError(f'Seed {seed} {phase} has no accepted terminal result')
booked = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
if (record['booked_gpu_seconds'] != booked or record['charged_gpu_seconds'] > booked
or record['result_sha256'] != validate_result(c, seed, phase)):
raise ValueError('Terminal result bytes or resource accounting differ')
return record
def all_checkpoints(c):
checkpoints = {}
for seed in c['protocol']['seeds']:
accepted_terminal(c, seed, 'fit')
root = run_root(seed, 'fit')
checkpoints[str(seed)] = dict(
result_sha256=sha(root / 'worker-records/result.json'),
terminal_sha256=sha(root / 'complete.json'),
checkpoint=read(root / 'worker-records/result.json')['checkpoint'])
return checkpoints
def dispatch(c, seed, phase):
began = time.monotonic()
unix = time.time()
authorize(c)
budget = c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
check_reservation(unix, budget)
seed_root = run_root(seed, phase).parent
seed_root.mkdir(exist_ok=True)
with (seed_root / 'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
roster = all_checkpoints(c) if phase == 'score' else None
root = run_root(seed, phase)
root.mkdir(exist_ok=False)
if roster is not None:
save(root / 'checkpoints-frozen.json', roster)
ticket = dict(seed=seed, phase=phase, ordinal=str(ORDINALS[str(seed)]),
booked_gpu_seconds=budget, unix=unix,
contract_sha256=sha(ROOT / 'contract.json'),
guard_started_monotonic=began, booked_deadline_monotonic=began + budget)
save(root / 'started.json', ticket)
command = [c['environment']['python'], str(SOURCES / 'track_b_gino_formal_worker.py'),
str(seed), phase]
def request_stop(signum, _frame):
raise KeyboardInterrupt(signal.Signals(signum).name)
previous = signal.signal(signal.SIGTERM, request_stop)
try:
limits = dict(c['limits'], cpu_seconds=c['limits']['cpu_seconds_per_gpu_second'] * budget)
result = guard(root, command, uuid=DEVICES[str(seed)], limits=limits,
deadline=began + budget, ticket=ticket,
validate_result=lambda: validate_result(c, seed, phase))
finally:
signal.signal(signal.SIGTERM, previous)
return 0 if result['status'] == 'COMPLETE' else 1
def pool(means, limits):
if len(means) != 3 or any(v is None for v in means):
return None, None
mean = math.fsum(means) / 3
return mean, limits[0] <= mean <= limits[1]
def collect(c):
seeds = {}
accounted = unsettled = 0
for seed in c['protocol']['seeds']:
rows = {}
for phase in ('fit', 'score'):
root = run_root(seed, phase)
terminal = root / 'complete.json'
if terminal.exists():
record = read(terminal)
if record['status'] == 'COMPLETE':
accepted_terminal(c, seed, phase)
accounted += record['charged_gpu_seconds']
rows[phase] = dict(status=record['status'], trigger=record['trigger'],
terminal_sha256=sha(terminal))
if phase == 'score' and record['status'] == 'COMPLETE':
rows['mean_percent'] = read(root / 'worker-records/result.json')['mean_percent']
elif (root / 'started.json').exists():
rows[phase] = dict(status='NO_TERMINAL_RECEIPT')
unsettled += c['formal']['budget_per_seed_stage_gpu_seconds'][str(seed)][phase]
else:
rows[phase] = dict(status='NOT_STARTED')
seeds[str(seed)] = rows
mean, verdict = pool([row.get('mean_percent') for row in seeds.values()],
c['protocol']['acceptance_percent'])
prior = c['formal']['timing']['prior_measurement_gpu_seconds']
return dict(seeds=seeds, mean_percent=mean, within_registered_range=verdict,
reproduction_identity=c['paper_compatibility']['identity'],
prior_measurement_gpu_seconds=prior, settled_formal_gpu_seconds=accounted,
settled_combined_gpu_seconds=accounted + prior,
unsettled_formal_bookings_gpu_seconds=unsettled, formal_gpu_cap_seconds=40000)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('fit', 'score', 'collect'))
parser.add_argument('--seed', type=int, choices=(0, 1, 2))
args = parser.parse_args()
c = contract()
if args.action == 'collect':
if args.seed is not None:
parser.error('collect reports all three seeds')
print(json.dumps(collect(c), indent=2, allow_nan=False))
else:
if args.seed is None:
parser.error('--seed is required for fit or score')
raise SystemExit(dispatch(c, args.seed, args.action))
+175
View File
@@ -0,0 +1,175 @@
"""Supervise a formal GINO fit or score on one physical V100."""
import os
import shutil
import subprocess
import threading
import time
from types import FunctionType, SimpleNamespace
import psutil
from s2_gpu_idle_gate import snapshot, wait_idle
from s2_main_memory_guard import PeakRecords
from s2_recovery_gpu_supervisor import stop_group, supervise
from track_b_gino_ordinal_recovery import dispatch_environment, monitor_identity
from track_b_gino_shapenet_entry import process_identity, save
def current_identity(pid):
try:
return process_identity(pid)
except psutil.NoSuchProcess:
return None
def process_tree(worker):
if current_identity(worker['pid']) != worker:
return []
try:
children = psutil.Process(worker['pid']).children(recursive=True)
except psutil.NoSuchProcess:
return []
result = []
for child in children:
try:
result.append(dict(pid=child.pid, created=child.create_time()))
except psutil.NoSuchProcess:
continue
return result
class OccupancyMonitor:
def __init__(self, root, uuid, worker, device_vram_bytes):
self.root = root
self.uuid = uuid
self.worker = worker
self.known = {worker['pid']: worker}
self.device_vram_bytes = device_vram_bytes
self.peak_device_bytes = 0
def check(self):
before = process_tree(self.worker)
state = snapshot(self.uuid)
after = process_tree(self.worker)
for identity in before + after:
if identity['pid'] != self.worker['pid']:
self.known[identity['pid']] = identity
foreign = []
for row in state['occupants']:
pid = row['pid']
current = current_identity(pid)
if pid not in self.known or (current is not None and current != self.known[pid]):
foreign.append(dict(row, observed_process_identity=current))
if foreign:
save(self.root / 'foreign-occupant.json', dict(
trigger='FOREIGN_GPU_OCCUPANT', unix=state['unix'], physical_uuid=self.uuid,
occupants=foreign, own_spawn=self.worker, tree_before=before, tree_after=after))
raise RuntimeError('FOREIGN_GPU_OCCUPANT')
self.peak_device_bytes = max(self.peak_device_bytes, state['memory_MiB'] * 2**20)
if self.peak_device_bytes > self.device_vram_bytes:
raise RuntimeError('DEVICE_VRAM_LIMIT')
def watch(root, uuid, worker, limits, done, tracker, state):
occupancy = OccupancyMonitor(root, uuid, worker, limits['device_vram_bytes'])
while not done.is_set():
try:
occupancy.check()
state['peak_device_bytes'] = occupancy.peak_device_bytes
tracker.scan(root / 'worker-records')
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
raise RuntimeError('RECORDED_VRAM_LIMIT')
monitor_identity(root, uuid)
except Exception as exc:
state['trigger'] = str(exc) or type(exc).__name__
# Signal only the recorded worker group while its leader has the same identity.
if current_identity(worker['pid']) == worker:
stop_group(worker['pid'])
return
done.wait(.5)
def guard(root, command, *, uuid, limits, deadline, ticket, validate_result):
began = time.monotonic()
admission = min(deadline - limits['cleanup_seconds'], began + limits['idle_wait_seconds'])
done = threading.Event()
tracker = PeakRecords()
state = dict(trigger=None, peak_device_bytes=0)
watcher = None
worker_receipt = None
status, trigger, result_hash = 'STOPPED', None, None
def spawn(args, **kwargs):
nonlocal watcher
env, inventory = dispatch_environment(uuid, kwargs['env'])
ordinal = env['CUDA_VISIBLE_DEVICES']
if ordinal != ticket['ordinal']:
raise RuntimeError('PINNED_PCI_ORDINAL_DIFFERS')
if not snapshot(uuid)['idle']:
raise RuntimeError('SELECTED_GPU_BECAME_BUSY')
save(root / 'ordinal-dispatch.json', dict(ordinal=ordinal, physical_uuid=uuid, inventory=inventory))
if time.monotonic() >= admission:
raise RuntimeError('ADMISSION_DEADLINE')
proc = subprocess.Popen(args, **dict(kwargs, env=env))
try:
worker = process_identity(proc.pid)
save(root / 'actual-worker-spawn.json', dict(process=worker, ordinal=ordinal, physical_uuid=uuid))
watcher = threading.Thread(target=watch, daemon=True,
args=(root, uuid, worker, limits, done, tracker, state))
watcher.start()
except (Exception, KeyboardInterrupt):
# The shared supervisor has not yet received the process to reap.
stop_group(proc.pid)
proc.wait()
raise
return proc
try:
if shutil.disk_usage(root).free < limits['disk_bytes']:
raise RuntimeError('DISK_HEADROOM')
idle = wait_idle(uuid, max(0., admission - time.monotonic()), lambda _: None, sample=snapshot)
save(root / 'idle-gate.json', idle)
if idle['status'] != 'IDLE_CONFIRMED' or time.monotonic() >= admission:
raise RuntimeError('IDLE_DEADLINE')
save(root / 'dispatch.json', dict(ticket, coordinator=process_identity(os.getpid()),
start_by_monotonic=admission))
remaining = deadline - time.monotonic() - limits['cleanup_seconds']
if remaining <= 0:
raise RuntimeError('INSUFFICIENT_REMAINING_BUDGET')
ns = dict(supervise.__globals__, subprocess=SimpleNamespace(Popen=spawn, STDOUT=subprocess.STDOUT))
run_worker = FunctionType(supervise.__code__, ns, supervise.__name__)
try:
worker_receipt = run_worker(command, root, 'gino', dict(
total_cpu_seconds=limits['cpu_seconds'], job_cpu_seconds=limits['cpu_seconds'],
job_wall_seconds=remaining, tree_rss_bytes=limits['rss_bytes'],
disk_bytes=limits['disk_bytes'], max_processes=limits['max_processes']),
poll=.2, gpu_uuid=uuid)
finally:
done.set()
if watcher is not None:
watcher.join(timeout=8)
if watcher is not None and watcher.is_alive():
raise RuntimeError('MONITOR_DID_NOT_EXIT')
tracker.scan(root / 'worker-records')
if state['trigger']:
raise RuntimeError(state['trigger'])
if max(tracker.maxima.values()) > limits['recorded_vram_bytes']:
raise RuntimeError('RECORDED_VRAM_LIMIT')
if worker_receipt['status'] != 'PASS' or worker_receipt['monitor_errors']:
raise RuntimeError(worker_receipt.get('stop_reason') or 'WORKER_FAILED')
if set(tracker.witnesses) != set(tracker.maxima):
raise RuntimeError('MISSING_RECORDED_MEMORY_PEAKS')
result_hash = validate_result()
status = 'COMPLETE'
except (Exception, KeyboardInterrupt) as exc:
trigger = str(exc) or type(exc).__name__
end = time.monotonic()
if end > deadline and status == 'COMPLETE':
status, trigger, result_hash = 'STOPPED', 'BOOKED_GPU_WALL_CAP', None
terminal = dict(ticket, status=status, trigger=trigger, result_sha256=result_hash, worker=worker_receipt,
physical_uuid=uuid, guard_wall_seconds=end - began, monitor=state,
charged_gpu_seconds=end - ticket['guard_started_monotonic'],
booked_gpu_cap_exceeded=end > deadline, recorded_peaks=tracker.maxima, no_retry=True)
save(root / 'complete.json', terminal)
return terminal
+143
View File
@@ -0,0 +1,143 @@
"""Run one formally bound GINO fit or the corresponding one-pass score."""
import argparse
import ctypes
import os
import signal
import sys
from pathlib import Path
from tempfile import TemporaryDirectory
import time
import track_b_gino_formal as formal
from track_b_gino_paper_keyword_patch import extract_patched_paper_code
from track_b_gino_shapenet_entry import checked, process_identity, read, save, sha
from track_b_gino_shapenet_worker import (
OriginalData, case_scores, cuda_peaks, paper_model, test_environment, train_seed,
)
def attach(root, seed, phase):
ticket = read(root / 'dispatch.json')
parent = os.getppid()
if (ticket['coordinator'] != process_identity(parent)
or ticket['contract_sha256'] != sha(formal.ROOT / 'contract.json')
or (ticket['seed'], ticket['phase']) != (seed, phase)
or time.monotonic() >= ticket['start_by_monotonic']):
raise PermissionError('Worker is not attached to the authorized guard')
if ctypes.CDLL(None, use_errno=True).prctl(1, signal.SIGKILL, 0, 0, 0) or os.getppid() != parent:
raise RuntimeError('Could not attach worker lifetime to its guard')
c = formal.contract()
formal.authorize(c)
if str(Path(sys.executable).absolute()) != c['environment']['python']:
raise PermissionError('Unreviewed interpreter')
return c, ticket
def bind_device(root, c, ticket, torch):
ordinal = ticket['ordinal']
uuid = formal.DEVICES[str(ticket['seed'])]
dispatch = read(root / 'ordinal-dispatch.json')
if (dispatch['ordinal'] != ordinal or dispatch['physical_uuid'] != uuid
or os.environ.get('CUDA_VISIBLE_DEVICES') != ordinal
or os.environ.get('CUDA_DEVICE_ORDER') != 'PCI_BUS_ID'
or torch.cuda.device_count() != 1 or torch.version.cuda != c['environment']['torch_cuda']):
raise PermissionError('One reviewed CUDA PCI ordinal is required')
torch.cuda.set_device(0)
name = torch.cuda.get_device_name(0)
if 'V100' not in name:
raise PermissionError('The reviewed device must be a V100')
probe = torch.empty(1, dtype=torch.uint8, device='cuda:0')
current = process_identity(os.getpid())
save(root / 'worker-device-witness.json', dict(
process=current, ordinal=ordinal, device_count=1, device_name=name,
physical_uuid_expected=uuid, first_allocation_bytes=probe.numel()))
verified = root / 'worker-device-verified.json'
while time.monotonic() < ticket['start_by_monotonic']:
if verified.exists():
record = read(verified)
expected = dict(process=current, physical_uuid=uuid, ordinal=ordinal, observed_uuids=[uuid])
if record != expected:
raise PermissionError('Actual worker physical-device witness differs')
return
time.sleep(.1)
raise RuntimeError('Physical device was not verified in the admission window')
def train(c, seed, data, root, records, torch):
learning = data.data(data.train_ids, 'train')
epochs = c['protocol']['epochs_per_seed']
model, times = train_seed(c, learning, seed, epochs, records / 'progress.json')
if len(times) != epochs:
raise ValueError('Full training did not finish')
formal.epoch100(root, seed, c)
path = root / 'epoch-100.pt'
temporary = root / 'epoch-100.pt.tmp'
with temporary.open('xb') as stream:
torch.save(dict(state=model.state_dict(), pressure_mean=data.normalizer.mean,
pressure_std=data.normalizer.std), stream)
temporary.rename(path)
return dict(checkpoint=dict(path=str(path), sha256=sha(path)), target_reads=0,
epochs=epochs, training_examples=500, epoch_seconds=times)
def score(c, seed, data, root, torch):
# Targets are still unopened while rechecking all three finished fits,
# initializing the exact model and loading its immutable checkpoint.
manifest = root / 'checkpoints-frozen.json'
manifest_sha = sha(manifest)
roster = read(manifest)
if roster != formal.all_checkpoints(c):
raise PermissionError('Three complete, pinned checkpoints must precede custody')
checkpoint = roster[str(seed)]['checkpoint']
state = torch.load(checked(checkpoint['path'], checkpoint['sha256']), map_location='cpu')
model = paper_model(c).to('cuda')
model.load_state_dict(state['state'], strict=True)
# The pinned normalizer's encode/decode and device methods use these
# fitted tensors and epsilon. No learning or target dataset is refitted.
normalizer = object.__new__(data.normalizer_class)
normalizer.mean = state['pressure_mean']
normalizer.std = state['pressure_std']
normalizer.eps = c['protocol']['pressure_epsilon']
if any(t.shape != (1,) or not bool(torch.isfinite(t).all())
for t in (normalizer.mean, normalizer.std)) or bool(normalizer.std < 0):
raise ValueError('Invalid frozen learning-only pressure statistics')
data.normalizer = normalizer
access = root / 'heldout-target-access.jsonl'
with access.open('x') as log:
heldout = data.data(data.test_ids, 'heldout', log)
rows = case_scores(model, heldout, data.test_ids, data.normalizer)
torch.cuda.synchronize()
return dict(cases=rows, mean_percent=formal.score_mean(rows, data.test_ids),
target_reads=len(rows), target_access_sha256=sha(access),
checkpoints_manifest_sha256=manifest_sha, checkpoint_sha256=checkpoint['sha256'])
def run(seed, phase):
root = formal.run_root(seed, phase)
c, ticket = attach(root, seed, phase)
test_environment(c)
import torch
torch.set_num_threads(c['limits']['torch_threads'])
torch.set_num_interop_threads(1)
bind_device(root, c, ticket, torch)
torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
records = root / 'worker-records'
records.mkdir(exist_ok=False)
with TemporaryDirectory(prefix='formal-gino-', dir=root) as scratch:
extract_patched_paper_code(c, scratch)
data = OriginalData(c, scratch)
result = (train(c, seed, data, root, records, torch) if phase == 'fit'
else score(c, seed, data, root, torch))
save(records / 'result.json', dict(result, phase=phase, seed=seed, status='COMPLETE', **cuda_peaks(torch)))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('seed', type=int, choices=(0, 1, 2))
parser.add_argument('phase', choices=('fit', 'score'))
args = parser.parse_args()
run(args.seed, args.phase)