Record reproduced fp16 overflow and freeze isolated FP32-slice pilot v2

This commit is contained in:
Frank14f committed 2026-11-15 06:27:43 +08:00
1 parent f502b7b9f8
commit 8a548cc259
28 files changed
+13918

No files matched your search

@@ -0,0 +1,9 @@
{
"status": "COMPLETE",
"decision": "FP32_FINITE_FP16_NONFINITE",
"total_wall_seconds": 262.80677626800025,
"limit_seconds": 900,
"separate_from_pilot": true,
"validation_scoring": false,
"test_data_read": false
}
@@ -0,0 +1,10 @@
{
"cpu_seconds_charged": 262.89331899999996,
"attempts": [
{
"name": "replay",
"status": "PASS",
"cpu_seconds": 262.89331899999996
}
]
}
@@ -0,0 +1 @@
{"status": "COMPLETE", "diagnostic_only": true, "decision": "FP32_FINITE_FP16_NONFINITE", "reproduced_original_failure": true, "case": "00000644_761a451dfea94bfa93e23cfe_step_000", "nodes": 81832, "epoch": 10, "step_zero_based": 371, "step_one_based": 372, "validation_arrays_read": false, "validation_scoring": false, "test_data_read": false, "pilot_run": false, "standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03", "wall_seconds": 258.6586924850126, "peak_allocated_bytes": 4850479616, "peak_reserved_bytes": 9663676416, "failure_state_sha256": "a7510d3a5a09dae348ec37c5c78251d2405a6414bb986c4443e03caa4087e200"}
@@ -0,0 +1,25 @@
{
"status": "PASS",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_encoder_diagnostic.py",
"artifacts/s2-encoder-diagnostic-2026-11-15",
"--worker"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 262.89331899999996,
"wall_seconds": 262.75584313995205,
"peak_tree_rss_bytes": 2277941248,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 8192,
"max_poll_gap_seconds": 0.5908525820123032,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
}
@@ -0,0 +1,17 @@
{
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_encoder_diagnostic.py",
"artifacts/s2-encoder-diagnostic-2026-11-15",
"--worker"
],
"limits": {
"total_cpu_seconds": 7200,
"job_cpu_seconds": 900,
"job_wall_seconds": 897.4562088979874,
"tree_rss_bytes": 17179869184,
"disk_bytes": 5368709120,
"max_processes": 2
},
"started_unix": 1794694802.046334
}
@@ -0,0 +1,34 @@
{
"status": "PASS",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_encoder_diagnostic.py",
"artifacts/s2-encoder-diagnostic-2026-11-15",
"--worker"
],
"returncode": 0,
"stop_reason": null,
"cpu_seconds": 262.89331899999996,
"wall_seconds": 262.75584313995205,
"peak_tree_rss_bytes": 2277941248,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 8192,
"max_poll_gap_seconds": 0.5908525820123032,
"poll_target_seconds": 0.1,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": true,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null,
"device_monitor": {
"peak_device_memory_bytes": 10079961088,
"peak_whole_run_tree_rss_bytes": 2686857216,
"max_poll_gap_seconds": 1.009083034005016,
"reason": null,
"samples": 446
},
"total_run_wall_seconds": 262.8028003940126,
"result_sha256": "69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41"
}
@@ -0,0 +1,13 @@
{
"epoch": 10,
"step_zero_based": 352,
"case": "00000164_91adbf180e114b698160c9a4_step_000",
"wall_seconds": 256.8595967130386,
"amp": {
"attempted_steps": 353,
"skipped_steps": 0,
"successful_updates": 353,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
}
}
@@ -0,0 +1,132 @@
[
{
"epoch": 0,
"training_global_weighted_mse": 0.03763483001738395,
"amp": {
"attempted_steps": 382,
"skipped_steps": 2,
"successful_updates": 380,
"skipped_fraction": 0.005235602094240838,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.03763483001738395,
"matches_original_training_summary": true
},
{
"epoch": 1,
"training_global_weighted_mse": 0.013855781360679905,
"amp": {
"attempted_steps": 382,
"skipped_steps": 0,
"successful_updates": 382,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.013855781360679905,
"matches_original_training_summary": true
},
{
"epoch": 2,
"training_global_weighted_mse": 0.014031890443820595,
"amp": {
"attempted_steps": 382,
"skipped_steps": 0,
"successful_updates": 382,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.014031890443820595,
"matches_original_training_summary": true
},
{
"epoch": 3,
"training_global_weighted_mse": 0.011857566869414183,
"amp": {
"attempted_steps": 382,
"skipped_steps": 0,
"successful_updates": 382,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.011857566869414183,
"matches_original_training_summary": true
},
{
"epoch": 4,
"training_global_weighted_mse": 0.008991813481875563,
"amp": {
"attempted_steps": 382,
"skipped_steps": 0,
"successful_updates": 382,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.008991813481875563,
"matches_original_training_summary": true
},
{
"epoch": 5,
"training_global_weighted_mse": 0.011181466419428661,
"amp": {
"attempted_steps": 382,
"skipped_steps": 0,
"successful_updates": 382,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.011181466419428661,
"matches_original_training_summary": true
},
{
"epoch": 6,
"training_global_weighted_mse": 0.011864404833579386,
"amp": {
"attempted_steps": 382,
"skipped_steps": 0,
"successful_updates": 382,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.011864404833579386,
"matches_original_training_summary": true
},
{
"epoch": 7,
"training_global_weighted_mse": 0.009409040550106135,
"amp": {
"attempted_steps": 382,
"skipped_steps": 0,
"successful_updates": 382,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.009409040550106135,
"matches_original_training_summary": true
},
{
"epoch": 8,
"training_global_weighted_mse": 0.008648868906570734,
"amp": {
"attempted_steps": 382,
"skipped_steps": 0,
"successful_updates": 382,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.008648868906570734,
"matches_original_training_summary": true
},
{
"epoch": 9,
"training_global_weighted_mse": 0.008291036860072012,
"amp": {
"attempted_steps": 382,
"skipped_steps": 0,
"successful_updates": 382,
"skipped_fraction": 0.0,
"consecutive_skipped": 0
},
"original_training_global_weighted_mse": 0.008291036860072012,
"matches_original_training_summary": true
}
]
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,10 @@
{
"started_unix": 1794694809.765883,
"contract_sha256": "718b6e445cf8dde502fd0e59933fe4b1bae83b46fc70b49be9655615d8a0bdd0",
"expected_case": "00000644_761a451dfea94bfa93e23cfe_step_000",
"expected_epoch": 10,
"expected_step_zero_based": 371,
"learning_parts": 382,
"validation_arrays_read": false,
"test_data_read": false
}
@@ -0,0 +1,5 @@
{
"started_unix": 1794694802.0002818,
"pid": 207587,
"contract_sha256": "718b6e445cf8dde502fd0e59933fe4b1bae83b46fc70b49be9655615d8a0bdd0"
}
@@ -0,0 +1,324 @@
{
"fold": 0,
"pool": 100,
"seed": 0,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"code_sha256": {
"tools/prepare_s2_recovered_pilot_training.py": "ab5a4274363598ff933ff873df2b00497f069b1fb45756a1d1dba8d723294f68",
"tools/run_s2_recovered_pilot.py": "56962009bf5eaaee599ffea6a3d454372e0aea2842fa79eac609fd4a24ecbb39",
"tools/run_s2_pilot_probe.py": "2360ab7b44b179144a5ecd2d691e33cd36a6ec4bfeac71452bf062e7f42f81eb",
"tools/run_s2_pilot_training.py": "d5e543478543243fe1db3475a4d4cd135fbdcf631d3bce48c4b5af8b2edafb22",
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"tools/s2_recovered_pilot_gpu_guard.py": "9a27f47918c865ce37e46b0be54144dc3f9ab1058e4eddc9f6fbc771f4959eb0",
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02",
"tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97",
"tools/s2_incident_recovery_manifest.py": "8cbb2e1067c2cc22762d8ae3cf192aa2c5fa7c7a199a22e7a39a25570ee1c3b1",
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"tools/prepare_s2_incident_recovery.py": "4f772573dedc2b7921986e25eaedb14f91d98b235421829c284f3b40b85710aa",
"tools/run_s2_recovery_chain.py": "76d0808bdb2d624cfbaadf5a3c5e7e9a0c2be5d5ac41d4ac3fd9ccd48743933c",
"tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
"tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
"tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
"tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
"tools/prepare_s2_precision_pilot.py": "c3a71b0ff58bcea4433ff93e18ef14d3a00521438a76baaa69830a5ee8ac098a",
"tools/run_s2_precision_pilot.py": "ac8347be1ca11f48cb67abbf5e5c78b39af46ed3a7c6f89b49098fd5e77a6904",
"tools/s2_precision_pilot_gpu_guard.py": "5cf0f41de34b28394dbee4672b5bd34eb9c4fe2344822aa3d648f20561d96328",
"tools/run_s2_pilot_probe_v2.py": "61c08499346d50405d0c254de54deea24b3522256bf1c8b0aec3e9f4ddab6390",
"tools/run_s2_pilot_training_v2.py": "f461cb43bc602a32677d7dcfb8a322e206531ad152fa33bb4e36ea8546a1c0f2",
"tools/run_s2_encoder_diagnostic.py": "8f31d312fb53bfdb9b1804af064e463d3989f1b33c4aa7279a967ca82dd2c0a4",
"tools/s2_models_v2/__init__.py": "0302d333bcd8b33e85aaa6a4e7822e7e1ccfc66343136107fea6f4ca283ba1ec",
"tools/s2_models_v2/attention.py": "3829e34ca30942c863699c4d43fb6fa104452c6c04c4d606c11019d3dde16ffa",
"tools/s2_models_v2/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"tools/s2_models_v2/network.py": "a443470caeecdf529504bffab3ca8cc0b70ce44b505bad60f9c63db9709dbb83"
},
"role_manifest": "artifacts/s2-pilot-inputs-2026-11-15/role-manifest.json",
"role_manifest_sha256": "a6dce3ebf875dbbe2df349f06e728960140f131d4d55ebd00af03945a46ac35c",
"success_manifest": "artifacts/s2-pilot-v2-2026-11-15/success-manifest.json",
"success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
"probe_cases": [
"00001260_a4715d363f88400c8f0238d1_step_003",
"00000290_58f8ff46b7564e989f9695b7_step_003",
"00001557_16c191aef5034779aaab6e3f_step_000",
"00000293_58f8ff46b7564e989f9695b7_step_006",
"00001193_46b17dd88ed24f55bca2eb69_step_000",
"00000105_52f7124261ae4c558f800775_step_002",
"00001592_35f4abaacc9447e4ac0b964f_step_001",
"00000140_2fc54fcd110d4f49969163c4_step_003",
"00000166_2246cab58dbe4a2fb00fed50_step_000",
"00001321_7cef9b79a6b04db894fd74e5_step_000",
"00000462_516b1784884d4facbc4c0d74_step_010",
"00000470_11951e51b89249dba38ded3e_step_005",
"00001495_6c8ef6d2f5564d139a5077f8_step_006",
"00001484_a45f6c99e2154f3b94955175_step_068",
"00000784_a25fb37392a44447b9684f16_step_009",
"00000314_3062bccff48e47a2b9de05e3_step_009",
"00001186_507b60b57791425bbb73273e_step_001",
"00001569_6fc579f3224d4d53a1b60710_step_000",
"00000617_c710d414bbbc4340b682c58a_step_003",
"00000666_6f67a59c16b949a4b66dcfe1_step_000"
],
"probe_selection": "20 equally spaced ranks in successful learning node-count order; includes min/max; tie by case",
"learning_nodes": 11863762,
"validation_nodes": 3330152,
"successful": {
"learning": {
"parts": 382,
"nodes": 11300812
},
"validation": {
"parts": 111,
"nodes": 3255106
}
},
"extrapolation": "Train rate * all 387 planned learning nodes + FP32 inference rate * all 112 planned validation nodes; conservative if failed parts dropped",
"runs": [
{
"name": "N128-seed0",
"arm": "N",
"width": 128,
"seed": 0
},
{
"name": "E128-seed0",
"arm": "E",
"width": 128,
"seed": 0
},
{
"name": "N256-seed0",
"arm": "N",
"width": 256,
"seed": 0
},
{
"name": "E256-seed0",
"arm": "E",
"width": 256,
"seed": 0
}
],
"limits": {
"total_gpu_wall_seconds": 7200,
"probe_wall_seconds": 300,
"run_wall_seconds": 1800,
"host_rss_bytes": 17179869184,
"vram_bytes": 16106127360,
"disk_bytes": 5368709120
},
"coverage_boundary": {
"planned_max_learning_nodes": 187345,
"max_successful_learning_nodes": 187345,
"failed_parts": [
{
"case": "00000575_5a4cef68211d4706b1ec8586_step_004",
"role": "validation",
"nodes": 75046,
"component_id": "component-3a3e340f4dfac391",
"guard": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/s2_pilot_prepare.py",
"artifacts/s2-pilot-inputs-2026-11-15",
"181"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1799.766116,
"wall_seconds": 1800.400393687014,
"peak_tree_rss_bytes": 602218496,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 1.128913332009688,
"poll_target_seconds": 0.25,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false
},
"failure": null
},
{
"case": "00000614_c710d414bbbc4340b682c58a_step_000",
"role": "learning",
"nodes": 145860,
"component_id": "component-4bd089778094e746",
"guard": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/s2_pilot_prepare.py",
"artifacts/s2-pilot-inputs-2026-11-15",
"201"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1799.796751,
"wall_seconds": 1800.0567526929663,
"peak_tree_rss_bytes": 629305344,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 1.1576595069491304,
"poll_target_seconds": 0.25,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false
},
"failure": null
},
{
"case": "00000625_6af98f5458b347bb843f0562_step_000",
"role": "learning",
"nodes": 145707,
"component_id": "component-4bd089778094e746",
"guard": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/s2_pilot_prepare.py",
"artifacts/s2-pilot-inputs-2026-11-15",
"211"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1799.8864629999998,
"wall_seconds": 1800.119237862993,
"peak_tree_rss_bytes": 647880704,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 1.159920622012578,
"poll_target_seconds": 0.25,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false
},
"failure": null
},
{
"case": "00001466_a45f6c99e2154f3b94955175_step_050",
"role": "learning",
"nodes": 77138,
"component_id": "component-7fe5173e77c86f9d",
"guard": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/s2_pilot_prepare.py",
"artifacts/s2-pilot-recovery-inputs-2026-11-15",
"83"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1799.926194,
"wall_seconds": 1800.0714844280155,
"peak_tree_rss_bytes": 383070208,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 1.0109735240112059,
"poll_target_seconds": 0.25,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
},
{
"case": "00001470_a45f6c99e2154f3b94955175_step_054",
"role": "learning",
"nodes": 124817,
"component_id": "component-7fe5173e77c86f9d",
"guard": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/s2_pilot_prepare.py",
"artifacts/s2-pilot-recovery-inputs-2026-11-15",
"87"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1799.664371,
"wall_seconds": 1800.0443400070071,
"peak_tree_rss_bytes": 578732032,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 1.007104828953743,
"poll_target_seconds": 0.25,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
},
{
"case": "00001473_a45f6c99e2154f3b94955175_step_057",
"role": "learning",
"nodes": 69428,
"component_id": "component-7fe5173e77c86f9d",
"guard": {
"status": "STOPPED_OR_FAILED",
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/s2_pilot_prepare.py",
"artifacts/s2-pilot-recovery-inputs-2026-11-15",
"90"
],
"returncode": -9,
"stop_reason": "wall_limit",
"cpu_seconds": 1799.751432,
"wall_seconds": 1800.2498029720155,
"peak_tree_rss_bytes": 438853632,
"peak_process_count": 1,
"output_disk_bytes_before_receipt": 4096,
"max_poll_gap_seconds": 0.8929723129840568,
"poll_target_seconds": 0.25,
"reaped_orphan_cpu_seconds": 0.0,
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
"gpu_visible": false,
"retry": false,
"monitor_errors": 0,
"last_monitor_error": null
},
"failure": null
}
],
"statement": "VRAM probe covers only the maximum successful learning part included; not failed/larger unprobed parts"
},
"authorizing_rulings": [
"1794680129.303719",
"1794681576.845789",
"1794681807.376669",
"1794689807.792819",
"1794694544.819729",
"1794694646.032819"
],
"test_labels_read": false,
"precision_variant": {
"version": "S2-pilot-v2-slice-FP32",
"changed": "Only shared N/E slice aggregation numerator, denominator and normalization in FP32 with autocast disabled",
"original_pilot": "artifacts/s2-pilot-training-2026-11-15",
"original_contract_sha256": "f6987431f478275485d18f62240144c2626cab3fbeb75c3e130b68b5d57a1fd2",
"original_status": "PILOT_INCOMPLETE",
"diagnostic_root": "artifacts/s2-encoder-diagnostic-2026-11-15",
"diagnostic_result_sha256": "69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41",
"diagnostic_guard_sha256": "ec7b55e2cbca465c29aec84630fc02c9cfd9fed71e98226a90eb5b3fb2b45537",
"from_scratch": true,
"original_weights_reused": false,
"preparation_repeated": false,
"reprobe_max_nodes": 187345,
"VRAM_limit_failure_no_batch_or_width_fallback": true
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,44 @@
{
"status": "PRELAUNCH_FIXED",
"observed_unix": 1794695249.0009365,
"original_source_pins_unchanged": true,
"worker_diff_only_model_import": true,
"blocks_byte_identical": true,
"preparation_repeated": false,
"original_pilot_status": "PILOT_INCOMPLETE",
"v2_plan_sha256": "5a127d5ddfab366da49475867c9f14f2854f5000a733c7a3211df61bd7acecae",
"v2_success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
"diagnostic_result_sha256": "69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41",
"diagnostic_complete_sha256": "624ccafdaff38e3e14d6438752bcb0545de67787a9f43bedd7cda4acbeeeeee6",
"diagnostic_guard_sha256": "ec7b55e2cbca465c29aec84630fc02c9cfd9fed71e98226a90eb5b3fb2b45537",
"file_sha256": {
"artifacts/s2-encoder-diagnostic-2026-11-15/complete.json": "624ccafdaff38e3e14d6438752bcb0545de67787a9f43bedd7cda4acbeeeeee6",
"artifacts/s2-encoder-diagnostic-2026-11-15/contract.json": "718b6e445cf8dde502fd0e59933fe4b1bae83b46fc70b49be9655615d8a0bdd0",
"artifacts/s2-encoder-diagnostic-2026-11-15/guards/ledger.json": "4651375ee58862562af2b119361d62a926405a0985e89bf24a82a542c9de16db",
"artifacts/s2-encoder-diagnostic-2026-11-15/guards/replay.log": "faadfb878964bf4d976d4242a0ce4b3389031cd2aac28c5f5976ac2102afa4c6",
"artifacts/s2-encoder-diagnostic-2026-11-15/guards/replay.receipt.json": "26de90165173bf65d288aedd1bc895de510f69ac4591d4e1fd9a89a5d99c2048",
"artifacts/s2-encoder-diagnostic-2026-11-15/guards/replay.started.json": "9cf48ebc8f195167664040ea027108a2c10ecb2460e369075591ed4c944e41be",
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/failure-state.pt": "a7510d3a5a09dae348ec37c5c78251d2405a6414bb986c4443e03caa4087e200",
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/progress.json": "79ea82472e504ae5d396f7028732ec1cf7883e96a7f1bd5fea5142df1f09199a",
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/replay-history.json": "9742e3d52d6296594f8fe021096424d04828da3f1a58d4815e0d1cea6afe6c8d",
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/result.json": "69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41",
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/started.json": "ab6ae7d09d6b06bf62290aa1c2e121332778978e47da1d6bfe793e2c296462f5",
"artifacts/s2-encoder-diagnostic-2026-11-15/replay-guard-receipt.json": "ec7b55e2cbca465c29aec84630fc02c9cfd9fed71e98226a90eb5b3fb2b45537",
"artifacts/s2-encoder-diagnostic-2026-11-15/started.json": "55c03424b933e8b1e9b90f2530232f42ebedcc287364eb9ab5999ba7e6f6ade6",
"artifacts/s2-encoder-diagnostic-2026-11-15/synthetic-check.json": "6de76241c39e70f3888e7937ed2b6255274d32f7fe9417ab0cb1c962f052a6c2",
"artifacts/s2-pilot-v2-2026-11-15/pilot-plan.json": "5a127d5ddfab366da49475867c9f14f2854f5000a733c7a3211df61bd7acecae",
"artifacts/s2-pilot-v2-2026-11-15/success-manifest.json": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
"artifacts/s2-precision-pilot-implementation-2026-11-15/synthetic-tests.txt": "b4786b068b4fcbd768942a1daa2229f6fe57e3dcc81792a565b61d0f2f7663d7",
"artifacts/s2-precision-pilot-implementation-2026-11-15/test-receipt.json": "56ee518976e55e4f265701368e661103fdcd4478f6bd45bf2f908024ac1b8b4d",
"tools/prepare_s2_precision_pilot.py": "c3a71b0ff58bcea4433ff93e18ef14d3a00521438a76baaa69830a5ee8ac098a",
"tools/run_s2_pilot_probe_v2.py": "61c08499346d50405d0c254de54deea24b3522256bf1c8b0aec3e9f4ddab6390",
"tools/run_s2_pilot_training_v2.py": "f461cb43bc602a32677d7dcfb8a322e206531ad152fa33bb4e36ea8546a1c0f2",
"tools/run_s2_precision_pilot.py": "ac8347be1ca11f48cb67abbf5e5c78b39af46ed3a7c6f89b49098fd5e77a6904",
"tools/s2_models_v2/__init__.py": "0302d333bcd8b33e85aaa6a4e7822e7e1ccfc66343136107fea6f4ca283ba1ec",
"tools/s2_models_v2/attention.py": "3829e34ca30942c863699c4d43fb6fa104452c6c04c4d606c11019d3dde16ffa",
"tools/s2_models_v2/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"tools/s2_models_v2/network.py": "a443470caeecdf529504bffab3ca8cc0b70ce44b505bad60f9c63db9709dbb83",
"tools/s2_precision_pilot_gpu_guard.py": "5cf0f41de34b28394dbee4672b5bd34eb9c4fe2344822aa3d648f20561d96328",
"tools/test_s2_precision_pilot.py": "5aa14d80f0eba7c26f812c08be0e6bbebde7144dbbcdf1356e8dd379f9802334"
}
}
@@ -0,0 +1,11 @@
test_fp32_model_outputs_and_gradients_unchanged_for_both_arms (test_s2_precision_pilot.PrecisionTests) ... ok
test_large_slice_fp16_overflow_removed_and_backward_finite (test_s2_precision_pilot.PrecisionTests) ... ok
test_exact_device_memory_units (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok
test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok
test_missing_or_ambiguous_counters_fail_closed (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok
test_vram_monitor_error_and_whole_host_rss_stop_without_second_run (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok
----------------------------------------------------------------------
Ran 6 tests in 0.950s
OK
@@ -0,0 +1,13 @@
{
"status": "PASS",
"tests_run": 6,
"synthetic_cpu_only": true,
"large_nodes": 100000,
"large_fp16_numerator_inf": true,
"FP32_reference_tolerance": {
"rtol": 0.005,
"atol": 0.005
},
"source_parse": "PASS",
"executed_unix": 1794695189.798009
}
@@ -0,0 +1,15 @@
# S2 编码器诊断与声明的精度修正 v2
原pilot固定5744cc119f638b46c50edf38579575ed5555318f,保持PILOT_INCOMPLETE、无选宽。Claude1794694646.032819已明确收到该固定阶段包,说明依赖MeltAstra stage-audit而未自行重验SHA;勿称其独立复核全部文件。
按Claude1794694544.819729,独立诊断固定f502b7b,session41846已完成PASS:262.806776秒/900秒,GPU设备峰值10,079,961,088字节、无监控异常,账本位于artifacts/s2-encoder-diagnostic-2026-11-15/guards。只加载382学习件,无验证数组/评分,无测试数据,不计作pilot。
十个完整epoch的训练loss及AMP摘要与原运行完全一致;重现epoch10(零起算)step371/第372步,case644(00000644_761a451dfea94bfa93e23cfe_step_000,81,832节点)。失败前参数全部有限。FP16在encoder.blocks.3.Attn.slice_numerator出现1个inf,继续传播;同一状态FP32全部层及最终输出有限,该分子最大绝对值68,573.625,大于FP16上限65,504,分母最大11,802.517578。结果69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41;完整failure-state.pt及逐层trace均保留。Claude1794695133.154989接受诊断并确认限定修正。
v2仅对N/E共用slice聚合分子、分母及归一化禁autocast、使用FP32。原源码包不改,新tools/s2_models_v2保留相同参数结构;blocks字节相同,network只改为引用新blocks,训练/probe副本只换模型导入。优化器、学习率、权重衰减、AMP skip政策、数据/划分、查询、模型宽度与其余计算保持原定义。
6项CPU合成检查PASS:N/E×128/256 FP32输出与梯度逐位等同;100,000节点且聚合分子超过65,504的样例v1半精度分子inf/输出非有限、v2输出有限且以rtol=atol=.005匹配FP32参考,反向梯度有限;新的guard副本在显存/主机内存/监控失败时停止且不进入下一run,设备可见性和计数器解析通过。检查原始输出在artifacts/s2-precision-pilot-implementation-2026-11-15。
新计划根artifacts/s2-pilot-v2-2026-11-15;成功manifest与原493件逐字节相同,没有重复准备或复用训练权重。计划绑定诊断/旧pilot/全部新旧源码SHA,四run从头。E256 probe重新计时并覆盖187,345节点最大件,15GiB VRAM超限报告,不调batch或width;新的E_max/共同cosine由新probe冻结。probe+设置+四run共7200秒,各run≤1800秒,125测试仍封存。
此文件记录启动前固定实现,实际启动、probe、训练和最终状态以新根orchestrator-started、guard、training-contract和complete/failure回执为准,不能把计划写成已完成。仅使用run_s2_precision_pilot.py唯一入口,不重启旧pilot或恢复队列。
+49
View File
@@ -0,0 +1,49 @@
"""Freeze a new pilot after the approved, evidenced forward-precision diagnosis."""
import argparse
import json
from pathlib import Path
from freeze_g1_components import sha
from prepare_s2_recovered_pilot_training import finalize
from s2_e1_supervisor import save
def prepare(original,diagnostic,destination):
result=json.loads((diagnostic/'replay/result.json').read_text())
complete=json.loads((diagnostic/'complete.json').read_text())
guard=json.loads((diagnostic/'replay-guard-receipt.json').read_text())
assert complete['status']=='COMPLETE' and complete['total_wall_seconds']<=900
assert guard['status']=='PASS' and guard['result_sha256']==sha(diagnostic/'replay/result.json')
assert result['decision']=='FP32_FINITE_FP16_NONFINITE' and result['reproduced_original_failure']
assert result['fp32']['finite'] and not result['fp16']['finite']
assert all(r['matches_original_training_summary'] for r in result['prior_epoch_training_summaries'])
assert all(r['finite'] for r in result['parameters'].values())
assert result['validation_scoring'] is False and result['test_data_read'] is False
assert sha(diagnostic/'replay/failure-state.pt')==result['failure_state_sha256']
old=json.loads((original/'training-contract.json').read_text())
for path,digest in old['code_sha256'].items():assert sha(path)==digest,path
old_status=json.loads((original/'pilot-complete.json').read_text())
assert old_status['status']=='PILOT_INCOMPLETE' and old_status['selected_global_width'] is None
manifest=json.loads((original/'success-manifest.json').read_text())
source=Path(old['role_manifest']).parent
plan=finalize(source,destination,old['gpu_uuid'],Path(manifest['preparation_terminal_manifest']))
assert sha(destination/'success-manifest.json')==sha(original/'success-manifest.json')
additions=['tools/prepare_s2_precision_pilot.py','tools/run_s2_precision_pilot.py',
'tools/s2_precision_pilot_gpu_guard.py','tools/run_s2_pilot_probe_v2.py',
'tools/run_s2_pilot_training_v2.py','tools/run_s2_encoder_diagnostic.py']
additions+=sorted(str(p) for p in Path('tools/s2_models_v2').glob('*.py'))
plan['code_sha256'].update({p:sha(p) for p in additions})
plan['authorizing_rulings']+=['1794694544.819729','1794694646.032819']
plan['precision_variant']=dict(version='S2-pilot-v2-slice-FP32',
changed='Only shared N/E slice aggregation numerator, denominator and normalization in FP32 with autocast disabled',
original_pilot=str(original),original_contract_sha256=sha(original/'training-contract.json'),
original_status='PILOT_INCOMPLETE',diagnostic_root=str(diagnostic),
diagnostic_result_sha256=sha(diagnostic/'replay/result.json'),diagnostic_guard_sha256=sha(diagnostic/'replay-guard-receipt.json'),
from_scratch=True,original_weights_reused=False,preparation_repeated=False,
reprobe_max_nodes=187345,VRAM_limit_failure_no_batch_or_width_fallback=True)
save(destination/'pilot-plan.json',plan)
return plan
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('original',type=Path);p.add_argument('diagnostic',type=Path);p.add_argument('destination',type=Path);a=p.parse_args()
prepare(a.original,a.diagnostic,a.destination)
+60
View File
@@ -0,0 +1,60 @@
"""One bounded E256 timing probe, learning geometry and synthetic zero targets only."""
import argparse
from collections import Counter
import json
from pathlib import Path
import time
import traceback
import numpy as np
import torch
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_models_v2.network import QueryModel
from s2_models.normalization import fit_base_standardizer
from s2_pilot_data import permitted_role
from s2_pilot_train import train_part,part_weights,SkipPolicy,common_epoch_limit
from run_s2_pilot_training import LearningBases,read_part_arrays,to_device
def run(root):
started=time.monotonic();dest=root/'timing-probe';dest.mkdir(exist_ok=False)
rows=[];policy=SkipPolicy()
try:
c=json.loads((root/'pilot-plan.json').read_text())
for p,h in c['code_sha256'].items():assert sha(p)==h,p
assert sha(c['success_manifest'])==c['success_manifest_sha256'] and sha(c['role_manifest'])==c['role_manifest_sha256']
roles=json.loads(Path(c['role_manifest']).read_text());records={r['case']:r for r in json.loads(Path(c['success_manifest']).read_text())['ready_parts']}
learning=sorted(p for p in records if permitted_role(roles,p)=='learning');probe=c['probe_cases']
assert len(probe)==len(set(probe))==20 and set(probe)<=set(learning)
torch.set_num_threads(1);torch.manual_seed(0);np.random.seed(0)
if not torch.cuda.is_available() or torch.cuda.device_count()!=1 or 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Exactly one visible V100 required')
device=torch.device('cuda:0');torch.cuda.manual_seed_all(0);torch.cuda.reset_peak_memory_stats();torch.backends.cuda.matmul.allow_tf32=False
stats=fit_base_standardizer(LearningBases(records,learning,roles),learning_part_ids=learning,fold='0',pool='100')
cache={case:read_part_arrays(records[case],roles,synthetic_targets=True) for case in probe}
model=QueryModel('E',256,standardizer=stats).to(device);optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
scaler=torch.amp.GradScaler('cuda');groups=Counter(records[p]['component_id'] for p in learning)
for case in probe:
torch.cuda.synchronize();start=time.monotonic();model.train();data=to_device(cache[case],device)
weights=torch.tensor(part_weights(data['counts'],groups[records[case]['component_id']],len(groups),len(learning)),dtype=torch.float32,device=device)
info={};train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=info)
policy.observe(info['step_skipped']);del data,weights;torch.cuda.synchronize();train_seconds=time.monotonic()-start
start=time.monotonic();model.eval()
with torch.no_grad(),torch.autocast(device_type='cuda',enabled=False):
data=to_device(cache[case],device);prediction=model(data['base'],data['ids'],data['gaps'],data['edge'])
# Include prediction device-to-host cost; discard values, no metrics.
prediction.float().cpu().numpy();del prediction,data
torch.cuda.synchronize();inference_seconds=time.monotonic()-start
rows.append(dict(case=case,nodes=records[case]['nodes'],training_seconds=train_seconds,inference_seconds=inference_seconds,amp=info))
save(dest/'progress.json',dict(rows=rows,amp=policy.summary(),elapsed_seconds=time.monotonic()-started))
nodes=sum(r['nodes'] for r in rows)
timing=common_epoch_limit(sum(r['training_seconds'] for r in rows)/nodes,sum(r['inference_seconds'] for r in rows)/nodes,c['learning_nodes'],c['validation_nodes'])
result=dict(status='COMPLETE',**timing,rows=rows,probe_nodes=nodes,amp=policy.summary(),standardizer_fingerprint=stats.fingerprint,
synthetic_zero_targets=True,labels_read=False,metrics_evaluated=False,model_discarded=True,
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
save(dest/'result.json',result);print(json.dumps({k:v for k,v in result.items() if k!='rows'}),flush=True)
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),amp=policy.summary(),rows=rows,wall_seconds=time.monotonic()-started));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
+150
View File
@@ -0,0 +1,150 @@
"""One fixed pilot arm/width. Must be launched inside the GPU resource guard."""
import argparse
from collections import Counter
from collections.abc import Mapping
import json
from pathlib import Path
import random
import time
import traceback
import numpy as np
import torch
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_models_v2.network import QueryModel,TRAINING_CONTRACT
from s2_models.normalization import BaseStandardizer,fit_base_standardizer
from s2_models.objectives import relative_l2
from s2_pilot_data import permitted_role
from s2_pilot_train import part_weights,train_part,component_score,SkipPolicy
class LearningBases(Mapping):
def __init__(self,records,ids,roles,cache=None):self.records=records;self.ids=tuple(ids);self.roles=roles;self.cache=cache
def __iter__(self):return iter(self.ids)
def __len__(self):return len(self.ids)
def __getitem__(self,case):
if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization')
if self.cache is not None:return self.cache[case]['base']
assert sha(Path(self.records[case]['preparation_root'])/'base.npy')==self.records[case]['input_sha256']['base.npy']
return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r')
def read_part_arrays(record,roles,*,synthetic_targets=False):
case=record['case'];role=permitted_role(roles,case)
if synthetic_targets and role!='learning':raise PermissionError('Timing probe accepts only learning parts')
root=Path(record['preparation_root']);label=Path(record['label_path'])
assert sha(root/'base.npy')==record['input_sha256']['base.npy']
assert sha(root/'queries.npz')==record['input_sha256']['queries.npz']
with np.load(root/'queries.npz',allow_pickle=False) as q:
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask]
if synthetic_targets:
target=np.zeros(len(ids));seconds=None
else:
assert sha(label)==record['input_sha256']['labels.npz']
with np.load(label,allow_pickle=False) as z:
assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity'])
target=z['target_t_star'];seconds=z['seconds']
base=np.load(root/'base.npy',allow_pickle=False)
if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch')
return dict(base=base,ids=ids,gaps=g,edge=edge,target=target,counts=counts,truth_seconds=seconds)
def to_device(data,device):
result={k:v for k,v in data.items() if k not in ['base','ids','gaps','edge','target']}
for key in ['base','ids','gaps','edge','target']:
result[key]=torch.as_tensor(np.array(data[key],copy=True),dtype=torch.long if key=='ids' else torch.float32,device=device)
return result
def load_part(record,roles,device):
return to_device(read_part_arrays(record,roles),device)
def run(root,index):
started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index]
dest=root/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
try:
for p,h in c['code_sha256'].items():assert sha(p)==h,p
source=Path(c['success_manifest']);assert sha(source)==c['success_manifest_sha256']
manifest=json.loads(source.read_text());records={r['case']:r for r in manifest['ready_parts']}
roles=json.loads(Path(c['role_manifest']).read_text());assert sha(c['role_manifest'])==c['role_manifest_sha256']
learning=sorted(p for p in records if permitted_role(roles,p)=='learning')
validation=sorted(p for p in records if permitted_role(roles,p)=='validation')
if not learning or not validation:raise ValueError('Empty successful learning/validation roster')
if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest')
assert run['seed']==0 and c['fold']==0 and c['pool']==100
torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed'])
if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required')
device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32=False
if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100')
# Check hashes once, cache raw FP64 arrays, and fit only the learning subset.
cache={case:read_part_arrays(records[case],roles) for case in sorted(records)}
stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold='0',pool='100')
assert stats.fingerprint==c['standardizer_fingerprint'],'Probe and actual runs must share learning statistics'
save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint))
shared=root/'shared-standardizer-fingerprint.json'
if shared.exists():assert json.loads(shared.read_text())['fingerprint']==stats.fingerprint
else:save(shared,dict(fingerprint=stats.fingerprint,learning_parts=learning))
model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device)
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
epoch_limit=int(c['E_max']);assert 1<=epoch_limit<=200 and c['cosine_T_max']==epoch_limit
assert sha(c['timing_probe_receipt'])==c['timing_probe_sha256']
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=epoch_limit)
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed']))
group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts)
val_groups={p:records[p]['component_id'] for p in validation}
best=float('inf');best_epoch=None;stale=0;history=[];updates=0;attempted_updates=0;skip_policy=SkipPolicy()
for epoch in range(epoch_limit):
model.train();loss_sum=0.;skip_policy.start_epoch()
order=[learning[int(j)] for j in rng.permutation(len(learning))]
for j,case in enumerate(order):
record=records[case];data=to_device(cache[case],device)
weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device)
step_info={}
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=step_info)
loss_sum+=loss/len(learning);attempted_updates+=1;updates+=int(not step_info['step_skipped'])
skip_policy.observe(step_info['step_skipped'])
del data,weights
if j%16==0:
torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
amp_summary=skip_policy.summary();save(dest/f'epoch-{epoch:03d}-amp.json',amp_summary)
skip_policy.finish_epoch(epoch)
model.eval();errors={}
with torch.no_grad():
for case in validation:
data=to_device(cache[case],device)
with torch.autocast(device_type='cuda',enabled=False):
pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None)
predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set']
score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool))
errors[case]=score['value'];del data,pred
value=component_score(errors,val_groups);torch.cuda.synchronize()
row=dict(epoch=epoch,amp=amp_summary,attempted_updates=attempted_updates,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
history.append(row);save(dest/'history.json',history)
if value<best:
best=value;best_epoch=epoch;stale=0
torch.save(dict(state_dict={k:v.detach().cpu() for k,v in model.state_dict().items()},arm=run['arm'],width=run['width'],seed=run['seed'],epoch=epoch,
standardizer=stats.as_dict(),standardizer_fingerprint=stats.fingerprint),dest/'best.pt')
save(dest/'best-validation-parts.json',errors)
else:stale+=1
scheduler.step()
save(dest/'progress.json',dict(stage='epoch_complete',**row,patience_count=stale))
if stale>=20:break
torch.cuda.synchronize()
result=dict(status='COMPLETE',run=run,E_max=epoch_limit,cosine_T_max=epoch_limit,epochs=len(history),attempted_updates=attempted_updates,updates=updates,best_epoch=best_epoch,best_validation_score=best,
learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())),
failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint,
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt'))
save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started,
epoch=locals().get('epoch'),amp=skip_policy.summary() if 'skip_policy' in locals() else None))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
+54
View File
@@ -0,0 +1,54 @@
"""One irreversible attempt: bounded timing probe, frozen E_max, then four runs."""
import argparse
import fcntl
import json
from pathlib import Path
import sys
import time
import traceback
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_precision_pilot_gpu_guard import guarded_worker,run as train_runs
from s2_pilot_train import common_epoch_limit
def freeze_training_contract(root,plan,probe):
assert probe['status']=='COMPLETE' and probe['labels_read'] is False and probe['metrics_evaluated'] is False
assert [r['case'] for r in probe['rows']]==plan['probe_cases']
nodes=sum(r['nodes'] for r in probe['rows']);assert nodes==probe['probe_nodes']
timing=common_epoch_limit(sum(r['training_seconds'] for r in probe['rows'])/nodes,
sum(r['inference_seconds'] for r in probe['rows'])/nodes,plan['learning_nodes'],plan['validation_nodes'])
assert all(probe[k]==v for k,v in timing.items())
c=dict(plan,**timing,timing_probe_receipt=str(root/'timing-probe/result.json'),timing_probe_sha256=sha(root/'timing-probe/result.json'),
timing_probe_guard_sha256=sha(root/'timing-probe-guard-receipt.json'),
probe_guard_wall_seconds=json.loads((root/'timing-probe-guard-receipt.json').read_text())['total_run_wall_seconds'],
standardizer_fingerprint=probe['standardizer_fingerprint'])
with (root/'training-contract.json').open('x') as f:json.dump(c,f,indent=2);f.write('\n')
return c
def run(root):
with (root/'orchestrator.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
assert not (root/'orchestrator-started.json').exists(),'No automatic probe/pilot retry'
plan=json.loads((root/'pilot-plan.json').read_text())
for p,h in plan['code_sha256'].items():assert sha(p)==h,p
start=time.monotonic();save(root/'orchestrator-started.json',dict(started_unix=time.time(),plan_sha256=sha(root/'pilot-plan.json')))
try:
receipt,probe=guarded_worker(root,dict(name='timing-probe'),-1,plan['gpu_uuid'],300,
[sys.executable,'tools/run_s2_pilot_probe_v2.py',str(root)])
if receipt['status']!='PASS' or not probe or probe['status']!='COMPLETE':raise RuntimeError('Timing probe failed; no training, no retry')
coverage=dict(**plan['coverage_boundary'],actual_probe_max_nodes=max(r['nodes'] for r in probe['rows']),probe_receipt_sha256=sha(root/'timing-probe/result.json'),probe_guard_sha256=sha(root/'timing-probe-guard-receipt.json'))
assert coverage['actual_probe_max_nodes']==coverage['max_successful_learning_nodes']
save(root/'timing-probe/coverage-addendum.json',coverage)
freeze_training_contract(root,plan,probe)
train_runs(root,prior_wall_seconds=time.monotonic()-start)
result=json.loads((root/'pilot-complete.json').read_text())
save(root/'orchestrator-complete.json',dict(status=result['status'],total_wall_seconds=time.monotonic()-start,
selected_global_width=result['selected_global_width'],test_labels_read=False,probe_included_in_7200_seconds=True))
except Exception as exc:
save(root/'orchestrator-failure.json',dict(status='FAILED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),total_wall_seconds=time.monotonic()-start));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
+1
View File
@@ -0,0 +1 @@
"""Declared S2 precision variant; original s2_models remains immutable."""
+62
View File
@@ -0,0 +1,62 @@
# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
import torch.nn as nn
import torch
class Physics_Attention_Irregular_Mesh(nn.Module):
## for irregular meshes in 1D, 2D or 3D space
def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64):
super().__init__()
inner_dim = dim_head * heads
self.dim_head = dim_head
self.heads = heads
self.scale = dim_head ** -0.5
self.softmax = nn.Softmax(dim=-1)
self.dropout = nn.Dropout(dropout)
self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
self.in_project_x = nn.Linear(dim, inner_dim)
self.in_project_fx = nn.Linear(dim, inner_dim)
self.in_project_slice = nn.Linear(dim_head, slice_num)
for l in [self.in_project_slice]:
torch.nn.init.orthogonal_(l.weight) # use a principled initialization
self.to_q = nn.Linear(dim_head, dim_head, bias=False)
self.to_k = nn.Linear(dim_head, dim_head, bias=False)
self.to_v = nn.Linear(dim_head, dim_head, bias=False)
self.to_out = nn.Sequential(
nn.Linear(inner_dim, dim),
nn.Dropout(dropout)
)
def forward(self, x):
# B N C
B, N, C = x.shape
### (1) Slice
fx_mid = self.in_project_fx(x).reshape(B, N, self.heads, self.dim_head) \
.permute(0, 2, 1, 3).contiguous() # B H N C
x_mid = self.in_project_x(x).reshape(B, N, self.heads, self.dim_head) \
.permute(0, 2, 1, 3).contiguous() # B H N C
slice_weights = self.softmax(self.in_project_slice(x_mid) / self.temperature) # B H N G
# Declared precision fix: the full-node sum can exceed fp16's range.
# Shared by N/E; projections, attention, deslicing and optimizer unchanged.
with torch.autocast(device_type=x.device.type, enabled=False):
slice_norm = slice_weights.float().sum(2) # B H G
slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid.float(), slice_weights.float())
slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
### (2) Attention among slice tokens
q_slice_token = self.to_q(slice_token)
k_slice_token = self.to_k(slice_token)
v_slice_token = self.to_v(slice_token)
dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
attn = self.softmax(dots)
attn = self.dropout(attn)
out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
### (3) Deslice
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
out_x = out_x.permute(0, 2, 1, 3).reshape(B, N, -1)
return self.to_out(out_x)
+71
View File
@@ -0,0 +1,71 @@
# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
import torch
import torch.nn as nn
from .attention import Physics_Attention_Irregular_Mesh
ACTIVATION = {'gelu': nn.GELU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid, 'relu': nn.ReLU, 'leaky_relu': nn.LeakyReLU(0.1),
'softplus': nn.Softplus, 'ELU': nn.ELU, 'silu': nn.SiLU}
class MLP(nn.Module):
def __init__(self, n_input, n_hidden, n_output, n_layers=1, act='gelu', res=True):
super(MLP, self).__init__()
if act in ACTIVATION.keys():
act = ACTIVATION[act]
else:
raise NotImplementedError
self.n_input = n_input
self.n_hidden = n_hidden
self.n_output = n_output
self.n_layers = n_layers
self.res = res
self.linear_pre = nn.Sequential(nn.Linear(n_input, n_hidden), act())
self.linear_post = nn.Linear(n_hidden, n_output)
self.linears = nn.ModuleList([nn.Sequential(nn.Linear(n_hidden, n_hidden), act()) for _ in range(n_layers)])
def forward(self, x):
x = self.linear_pre(x)
for i in range(self.n_layers):
if self.res:
x = self.linears[i](x) + x
else:
x = self.linears[i](x)
x = self.linear_post(x)
return x
class Transolver_block(nn.Module):
"""Transformer encoder block."""
def __init__(
self,
num_heads: int,
hidden_dim: int,
dropout: float,
act='gelu',
mlp_ratio=4,
last_layer=False,
out_dim=1,
slice_num=32,
):
super().__init__()
self.last_layer = last_layer
self.ln_1 = nn.LayerNorm(hidden_dim)
self.Attn = Physics_Attention_Irregular_Mesh(hidden_dim, heads=num_heads, dim_head=hidden_dim // num_heads,
dropout=dropout, slice_num=slice_num)
self.ln_2 = nn.LayerNorm(hidden_dim)
self.mlp = MLP(hidden_dim, hidden_dim * mlp_ratio, hidden_dim, n_layers=0, res=False, act=act)
if self.last_layer:
self.ln_3 = nn.LayerNorm(hidden_dim)
self.mlp2 = nn.Linear(hidden_dim, out_dim)
def forward(self, fx):
fx = self.Attn(self.ln_1(fx)) + fx
fx = self.mlp(self.ln_2(fx)) + fx
if self.last_layer:
return self.mlp2(self.ln_3(fx))
else:
return fx
+119
View File
@@ -0,0 +1,119 @@
"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
No optimiser, training loop, data reader or GPU invocation is provided.
"""
import torch
from torch import nn
from .blocks import MLP, Transolver_block
from s2_models.features import BASE_COLUMNS
from s2_models.normalization import BaseStandardizer
TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
schedule="cosine", batch_parts=1, max_epochs=200,
patience=20, seeds=[0, 1, 2], widths=[128, 256])
def check_tensor(tensor, shape, name):
if tuple(tensor.shape) != tuple(shape) or not torch.isfinite(tensor).all():
raise ValueError(f"invalid {name}")
class WholePartEncoder(nn.Module):
def __init__(self, width, standardizer):
super().__init__()
if width not in (128, 256):
raise ValueError("S2 permits widths 128 or 256 only")
if not isinstance(standardizer, BaseStandardizer):
raise ValueError("frozen learning-pool standardizer required")
self.standardizer = standardizer
self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
dropout=0., mlp_ratio=1, slice_num=32,
last_layer=False) for _ in range(4)])
self.apply(self._initialize)
self.placeholder = nn.Parameter(torch.rand(width)/width)
@staticmethod
def _initialize(module):
# torch.nn.init replaces timm's same trunc_normal_ defaults; no timm dependency.
if isinstance(module, nn.Linear):
nn.init.trunc_normal_(module.weight, std=.02)
if module.bias is not None:
nn.init.zeros_(module.bias)
elif isinstance(module, nn.LayerNorm):
nn.init.zeros_(module.bias)
nn.init.ones_(module.weight)
def forward(self, base):
if base.ndim != 2 or base.shape[0] == 0:
raise ValueError("one whole part with all tet nodes required")
check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
# Keep normalisation in FP32 even under future mixed-precision training.
base = (base.float()-self.base_mean.float()) / self.base_scale.float()
if not torch.isfinite(base).all():
raise ValueError("nonfinite standardized base input")
fx = self.preprocess(base[None]) + self.placeholder[None, None]
for block in self.blocks:
fx = block(fx)
if not torch.isfinite(fx).all():
raise ValueError("nonfinite encoder output")
return fx[0]
class QueryModel(nn.Module):
def __init__(self, arm, width=128, *, standardizer):
super().__init__()
if arm not in ("N", "E"):
raise ValueError("arm must be N or E")
self.arm, self.width = arm, width
self.encoder = WholePartEncoder(width, standardizer)
query_dim = 2 if arm == "N" else 6
self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
self.decoder.apply(WholePartEncoder._initialize)
def forward(self, base, query_ids, gaps, edge_q=None):
if query_ids.ndim != 1 or query_ids.dtype != torch.long or query_ids.numel() == 0:
raise ValueError("nonempty int64 original-node query IDs required")
if int(query_ids.min()) < 0 or int(query_ids.max()) >= len(base):
raise ValueError("query node outside whole part")
check_tensor(gaps, (len(query_ids), 2), "g12/g23")
if self.arm == "E":
if edge_q is None:
raise ValueError("E requires all four edge summaries")
check_tensor(edge_q, (len(query_ids), 4), "edge summaries")
local = torch.cat([gaps, edge_q], dim=-1)
else:
if edge_q is not None:
raise ValueError("N must not receive edge summaries")
local = gaps
embedding = self.encoder(base)[query_ids]
y = self.decoder(torch.cat([embedding, local], dim=-1)).squeeze(-1)
if not torch.isfinite(y).all():
raise ValueError("nonfinite decoder output")
return y
def parameter_counts(self):
return {"encoder": sum(p.numel() for p in self.encoder.parameters()),
"decoder": sum(p.numel() for p in self.decoder.parameters()),
"total": sum(p.numel() for p in self.parameters())}
def weighted_mse(prediction, target, global_weights):
"""Sum of contributions with already globally-normalised hierarchy weights.
Do not renormalise a single part's weights: that would erase component balance.
"""
if prediction.ndim != 1 or prediction.numel() == 0:
raise ValueError("nonempty query predictions required")
check_tensor(prediction, prediction.shape, "prediction")
check_tensor(target, prediction.shape, "target")
check_tensor(global_weights, prediction.shape, "weights")
if (global_weights <= 0).any():
raise ValueError("positive hierarchy weights required")
# FP32 arithmetic under future AMP; do not form residuals in float16.
return torch.sum(global_weights.float() * (prediction.float()-target.float()).square())
+114
View File
@@ -0,0 +1,114 @@
"""Sequential four-run pilot guard: process-tree RSS/wall plus device VRAM."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import psutil
from freeze_g1_components import sha
from s2_recovery_gpu_supervisor import supervise,save,stop_group,disk_usage,tree_usage
from s2_pilot_train import select_capacity
def parse_device_usage(text):
lines=text.strip().splitlines()
if len(lines)!=1:raise ValueError('Expected exactly one GPU')
fields=lines[0].split(',')
if len(fields)!=2:raise ValueError('Malformed GPU usage row')
memory,util=map(lambda s:int(s.strip()),fields)
if memory<0 or not 0<=util<=100:raise ValueError('Invalid GPU counters')
return memory*2**20,util
def device_usage(uuid):
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=2)
return parse_device_usage(r.stdout)
def active_training_pids(parent,root,index):
ids=[]
try:children=psutil.Process(parent).children(recursive=True)
except psutil.NoSuchProcess:return ids
for p in children:
try:
cmd=p.cmdline()
if str(root) in cmd and (('tools/run_s2_pilot_training_v2.py' in cmd and str(index)==cmd[-1]) or (index==-1 and 'tools/run_s2_pilot_probe_v2.py' in cmd)):ids.append(p.pid)
except psutil.NoSuchProcess:pass
return ids
def guarded_worker(root,spec,i,uuid,wall_limit,command):
run_start=time.monotonic();initial,util=device_usage(uuid)
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
done=threading.Event();monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,max_poll_gap_seconds=0.,reason=None,samples=0)
def watch():
previous=time.monotonic()
while not done.is_set():
now=time.monotonic();monitor['max_poll_gap_seconds']=max(monitor['max_poll_gap_seconds'],now-previous);previous=now
try:
used,_=device_usage(uuid);monitor['samples']+=1;monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
if used>15*2**30:monitor['reason']='DEVICE_VRAM_LIMIT'
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
if rss>16*2**30:monitor['reason']='WHOLE_RUN_HOST_RSS_LIMIT'
if disk_usage(root)>=5*2**30:monitor['reason']='PILOT_DISK_LIMIT'
except Exception as exc:monitor['reason']='GPU_MONITOR_FAILED';monitor['error']=str(exc)
if monitor['reason']:
for pid in active_training_pids(os.getpid(),root,i):stop_group(pid)
done.wait(.5)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
wall=wall_limit-(time.monotonic()-run_start)-2.5
if wall<=0:raise RuntimeError('No budget left before spawn')
r=supervise(command,root/'guards',spec['name'],dict(total_cpu_seconds=7200,job_cpu_seconds=wall_limit,job_wall_seconds=wall,
tree_rss_bytes=16*2**30,disk_bytes=5*2**30,max_processes=2),poll=.1,gpu_uuid=uuid)
finally:
done.set();thread.join(timeout=3)
total_wall=time.monotonic()-run_start
result_path=root/spec['name']/'result.json'
result=json.loads(result_path.read_text()) if result_path.exists() else None
if monitor['reason']:r.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reason'])
if total_wall>wall_limit:r.update(status='STOPPED_OR_FAILED',stop_reason='TOTAL_RUN_WALL_LIMIT')
if result and max(result['peak_allocated_bytes'],result['peak_reserved_bytes'])>15*2**30:r.update(status='STOPPED_OR_FAILED',stop_reason='RECORDED_VRAM_LIMIT')
r.update(gpu_visible=True,gpu_uuid=uuid,device_monitor=monitor,total_run_wall_seconds=total_wall,
result_sha256=sha(result_path) if result else None)
save(root/(spec['name']+'-guard-receipt.json'),r)
return r,result
def run(root,prior_wall_seconds=0.):
with (root/'pilot.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root,prior_wall_seconds)
def run_locked(root,prior_wall_seconds=0.):
c=json.loads((root/'training-contract.json').read_text());uuid=c['gpu_uuid']
prior_wall_seconds=max(prior_wall_seconds,c.get('probe_guard_wall_seconds',0.))
assert [(r['arm'],r['width'],r['seed']) for r in c['runs']]==[('N',128,0),('E',128,0),('N',256,0),('E',256,0)]
for p,h in c['code_sha256'].items():assert sha(p)==h,p
assert not (root/'pilot-started.json').exists(),'No automatic pilot retry'
start=time.monotonic();save(root/'pilot-started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
rows=[];scores={};reason=None
for i,spec in enumerate(c['runs']):
remaining=7200-prior_wall_seconds-(time.monotonic()-start)
if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break
command=[sys.executable,'tools/run_s2_pilot_training_v2.py',str(root),str(i)]
try:r,result=guarded_worker(root,spec,i,uuid,min(1800,remaining),command)
except Exception as exc:reason=str(exc);break
rows.append(dict(run=spec,**r))
if r['status']!='PASS' or not result or result['status']!='COMPLETE':reason='PILOT_RUN_FAILED_NO_RETRY';break
scores[(spec['arm'],spec['width'])]=result['best_validation_score']
if prior_wall_seconds+time.monotonic()-start>7200:reason='TOTAL_PILOT_WALL_LIMIT_AFTER_EXIT'
selected=select_capacity(scores) if len(scores)==4 and reason is None else None
summary=dict(status='FOUR_RUNS_COMPLETE' if selected else 'PILOT_INCOMPLETE',reason=reason,runs=rows,selected_global_width=selected,
selection_rule='mean of N/E validation error at each width, ties smaller; all four runs required',
pilot_wall_seconds=prior_wall_seconds+time.monotonic()-start,probe_and_setup_wall_seconds=prior_wall_seconds,test_labels_read=False,test_predictions=False,retries=0)
save(root/'pilot-complete.json',summary);print(json.dumps(summary),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
+50
View File
@@ -0,0 +1,50 @@
"""Synthetic precision regression checks, without corpus or GPU use."""
import unittest
import torch
from s2_models.network import QueryModel as OriginalModel
from s2_models_v2.network import QueryModel as PrecisionModel
from s2_models.normalization import BaseStandardizer
from s2_models.attention import Physics_Attention_Irregular_Mesh as OriginalAttention
from s2_models_v2.attention import Physics_Attention_Irregular_Mesh as PrecisionAttention
class PrecisionTests(unittest.TestCase):
def setUp(self):
torch.set_num_threads(1);torch.manual_seed(9)
def test_fp32_model_outputs_and_gradients_unchanged_for_both_arms(self):
stats=BaseStandardizer('0','100',('synthetic',),32,(0.,)*10,(1.,)*10)
for arm in ['N','E']:
for width in [128,256]:
with self.subTest(arm=arm,width=width):
original=OriginalModel(arm,width,standardizer=stats)
fixed=PrecisionModel(arm,width,standardizer=stats)
fixed.load_state_dict(original.state_dict())
base=torch.randn(32,10);ids=torch.arange(8);gaps=torch.randn(8,2);edge=torch.randn(8,4) if arm=='E' else None
a=original(base,ids,gaps,edge);b=fixed(base,ids,gaps,edge)
self.assertTrue(torch.equal(a,b));a.square().sum().backward();b.square().sum().backward()
self.assertEqual(original.parameter_counts(),fixed.parameter_counts())
for p,q in zip(original.parameters(),fixed.parameters()):self.assertTrue(torch.equal(p.grad,q.grad))
def test_large_slice_fp16_overflow_removed_and_backward_finite(self):
original=OriginalAttention(8,heads=1,dim_head=8,slice_num=1)
fixed=PrecisionAttention(8,heads=1,dim_head=8,slice_num=1)
with torch.no_grad():
original.in_project_fx.weight.zero_();original.in_project_fx.bias.fill_(8.)
fixed.load_state_dict(original.state_dict())
x=torch.ones(1,100000,8,requires_grad=True)
with torch.no_grad():reference=original(x)
numerator=torch.einsum('bhnc,bhng->bhgc',torch.full((1,1,100000,8),8.,dtype=torch.float16),torch.ones(1,1,100000,1,dtype=torch.float16))
self.assertTrue(bool(torch.isinf(numerator).all()))
with torch.autocast('cpu',dtype=torch.float16):
a=original(x);b=fixed(x)
self.assertFalse(bool(torch.isfinite(a).all()))
self.assertTrue(bool(torch.isfinite(b).all()))
torch.testing.assert_close(b.float(),reference,rtol=5e-3,atol=5e-3)
b.float().square().mean().backward()
self.assertTrue(bool(torch.isfinite(x.grad).all()))
for p in fixed.parameters():
if p.grad is not None:self.assertTrue(bool(torch.isfinite(p.grad).all()))
if __name__=='__main__':unittest.main()