Record reproduced fp16 overflow and freeze isolated FP32-slice pilot v2
This commit is contained in:
1 parent
f502b7b9f8
commit
8a548cc259
28 files changed
+13918
No files matched your search
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"status": "COMPLETE",
|
||||
"decision": "FP32_FINITE_FP16_NONFINITE",
|
||||
"total_wall_seconds": 262.80677626800025,
|
||||
"limit_seconds": 900,
|
||||
"separate_from_pilot": true,
|
||||
"validation_scoring": false,
|
||||
"test_data_read": false
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cpu_seconds_charged": 262.89331899999996,
|
||||
"attempts": [
|
||||
{
|
||||
"name": "replay",
|
||||
"status": "PASS",
|
||||
"cpu_seconds": 262.89331899999996
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"status": "COMPLETE", "diagnostic_only": true, "decision": "FP32_FINITE_FP16_NONFINITE", "reproduced_original_failure": true, "case": "00000644_761a451dfea94bfa93e23cfe_step_000", "nodes": 81832, "epoch": 10, "step_zero_based": 371, "step_one_based": 372, "validation_arrays_read": false, "validation_scoring": false, "test_data_read": false, "pilot_run": false, "standardizer_fingerprint": "93ca89996de50e130d241a674591ad443fdbf334f965d52fdfe9574d10939b03", "wall_seconds": 258.6586924850126, "peak_allocated_bytes": 4850479616, "peak_reserved_bytes": 9663676416, "failure_state_sha256": "a7510d3a5a09dae348ec37c5c78251d2405a6414bb986c4443e03caa4087e200"}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_encoder_diagnostic.py",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15",
|
||||
"--worker"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 262.89331899999996,
|
||||
"wall_seconds": 262.75584313995205,
|
||||
"peak_tree_rss_bytes": 2277941248,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 8192,
|
||||
"max_poll_gap_seconds": 0.5908525820123032,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_encoder_diagnostic.py",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15",
|
||||
"--worker"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 7200,
|
||||
"job_cpu_seconds": 900,
|
||||
"job_wall_seconds": 897.4562088979874,
|
||||
"tree_rss_bytes": 17179869184,
|
||||
"disk_bytes": 5368709120,
|
||||
"max_processes": 2
|
||||
},
|
||||
"started_unix": 1794694802.046334
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_encoder_diagnostic.py",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15",
|
||||
"--worker"
|
||||
],
|
||||
"returncode": 0,
|
||||
"stop_reason": null,
|
||||
"cpu_seconds": 262.89331899999996,
|
||||
"wall_seconds": 262.75584313995205,
|
||||
"peak_tree_rss_bytes": 2277941248,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 8192,
|
||||
"max_poll_gap_seconds": 0.5908525820123032,
|
||||
"poll_target_seconds": 0.1,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": true,
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null,
|
||||
"device_monitor": {
|
||||
"peak_device_memory_bytes": 10079961088,
|
||||
"peak_whole_run_tree_rss_bytes": 2686857216,
|
||||
"max_poll_gap_seconds": 1.009083034005016,
|
||||
"reason": null,
|
||||
"samples": 446
|
||||
},
|
||||
"total_run_wall_seconds": 262.8028003940126,
|
||||
"result_sha256": "69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41"
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"epoch": 10,
|
||||
"step_zero_based": 352,
|
||||
"case": "00000164_91adbf180e114b698160c9a4_step_000",
|
||||
"wall_seconds": 256.8595967130386,
|
||||
"amp": {
|
||||
"attempted_steps": 353,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 353,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
[
|
||||
{
|
||||
"epoch": 0,
|
||||
"training_global_weighted_mse": 0.03763483001738395,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 2,
|
||||
"successful_updates": 380,
|
||||
"skipped_fraction": 0.005235602094240838,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.03763483001738395,
|
||||
"matches_original_training_summary": true
|
||||
},
|
||||
{
|
||||
"epoch": 1,
|
||||
"training_global_weighted_mse": 0.013855781360679905,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 382,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.013855781360679905,
|
||||
"matches_original_training_summary": true
|
||||
},
|
||||
{
|
||||
"epoch": 2,
|
||||
"training_global_weighted_mse": 0.014031890443820595,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 382,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.014031890443820595,
|
||||
"matches_original_training_summary": true
|
||||
},
|
||||
{
|
||||
"epoch": 3,
|
||||
"training_global_weighted_mse": 0.011857566869414183,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 382,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.011857566869414183,
|
||||
"matches_original_training_summary": true
|
||||
},
|
||||
{
|
||||
"epoch": 4,
|
||||
"training_global_weighted_mse": 0.008991813481875563,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 382,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.008991813481875563,
|
||||
"matches_original_training_summary": true
|
||||
},
|
||||
{
|
||||
"epoch": 5,
|
||||
"training_global_weighted_mse": 0.011181466419428661,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 382,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.011181466419428661,
|
||||
"matches_original_training_summary": true
|
||||
},
|
||||
{
|
||||
"epoch": 6,
|
||||
"training_global_weighted_mse": 0.011864404833579386,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 382,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.011864404833579386,
|
||||
"matches_original_training_summary": true
|
||||
},
|
||||
{
|
||||
"epoch": 7,
|
||||
"training_global_weighted_mse": 0.009409040550106135,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 382,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.009409040550106135,
|
||||
"matches_original_training_summary": true
|
||||
},
|
||||
{
|
||||
"epoch": 8,
|
||||
"training_global_weighted_mse": 0.008648868906570734,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 382,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.008648868906570734,
|
||||
"matches_original_training_summary": true
|
||||
},
|
||||
{
|
||||
"epoch": 9,
|
||||
"training_global_weighted_mse": 0.008291036860072012,
|
||||
"amp": {
|
||||
"attempted_steps": 382,
|
||||
"skipped_steps": 0,
|
||||
"successful_updates": 382,
|
||||
"skipped_fraction": 0.0,
|
||||
"consecutive_skipped": 0
|
||||
},
|
||||
"original_training_global_weighted_mse": 0.008291036860072012,
|
||||
"matches_original_training_summary": true
|
||||
}
|
||||
]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"started_unix": 1794694809.765883,
|
||||
"contract_sha256": "718b6e445cf8dde502fd0e59933fe4b1bae83b46fc70b49be9655615d8a0bdd0",
|
||||
"expected_case": "00000644_761a451dfea94bfa93e23cfe_step_000",
|
||||
"expected_epoch": 10,
|
||||
"expected_step_zero_based": 371,
|
||||
"learning_parts": 382,
|
||||
"validation_arrays_read": false,
|
||||
"test_data_read": false
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"started_unix": 1794694802.0002818,
|
||||
"pid": 207587,
|
||||
"contract_sha256": "718b6e445cf8dde502fd0e59933fe4b1bae83b46fc70b49be9655615d8a0bdd0"
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
{
|
||||
"fold": 0,
|
||||
"pool": 100,
|
||||
"seed": 0,
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"code_sha256": {
|
||||
"tools/prepare_s2_recovered_pilot_training.py": "ab5a4274363598ff933ff873df2b00497f069b1fb45756a1d1dba8d723294f68",
|
||||
"tools/run_s2_recovered_pilot.py": "56962009bf5eaaee599ffea6a3d454372e0aea2842fa79eac609fd4a24ecbb39",
|
||||
"tools/run_s2_pilot_probe.py": "2360ab7b44b179144a5ecd2d691e33cd36a6ec4bfeac71452bf062e7f42f81eb",
|
||||
"tools/run_s2_pilot_training.py": "d5e543478543243fe1db3475a4d4cd135fbdcf631d3bce48c4b5af8b2edafb22",
|
||||
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"tools/s2_recovered_pilot_gpu_guard.py": "9a27f47918c865ce37e46b0be54144dc3f9ab1058e4eddc9f6fbc771f4959eb0",
|
||||
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02",
|
||||
"tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97",
|
||||
"tools/s2_incident_recovery_manifest.py": "8cbb2e1067c2cc22762d8ae3cf192aa2c5fa7c7a199a22e7a39a25570ee1c3b1",
|
||||
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"tools/prepare_s2_incident_recovery.py": "4f772573dedc2b7921986e25eaedb14f91d98b235421829c284f3b40b85710aa",
|
||||
"tools/run_s2_recovery_chain.py": "76d0808bdb2d624cfbaadf5a3c5e7e9a0c2be5d5ac41d4ac3fd9ccd48743933c",
|
||||
"tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"tools/prepare_s2_precision_pilot.py": "c3a71b0ff58bcea4433ff93e18ef14d3a00521438a76baaa69830a5ee8ac098a",
|
||||
"tools/run_s2_precision_pilot.py": "ac8347be1ca11f48cb67abbf5e5c78b39af46ed3a7c6f89b49098fd5e77a6904",
|
||||
"tools/s2_precision_pilot_gpu_guard.py": "5cf0f41de34b28394dbee4672b5bd34eb9c4fe2344822aa3d648f20561d96328",
|
||||
"tools/run_s2_pilot_probe_v2.py": "61c08499346d50405d0c254de54deea24b3522256bf1c8b0aec3e9f4ddab6390",
|
||||
"tools/run_s2_pilot_training_v2.py": "f461cb43bc602a32677d7dcfb8a322e206531ad152fa33bb4e36ea8546a1c0f2",
|
||||
"tools/run_s2_encoder_diagnostic.py": "8f31d312fb53bfdb9b1804af064e463d3989f1b33c4aa7279a967ca82dd2c0a4",
|
||||
"tools/s2_models_v2/__init__.py": "0302d333bcd8b33e85aaa6a4e7822e7e1ccfc66343136107fea6f4ca283ba1ec",
|
||||
"tools/s2_models_v2/attention.py": "3829e34ca30942c863699c4d43fb6fa104452c6c04c4d606c11019d3dde16ffa",
|
||||
"tools/s2_models_v2/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"tools/s2_models_v2/network.py": "a443470caeecdf529504bffab3ca8cc0b70ce44b505bad60f9c63db9709dbb83"
|
||||
},
|
||||
"role_manifest": "artifacts/s2-pilot-inputs-2026-11-15/role-manifest.json",
|
||||
"role_manifest_sha256": "a6dce3ebf875dbbe2df349f06e728960140f131d4d55ebd00af03945a46ac35c",
|
||||
"success_manifest": "artifacts/s2-pilot-v2-2026-11-15/success-manifest.json",
|
||||
"success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
|
||||
"probe_cases": [
|
||||
"00001260_a4715d363f88400c8f0238d1_step_003",
|
||||
"00000290_58f8ff46b7564e989f9695b7_step_003",
|
||||
"00001557_16c191aef5034779aaab6e3f_step_000",
|
||||
"00000293_58f8ff46b7564e989f9695b7_step_006",
|
||||
"00001193_46b17dd88ed24f55bca2eb69_step_000",
|
||||
"00000105_52f7124261ae4c558f800775_step_002",
|
||||
"00001592_35f4abaacc9447e4ac0b964f_step_001",
|
||||
"00000140_2fc54fcd110d4f49969163c4_step_003",
|
||||
"00000166_2246cab58dbe4a2fb00fed50_step_000",
|
||||
"00001321_7cef9b79a6b04db894fd74e5_step_000",
|
||||
"00000462_516b1784884d4facbc4c0d74_step_010",
|
||||
"00000470_11951e51b89249dba38ded3e_step_005",
|
||||
"00001495_6c8ef6d2f5564d139a5077f8_step_006",
|
||||
"00001484_a45f6c99e2154f3b94955175_step_068",
|
||||
"00000784_a25fb37392a44447b9684f16_step_009",
|
||||
"00000314_3062bccff48e47a2b9de05e3_step_009",
|
||||
"00001186_507b60b57791425bbb73273e_step_001",
|
||||
"00001569_6fc579f3224d4d53a1b60710_step_000",
|
||||
"00000617_c710d414bbbc4340b682c58a_step_003",
|
||||
"00000666_6f67a59c16b949a4b66dcfe1_step_000"
|
||||
],
|
||||
"probe_selection": "20 equally spaced ranks in successful learning node-count order; includes min/max; tie by case",
|
||||
"learning_nodes": 11863762,
|
||||
"validation_nodes": 3330152,
|
||||
"successful": {
|
||||
"learning": {
|
||||
"parts": 382,
|
||||
"nodes": 11300812
|
||||
},
|
||||
"validation": {
|
||||
"parts": 111,
|
||||
"nodes": 3255106
|
||||
}
|
||||
},
|
||||
"extrapolation": "Train rate * all 387 planned learning nodes + FP32 inference rate * all 112 planned validation nodes; conservative if failed parts dropped",
|
||||
"runs": [
|
||||
{
|
||||
"name": "N128-seed0",
|
||||
"arm": "N",
|
||||
"width": 128,
|
||||
"seed": 0
|
||||
},
|
||||
{
|
||||
"name": "E128-seed0",
|
||||
"arm": "E",
|
||||
"width": 128,
|
||||
"seed": 0
|
||||
},
|
||||
{
|
||||
"name": "N256-seed0",
|
||||
"arm": "N",
|
||||
"width": 256,
|
||||
"seed": 0
|
||||
},
|
||||
{
|
||||
"name": "E256-seed0",
|
||||
"arm": "E",
|
||||
"width": 256,
|
||||
"seed": 0
|
||||
}
|
||||
],
|
||||
"limits": {
|
||||
"total_gpu_wall_seconds": 7200,
|
||||
"probe_wall_seconds": 300,
|
||||
"run_wall_seconds": 1800,
|
||||
"host_rss_bytes": 17179869184,
|
||||
"vram_bytes": 16106127360,
|
||||
"disk_bytes": 5368709120
|
||||
},
|
||||
"coverage_boundary": {
|
||||
"planned_max_learning_nodes": 187345,
|
||||
"max_successful_learning_nodes": 187345,
|
||||
"failed_parts": [
|
||||
{
|
||||
"case": "00000575_5a4cef68211d4706b1ec8586_step_004",
|
||||
"role": "validation",
|
||||
"nodes": 75046,
|
||||
"component_id": "component-3a3e340f4dfac391",
|
||||
"guard": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/s2_pilot_prepare.py",
|
||||
"artifacts/s2-pilot-inputs-2026-11-15",
|
||||
"181"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": "wall_limit",
|
||||
"cpu_seconds": 1799.766116,
|
||||
"wall_seconds": 1800.400393687014,
|
||||
"peak_tree_rss_bytes": 602218496,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 1.128913332009688,
|
||||
"poll_target_seconds": 0.25,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"retry": false
|
||||
},
|
||||
"failure": null
|
||||
},
|
||||
{
|
||||
"case": "00000614_c710d414bbbc4340b682c58a_step_000",
|
||||
"role": "learning",
|
||||
"nodes": 145860,
|
||||
"component_id": "component-4bd089778094e746",
|
||||
"guard": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/s2_pilot_prepare.py",
|
||||
"artifacts/s2-pilot-inputs-2026-11-15",
|
||||
"201"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": "wall_limit",
|
||||
"cpu_seconds": 1799.796751,
|
||||
"wall_seconds": 1800.0567526929663,
|
||||
"peak_tree_rss_bytes": 629305344,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 1.1576595069491304,
|
||||
"poll_target_seconds": 0.25,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"retry": false
|
||||
},
|
||||
"failure": null
|
||||
},
|
||||
{
|
||||
"case": "00000625_6af98f5458b347bb843f0562_step_000",
|
||||
"role": "learning",
|
||||
"nodes": 145707,
|
||||
"component_id": "component-4bd089778094e746",
|
||||
"guard": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/s2_pilot_prepare.py",
|
||||
"artifacts/s2-pilot-inputs-2026-11-15",
|
||||
"211"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": "wall_limit",
|
||||
"cpu_seconds": 1799.8864629999998,
|
||||
"wall_seconds": 1800.119237862993,
|
||||
"peak_tree_rss_bytes": 647880704,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 1.159920622012578,
|
||||
"poll_target_seconds": 0.25,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"retry": false
|
||||
},
|
||||
"failure": null
|
||||
},
|
||||
{
|
||||
"case": "00001466_a45f6c99e2154f3b94955175_step_050",
|
||||
"role": "learning",
|
||||
"nodes": 77138,
|
||||
"component_id": "component-7fe5173e77c86f9d",
|
||||
"guard": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/s2_pilot_prepare.py",
|
||||
"artifacts/s2-pilot-recovery-inputs-2026-11-15",
|
||||
"83"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": "wall_limit",
|
||||
"cpu_seconds": 1799.926194,
|
||||
"wall_seconds": 1800.0714844280155,
|
||||
"peak_tree_rss_bytes": 383070208,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 1.0109735240112059,
|
||||
"poll_target_seconds": 0.25,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
},
|
||||
"failure": null
|
||||
},
|
||||
{
|
||||
"case": "00001470_a45f6c99e2154f3b94955175_step_054",
|
||||
"role": "learning",
|
||||
"nodes": 124817,
|
||||
"component_id": "component-7fe5173e77c86f9d",
|
||||
"guard": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/s2_pilot_prepare.py",
|
||||
"artifacts/s2-pilot-recovery-inputs-2026-11-15",
|
||||
"87"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": "wall_limit",
|
||||
"cpu_seconds": 1799.664371,
|
||||
"wall_seconds": 1800.0443400070071,
|
||||
"peak_tree_rss_bytes": 578732032,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 1.007104828953743,
|
||||
"poll_target_seconds": 0.25,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
},
|
||||
"failure": null
|
||||
},
|
||||
{
|
||||
"case": "00001473_a45f6c99e2154f3b94955175_step_057",
|
||||
"role": "learning",
|
||||
"nodes": 69428,
|
||||
"component_id": "component-7fe5173e77c86f9d",
|
||||
"guard": {
|
||||
"status": "STOPPED_OR_FAILED",
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/s2_pilot_prepare.py",
|
||||
"artifacts/s2-pilot-recovery-inputs-2026-11-15",
|
||||
"90"
|
||||
],
|
||||
"returncode": -9,
|
||||
"stop_reason": "wall_limit",
|
||||
"cpu_seconds": 1799.751432,
|
||||
"wall_seconds": 1800.2498029720155,
|
||||
"peak_tree_rss_bytes": 438853632,
|
||||
"peak_process_count": 1,
|
||||
"output_disk_bytes_before_receipt": 4096,
|
||||
"max_poll_gap_seconds": 0.8929723129840568,
|
||||
"poll_target_seconds": 0.25,
|
||||
"reaped_orphan_cpu_seconds": 0.0,
|
||||
"supervision": "sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure",
|
||||
"gpu_visible": false,
|
||||
"retry": false,
|
||||
"monitor_errors": 0,
|
||||
"last_monitor_error": null
|
||||
},
|
||||
"failure": null
|
||||
}
|
||||
],
|
||||
"statement": "VRAM probe covers only the maximum successful learning part included; not failed/larger unprobed parts"
|
||||
},
|
||||
"authorizing_rulings": [
|
||||
"1794680129.303719",
|
||||
"1794681576.845789",
|
||||
"1794681807.376669",
|
||||
"1794689807.792819",
|
||||
"1794694544.819729",
|
||||
"1794694646.032819"
|
||||
],
|
||||
"test_labels_read": false,
|
||||
"precision_variant": {
|
||||
"version": "S2-pilot-v2-slice-FP32",
|
||||
"changed": "Only shared N/E slice aggregation numerator, denominator and normalization in FP32 with autocast disabled",
|
||||
"original_pilot": "artifacts/s2-pilot-training-2026-11-15",
|
||||
"original_contract_sha256": "f6987431f478275485d18f62240144c2626cab3fbeb75c3e130b68b5d57a1fd2",
|
||||
"original_status": "PILOT_INCOMPLETE",
|
||||
"diagnostic_root": "artifacts/s2-encoder-diagnostic-2026-11-15",
|
||||
"diagnostic_result_sha256": "69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41",
|
||||
"diagnostic_guard_sha256": "ec7b55e2cbca465c29aec84630fc02c9cfd9fed71e98226a90eb5b3fb2b45537",
|
||||
"from_scratch": true,
|
||||
"original_weights_reused": false,
|
||||
"preparation_repeated": false,
|
||||
"reprobe_max_nodes": 187345,
|
||||
"VRAM_limit_failure_no_batch_or_width_fallback": true
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"status": "PRELAUNCH_FIXED",
|
||||
"observed_unix": 1794695249.0009365,
|
||||
"original_source_pins_unchanged": true,
|
||||
"worker_diff_only_model_import": true,
|
||||
"blocks_byte_identical": true,
|
||||
"preparation_repeated": false,
|
||||
"original_pilot_status": "PILOT_INCOMPLETE",
|
||||
"v2_plan_sha256": "5a127d5ddfab366da49475867c9f14f2854f5000a733c7a3211df61bd7acecae",
|
||||
"v2_success_manifest_sha256": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
|
||||
"diagnostic_result_sha256": "69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41",
|
||||
"diagnostic_complete_sha256": "624ccafdaff38e3e14d6438752bcb0545de67787a9f43bedd7cda4acbeeeeee6",
|
||||
"diagnostic_guard_sha256": "ec7b55e2cbca465c29aec84630fc02c9cfd9fed71e98226a90eb5b3fb2b45537",
|
||||
"file_sha256": {
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/complete.json": "624ccafdaff38e3e14d6438752bcb0545de67787a9f43bedd7cda4acbeeeeee6",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/contract.json": "718b6e445cf8dde502fd0e59933fe4b1bae83b46fc70b49be9655615d8a0bdd0",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/guards/ledger.json": "4651375ee58862562af2b119361d62a926405a0985e89bf24a82a542c9de16db",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/guards/replay.log": "faadfb878964bf4d976d4242a0ce4b3389031cd2aac28c5f5976ac2102afa4c6",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/guards/replay.receipt.json": "26de90165173bf65d288aedd1bc895de510f69ac4591d4e1fd9a89a5d99c2048",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/guards/replay.started.json": "9cf48ebc8f195167664040ea027108a2c10ecb2460e369075591ed4c944e41be",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/failure-state.pt": "a7510d3a5a09dae348ec37c5c78251d2405a6414bb986c4443e03caa4087e200",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/progress.json": "79ea82472e504ae5d396f7028732ec1cf7883e96a7f1bd5fea5142df1f09199a",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/replay-history.json": "9742e3d52d6296594f8fe021096424d04828da3f1a58d4815e0d1cea6afe6c8d",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/result.json": "69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/replay/started.json": "ab6ae7d09d6b06bf62290aa1c2e121332778978e47da1d6bfe793e2c296462f5",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/replay-guard-receipt.json": "ec7b55e2cbca465c29aec84630fc02c9cfd9fed71e98226a90eb5b3fb2b45537",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/started.json": "55c03424b933e8b1e9b90f2530232f42ebedcc287364eb9ab5999ba7e6f6ade6",
|
||||
"artifacts/s2-encoder-diagnostic-2026-11-15/synthetic-check.json": "6de76241c39e70f3888e7937ed2b6255274d32f7fe9417ab0cb1c962f052a6c2",
|
||||
"artifacts/s2-pilot-v2-2026-11-15/pilot-plan.json": "5a127d5ddfab366da49475867c9f14f2854f5000a733c7a3211df61bd7acecae",
|
||||
"artifacts/s2-pilot-v2-2026-11-15/success-manifest.json": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3",
|
||||
"artifacts/s2-precision-pilot-implementation-2026-11-15/synthetic-tests.txt": "b4786b068b4fcbd768942a1daa2229f6fe57e3dcc81792a565b61d0f2f7663d7",
|
||||
"artifacts/s2-precision-pilot-implementation-2026-11-15/test-receipt.json": "56ee518976e55e4f265701368e661103fdcd4478f6bd45bf2f908024ac1b8b4d",
|
||||
"tools/prepare_s2_precision_pilot.py": "c3a71b0ff58bcea4433ff93e18ef14d3a00521438a76baaa69830a5ee8ac098a",
|
||||
"tools/run_s2_pilot_probe_v2.py": "61c08499346d50405d0c254de54deea24b3522256bf1c8b0aec3e9f4ddab6390",
|
||||
"tools/run_s2_pilot_training_v2.py": "f461cb43bc602a32677d7dcfb8a322e206531ad152fa33bb4e36ea8546a1c0f2",
|
||||
"tools/run_s2_precision_pilot.py": "ac8347be1ca11f48cb67abbf5e5c78b39af46ed3a7c6f89b49098fd5e77a6904",
|
||||
"tools/s2_models_v2/__init__.py": "0302d333bcd8b33e85aaa6a4e7822e7e1ccfc66343136107fea6f4ca283ba1ec",
|
||||
"tools/s2_models_v2/attention.py": "3829e34ca30942c863699c4d43fb6fa104452c6c04c4d606c11019d3dde16ffa",
|
||||
"tools/s2_models_v2/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"tools/s2_models_v2/network.py": "a443470caeecdf529504bffab3ca8cc0b70ce44b505bad60f9c63db9709dbb83",
|
||||
"tools/s2_precision_pilot_gpu_guard.py": "5cf0f41de34b28394dbee4672b5bd34eb9c4fe2344822aa3d648f20561d96328",
|
||||
"tools/test_s2_precision_pilot.py": "5aa14d80f0eba7c26f812c08be0e6bbebde7144dbbcdf1356e8dd379f9802334"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
test_fp32_model_outputs_and_gradients_unchanged_for_both_arms (test_s2_precision_pilot.PrecisionTests) ... ok
|
||||
test_large_slice_fp16_overflow_removed_and_backward_finite (test_s2_precision_pilot.PrecisionTests) ... ok
|
||||
test_exact_device_memory_units (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok
|
||||
test_explicit_visibility_reaches_worker_and_receipt_without_gpu_use (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok
|
||||
test_missing_or_ambiguous_counters_fail_closed (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok
|
||||
test_vram_monitor_error_and_whole_host_rss_stop_without_second_run (test_s2_pilot_gpu_guard.GPUUsageTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 6 tests in 0.950s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests_run": 6,
|
||||
"synthetic_cpu_only": true,
|
||||
"large_nodes": 100000,
|
||||
"large_fp16_numerator_inf": true,
|
||||
"FP32_reference_tolerance": {
|
||||
"rtol": 0.005,
|
||||
"atol": 0.005
|
||||
},
|
||||
"source_parse": "PASS",
|
||||
"executed_unix": 1794695189.798009
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
# S2 编码器诊断与声明的精度修正 v2
|
||||
|
||||
原pilot固定5744cc119f638b46c50edf38579575ed5555318f,保持PILOT_INCOMPLETE、无选宽。Claude1794694646.032819已明确收到该固定阶段包,说明依赖MeltAstra stage-audit而未自行重验SHA;勿称其独立复核全部文件。
|
||||
|
||||
按Claude1794694544.819729,独立诊断固定f502b7b,session41846已完成PASS:262.806776秒/900秒,GPU设备峰值10,079,961,088字节、无监控异常,账本位于artifacts/s2-encoder-diagnostic-2026-11-15/guards。只加载382学习件,无验证数组/评分,无测试数据,不计作pilot。
|
||||
|
||||
十个完整epoch的训练loss及AMP摘要与原运行完全一致;重现epoch10(零起算)step371/第372步,case644(00000644_761a451dfea94bfa93e23cfe_step_000,81,832节点)。失败前参数全部有限。FP16在encoder.blocks.3.Attn.slice_numerator出现1个inf,继续传播;同一状态FP32全部层及最终输出有限,该分子最大绝对值68,573.625,大于FP16上限65,504,分母最大11,802.517578。结果69132454710b5e063567cd7424a87c23611719147ab45557f205855493ec3e41;完整failure-state.pt及逐层trace均保留。Claude1794695133.154989接受诊断并确认限定修正。
|
||||
|
||||
v2仅对N/E共用slice聚合分子、分母及归一化禁autocast、使用FP32。原源码包不改,新tools/s2_models_v2保留相同参数结构;blocks字节相同,network只改为引用新blocks,训练/probe副本只换模型导入。优化器、学习率、权重衰减、AMP skip政策、数据/划分、查询、模型宽度与其余计算保持原定义。
|
||||
|
||||
6项CPU合成检查PASS:N/E×128/256 FP32输出与梯度逐位等同;100,000节点且聚合分子超过65,504的样例v1半精度分子inf/输出非有限、v2输出有限且以rtol=atol=.005匹配FP32参考,反向梯度有限;新的guard副本在显存/主机内存/监控失败时停止且不进入下一run,设备可见性和计数器解析通过。检查原始输出在artifacts/s2-precision-pilot-implementation-2026-11-15。
|
||||
|
||||
新计划根artifacts/s2-pilot-v2-2026-11-15;成功manifest与原493件逐字节相同,没有重复准备或复用训练权重。计划绑定诊断/旧pilot/全部新旧源码SHA,四run从头。E256 probe重新计时并覆盖187,345节点最大件,15GiB VRAM超限报告,不调batch或width;新的E_max/共同cosine由新probe冻结。probe+设置+四run共7200秒,各run≤1800秒,125测试仍封存。
|
||||
|
||||
此文件记录启动前固定实现,实际启动、probe、训练和最终状态以新根orchestrator-started、guard、training-contract和complete/failure回执为准,不能把计划写成已完成。仅使用run_s2_precision_pilot.py唯一入口,不重启旧pilot或恢复队列。
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Freeze a new pilot after the approved, evidenced forward-precision diagnosis."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from freeze_g1_components import sha
|
||||
from prepare_s2_recovered_pilot_training import finalize
|
||||
from s2_e1_supervisor import save
|
||||
|
||||
|
||||
def prepare(original,diagnostic,destination):
|
||||
result=json.loads((diagnostic/'replay/result.json').read_text())
|
||||
complete=json.loads((diagnostic/'complete.json').read_text())
|
||||
guard=json.loads((diagnostic/'replay-guard-receipt.json').read_text())
|
||||
assert complete['status']=='COMPLETE' and complete['total_wall_seconds']<=900
|
||||
assert guard['status']=='PASS' and guard['result_sha256']==sha(diagnostic/'replay/result.json')
|
||||
assert result['decision']=='FP32_FINITE_FP16_NONFINITE' and result['reproduced_original_failure']
|
||||
assert result['fp32']['finite'] and not result['fp16']['finite']
|
||||
assert all(r['matches_original_training_summary'] for r in result['prior_epoch_training_summaries'])
|
||||
assert all(r['finite'] for r in result['parameters'].values())
|
||||
assert result['validation_scoring'] is False and result['test_data_read'] is False
|
||||
assert sha(diagnostic/'replay/failure-state.pt')==result['failure_state_sha256']
|
||||
old=json.loads((original/'training-contract.json').read_text())
|
||||
for path,digest in old['code_sha256'].items():assert sha(path)==digest,path
|
||||
old_status=json.loads((original/'pilot-complete.json').read_text())
|
||||
assert old_status['status']=='PILOT_INCOMPLETE' and old_status['selected_global_width'] is None
|
||||
manifest=json.loads((original/'success-manifest.json').read_text())
|
||||
source=Path(old['role_manifest']).parent
|
||||
plan=finalize(source,destination,old['gpu_uuid'],Path(manifest['preparation_terminal_manifest']))
|
||||
assert sha(destination/'success-manifest.json')==sha(original/'success-manifest.json')
|
||||
additions=['tools/prepare_s2_precision_pilot.py','tools/run_s2_precision_pilot.py',
|
||||
'tools/s2_precision_pilot_gpu_guard.py','tools/run_s2_pilot_probe_v2.py',
|
||||
'tools/run_s2_pilot_training_v2.py','tools/run_s2_encoder_diagnostic.py']
|
||||
additions+=sorted(str(p) for p in Path('tools/s2_models_v2').glob('*.py'))
|
||||
plan['code_sha256'].update({p:sha(p) for p in additions})
|
||||
plan['authorizing_rulings']+=['1794694544.819729','1794694646.032819']
|
||||
plan['precision_variant']=dict(version='S2-pilot-v2-slice-FP32',
|
||||
changed='Only shared N/E slice aggregation numerator, denominator and normalization in FP32 with autocast disabled',
|
||||
original_pilot=str(original),original_contract_sha256=sha(original/'training-contract.json'),
|
||||
original_status='PILOT_INCOMPLETE',diagnostic_root=str(diagnostic),
|
||||
diagnostic_result_sha256=sha(diagnostic/'replay/result.json'),diagnostic_guard_sha256=sha(diagnostic/'replay-guard-receipt.json'),
|
||||
from_scratch=True,original_weights_reused=False,preparation_repeated=False,
|
||||
reprobe_max_nodes=187345,VRAM_limit_failure_no_batch_or_width_fallback=True)
|
||||
save(destination/'pilot-plan.json',plan)
|
||||
return plan
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('original',type=Path);p.add_argument('diagnostic',type=Path);p.add_argument('destination',type=Path);a=p.parse_args()
|
||||
prepare(a.original,a.diagnostic,a.destination)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""One bounded E256 timing probe, learning geometry and synthetic zero targets only."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
import torch
|
||||
from freeze_g1_components import sha
|
||||
from s2_e1_supervisor import save
|
||||
from s2_models_v2.network import QueryModel
|
||||
from s2_models.normalization import fit_base_standardizer
|
||||
from s2_pilot_data import permitted_role
|
||||
from s2_pilot_train import train_part,part_weights,SkipPolicy,common_epoch_limit
|
||||
from run_s2_pilot_training import LearningBases,read_part_arrays,to_device
|
||||
|
||||
|
||||
def run(root):
|
||||
started=time.monotonic();dest=root/'timing-probe';dest.mkdir(exist_ok=False)
|
||||
rows=[];policy=SkipPolicy()
|
||||
try:
|
||||
c=json.loads((root/'pilot-plan.json').read_text())
|
||||
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
||||
assert sha(c['success_manifest'])==c['success_manifest_sha256'] and sha(c['role_manifest'])==c['role_manifest_sha256']
|
||||
roles=json.loads(Path(c['role_manifest']).read_text());records={r['case']:r for r in json.loads(Path(c['success_manifest']).read_text())['ready_parts']}
|
||||
learning=sorted(p for p in records if permitted_role(roles,p)=='learning');probe=c['probe_cases']
|
||||
assert len(probe)==len(set(probe))==20 and set(probe)<=set(learning)
|
||||
torch.set_num_threads(1);torch.manual_seed(0);np.random.seed(0)
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count()!=1 or 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Exactly one visible V100 required')
|
||||
device=torch.device('cuda:0');torch.cuda.manual_seed_all(0);torch.cuda.reset_peak_memory_stats();torch.backends.cuda.matmul.allow_tf32=False
|
||||
stats=fit_base_standardizer(LearningBases(records,learning,roles),learning_part_ids=learning,fold='0',pool='100')
|
||||
cache={case:read_part_arrays(records[case],roles,synthetic_targets=True) for case in probe}
|
||||
model=QueryModel('E',256,standardizer=stats).to(device);optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
|
||||
scaler=torch.amp.GradScaler('cuda');groups=Counter(records[p]['component_id'] for p in learning)
|
||||
for case in probe:
|
||||
torch.cuda.synchronize();start=time.monotonic();model.train();data=to_device(cache[case],device)
|
||||
weights=torch.tensor(part_weights(data['counts'],groups[records[case]['component_id']],len(groups),len(learning)),dtype=torch.float32,device=device)
|
||||
info={};train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=info)
|
||||
policy.observe(info['step_skipped']);del data,weights;torch.cuda.synchronize();train_seconds=time.monotonic()-start
|
||||
start=time.monotonic();model.eval()
|
||||
with torch.no_grad(),torch.autocast(device_type='cuda',enabled=False):
|
||||
data=to_device(cache[case],device);prediction=model(data['base'],data['ids'],data['gaps'],data['edge'])
|
||||
# Include prediction device-to-host cost; discard values, no metrics.
|
||||
prediction.float().cpu().numpy();del prediction,data
|
||||
torch.cuda.synchronize();inference_seconds=time.monotonic()-start
|
||||
rows.append(dict(case=case,nodes=records[case]['nodes'],training_seconds=train_seconds,inference_seconds=inference_seconds,amp=info))
|
||||
save(dest/'progress.json',dict(rows=rows,amp=policy.summary(),elapsed_seconds=time.monotonic()-started))
|
||||
nodes=sum(r['nodes'] for r in rows)
|
||||
timing=common_epoch_limit(sum(r['training_seconds'] for r in rows)/nodes,sum(r['inference_seconds'] for r in rows)/nodes,c['learning_nodes'],c['validation_nodes'])
|
||||
result=dict(status='COMPLETE',**timing,rows=rows,probe_nodes=nodes,amp=policy.summary(),standardizer_fingerprint=stats.fingerprint,
|
||||
synthetic_zero_targets=True,labels_read=False,metrics_evaluated=False,model_discarded=True,
|
||||
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
||||
save(dest/'result.json',result);print(json.dumps({k:v for k,v in result.items() if k!='rows'}),flush=True)
|
||||
except Exception as exc:
|
||||
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),amp=policy.summary(),rows=rows,wall_seconds=time.monotonic()-started));raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
|
||||
@@ -0,0 +1,150 @@
|
||||
"""One fixed pilot arm/width. Must be launched inside the GPU resource guard."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from collections.abc import Mapping
|
||||
import json
|
||||
from pathlib import Path
|
||||
import random
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
import torch
|
||||
from freeze_g1_components import sha
|
||||
from s2_e1_supervisor import save
|
||||
from s2_models_v2.network import QueryModel,TRAINING_CONTRACT
|
||||
from s2_models.normalization import BaseStandardizer,fit_base_standardizer
|
||||
from s2_models.objectives import relative_l2
|
||||
from s2_pilot_data import permitted_role
|
||||
from s2_pilot_train import part_weights,train_part,component_score,SkipPolicy
|
||||
|
||||
|
||||
class LearningBases(Mapping):
|
||||
def __init__(self,records,ids,roles,cache=None):self.records=records;self.ids=tuple(ids);self.roles=roles;self.cache=cache
|
||||
def __iter__(self):return iter(self.ids)
|
||||
def __len__(self):return len(self.ids)
|
||||
def __getitem__(self,case):
|
||||
if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization')
|
||||
if self.cache is not None:return self.cache[case]['base']
|
||||
assert sha(Path(self.records[case]['preparation_root'])/'base.npy')==self.records[case]['input_sha256']['base.npy']
|
||||
return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r')
|
||||
|
||||
|
||||
def read_part_arrays(record,roles,*,synthetic_targets=False):
|
||||
case=record['case'];role=permitted_role(roles,case)
|
||||
if synthetic_targets and role!='learning':raise PermissionError('Timing probe accepts only learning parts')
|
||||
root=Path(record['preparation_root']);label=Path(record['label_path'])
|
||||
assert sha(root/'base.npy')==record['input_sha256']['base.npy']
|
||||
assert sha(root/'queries.npz')==record['input_sha256']['queries.npz']
|
||||
with np.load(root/'queries.npz',allow_pickle=False) as q:
|
||||
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask]
|
||||
if synthetic_targets:
|
||||
target=np.zeros(len(ids));seconds=None
|
||||
else:
|
||||
assert sha(label)==record['input_sha256']['labels.npz']
|
||||
with np.load(label,allow_pickle=False) as z:
|
||||
assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity'])
|
||||
target=z['target_t_star'];seconds=z['seconds']
|
||||
base=np.load(root/'base.npy',allow_pickle=False)
|
||||
if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch')
|
||||
return dict(base=base,ids=ids,gaps=g,edge=edge,target=target,counts=counts,truth_seconds=seconds)
|
||||
|
||||
|
||||
def to_device(data,device):
|
||||
result={k:v for k,v in data.items() if k not in ['base','ids','gaps','edge','target']}
|
||||
for key in ['base','ids','gaps','edge','target']:
|
||||
result[key]=torch.as_tensor(np.array(data[key],copy=True),dtype=torch.long if key=='ids' else torch.float32,device=device)
|
||||
return result
|
||||
|
||||
|
||||
def load_part(record,roles,device):
|
||||
return to_device(read_part_arrays(record,roles),device)
|
||||
|
||||
|
||||
def run(root,index):
|
||||
started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index]
|
||||
dest=root/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
||||
try:
|
||||
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
||||
source=Path(c['success_manifest']);assert sha(source)==c['success_manifest_sha256']
|
||||
manifest=json.loads(source.read_text());records={r['case']:r for r in manifest['ready_parts']}
|
||||
roles=json.loads(Path(c['role_manifest']).read_text());assert sha(c['role_manifest'])==c['role_manifest_sha256']
|
||||
learning=sorted(p for p in records if permitted_role(roles,p)=='learning')
|
||||
validation=sorted(p for p in records if permitted_role(roles,p)=='validation')
|
||||
if not learning or not validation:raise ValueError('Empty successful learning/validation roster')
|
||||
if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest')
|
||||
assert run['seed']==0 and c['fold']==0 and c['pool']==100
|
||||
torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed'])
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required')
|
||||
device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats()
|
||||
torch.backends.cuda.matmul.allow_tf32=False
|
||||
if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100')
|
||||
# Check hashes once, cache raw FP64 arrays, and fit only the learning subset.
|
||||
cache={case:read_part_arrays(records[case],roles) for case in sorted(records)}
|
||||
stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold='0',pool='100')
|
||||
assert stats.fingerprint==c['standardizer_fingerprint'],'Probe and actual runs must share learning statistics'
|
||||
save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint))
|
||||
shared=root/'shared-standardizer-fingerprint.json'
|
||||
if shared.exists():assert json.loads(shared.read_text())['fingerprint']==stats.fingerprint
|
||||
else:save(shared,dict(fingerprint=stats.fingerprint,learning_parts=learning))
|
||||
model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device)
|
||||
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
|
||||
epoch_limit=int(c['E_max']);assert 1<=epoch_limit<=200 and c['cosine_T_max']==epoch_limit
|
||||
assert sha(c['timing_probe_receipt'])==c['timing_probe_sha256']
|
||||
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=epoch_limit)
|
||||
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed']))
|
||||
group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts)
|
||||
val_groups={p:records[p]['component_id'] for p in validation}
|
||||
best=float('inf');best_epoch=None;stale=0;history=[];updates=0;attempted_updates=0;skip_policy=SkipPolicy()
|
||||
for epoch in range(epoch_limit):
|
||||
model.train();loss_sum=0.;skip_policy.start_epoch()
|
||||
order=[learning[int(j)] for j in rng.permutation(len(learning))]
|
||||
for j,case in enumerate(order):
|
||||
record=records[case];data=to_device(cache[case],device)
|
||||
weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device)
|
||||
step_info={}
|
||||
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=step_info)
|
||||
loss_sum+=loss/len(learning);attempted_updates+=1;updates+=int(not step_info['step_skipped'])
|
||||
skip_policy.observe(step_info['step_skipped'])
|
||||
del data,weights
|
||||
if j%16==0:
|
||||
torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started,
|
||||
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
|
||||
amp_summary=skip_policy.summary();save(dest/f'epoch-{epoch:03d}-amp.json',amp_summary)
|
||||
skip_policy.finish_epoch(epoch)
|
||||
model.eval();errors={}
|
||||
with torch.no_grad():
|
||||
for case in validation:
|
||||
data=to_device(cache[case],device)
|
||||
with torch.autocast(device_type='cuda',enabled=False):
|
||||
pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None)
|
||||
predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set']
|
||||
score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool))
|
||||
errors[case]=score['value'];del data,pred
|
||||
value=component_score(errors,val_groups);torch.cuda.synchronize()
|
||||
row=dict(epoch=epoch,amp=amp_summary,attempted_updates=attempted_updates,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started,
|
||||
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
||||
history.append(row);save(dest/'history.json',history)
|
||||
if value<best:
|
||||
best=value;best_epoch=epoch;stale=0
|
||||
torch.save(dict(state_dict={k:v.detach().cpu() for k,v in model.state_dict().items()},arm=run['arm'],width=run['width'],seed=run['seed'],epoch=epoch,
|
||||
standardizer=stats.as_dict(),standardizer_fingerprint=stats.fingerprint),dest/'best.pt')
|
||||
save(dest/'best-validation-parts.json',errors)
|
||||
else:stale+=1
|
||||
scheduler.step()
|
||||
save(dest/'progress.json',dict(stage='epoch_complete',**row,patience_count=stale))
|
||||
if stale>=20:break
|
||||
torch.cuda.synchronize()
|
||||
result=dict(status='COMPLETE',run=run,E_max=epoch_limit,cosine_T_max=epoch_limit,epochs=len(history),attempted_updates=attempted_updates,updates=updates,best_epoch=best_epoch,best_validation_score=best,
|
||||
learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())),
|
||||
failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint,
|
||||
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
|
||||
test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt'))
|
||||
save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0
|
||||
except Exception as exc:
|
||||
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started,
|
||||
epoch=locals().get('epoch'),amp=skip_policy.summary() if 'skip_policy' in locals() else None))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
|
||||
@@ -0,0 +1,54 @@
|
||||
"""One irreversible attempt: bounded timing probe, frozen E_max, then four runs."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from freeze_g1_components import sha
|
||||
from s2_e1_supervisor import save
|
||||
from s2_precision_pilot_gpu_guard import guarded_worker,run as train_runs
|
||||
from s2_pilot_train import common_epoch_limit
|
||||
|
||||
|
||||
def freeze_training_contract(root,plan,probe):
|
||||
assert probe['status']=='COMPLETE' and probe['labels_read'] is False and probe['metrics_evaluated'] is False
|
||||
assert [r['case'] for r in probe['rows']]==plan['probe_cases']
|
||||
nodes=sum(r['nodes'] for r in probe['rows']);assert nodes==probe['probe_nodes']
|
||||
timing=common_epoch_limit(sum(r['training_seconds'] for r in probe['rows'])/nodes,
|
||||
sum(r['inference_seconds'] for r in probe['rows'])/nodes,plan['learning_nodes'],plan['validation_nodes'])
|
||||
assert all(probe[k]==v for k,v in timing.items())
|
||||
c=dict(plan,**timing,timing_probe_receipt=str(root/'timing-probe/result.json'),timing_probe_sha256=sha(root/'timing-probe/result.json'),
|
||||
timing_probe_guard_sha256=sha(root/'timing-probe-guard-receipt.json'),
|
||||
probe_guard_wall_seconds=json.loads((root/'timing-probe-guard-receipt.json').read_text())['total_run_wall_seconds'],
|
||||
standardizer_fingerprint=probe['standardizer_fingerprint'])
|
||||
with (root/'training-contract.json').open('x') as f:json.dump(c,f,indent=2);f.write('\n')
|
||||
return c
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'orchestrator.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
assert not (root/'orchestrator-started.json').exists(),'No automatic probe/pilot retry'
|
||||
plan=json.loads((root/'pilot-plan.json').read_text())
|
||||
for p,h in plan['code_sha256'].items():assert sha(p)==h,p
|
||||
start=time.monotonic();save(root/'orchestrator-started.json',dict(started_unix=time.time(),plan_sha256=sha(root/'pilot-plan.json')))
|
||||
try:
|
||||
receipt,probe=guarded_worker(root,dict(name='timing-probe'),-1,plan['gpu_uuid'],300,
|
||||
[sys.executable,'tools/run_s2_pilot_probe_v2.py',str(root)])
|
||||
if receipt['status']!='PASS' or not probe or probe['status']!='COMPLETE':raise RuntimeError('Timing probe failed; no training, no retry')
|
||||
coverage=dict(**plan['coverage_boundary'],actual_probe_max_nodes=max(r['nodes'] for r in probe['rows']),probe_receipt_sha256=sha(root/'timing-probe/result.json'),probe_guard_sha256=sha(root/'timing-probe-guard-receipt.json'))
|
||||
assert coverage['actual_probe_max_nodes']==coverage['max_successful_learning_nodes']
|
||||
save(root/'timing-probe/coverage-addendum.json',coverage)
|
||||
freeze_training_contract(root,plan,probe)
|
||||
train_runs(root,prior_wall_seconds=time.monotonic()-start)
|
||||
result=json.loads((root/'pilot-complete.json').read_text())
|
||||
save(root/'orchestrator-complete.json',dict(status=result['status'],total_wall_seconds=time.monotonic()-start,
|
||||
selected_global_width=result['selected_global_width'],test_labels_read=False,probe_included_in_7200_seconds=True))
|
||||
except Exception as exc:
|
||||
save(root/'orchestrator-failure.json',dict(status='FAILED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),total_wall_seconds=time.monotonic()-start));raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
|
||||
@@ -0,0 +1 @@
|
||||
"""Declared S2 precision variant; original s2_models remains immutable."""
|
||||
@@ -0,0 +1,62 @@
|
||||
# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
|
||||
import torch.nn as nn
|
||||
import torch
|
||||
|
||||
|
||||
class Physics_Attention_Irregular_Mesh(nn.Module):
|
||||
## for irregular meshes in 1D, 2D or 3D space
|
||||
def __init__(self, dim, heads=8, dim_head=64, dropout=0., slice_num=64):
|
||||
super().__init__()
|
||||
inner_dim = dim_head * heads
|
||||
self.dim_head = dim_head
|
||||
self.heads = heads
|
||||
self.scale = dim_head ** -0.5
|
||||
self.softmax = nn.Softmax(dim=-1)
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.temperature = nn.Parameter(torch.ones([1, heads, 1, 1]) * 0.5)
|
||||
|
||||
self.in_project_x = nn.Linear(dim, inner_dim)
|
||||
self.in_project_fx = nn.Linear(dim, inner_dim)
|
||||
self.in_project_slice = nn.Linear(dim_head, slice_num)
|
||||
for l in [self.in_project_slice]:
|
||||
torch.nn.init.orthogonal_(l.weight) # use a principled initialization
|
||||
self.to_q = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_k = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_v = nn.Linear(dim_head, dim_head, bias=False)
|
||||
self.to_out = nn.Sequential(
|
||||
nn.Linear(inner_dim, dim),
|
||||
nn.Dropout(dropout)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
# B N C
|
||||
B, N, C = x.shape
|
||||
|
||||
### (1) Slice
|
||||
fx_mid = self.in_project_fx(x).reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
x_mid = self.in_project_x(x).reshape(B, N, self.heads, self.dim_head) \
|
||||
.permute(0, 2, 1, 3).contiguous() # B H N C
|
||||
slice_weights = self.softmax(self.in_project_slice(x_mid) / self.temperature) # B H N G
|
||||
# Declared precision fix: the full-node sum can exceed fp16's range.
|
||||
# Shared by N/E; projections, attention, deslicing and optimizer unchanged.
|
||||
with torch.autocast(device_type=x.device.type, enabled=False):
|
||||
slice_norm = slice_weights.float().sum(2) # B H G
|
||||
slice_token = torch.einsum("bhnc,bhng->bhgc", fx_mid.float(), slice_weights.float())
|
||||
slice_token = slice_token / ((slice_norm + 1e-5)[:, :, :, None].repeat(1, 1, 1, self.dim_head))
|
||||
|
||||
### (2) Attention among slice tokens
|
||||
q_slice_token = self.to_q(slice_token)
|
||||
k_slice_token = self.to_k(slice_token)
|
||||
v_slice_token = self.to_v(slice_token)
|
||||
dots = torch.matmul(q_slice_token, k_slice_token.transpose(-1, -2)) * self.scale
|
||||
attn = self.softmax(dots)
|
||||
attn = self.dropout(attn)
|
||||
out_slice_token = torch.matmul(attn, v_slice_token) # B H G D
|
||||
|
||||
### (3) Deslice
|
||||
out_x = torch.einsum("bhgc,bhng->bhnc", out_slice_token, slice_weights)
|
||||
out_x = out_x.permute(0, 2, 1, 3).reshape(B, N, -1)
|
||||
return self.to_out(out_x)
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# Adapted from Transolver 75e0f676; MIT, see upstream/LICENSE.
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from .attention import Physics_Attention_Irregular_Mesh
|
||||
|
||||
ACTIVATION = {'gelu': nn.GELU, 'tanh': nn.Tanh, 'sigmoid': nn.Sigmoid, 'relu': nn.ReLU, 'leaky_relu': nn.LeakyReLU(0.1),
|
||||
'softplus': nn.Softplus, 'ELU': nn.ELU, 'silu': nn.SiLU}
|
||||
|
||||
|
||||
class MLP(nn.Module):
|
||||
def __init__(self, n_input, n_hidden, n_output, n_layers=1, act='gelu', res=True):
|
||||
super(MLP, self).__init__()
|
||||
|
||||
if act in ACTIVATION.keys():
|
||||
act = ACTIVATION[act]
|
||||
else:
|
||||
raise NotImplementedError
|
||||
self.n_input = n_input
|
||||
self.n_hidden = n_hidden
|
||||
self.n_output = n_output
|
||||
self.n_layers = n_layers
|
||||
self.res = res
|
||||
self.linear_pre = nn.Sequential(nn.Linear(n_input, n_hidden), act())
|
||||
self.linear_post = nn.Linear(n_hidden, n_output)
|
||||
self.linears = nn.ModuleList([nn.Sequential(nn.Linear(n_hidden, n_hidden), act()) for _ in range(n_layers)])
|
||||
|
||||
def forward(self, x):
|
||||
x = self.linear_pre(x)
|
||||
for i in range(self.n_layers):
|
||||
if self.res:
|
||||
x = self.linears[i](x) + x
|
||||
else:
|
||||
x = self.linears[i](x)
|
||||
x = self.linear_post(x)
|
||||
return x
|
||||
|
||||
|
||||
class Transolver_block(nn.Module):
|
||||
"""Transformer encoder block."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
hidden_dim: int,
|
||||
dropout: float,
|
||||
act='gelu',
|
||||
mlp_ratio=4,
|
||||
last_layer=False,
|
||||
out_dim=1,
|
||||
slice_num=32,
|
||||
):
|
||||
super().__init__()
|
||||
self.last_layer = last_layer
|
||||
self.ln_1 = nn.LayerNorm(hidden_dim)
|
||||
self.Attn = Physics_Attention_Irregular_Mesh(hidden_dim, heads=num_heads, dim_head=hidden_dim // num_heads,
|
||||
dropout=dropout, slice_num=slice_num)
|
||||
self.ln_2 = nn.LayerNorm(hidden_dim)
|
||||
self.mlp = MLP(hidden_dim, hidden_dim * mlp_ratio, hidden_dim, n_layers=0, res=False, act=act)
|
||||
if self.last_layer:
|
||||
self.ln_3 = nn.LayerNorm(hidden_dim)
|
||||
self.mlp2 = nn.Linear(hidden_dim, out_dim)
|
||||
|
||||
def forward(self, fx):
|
||||
fx = self.Attn(self.ln_1(fx)) + fx
|
||||
fx = self.mlp(self.ln_2(fx)) + fx
|
||||
if self.last_layer:
|
||||
return self.mlp2(self.ln_3(fx))
|
||||
else:
|
||||
return fx
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Transolver 75e0f676 encoder adaptation + S2 query-only N/E decoders.
|
||||
|
||||
No optimiser, training loop, data reader or GPU invocation is provided.
|
||||
"""
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from .blocks import MLP, Transolver_block
|
||||
from s2_models.features import BASE_COLUMNS
|
||||
from s2_models.normalization import BaseStandardizer
|
||||
|
||||
|
||||
TRAINING_CONTRACT = dict(optimizer="AdamW", learning_rate=1e-3, weight_decay=1e-5,
|
||||
schedule="cosine", batch_parts=1, max_epochs=200,
|
||||
patience=20, seeds=[0, 1, 2], widths=[128, 256])
|
||||
|
||||
|
||||
def check_tensor(tensor, shape, name):
|
||||
if tuple(tensor.shape) != tuple(shape) or not torch.isfinite(tensor).all():
|
||||
raise ValueError(f"invalid {name}")
|
||||
|
||||
|
||||
class WholePartEncoder(nn.Module):
|
||||
def __init__(self, width, standardizer):
|
||||
super().__init__()
|
||||
if width not in (128, 256):
|
||||
raise ValueError("S2 permits widths 128 or 256 only")
|
||||
if not isinstance(standardizer, BaseStandardizer):
|
||||
raise ValueError("frozen learning-pool standardizer required")
|
||||
self.standardizer = standardizer
|
||||
self.register_buffer("base_mean", torch.tensor(standardizer.mean, dtype=torch.float64))
|
||||
self.register_buffer("base_scale", torch.tensor(standardizer.scale, dtype=torch.float64))
|
||||
self.preprocess = MLP(len(BASE_COLUMNS), width*2, width, n_layers=0, res=False)
|
||||
self.blocks = nn.ModuleList([Transolver_block(num_heads=8, hidden_dim=width,
|
||||
dropout=0., mlp_ratio=1, slice_num=32,
|
||||
last_layer=False) for _ in range(4)])
|
||||
self.apply(self._initialize)
|
||||
self.placeholder = nn.Parameter(torch.rand(width)/width)
|
||||
|
||||
@staticmethod
|
||||
def _initialize(module):
|
||||
# torch.nn.init replaces timm's same trunc_normal_ defaults; no timm dependency.
|
||||
if isinstance(module, nn.Linear):
|
||||
nn.init.trunc_normal_(module.weight, std=.02)
|
||||
if module.bias is not None:
|
||||
nn.init.zeros_(module.bias)
|
||||
elif isinstance(module, nn.LayerNorm):
|
||||
nn.init.zeros_(module.bias)
|
||||
nn.init.ones_(module.weight)
|
||||
|
||||
def forward(self, base):
|
||||
if base.ndim != 2 or base.shape[0] == 0:
|
||||
raise ValueError("one whole part with all tet nodes required")
|
||||
check_tensor(base, (base.shape[0], len(BASE_COLUMNS)), "base inputs")
|
||||
# Keep normalisation in FP32 even under future mixed-precision training.
|
||||
base = (base.float()-self.base_mean.float()) / self.base_scale.float()
|
||||
if not torch.isfinite(base).all():
|
||||
raise ValueError("nonfinite standardized base input")
|
||||
fx = self.preprocess(base[None]) + self.placeholder[None, None]
|
||||
for block in self.blocks:
|
||||
fx = block(fx)
|
||||
if not torch.isfinite(fx).all():
|
||||
raise ValueError("nonfinite encoder output")
|
||||
return fx[0]
|
||||
|
||||
|
||||
class QueryModel(nn.Module):
|
||||
def __init__(self, arm, width=128, *, standardizer):
|
||||
super().__init__()
|
||||
if arm not in ("N", "E"):
|
||||
raise ValueError("arm must be N or E")
|
||||
self.arm, self.width = arm, width
|
||||
self.encoder = WholePartEncoder(width, standardizer)
|
||||
query_dim = 2 if arm == "N" else 6
|
||||
self.decoder = nn.Sequential(nn.Linear(width+query_dim, width), nn.GELU(),
|
||||
nn.Linear(width, width), nn.GELU(), nn.Linear(width, 1))
|
||||
self.decoder.apply(WholePartEncoder._initialize)
|
||||
|
||||
def forward(self, base, query_ids, gaps, edge_q=None):
|
||||
if query_ids.ndim != 1 or query_ids.dtype != torch.long or query_ids.numel() == 0:
|
||||
raise ValueError("nonempty int64 original-node query IDs required")
|
||||
if int(query_ids.min()) < 0 or int(query_ids.max()) >= len(base):
|
||||
raise ValueError("query node outside whole part")
|
||||
check_tensor(gaps, (len(query_ids), 2), "g12/g23")
|
||||
if self.arm == "E":
|
||||
if edge_q is None:
|
||||
raise ValueError("E requires all four edge summaries")
|
||||
check_tensor(edge_q, (len(query_ids), 4), "edge summaries")
|
||||
local = torch.cat([gaps, edge_q], dim=-1)
|
||||
else:
|
||||
if edge_q is not None:
|
||||
raise ValueError("N must not receive edge summaries")
|
||||
local = gaps
|
||||
embedding = self.encoder(base)[query_ids]
|
||||
y = self.decoder(torch.cat([embedding, local], dim=-1)).squeeze(-1)
|
||||
if not torch.isfinite(y).all():
|
||||
raise ValueError("nonfinite decoder output")
|
||||
return y
|
||||
|
||||
def parameter_counts(self):
|
||||
return {"encoder": sum(p.numel() for p in self.encoder.parameters()),
|
||||
"decoder": sum(p.numel() for p in self.decoder.parameters()),
|
||||
"total": sum(p.numel() for p in self.parameters())}
|
||||
|
||||
|
||||
def weighted_mse(prediction, target, global_weights):
|
||||
"""Sum of contributions with already globally-normalised hierarchy weights.
|
||||
|
||||
Do not renormalise a single part's weights: that would erase component balance.
|
||||
"""
|
||||
if prediction.ndim != 1 or prediction.numel() == 0:
|
||||
raise ValueError("nonempty query predictions required")
|
||||
check_tensor(prediction, prediction.shape, "prediction")
|
||||
check_tensor(target, prediction.shape, "target")
|
||||
check_tensor(global_weights, prediction.shape, "weights")
|
||||
if (global_weights <= 0).any():
|
||||
raise ValueError("positive hierarchy weights required")
|
||||
# FP32 arithmetic under future AMP; do not form residuals in float16.
|
||||
return torch.sum(global_weights.float() * (prediction.float()-target.float()).square())
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Sequential four-run pilot guard: process-tree RSS/wall plus device VRAM."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_gpu_supervisor import supervise,save,stop_group,disk_usage,tree_usage
|
||||
from s2_pilot_train import select_capacity
|
||||
|
||||
|
||||
def parse_device_usage(text):
|
||||
lines=text.strip().splitlines()
|
||||
if len(lines)!=1:raise ValueError('Expected exactly one GPU')
|
||||
fields=lines[0].split(',')
|
||||
if len(fields)!=2:raise ValueError('Malformed GPU usage row')
|
||||
memory,util=map(lambda s:int(s.strip()),fields)
|
||||
if memory<0 or not 0<=util<=100:raise ValueError('Invalid GPU counters')
|
||||
return memory*2**20,util
|
||||
|
||||
|
||||
def device_usage(uuid):
|
||||
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=2)
|
||||
return parse_device_usage(r.stdout)
|
||||
|
||||
|
||||
def active_training_pids(parent,root,index):
|
||||
ids=[]
|
||||
try:children=psutil.Process(parent).children(recursive=True)
|
||||
except psutil.NoSuchProcess:return ids
|
||||
for p in children:
|
||||
try:
|
||||
cmd=p.cmdline()
|
||||
if str(root) in cmd and (('tools/run_s2_pilot_training_v2.py' in cmd and str(index)==cmd[-1]) or (index==-1 and 'tools/run_s2_pilot_probe_v2.py' in cmd)):ids.append(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
return ids
|
||||
|
||||
|
||||
def guarded_worker(root,spec,i,uuid,wall_limit,command):
|
||||
run_start=time.monotonic();initial,util=device_usage(uuid)
|
||||
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
|
||||
done=threading.Event();monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,max_poll_gap_seconds=0.,reason=None,samples=0)
|
||||
def watch():
|
||||
previous=time.monotonic()
|
||||
while not done.is_set():
|
||||
now=time.monotonic();monitor['max_poll_gap_seconds']=max(monitor['max_poll_gap_seconds'],now-previous);previous=now
|
||||
try:
|
||||
used,_=device_usage(uuid);monitor['samples']+=1;monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
|
||||
if used>15*2**30:monitor['reason']='DEVICE_VRAM_LIMIT'
|
||||
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
|
||||
if rss>16*2**30:monitor['reason']='WHOLE_RUN_HOST_RSS_LIMIT'
|
||||
if disk_usage(root)>=5*2**30:monitor['reason']='PILOT_DISK_LIMIT'
|
||||
except Exception as exc:monitor['reason']='GPU_MONITOR_FAILED';monitor['error']=str(exc)
|
||||
if monitor['reason']:
|
||||
for pid in active_training_pids(os.getpid(),root,i):stop_group(pid)
|
||||
done.wait(.5)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
wall=wall_limit-(time.monotonic()-run_start)-2.5
|
||||
if wall<=0:raise RuntimeError('No budget left before spawn')
|
||||
r=supervise(command,root/'guards',spec['name'],dict(total_cpu_seconds=7200,job_cpu_seconds=wall_limit,job_wall_seconds=wall,
|
||||
tree_rss_bytes=16*2**30,disk_bytes=5*2**30,max_processes=2),poll=.1,gpu_uuid=uuid)
|
||||
finally:
|
||||
done.set();thread.join(timeout=3)
|
||||
total_wall=time.monotonic()-run_start
|
||||
result_path=root/spec['name']/'result.json'
|
||||
result=json.loads(result_path.read_text()) if result_path.exists() else None
|
||||
if monitor['reason']:r.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reason'])
|
||||
if total_wall>wall_limit:r.update(status='STOPPED_OR_FAILED',stop_reason='TOTAL_RUN_WALL_LIMIT')
|
||||
if result and max(result['peak_allocated_bytes'],result['peak_reserved_bytes'])>15*2**30:r.update(status='STOPPED_OR_FAILED',stop_reason='RECORDED_VRAM_LIMIT')
|
||||
r.update(gpu_visible=True,gpu_uuid=uuid,device_monitor=monitor,total_run_wall_seconds=total_wall,
|
||||
result_sha256=sha(result_path) if result else None)
|
||||
save(root/(spec['name']+'-guard-receipt.json'),r)
|
||||
return r,result
|
||||
|
||||
|
||||
def run(root,prior_wall_seconds=0.):
|
||||
with (root/'pilot.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
return run_locked(root,prior_wall_seconds)
|
||||
|
||||
|
||||
def run_locked(root,prior_wall_seconds=0.):
|
||||
c=json.loads((root/'training-contract.json').read_text());uuid=c['gpu_uuid']
|
||||
prior_wall_seconds=max(prior_wall_seconds,c.get('probe_guard_wall_seconds',0.))
|
||||
assert [(r['arm'],r['width'],r['seed']) for r in c['runs']]==[('N',128,0),('E',128,0),('N',256,0),('E',256,0)]
|
||||
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
||||
assert not (root/'pilot-started.json').exists(),'No automatic pilot retry'
|
||||
start=time.monotonic();save(root/'pilot-started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
||||
rows=[];scores={};reason=None
|
||||
for i,spec in enumerate(c['runs']):
|
||||
remaining=7200-prior_wall_seconds-(time.monotonic()-start)
|
||||
if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break
|
||||
command=[sys.executable,'tools/run_s2_pilot_training_v2.py',str(root),str(i)]
|
||||
try:r,result=guarded_worker(root,spec,i,uuid,min(1800,remaining),command)
|
||||
except Exception as exc:reason=str(exc);break
|
||||
rows.append(dict(run=spec,**r))
|
||||
if r['status']!='PASS' or not result or result['status']!='COMPLETE':reason='PILOT_RUN_FAILED_NO_RETRY';break
|
||||
scores[(spec['arm'],spec['width'])]=result['best_validation_score']
|
||||
if prior_wall_seconds+time.monotonic()-start>7200:reason='TOTAL_PILOT_WALL_LIMIT_AFTER_EXIT'
|
||||
selected=select_capacity(scores) if len(scores)==4 and reason is None else None
|
||||
summary=dict(status='FOUR_RUNS_COMPLETE' if selected else 'PILOT_INCOMPLETE',reason=reason,runs=rows,selected_global_width=selected,
|
||||
selection_rule='mean of N/E validation error at each width, ties smaller; all four runs required',
|
||||
pilot_wall_seconds=prior_wall_seconds+time.monotonic()-start,probe_and_setup_wall_seconds=prior_wall_seconds,test_labels_read=False,test_predictions=False,retries=0)
|
||||
save(root/'pilot-complete.json',summary);print(json.dumps(summary),flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Synthetic precision regression checks, without corpus or GPU use."""
|
||||
import unittest
|
||||
import torch
|
||||
from s2_models.network import QueryModel as OriginalModel
|
||||
from s2_models_v2.network import QueryModel as PrecisionModel
|
||||
from s2_models.normalization import BaseStandardizer
|
||||
from s2_models.attention import Physics_Attention_Irregular_Mesh as OriginalAttention
|
||||
from s2_models_v2.attention import Physics_Attention_Irregular_Mesh as PrecisionAttention
|
||||
|
||||
|
||||
class PrecisionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
torch.set_num_threads(1);torch.manual_seed(9)
|
||||
|
||||
def test_fp32_model_outputs_and_gradients_unchanged_for_both_arms(self):
|
||||
stats=BaseStandardizer('0','100',('synthetic',),32,(0.,)*10,(1.,)*10)
|
||||
for arm in ['N','E']:
|
||||
for width in [128,256]:
|
||||
with self.subTest(arm=arm,width=width):
|
||||
original=OriginalModel(arm,width,standardizer=stats)
|
||||
fixed=PrecisionModel(arm,width,standardizer=stats)
|
||||
fixed.load_state_dict(original.state_dict())
|
||||
base=torch.randn(32,10);ids=torch.arange(8);gaps=torch.randn(8,2);edge=torch.randn(8,4) if arm=='E' else None
|
||||
a=original(base,ids,gaps,edge);b=fixed(base,ids,gaps,edge)
|
||||
self.assertTrue(torch.equal(a,b));a.square().sum().backward();b.square().sum().backward()
|
||||
self.assertEqual(original.parameter_counts(),fixed.parameter_counts())
|
||||
for p,q in zip(original.parameters(),fixed.parameters()):self.assertTrue(torch.equal(p.grad,q.grad))
|
||||
|
||||
def test_large_slice_fp16_overflow_removed_and_backward_finite(self):
|
||||
original=OriginalAttention(8,heads=1,dim_head=8,slice_num=1)
|
||||
fixed=PrecisionAttention(8,heads=1,dim_head=8,slice_num=1)
|
||||
with torch.no_grad():
|
||||
original.in_project_fx.weight.zero_();original.in_project_fx.bias.fill_(8.)
|
||||
fixed.load_state_dict(original.state_dict())
|
||||
x=torch.ones(1,100000,8,requires_grad=True)
|
||||
with torch.no_grad():reference=original(x)
|
||||
numerator=torch.einsum('bhnc,bhng->bhgc',torch.full((1,1,100000,8),8.,dtype=torch.float16),torch.ones(1,1,100000,1,dtype=torch.float16))
|
||||
self.assertTrue(bool(torch.isinf(numerator).all()))
|
||||
with torch.autocast('cpu',dtype=torch.float16):
|
||||
a=original(x);b=fixed(x)
|
||||
self.assertFalse(bool(torch.isfinite(a).all()))
|
||||
self.assertTrue(bool(torch.isfinite(b).all()))
|
||||
torch.testing.assert_close(b.float(),reference,rtol=5e-3,atol=5e-3)
|
||||
b.float().square().mean().backward()
|
||||
self.assertTrue(bool(torch.isfinite(x.grad).all()))
|
||||
for p in fixed.parameters():
|
||||
if p.grad is not None:self.assertTrue(bool(torch.isfinite(p.grad).all()))
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user