diff --git a/AGENTS.md b/AGENTS.md index 43109217f..4db034c43 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,3 +30,5 @@ - Latest actual launch (2026-11-15 07:51 UTC+8): Claude1794699027.366469 approved budgets/execution. Sole active131 preparation is session32694/coordinatorPID220180/root `artifacts/s2-main-preparation-2026-11-15`, fixedimplementationbf5203c, contract0a3c628c81c008eab53e9337406b84ec2063edcd49529bcc7a82e3e4a3d719a3. Host verified16guards+16geometryworkers. Do not restart or mutate pinned sources. 493successes reuse only; all131 label-free;6×60min+125×30min/1.25GiB;16workers;new20workerCPU-h with old34.75965h separate; cumulative100GiB.10syntheticchecksPASS. Automatic roster freeze only after131terminal withoutglobalstop. Then90fresh width128/Emax42 with46GPUh includingidle/setup,30min/run,up to4V100,N/Eadjacent,first8projection;R2singlefit30CPUmin,total100CPUh,leaveGPUfeedersfree. Currentmatrix/R2notstarted;all90+15terminal/checkpointsfrozen beforeunifiedcustodypredictions/scoring. Read leading handoff and actual-start-audit; older pending/running entries are historical. - Latest terminal (09:25): preparation32694/PID220180 exited,131=124PASS+7wall_limit; no globalstop,monitor_errors0. Final617 roster SHA fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e automatically frozen;7globallyexcluded,493oldsuccesses reused. Newworker13.8436103492h/20h; cumulativehistory48.6032611436h. Do not restart preparation. Read final-roster/roster-receipt/terminal-audit/final-fold-pool-counts in main-preparation root. Matrix implementation and synthetic acceptance now proceeding under Claude1794699027; no training launch yet. + +- Latest matrix launch (09:36,09:37 actual worker evidence): sole session92182/coordinatorPID223006/root artifacts/s2-main-matrix-2026-11-15; implementatione3fa0af, contract198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56,617rosterfb0c984f. Four V100 training workers223127/223141/223131/223145 observed epoch1; initialN f0pool25seed0/1/2 and f0pool50seed0, each followed byE.17syntheticchecksPASS. First4blocks/8runs barrier and46GPUhprojection automatic; allidle/setup/failure charged,30min/run,no retries,pins immutable. CPUaffinity0-7 GPUfeeders;8-31availableforR2. R2 notyetimplemented/started; all90+15terminal/checkpointsfrozen beforecustody. Check actual-start-audit and live receipts; do not duplicate matrix or restart completed131 preparation. diff --git a/artifacts/s2-main-matrix-2026-11-15/actual-start-audit.json b/artifacts/s2-main-matrix-2026-11-15/actual-start-audit.json new file mode 100644 index 000000000..b792a5caf --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/actual-start-audit.json @@ -0,0 +1,219 @@ +{ + "status": "FOUR_GPU_TRAINING_OBSERVED_NOT_COMPLETE", + "observed_unix": 1794706688.0787027, + "session_id": 92182, + "coordinator_pid": 223006, + "implementation_commit": "e3fa0af79a9bd4ecf74015e14a706fab0ea231a7", + "contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56", + "roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e", + "live_training_workers": [ + { + "pid": 223145, + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_main_training.py", + "artifacts/s2-main-matrix-2026-11-15", + "6" + ], + "cpu_affinity": [ + 6, + 7 + ], + "status": "running" + }, + { + "pid": 223131, + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_main_training.py", + "artifacts/s2-main-matrix-2026-11-15", + "4" + ], + "cpu_affinity": [ + 4, + 5 + ], + "status": "running" + }, + { + "pid": 223141, + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_main_training.py", + "artifacts/s2-main-matrix-2026-11-15", + "2" + ], + "cpu_affinity": [ + 2, + 3 + ], + "status": "running" + }, + { + "pid": 223127, + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_main_training.py", + "artifacts/s2-main-matrix-2026-11-15", + "0" + ], + "cpu_affinity": [ + 0, + 1 + ], + "status": "running" + } + ], + "training_progress": { + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/progress.json": { + "stage": "training", + "epoch": 6, + "parts_done": 33, + "parts_total": 145, + "updates": 901, + "elapsed_seconds": 79.2007255940116, + "peak_allocated_bytes": 3137353216, + "peak_reserved_bytes": 3292528640 + }, + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/progress.json": { + "stage": "training", + "epoch": 6, + "parts_done": 49, + "parts_total": 145, + "updates": 917, + "elapsed_seconds": 80.08159776800312, + "peak_allocated_bytes": 3137353216, + "peak_reserved_bytes": 3292528640 + }, + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/progress.json": { + "stage": "training", + "epoch": 6, + "parts_done": 49, + "parts_total": 145, + "updates": 916, + "elapsed_seconds": 79.6935267400113, + "peak_allocated_bytes": 3137353216, + "peak_reserved_bytes": 3271557120 + }, + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/progress.json": { + "stage": "training", + "epoch": 3, + "parts_done": 241, + "parts_total": 254, + "updates": 1001, + "elapsed_seconds": 77.72405460698064, + "peak_allocated_bytes": 3644658176, + "peak_reserved_bytes": 3797942272 + } + }, + "matrix_progress": { + "completed_blocks": 0, + "terminal_runs": 0, + "active_blocks": { + "0": 0, + "1": 1, + "2": 2, + "3": 3 + }, + "next_block": 4, + "settled_gpu_seconds": 0.0, + "active_gpu_seconds": 424.2205448030145, + "first_eight_passed": false, + "stop_reason": null, + "elapsed_seconds": 106.05700458900537 + }, + "idle_gate_receipts": { + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/idle-gates/before-run.json": { + "status": "IDLE_CONFIRMED", + "uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "polls": 3, + "consecutive_idle": 3, + "wall_seconds": 20.291602034994867, + "last": { + "unix": 1794706605.055837, + "uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "memory_MiB": 5, + "utilization_percent": 0, + "occupants": [], + "idle": true + } + }, + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/idle-gates/before-run.json": { + "status": "IDLE_CONFIRMED", + "uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36", + "polls": 3, + "consecutive_idle": 3, + "wall_seconds": 20.30101585999364, + "last": { + "unix": 1794706605.1271734, + "uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36", + "memory_MiB": 5, + "utilization_percent": 0, + "occupants": [], + "idle": true + } + }, + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/idle-gates/before-run.json": { + "status": "IDLE_CONFIRMED", + "uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "polls": 3, + "consecutive_idle": 3, + "wall_seconds": 20.28960164700402, + "last": { + "unix": 1794706605.0470319, + "uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "memory_MiB": 5, + "utilization_percent": 0, + "occupants": [], + "idle": true + } + }, + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/idle-gates/before-run.json": { + "status": "IDLE_CONFIRMED", + "uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "polls": 3, + "consecutive_idle": 3, + "wall_seconds": 20.291276325995568, + "last": { + "unix": 1794706605.1642594, + "uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "memory_MiB": 5, + "utilization_percent": 0, + "occupants": [], + "idle": true + } + } + }, + "target_access": { + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/target-access.json": { + "sha256": "ab03ec76b7e4170f30d192f3a2df70abd3d1286dac3d24e5d9452d4bb0f1c3e7", + "fold": 0, + "pool": 25, + "count": 256 + }, + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/target-access.json": { + "sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272", + "fold": 0, + "pool": 25, + "count": 256 + }, + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/target-access.json": { + "sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272", + "fold": 0, + "pool": 25, + "count": 256 + }, + "artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/target-access.json": { + "sha256": "d2f94029612562b1edea866c262dee8cc75205fa3a609ccb9df270dc7e724f37", + "fold": 0, + "pool": 50, + "count": 365 + } + }, + "source_pins_match": true, + "gpu_process_inventory": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 223127, 3528 MiB\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 223141, 3528 MiB\nGPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76, 223131, 3508 MiB\nGPU-8b71b032-1b75-2891-935c-ce106a53034a, 223145, 4010 MiB\n", + "failure_paths": [], + "r2_started": false, + "test_predictions": false, + "test_scoring": false +} diff --git a/artifacts/s2-main-matrix-2026-11-15/blocks/block-000/started.json b/artifacts/s2-main-matrix-2026-11-15/blocks/block-000/started.json new file mode 100644 index 000000000..df8fd1f39 --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/blocks/block-000/started.json @@ -0,0 +1,15 @@ +{ + "pid": 223038, + "started_unix": 1794706584.7555358, + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "block": 0, + "run_indices": [ + 0, + 1 + ], + "cpu_affinity": [ + 0, + 1 + ], + "contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56" +} diff --git a/artifacts/s2-main-matrix-2026-11-15/blocks/block-001/started.json b/artifacts/s2-main-matrix-2026-11-15/blocks/block-001/started.json new file mode 100644 index 000000000..1c88affd0 --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/blocks/block-001/started.json @@ -0,0 +1,15 @@ +{ + "pid": 223039, + "started_unix": 1794706584.824267, + "gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36", + "block": 1, + "run_indices": [ + 2, + 3 + ], + "cpu_affinity": [ + 2, + 3 + ], + "contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56" +} diff --git a/artifacts/s2-main-matrix-2026-11-15/blocks/block-002/started.json b/artifacts/s2-main-matrix-2026-11-15/blocks/block-002/started.json new file mode 100644 index 000000000..61ddf8023 --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/blocks/block-002/started.json @@ -0,0 +1,15 @@ +{ + "pid": 223040, + "started_unix": 1794706584.7622948, + "gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76", + "block": 2, + "run_indices": [ + 4, + 5 + ], + "cpu_affinity": [ + 4, + 5 + ], + "contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56" +} diff --git a/artifacts/s2-main-matrix-2026-11-15/blocks/block-003/started.json b/artifacts/s2-main-matrix-2026-11-15/blocks/block-003/started.json new file mode 100644 index 000000000..2c0c1670f --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/blocks/block-003/started.json @@ -0,0 +1,15 @@ +{ + "pid": 223041, + "started_unix": 1794706584.8710842, + "gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a", + "block": 3, + "run_indices": [ + 6, + 7 + ], + "cpu_affinity": [ + 6, + 7 + ], + "contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56" +} diff --git a/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/guards/fold0-pool25-N128-seed0.started.json b/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/guards/fold0-pool25-N128-seed0.started.json new file mode 100644 index 000000000..ced8def23 --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/guards/fold0-pool25-N128-seed0.started.json @@ -0,0 +1,17 @@ +{ + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_main_training.py", + "artifacts/s2-main-matrix-2026-11-15", + "0" + ], + "limits": { + "total_cpu_seconds": 1801, + "job_cpu_seconds": 1800, + "job_wall_seconds": 1797.454129953985, + "tree_rss_bytes": 17179869184, + "disk_bytes": 5368709120, + "max_processes": 2 + }, + "started_unix": 1794706605.095093 +} diff --git a/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/guards/fold0-pool25-N128-seed1.started.json b/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/guards/fold0-pool25-N128-seed1.started.json new file mode 100644 index 000000000..8a373243b --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/guards/fold0-pool25-N128-seed1.started.json @@ -0,0 +1,17 @@ +{ + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_main_training.py", + "artifacts/s2-main-matrix-2026-11-15", + "2" + ], + "limits": { + "total_cpu_seconds": 1801, + "job_cpu_seconds": 1800, + "job_wall_seconds": 1797.4547854439588, + "tree_rss_bytes": 17179869184, + "disk_bytes": 5368709120, + "max_processes": 2 + }, + "started_unix": 1794706605.177211 +} diff --git a/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/guards/fold0-pool25-N128-seed2.started.json b/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/guards/fold0-pool25-N128-seed2.started.json new file mode 100644 index 000000000..e0e87ae85 --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/guards/fold0-pool25-N128-seed2.started.json @@ -0,0 +1,17 @@ +{ + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_main_training.py", + "artifacts/s2-main-matrix-2026-11-15", + "4" + ], + "limits": { + "total_cpu_seconds": 1801, + "job_cpu_seconds": 1800, + "job_wall_seconds": 1797.453511872969, + "tree_rss_bytes": 17179869184, + "disk_bytes": 5368709120, + "max_processes": 2 + }, + "started_unix": 1794706605.1072228 +} diff --git a/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/guards/fold0-pool50-N128-seed0.started.json b/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/guards/fold0-pool50-N128-seed0.started.json new file mode 100644 index 000000000..668e2ff52 --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/guards/fold0-pool50-N128-seed0.started.json @@ -0,0 +1,17 @@ +{ + "command": [ + "/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python", + "tools/run_s2_main_training.py", + "artifacts/s2-main-matrix-2026-11-15", + "6" + ], + "limits": { + "total_cpu_seconds": 1801, + "job_cpu_seconds": 1800, + "job_wall_seconds": 1797.4575307809864, + "tree_rss_bytes": 17179869184, + "disk_bytes": 5368709120, + "max_processes": 2 + }, + "started_unix": 1794706605.2087455 +} diff --git a/artifacts/s2-main-matrix-2026-11-15/matrix-started.json b/artifacts/s2-main-matrix-2026-11-15/matrix-started.json new file mode 100644 index 000000000..4477a6bfd --- /dev/null +++ b/artifacts/s2-main-matrix-2026-11-15/matrix-started.json @@ -0,0 +1,5 @@ +{ + "pid": 223006, + "started_unix": 1794706581.995757, + "contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56" +} diff --git a/docs/s2-pilot-live-handoff-2026-11-15.md b/docs/s2-pilot-live-handoff-2026-11-15.md index 62bfc8dcd..762f2a8c7 100644 --- a/docs/s2-pilot-live-handoff-2026-11-15.md +++ b/docs/s2-pilot-live-handoff-2026-11-15.md @@ -1,5 +1,11 @@ # S2 当前执行接续记录 +## 当前唯一主矩阵:四V100真实训练已启动(09:36启动、09:37实证) + +固定执行器 `e3fa0af79a9bd4ecf74015e14a706fab0ea231a7`,17项合成检查PASS。唯一session92182/coordinatorPID223006,根 `artifacts/s2-main-matrix-2026-11-15`;合同SHA `198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56`,绑定617 roster SHA `fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e`。四真实GPU工作者223127/223141/223131/223145已观察到epoch1训练;依次f0pool25seed0/1/2的N128与f0pool50seed0的N128,每块随后执行配对E。各GPU亲和[0,1]/[2,3]/[4,5]/[6,7],CPU8–31保留给后续R2。不能只凭入口标记断言模型完成,当前仍0终态块。 + +首4块8run全终态前不派第5块;自动投影≤46聚合GPUh才继续,等待/setup/失败均计费;每run≤1800s,AMP/宽度/Emax冻结,失败NA且不重试,源码pin不可改。实际证据见matrix根actual-start-audit.json和实时matrix-progress.json;严禁重复启动。R2尚未实现/启动,是后续待办,所有90+15终态及检查点冻结前不做测试预测评分。本节取代下方“矩阵未启动”历史状态;131准备已经结束,绝不重启。 + ## 最新终态:131全部结束,617件统一名单已冻结(09:25核查) 原session32694/PID220180已退出,131=124 PASS+7 wall_limit,无全局停止,无监测错误。自动final-roster.json/roster-receipt.json已生成:617 included/7 excluded,SHA `fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e`。原493复用,七失败跨五折统一排除,原组件身份保持。新增worker13.8436103492h/20h,累计历史worker48.6032611436h;guard3770.200695s/coordinator2408.510226s另列,wall5378.170096s,冻结前磁盘8099577856B。五折实际计数与成本在新准备根final-fold-pool-counts.csv、terminal-audit.json,原回执及无标签输出已核验。不得重启准备。现接续90fresh矩阵执行器实现与合成验收,矩阵/R2尚未启动;沿Claude1794699027既定预算与托管顺序,无需再次用户go。