Record four-V100 fresh matrix training launch and live handoff

This commit is contained in:
Frank14f committed 2026-11-15 09:38:08 +08:00
1 parent e3fa0af79a
commit 5034de7377
12 files changed
+360

No files matched your search

+2
View File
@@ -30,3 +30,5 @@
- Latest actual launch (2026-11-15 07:51 UTC+8): Claude1794699027.366469 approved budgets/execution. Sole active131 preparation is session32694/coordinatorPID220180/root `artifacts/s2-main-preparation-2026-11-15`, fixedimplementationbf5203c, contract0a3c628c81c008eab53e9337406b84ec2063edcd49529bcc7a82e3e4a3d719a3. Host verified16guards+16geometryworkers. Do not restart or mutate pinned sources. 493successes reuse only; all131 label-free;6×60min+125×30min/1.25GiB;16workers;new20workerCPU-h with old34.75965h separate; cumulative100GiB.10syntheticchecksPASS. Automatic roster freeze only after131terminal withoutglobalstop. Then90fresh width128/Emax42 with46GPUh includingidle/setup,30min/run,up to4V100,N/Eadjacent,first8projection;R2singlefit30CPUmin,total100CPUh,leaveGPUfeedersfree. Currentmatrix/R2notstarted;all90+15terminal/checkpointsfrozen beforeunifiedcustodypredictions/scoring. Read leading handoff and actual-start-audit; older pending/running entries are historical.
- Latest terminal (09:25): preparation32694/PID220180 exited,131=124PASS+7wall_limit; no globalstop,monitor_errors0. Final617 roster SHA fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e automatically frozen;7globallyexcluded,493oldsuccesses reused. Newworker13.8436103492h/20h; cumulativehistory48.6032611436h. Do not restart preparation. Read final-roster/roster-receipt/terminal-audit/final-fold-pool-counts in main-preparation root. Matrix implementation and synthetic acceptance now proceeding under Claude1794699027; no training launch yet.
- Latest matrix launch (09:36,09:37 actual worker evidence): sole session92182/coordinatorPID223006/root artifacts/s2-main-matrix-2026-11-15; implementatione3fa0af, contract198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56,617rosterfb0c984f. Four V100 training workers223127/223141/223131/223145 observed epoch1; initialN f0pool25seed0/1/2 and f0pool50seed0, each followed byE.17syntheticchecksPASS. First4blocks/8runs barrier and46GPUhprojection automatic; allidle/setup/failure charged,30min/run,no retries,pins immutable. CPUaffinity0-7 GPUfeeders;8-31availableforR2. R2 notyetimplemented/started; all90+15terminal/checkpointsfrozen beforecustody. Check actual-start-audit and live receipts; do not duplicate matrix or restart completed131 preparation.
@@ -0,0 +1,219 @@
{
"status": "FOUR_GPU_TRAINING_OBSERVED_NOT_COMPLETE",
"observed_unix": 1794706688.0787027,
"session_id": 92182,
"coordinator_pid": 223006,
"implementation_commit": "e3fa0af79a9bd4ecf74015e14a706fab0ea231a7",
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56",
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
"live_training_workers": [
{
"pid": 223145,
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"6"
],
"cpu_affinity": [
6,
7
],
"status": "running"
},
{
"pid": 223131,
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"4"
],
"cpu_affinity": [
4,
5
],
"status": "running"
},
{
"pid": 223141,
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"2"
],
"cpu_affinity": [
2,
3
],
"status": "running"
},
{
"pid": 223127,
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"0"
],
"cpu_affinity": [
0,
1
],
"status": "running"
}
],
"training_progress": {
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/progress.json": {
"stage": "training",
"epoch": 6,
"parts_done": 33,
"parts_total": 145,
"updates": 901,
"elapsed_seconds": 79.2007255940116,
"peak_allocated_bytes": 3137353216,
"peak_reserved_bytes": 3292528640
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/progress.json": {
"stage": "training",
"epoch": 6,
"parts_done": 49,
"parts_total": 145,
"updates": 917,
"elapsed_seconds": 80.08159776800312,
"peak_allocated_bytes": 3137353216,
"peak_reserved_bytes": 3292528640
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/progress.json": {
"stage": "training",
"epoch": 6,
"parts_done": 49,
"parts_total": 145,
"updates": 916,
"elapsed_seconds": 79.6935267400113,
"peak_allocated_bytes": 3137353216,
"peak_reserved_bytes": 3271557120
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/progress.json": {
"stage": "training",
"epoch": 3,
"parts_done": 241,
"parts_total": 254,
"updates": 1001,
"elapsed_seconds": 77.72405460698064,
"peak_allocated_bytes": 3644658176,
"peak_reserved_bytes": 3797942272
}
},
"matrix_progress": {
"completed_blocks": 0,
"terminal_runs": 0,
"active_blocks": {
"0": 0,
"1": 1,
"2": 2,
"3": 3
},
"next_block": 4,
"settled_gpu_seconds": 0.0,
"active_gpu_seconds": 424.2205448030145,
"first_eight_passed": false,
"stop_reason": null,
"elapsed_seconds": 106.05700458900537
},
"idle_gate_receipts": {
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/idle-gates/before-run.json": {
"status": "IDLE_CONFIRMED",
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"polls": 3,
"consecutive_idle": 3,
"wall_seconds": 20.291602034994867,
"last": {
"unix": 1794706605.055837,
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
}
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/idle-gates/before-run.json": {
"status": "IDLE_CONFIRMED",
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"polls": 3,
"consecutive_idle": 3,
"wall_seconds": 20.30101585999364,
"last": {
"unix": 1794706605.1271734,
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
}
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/idle-gates/before-run.json": {
"status": "IDLE_CONFIRMED",
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"polls": 3,
"consecutive_idle": 3,
"wall_seconds": 20.28960164700402,
"last": {
"unix": 1794706605.0470319,
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
}
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/idle-gates/before-run.json": {
"status": "IDLE_CONFIRMED",
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"polls": 3,
"consecutive_idle": 3,
"wall_seconds": 20.291276325995568,
"last": {
"unix": 1794706605.1642594,
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"memory_MiB": 5,
"utilization_percent": 0,
"occupants": [],
"idle": true
}
}
},
"target_access": {
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/target-access.json": {
"sha256": "ab03ec76b7e4170f30d192f3a2df70abd3d1286dac3d24e5d9452d4bb0f1c3e7",
"fold": 0,
"pool": 25,
"count": 256
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/target-access.json": {
"sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272",
"fold": 0,
"pool": 25,
"count": 256
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/target-access.json": {
"sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272",
"fold": 0,
"pool": 25,
"count": 256
},
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/target-access.json": {
"sha256": "d2f94029612562b1edea866c262dee8cc75205fa3a609ccb9df270dc7e724f37",
"fold": 0,
"pool": 50,
"count": 365
}
},
"source_pins_match": true,
"gpu_process_inventory": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 223127, 3528 MiB\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 223141, 3528 MiB\nGPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76, 223131, 3508 MiB\nGPU-8b71b032-1b75-2891-935c-ce106a53034a, 223145, 4010 MiB\n",
"failure_paths": [],
"r2_started": false,
"test_predictions": false,
"test_scoring": false
}
@@ -0,0 +1,15 @@
{
"pid": 223038,
"started_unix": 1794706584.7555358,
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
"block": 0,
"run_indices": [
0,
1
],
"cpu_affinity": [
0,
1
],
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
}
@@ -0,0 +1,15 @@
{
"pid": 223039,
"started_unix": 1794706584.824267,
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
"block": 1,
"run_indices": [
2,
3
],
"cpu_affinity": [
2,
3
],
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
}
@@ -0,0 +1,15 @@
{
"pid": 223040,
"started_unix": 1794706584.7622948,
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
"block": 2,
"run_indices": [
4,
5
],
"cpu_affinity": [
4,
5
],
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
}
@@ -0,0 +1,15 @@
{
"pid": 223041,
"started_unix": 1794706584.8710842,
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
"block": 3,
"run_indices": [
6,
7
],
"cpu_affinity": [
6,
7
],
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
}
@@ -0,0 +1,17 @@
{
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"0"
],
"limits": {
"total_cpu_seconds": 1801,
"job_cpu_seconds": 1800,
"job_wall_seconds": 1797.454129953985,
"tree_rss_bytes": 17179869184,
"disk_bytes": 5368709120,
"max_processes": 2
},
"started_unix": 1794706605.095093
}
@@ -0,0 +1,17 @@
{
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"2"
],
"limits": {
"total_cpu_seconds": 1801,
"job_cpu_seconds": 1800,
"job_wall_seconds": 1797.4547854439588,
"tree_rss_bytes": 17179869184,
"disk_bytes": 5368709120,
"max_processes": 2
},
"started_unix": 1794706605.177211
}
@@ -0,0 +1,17 @@
{
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"4"
],
"limits": {
"total_cpu_seconds": 1801,
"job_cpu_seconds": 1800,
"job_wall_seconds": 1797.453511872969,
"tree_rss_bytes": 17179869184,
"disk_bytes": 5368709120,
"max_processes": 2
},
"started_unix": 1794706605.1072228
}
@@ -0,0 +1,17 @@
{
"command": [
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
"tools/run_s2_main_training.py",
"artifacts/s2-main-matrix-2026-11-15",
"6"
],
"limits": {
"total_cpu_seconds": 1801,
"job_cpu_seconds": 1800,
"job_wall_seconds": 1797.4575307809864,
"tree_rss_bytes": 17179869184,
"disk_bytes": 5368709120,
"max_processes": 2
},
"started_unix": 1794706605.2087455
}
@@ -0,0 +1,5 @@
{
"pid": 223006,
"started_unix": 1794706581.995757,
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
}
+6
View File
@@ -1,5 +1,11 @@
# S2 当前执行接续记录
## 当前唯一主矩阵:四V100真实训练已启动(09:36启动、09:37实证)
固定执行器 `e3fa0af79a9bd4ecf74015e14a706fab0ea231a7`,17项合成检查PASS。唯一session92182/coordinatorPID223006,根 `artifacts/s2-main-matrix-2026-11-15`;合同SHA `198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56`,绑定617 roster SHA `fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e`。四真实GPU工作者223127/223141/223131/223145已观察到epoch1训练;依次f0pool25seed0/1/2的N128与f0pool50seed0的N128,每块随后执行配对E。各GPU亲和[0,1]/[2,3]/[4,5]/[6,7],CPU8–31保留给后续R2。不能只凭入口标记断言模型完成,当前仍0终态块。
首4块8run全终态前不派第5块;自动投影≤46聚合GPUh才继续,等待/setup/失败均计费;每run≤1800s,AMP/宽度/Emax冻结,失败NA且不重试,源码pin不可改。实际证据见matrix根actual-start-audit.json和实时matrix-progress.json;严禁重复启动。R2尚未实现/启动,是后续待办,所有90+15终态及检查点冻结前不做测试预测评分。本节取代下方“矩阵未启动”历史状态;131准备已经结束,绝不重启。
## 最新终态:131全部结束,617件统一名单已冻结(09:25核查)
原session32694/PID220180已退出,131=124 PASS+7 wall_limit,无全局停止,无监测错误。自动final-roster.json/roster-receipt.json已生成:617 included/7 excluded,SHA `fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e`。原493复用,七失败跨五折统一排除,原组件身份保持。新增worker13.8436103492h/20h,累计历史worker48.6032611436h;guard3770.200695s/coordinator2408.510226s另列,wall5378.170096s,冻结前磁盘8099577856B。五折实际计数与成本在新准备根final-fold-pool-counts.csv、terminal-audit.json,原回执及无标签输出已核验。不得重启准备。现接续90fresh矩阵执行器实现与合成验收,矩阵/R2尚未启动;沿Claude1794699027既定预算与托管顺序,无需再次用户go。