Record four-V100 fresh matrix training launch and live handoff
This commit is contained in:
1 parent
e3fa0af79a
commit
5034de7377
12 files changed
+360
No files matched your search
@@ -30,3 +30,5 @@
|
||||
- Latest actual launch (2026-11-15 07:51 UTC+8): Claude1794699027.366469 approved budgets/execution. Sole active131 preparation is session32694/coordinatorPID220180/root `artifacts/s2-main-preparation-2026-11-15`, fixedimplementationbf5203c, contract0a3c628c81c008eab53e9337406b84ec2063edcd49529bcc7a82e3e4a3d719a3. Host verified16guards+16geometryworkers. Do not restart or mutate pinned sources. 493successes reuse only; all131 label-free;6×60min+125×30min/1.25GiB;16workers;new20workerCPU-h with old34.75965h separate; cumulative100GiB.10syntheticchecksPASS. Automatic roster freeze only after131terminal withoutglobalstop. Then90fresh width128/Emax42 with46GPUh includingidle/setup,30min/run,up to4V100,N/Eadjacent,first8projection;R2singlefit30CPUmin,total100CPUh,leaveGPUfeedersfree. Currentmatrix/R2notstarted;all90+15terminal/checkpointsfrozen beforeunifiedcustodypredictions/scoring. Read leading handoff and actual-start-audit; older pending/running entries are historical.
|
||||
|
||||
- Latest terminal (09:25): preparation32694/PID220180 exited,131=124PASS+7wall_limit; no globalstop,monitor_errors0. Final617 roster SHA fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e automatically frozen;7globallyexcluded,493oldsuccesses reused. Newworker13.8436103492h/20h; cumulativehistory48.6032611436h. Do not restart preparation. Read final-roster/roster-receipt/terminal-audit/final-fold-pool-counts in main-preparation root. Matrix implementation and synthetic acceptance now proceeding under Claude1794699027; no training launch yet.
|
||||
|
||||
- Latest matrix launch (09:36,09:37 actual worker evidence): sole session92182/coordinatorPID223006/root artifacts/s2-main-matrix-2026-11-15; implementatione3fa0af, contract198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56,617rosterfb0c984f. Four V100 training workers223127/223141/223131/223145 observed epoch1; initialN f0pool25seed0/1/2 and f0pool50seed0, each followed byE.17syntheticchecksPASS. First4blocks/8runs barrier and46GPUhprojection automatic; allidle/setup/failure charged,30min/run,no retries,pins immutable. CPUaffinity0-7 GPUfeeders;8-31availableforR2. R2 notyetimplemented/started; all90+15terminal/checkpointsfrozen beforecustody. Check actual-start-audit and live receipts; do not duplicate matrix or restart completed131 preparation.
|
||||
@@ -0,0 +1,219 @@
|
||||
{
|
||||
"status": "FOUR_GPU_TRAINING_OBSERVED_NOT_COMPLETE",
|
||||
"observed_unix": 1794706688.0787027,
|
||||
"session_id": 92182,
|
||||
"coordinator_pid": 223006,
|
||||
"implementation_commit": "e3fa0af79a9bd4ecf74015e14a706fab0ea231a7",
|
||||
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56",
|
||||
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
|
||||
"live_training_workers": [
|
||||
{
|
||||
"pid": 223145,
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_main_training.py",
|
||||
"artifacts/s2-main-matrix-2026-11-15",
|
||||
"6"
|
||||
],
|
||||
"cpu_affinity": [
|
||||
6,
|
||||
7
|
||||
],
|
||||
"status": "running"
|
||||
},
|
||||
{
|
||||
"pid": 223131,
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_main_training.py",
|
||||
"artifacts/s2-main-matrix-2026-11-15",
|
||||
"4"
|
||||
],
|
||||
"cpu_affinity": [
|
||||
4,
|
||||
5
|
||||
],
|
||||
"status": "running"
|
||||
},
|
||||
{
|
||||
"pid": 223141,
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_main_training.py",
|
||||
"artifacts/s2-main-matrix-2026-11-15",
|
||||
"2"
|
||||
],
|
||||
"cpu_affinity": [
|
||||
2,
|
||||
3
|
||||
],
|
||||
"status": "running"
|
||||
},
|
||||
{
|
||||
"pid": 223127,
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_main_training.py",
|
||||
"artifacts/s2-main-matrix-2026-11-15",
|
||||
"0"
|
||||
],
|
||||
"cpu_affinity": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"status": "running"
|
||||
}
|
||||
],
|
||||
"training_progress": {
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/progress.json": {
|
||||
"stage": "training",
|
||||
"epoch": 6,
|
||||
"parts_done": 33,
|
||||
"parts_total": 145,
|
||||
"updates": 901,
|
||||
"elapsed_seconds": 79.2007255940116,
|
||||
"peak_allocated_bytes": 3137353216,
|
||||
"peak_reserved_bytes": 3292528640
|
||||
},
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/progress.json": {
|
||||
"stage": "training",
|
||||
"epoch": 6,
|
||||
"parts_done": 49,
|
||||
"parts_total": 145,
|
||||
"updates": 917,
|
||||
"elapsed_seconds": 80.08159776800312,
|
||||
"peak_allocated_bytes": 3137353216,
|
||||
"peak_reserved_bytes": 3292528640
|
||||
},
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/progress.json": {
|
||||
"stage": "training",
|
||||
"epoch": 6,
|
||||
"parts_done": 49,
|
||||
"parts_total": 145,
|
||||
"updates": 916,
|
||||
"elapsed_seconds": 79.6935267400113,
|
||||
"peak_allocated_bytes": 3137353216,
|
||||
"peak_reserved_bytes": 3271557120
|
||||
},
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/progress.json": {
|
||||
"stage": "training",
|
||||
"epoch": 3,
|
||||
"parts_done": 241,
|
||||
"parts_total": 254,
|
||||
"updates": 1001,
|
||||
"elapsed_seconds": 77.72405460698064,
|
||||
"peak_allocated_bytes": 3644658176,
|
||||
"peak_reserved_bytes": 3797942272
|
||||
}
|
||||
},
|
||||
"matrix_progress": {
|
||||
"completed_blocks": 0,
|
||||
"terminal_runs": 0,
|
||||
"active_blocks": {
|
||||
"0": 0,
|
||||
"1": 1,
|
||||
"2": 2,
|
||||
"3": 3
|
||||
},
|
||||
"next_block": 4,
|
||||
"settled_gpu_seconds": 0.0,
|
||||
"active_gpu_seconds": 424.2205448030145,
|
||||
"first_eight_passed": false,
|
||||
"stop_reason": null,
|
||||
"elapsed_seconds": 106.05700458900537
|
||||
},
|
||||
"idle_gate_receipts": {
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/idle-gates/before-run.json": {
|
||||
"status": "IDLE_CONFIRMED",
|
||||
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"polls": 3,
|
||||
"consecutive_idle": 3,
|
||||
"wall_seconds": 20.291602034994867,
|
||||
"last": {
|
||||
"unix": 1794706605.055837,
|
||||
"uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"memory_MiB": 5,
|
||||
"utilization_percent": 0,
|
||||
"occupants": [],
|
||||
"idle": true
|
||||
}
|
||||
},
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/idle-gates/before-run.json": {
|
||||
"status": "IDLE_CONFIRMED",
|
||||
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"polls": 3,
|
||||
"consecutive_idle": 3,
|
||||
"wall_seconds": 20.30101585999364,
|
||||
"last": {
|
||||
"unix": 1794706605.1271734,
|
||||
"uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"memory_MiB": 5,
|
||||
"utilization_percent": 0,
|
||||
"occupants": [],
|
||||
"idle": true
|
||||
}
|
||||
},
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/idle-gates/before-run.json": {
|
||||
"status": "IDLE_CONFIRMED",
|
||||
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"polls": 3,
|
||||
"consecutive_idle": 3,
|
||||
"wall_seconds": 20.28960164700402,
|
||||
"last": {
|
||||
"unix": 1794706605.0470319,
|
||||
"uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"memory_MiB": 5,
|
||||
"utilization_percent": 0,
|
||||
"occupants": [],
|
||||
"idle": true
|
||||
}
|
||||
},
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/idle-gates/before-run.json": {
|
||||
"status": "IDLE_CONFIRMED",
|
||||
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"polls": 3,
|
||||
"consecutive_idle": 3,
|
||||
"wall_seconds": 20.291276325995568,
|
||||
"last": {
|
||||
"unix": 1794706605.1642594,
|
||||
"uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"memory_MiB": 5,
|
||||
"utilization_percent": 0,
|
||||
"occupants": [],
|
||||
"idle": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"target_access": {
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed2/fold0-pool25-N128-seed2/target-access.json": {
|
||||
"sha256": "ab03ec76b7e4170f30d192f3a2df70abd3d1286dac3d24e5d9452d4bb0f1c3e7",
|
||||
"fold": 0,
|
||||
"pool": 25,
|
||||
"count": 256
|
||||
},
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed1/fold0-pool25-N128-seed1/target-access.json": {
|
||||
"sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272",
|
||||
"fold": 0,
|
||||
"pool": 25,
|
||||
"count": 256
|
||||
},
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool25-N128-seed0/fold0-pool25-N128-seed0/target-access.json": {
|
||||
"sha256": "755570593fe0f114f4efbf8c7d648e320d45f1ec0cc9487cd2903939558a9272",
|
||||
"fold": 0,
|
||||
"pool": 25,
|
||||
"count": 256
|
||||
},
|
||||
"artifacts/s2-main-matrix-2026-11-15/cells/fold0-pool50-N128-seed0/fold0-pool50-N128-seed0/target-access.json": {
|
||||
"sha256": "d2f94029612562b1edea866c262dee8cc75205fa3a609ccb9df270dc7e724f37",
|
||||
"fold": 0,
|
||||
"pool": 50,
|
||||
"count": 365
|
||||
}
|
||||
},
|
||||
"source_pins_match": true,
|
||||
"gpu_process_inventory": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c, 223127, 3528 MiB\nGPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36, 223141, 3528 MiB\nGPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76, 223131, 3508 MiB\nGPU-8b71b032-1b75-2891-935c-ce106a53034a, 223145, 4010 MiB\n",
|
||||
"failure_paths": [],
|
||||
"r2_started": false,
|
||||
"test_predictions": false,
|
||||
"test_scoring": false
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"pid": 223038,
|
||||
"started_unix": 1794706584.7555358,
|
||||
"gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c",
|
||||
"block": 0,
|
||||
"run_indices": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"cpu_affinity": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"pid": 223039,
|
||||
"started_unix": 1794706584.824267,
|
||||
"gpu_uuid": "GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36",
|
||||
"block": 1,
|
||||
"run_indices": [
|
||||
2,
|
||||
3
|
||||
],
|
||||
"cpu_affinity": [
|
||||
2,
|
||||
3
|
||||
],
|
||||
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"pid": 223040,
|
||||
"started_unix": 1794706584.7622948,
|
||||
"gpu_uuid": "GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76",
|
||||
"block": 2,
|
||||
"run_indices": [
|
||||
4,
|
||||
5
|
||||
],
|
||||
"cpu_affinity": [
|
||||
4,
|
||||
5
|
||||
],
|
||||
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"pid": 223041,
|
||||
"started_unix": 1794706584.8710842,
|
||||
"gpu_uuid": "GPU-8b71b032-1b75-2891-935c-ce106a53034a",
|
||||
"block": 3,
|
||||
"run_indices": [
|
||||
6,
|
||||
7
|
||||
],
|
||||
"cpu_affinity": [
|
||||
6,
|
||||
7
|
||||
],
|
||||
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_main_training.py",
|
||||
"artifacts/s2-main-matrix-2026-11-15",
|
||||
"0"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 1801,
|
||||
"job_cpu_seconds": 1800,
|
||||
"job_wall_seconds": 1797.454129953985,
|
||||
"tree_rss_bytes": 17179869184,
|
||||
"disk_bytes": 5368709120,
|
||||
"max_processes": 2
|
||||
},
|
||||
"started_unix": 1794706605.095093
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_main_training.py",
|
||||
"artifacts/s2-main-matrix-2026-11-15",
|
||||
"2"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 1801,
|
||||
"job_cpu_seconds": 1800,
|
||||
"job_wall_seconds": 1797.4547854439588,
|
||||
"tree_rss_bytes": 17179869184,
|
||||
"disk_bytes": 5368709120,
|
||||
"max_processes": 2
|
||||
},
|
||||
"started_unix": 1794706605.177211
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_main_training.py",
|
||||
"artifacts/s2-main-matrix-2026-11-15",
|
||||
"4"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 1801,
|
||||
"job_cpu_seconds": 1800,
|
||||
"job_wall_seconds": 1797.453511872969,
|
||||
"tree_rss_bytes": 17179869184,
|
||||
"disk_bytes": 5368709120,
|
||||
"max_processes": 2
|
||||
},
|
||||
"started_unix": 1794706605.1072228
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": [
|
||||
"/home/frank14f/anaconda3/envs/pycuda_3_10/bin/python",
|
||||
"tools/run_s2_main_training.py",
|
||||
"artifacts/s2-main-matrix-2026-11-15",
|
||||
"6"
|
||||
],
|
||||
"limits": {
|
||||
"total_cpu_seconds": 1801,
|
||||
"job_cpu_seconds": 1800,
|
||||
"job_wall_seconds": 1797.4575307809864,
|
||||
"tree_rss_bytes": 17179869184,
|
||||
"disk_bytes": 5368709120,
|
||||
"max_processes": 2
|
||||
},
|
||||
"started_unix": 1794706605.2087455
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"pid": 223006,
|
||||
"started_unix": 1794706581.995757,
|
||||
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56"
|
||||
}
|
||||
@@ -1,5 +1,11 @@
|
||||
# S2 当前执行接续记录
|
||||
|
||||
## 当前唯一主矩阵:四V100真实训练已启动(09:36启动、09:37实证)
|
||||
|
||||
固定执行器 `e3fa0af79a9bd4ecf74015e14a706fab0ea231a7`,17项合成检查PASS。唯一session92182/coordinatorPID223006,根 `artifacts/s2-main-matrix-2026-11-15`;合同SHA `198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56`,绑定617 roster SHA `fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e`。四真实GPU工作者223127/223141/223131/223145已观察到epoch1训练;依次f0pool25seed0/1/2的N128与f0pool50seed0的N128,每块随后执行配对E。各GPU亲和[0,1]/[2,3]/[4,5]/[6,7],CPU8–31保留给后续R2。不能只凭入口标记断言模型完成,当前仍0终态块。
|
||||
|
||||
首4块8run全终态前不派第5块;自动投影≤46聚合GPUh才继续,等待/setup/失败均计费;每run≤1800s,AMP/宽度/Emax冻结,失败NA且不重试,源码pin不可改。实际证据见matrix根actual-start-audit.json和实时matrix-progress.json;严禁重复启动。R2尚未实现/启动,是后续待办,所有90+15终态及检查点冻结前不做测试预测评分。本节取代下方“矩阵未启动”历史状态;131准备已经结束,绝不重启。
|
||||
|
||||
## 最新终态:131全部结束,617件统一名单已冻结(09:25核查)
|
||||
|
||||
原session32694/PID220180已退出,131=124 PASS+7 wall_limit,无全局停止,无监测错误。自动final-roster.json/roster-receipt.json已生成:617 included/7 excluded,SHA `fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e`。原493复用,七失败跨五折统一排除,原组件身份保持。新增worker13.8436103492h/20h,累计历史worker48.6032611436h;guard3770.200695s/coordinator2408.510226s另列,wall5378.170096s,冻结前磁盘8099577856B。五折实际计数与成本在新准备根final-fold-pool-counts.csv、terminal-audit.json,原回执及无标签输出已核验。不得重启准备。现接续90fresh矩阵执行器实现与合成验收,矩阵/R2尚未启动;沿Claude1794699027既定预算与托管顺序,无需再次用户go。
|
||||
|
||||
Reference in new issue
Block a user