Add pinned continuation from completed preparation to guarded pilot
This commit is contained in:
1 parent
8f096a4e3b
commit
f771dea144
5 files changed
+122
No files matched your search
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"status": "PASS_SYNTHETIC_WAITER_GATES",
|
||||
"tests": 2,
|
||||
"passed": 2,
|
||||
"seconds": 0.01,
|
||||
"source_sha256": {
|
||||
"tools/run_s2_pilot_when_ready.py": "aa4ee01fefe406756da88a0470d382519f2ccddbf5132d42e09a202b01466858",
|
||||
"tools/test_s2_pilot_waiter.py": "084d5d6adc611cf8c34475f04bec493eb04163751a80d56124d8fff04419c636"
|
||||
},
|
||||
"log_sha256": "7d1f917fc12cfe22aaa86bba19e121d3aa2d22c054be3b3cd8327c4b2c79741b",
|
||||
"GPU_started": false,
|
||||
"real_preparation_not_restarted": true
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
test_stopped_preparation_does_not_finalize_or_use_gpu (test_s2_pilot_waiter.WaiterTests) ... ok
|
||||
test_terminal_gate_requires_full_roster_no_global_stop (test_s2_pilot_waiter.WaiterTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 2 tests in 0.010s
|
||||
|
||||
OK
|
||||
@@ -15,3 +15,7 @@ Claude 1794682712.505709补充将检查点继续上限设为80worker CPU-h;该
|
||||
pilot仅通过run_s2_pilot.py启动:E256 20件学习几何合成目标计时probe≤5min;训练和FP32推理各自按节点总量外推,原学习11,863,762节点/验证3,330,152节点;共同E_max=min(200,floor(1500/epoch秒))且cosine一致。probe和设置计入单卡四run总7200s,各run≤1800s。probe/任意run失败停止、不重试。四个run全部成功才选一次全局宽。
|
||||
|
||||
运行入口需要可见V100的主机环境;普通受限环境无法可靠读取GPU。用户已授权原计算范围与可控Git/Slack分享,不再要求重复授权。主分工保持:Claude整合/裁量,MeltAstra执行证据;用户休息勿扰。
|
||||
|
||||
受控接续入口run_s2_pilot_when_ready.py另有2项合成门控检查PASS(.010s),不重复前26项验收。入口只等待现有队列,不启动准备;499个唯一终态、无全局停止/未尝试项、源码和原契约pin一致后才调用成功清单核验和既有GPU监督入口。等待不使用GPU,实际probe和四run的7200秒另从GPU入口计时。任何准备停止/不完整或pin变化都留失败记录并停止,不自动生成80h续队列。
|
||||
|
||||
此版本只固定接续实现;是否实际启动以artifacts/s2-pilot-continuation-2026-11-15/started.json为准。原准备session16682和现有batch-started.json仍是唯一准备队列;不能因为接续入口在等待而重复启动。
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Wait for the existing preparation attempt, then run the already bounded pilot.
|
||||
|
||||
Never launches/restarts preparation. Waiting uses no CUDA; all GPU work remains
|
||||
inside run_s2_pilot's single-card probe-plus-four-run 7200-second guard.
|
||||
"""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import traceback
|
||||
from freeze_g1_components import sha
|
||||
from s2_e1_supervisor import save
|
||||
from prepare_s2_pilot_training import finalize
|
||||
from run_s2_pilot import run as pilot
|
||||
|
||||
|
||||
def preparation_terminal(source):
|
||||
path=source/'batch-complete.json'
|
||||
if not path.exists():return False
|
||||
batch=json.loads(path.read_text())
|
||||
if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']:
|
||||
raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch')
|
||||
if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)):
|
||||
raise ValueError('Exactly 499 distinct terminal attempts required')
|
||||
return True
|
||||
|
||||
|
||||
def run(source,root,watch,gpu_uuid):
|
||||
watch.mkdir(exist_ok=True)
|
||||
with (watch/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (watch/'started.json').exists():raise FileExistsError('Continuation already attempted; inspect evidence, do not restart')
|
||||
assert (source/'batch-started.json').exists(),'Existing preparation must already be running'
|
||||
assert not root.exists(),'No overwrite of a prior pilot'
|
||||
paths=[str(p) for p in Path('tools').glob('*s2*pilot*.py')]+[str(p) for p in Path('tools/s2_models').glob('*.py')]
|
||||
paths+=['tools/s2_gpu_process_supervisor.py','tools/s2_e1_supervisor.py','tools/freeze_g1_components.py']
|
||||
frozen={p:sha(p) for p in sorted(set(paths))}
|
||||
expected=sha(source/'contract.json');start=time.monotonic()
|
||||
save(watch/'started.json',dict(started_unix=time.time(),preparation_root=str(source),preparation_contract_sha256=expected,
|
||||
pilot_root=str(root),gpu_uuid=gpu_uuid,code_sha256=frozen,waiting_uses_GPU=False,
|
||||
authorizing_rulings=['1794681576.845789','1794681807.376669','1794683016.134929']))
|
||||
try:
|
||||
while not preparation_terminal(source):
|
||||
progress=json.loads((source/'progress.json').read_text())
|
||||
save(watch/'progress.json',dict(stage='WAITING_FOR_EXISTING_PREPARATION',completed=progress['completed'],
|
||||
waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True))
|
||||
time.sleep(10)
|
||||
assert sha(source/'contract.json')==expected
|
||||
for p,h in frozen.items():assert sha(p)==h,p
|
||||
plan=finalize(source,root,gpu_uuid)
|
||||
save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED',
|
||||
batch_sha256=sha(source/'batch-complete.json'),success_manifest_sha256=plan['success_manifest_sha256'],
|
||||
successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),
|
||||
pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start))
|
||||
save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),
|
||||
total_GPU_wall_seconds_limit=7200,preparation_wait_excluded_from_GPU_time=True))
|
||||
pilot(root)
|
||||
result=json.loads((root/'orchestrator-complete.json').read_text())
|
||||
save(watch/'complete.json',dict(**result,preparation_wait_and_pilot_wall_seconds=time.monotonic()-start))
|
||||
except Exception as exc:
|
||||
save(watch/'failure.json',dict(status='STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),
|
||||
wall_seconds=time.monotonic()-start,preparation_not_restarted=True));raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('source',type=Path);p.add_argument('root',type=Path);p.add_argument('watch',type=Path);p.add_argument('gpu_uuid');a=p.parse_args()
|
||||
run(a.source,a.root,a.watch,a.gpu_uuid)
|
||||
@@ -0,0 +1,30 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import run_s2_pilot_when_ready as waiter
|
||||
|
||||
|
||||
class WaiterTests(unittest.TestCase):
|
||||
def test_terminal_gate_requires_full_roster_no_global_stop(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
source=Path(tmp);self.assertFalse(waiter.preparation_terminal(source))
|
||||
batch=dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=[dict(index=i) for i in range(499)])
|
||||
p=source/'batch-complete.json';p.write_text(json.dumps(batch));self.assertTrue(waiter.preparation_terminal(source))
|
||||
for change in [dict(status='STOPPED'),dict(stop_reason='GLOBAL_CPU_LIMIT'),dict(not_run_indices=[498]),dict(jobs=batch['jobs'][:-1]),dict(jobs=[dict(index=0)]*499)]:
|
||||
p.write_text(json.dumps(dict(batch,**change)))
|
||||
with self.assertRaises((RuntimeError,ValueError)):waiter.preparation_terminal(source)
|
||||
def test_stopped_preparation_does_not_finalize_or_use_gpu(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
source=Path(tmp)/'source';source.mkdir();(source/'batch-started.json').write_text('{}');(source/'contract.json').write_text('{}')
|
||||
(source/'batch-complete.json').write_text('{"status":"STOPPED"}')
|
||||
watch=Path(tmp)/'watch';root=Path(tmp)/'pilot'
|
||||
with patch.object(waiter,'finalize') as finalize,patch.object(waiter,'pilot') as pilot:
|
||||
with self.assertRaises(RuntimeError):waiter.run(source,root,watch,'synthetic')
|
||||
finalize.assert_not_called();pilot.assert_not_called()
|
||||
with self.assertRaises(FileExistsError):waiter.run(source,root,watch,'synthetic')
|
||||
self.assertTrue((watch/'failure.json').exists());self.assertFalse(root.exists())
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user