diff --git a/artifacts/s2-pilot-implementation-2026-11-15/waiter-receipt.json b/artifacts/s2-pilot-implementation-2026-11-15/waiter-receipt.json new file mode 100644 index 000000000..4b1a997f3 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/waiter-receipt.json @@ -0,0 +1,13 @@ +{ + "status": "PASS_SYNTHETIC_WAITER_GATES", + "tests": 2, + "passed": 2, + "seconds": 0.01, + "source_sha256": { + "tools/run_s2_pilot_when_ready.py": "aa4ee01fefe406756da88a0470d382519f2ccddbf5132d42e09a202b01466858", + "tools/test_s2_pilot_waiter.py": "084d5d6adc611cf8c34475f04bec493eb04163751a80d56124d8fff04419c636" + }, + "log_sha256": "7d1f917fc12cfe22aaa86bba19e121d3aa2d22c054be3b3cd8327c4b2c79741b", + "GPU_started": false, + "real_preparation_not_restarted": true +} diff --git a/artifacts/s2-pilot-implementation-2026-11-15/waiter-tests.txt b/artifacts/s2-pilot-implementation-2026-11-15/waiter-tests.txt new file mode 100644 index 000000000..fdc9d18d3 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/waiter-tests.txt @@ -0,0 +1,7 @@ +test_stopped_preparation_does_not_finalize_or_use_gpu (test_s2_pilot_waiter.WaiterTests) ... ok +test_terminal_gate_requires_full_roster_no_global_stop (test_s2_pilot_waiter.WaiterTests) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 0.010s + +OK diff --git a/docs/s2-pilot-live-handoff-2026-11-15.md b/docs/s2-pilot-live-handoff-2026-11-15.md index a70602880..c1357d38d 100644 --- a/docs/s2-pilot-live-handoff-2026-11-15.md +++ b/docs/s2-pilot-live-handoff-2026-11-15.md @@ -15,3 +15,7 @@ Claude 1794682712.505709补充将检查点继续上限设为80worker CPU-h;该 pilot仅通过run_s2_pilot.py启动:E256 20件学习几何合成目标计时probe≤5min;训练和FP32推理各自按节点总量外推,原学习11,863,762节点/验证3,330,152节点;共同E_max=min(200,floor(1500/epoch秒))且cosine一致。probe和设置计入单卡四run总7200s,各run≤1800s。probe/任意run失败停止、不重试。四个run全部成功才选一次全局宽。 运行入口需要可见V100的主机环境;普通受限环境无法可靠读取GPU。用户已授权原计算范围与可控Git/Slack分享,不再要求重复授权。主分工保持:Claude整合/裁量,MeltAstra执行证据;用户休息勿扰。 + +受控接续入口run_s2_pilot_when_ready.py另有2项合成门控检查PASS(.010s),不重复前26项验收。入口只等待现有队列,不启动准备;499个唯一终态、无全局停止/未尝试项、源码和原契约pin一致后才调用成功清单核验和既有GPU监督入口。等待不使用GPU,实际probe和四run的7200秒另从GPU入口计时。任何准备停止/不完整或pin变化都留失败记录并停止,不自动生成80h续队列。 + +此版本只固定接续实现;是否实际启动以artifacts/s2-pilot-continuation-2026-11-15/started.json为准。原准备session16682和现有batch-started.json仍是唯一准备队列;不能因为接续入口在等待而重复启动。 diff --git a/tools/run_s2_pilot_when_ready.py b/tools/run_s2_pilot_when_ready.py new file mode 100644 index 000000000..90ecab5f0 --- /dev/null +++ b/tools/run_s2_pilot_when_ready.py @@ -0,0 +1,68 @@ +"""Wait for the existing preparation attempt, then run the already bounded pilot. + +Never launches/restarts preparation. Waiting uses no CUDA; all GPU work remains +inside run_s2_pilot's single-card probe-plus-four-run 7200-second guard. +""" +import argparse +import fcntl +import json +from pathlib import Path +import time +import traceback +from freeze_g1_components import sha +from s2_e1_supervisor import save +from prepare_s2_pilot_training import finalize +from run_s2_pilot import run as pilot + + +def preparation_terminal(source): + path=source/'batch-complete.json' + if not path.exists():return False + batch=json.loads(path.read_text()) + if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']: + raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch') + if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)): + raise ValueError('Exactly 499 distinct terminal attempts required') + return True + + +def run(source,root,watch,gpu_uuid): + watch.mkdir(exist_ok=True) + with (watch/'execution.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (watch/'started.json').exists():raise FileExistsError('Continuation already attempted; inspect evidence, do not restart') + assert (source/'batch-started.json').exists(),'Existing preparation must already be running' + assert not root.exists(),'No overwrite of a prior pilot' + paths=[str(p) for p in Path('tools').glob('*s2*pilot*.py')]+[str(p) for p in Path('tools/s2_models').glob('*.py')] + paths+=['tools/s2_gpu_process_supervisor.py','tools/s2_e1_supervisor.py','tools/freeze_g1_components.py'] + frozen={p:sha(p) for p in sorted(set(paths))} + expected=sha(source/'contract.json');start=time.monotonic() + save(watch/'started.json',dict(started_unix=time.time(),preparation_root=str(source),preparation_contract_sha256=expected, + pilot_root=str(root),gpu_uuid=gpu_uuid,code_sha256=frozen,waiting_uses_GPU=False, + authorizing_rulings=['1794681576.845789','1794681807.376669','1794683016.134929'])) + try: + while not preparation_terminal(source): + progress=json.loads((source/'progress.json').read_text()) + save(watch/'progress.json',dict(stage='WAITING_FOR_EXISTING_PREPARATION',completed=progress['completed'], + waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True)) + time.sleep(10) + assert sha(source/'contract.json')==expected + for p,h in frozen.items():assert sha(p)==h,p + plan=finalize(source,root,gpu_uuid) + save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED', + batch_sha256=sha(source/'batch-complete.json'),success_manifest_sha256=plan['success_manifest_sha256'], + successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()), + pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start)) + save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root), + total_GPU_wall_seconds_limit=7200,preparation_wait_excluded_from_GPU_time=True)) + pilot(root) + result=json.loads((root/'orchestrator-complete.json').read_text()) + save(watch/'complete.json',dict(**result,preparation_wait_and_pilot_wall_seconds=time.monotonic()-start)) + except Exception as exc: + save(watch/'failure.json',dict(status='STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(), + wall_seconds=time.monotonic()-start,preparation_not_restarted=True));raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('source',type=Path);p.add_argument('root',type=Path);p.add_argument('watch',type=Path);p.add_argument('gpu_uuid');a=p.parse_args() + run(a.source,a.root,a.watch,a.gpu_uuid) diff --git a/tools/test_s2_pilot_waiter.py b/tools/test_s2_pilot_waiter.py new file mode 100644 index 000000000..5ca2efa87 --- /dev/null +++ b/tools/test_s2_pilot_waiter.py @@ -0,0 +1,30 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +import run_s2_pilot_when_ready as waiter + + +class WaiterTests(unittest.TestCase): + def test_terminal_gate_requires_full_roster_no_global_stop(self): + with tempfile.TemporaryDirectory() as tmp: + source=Path(tmp);self.assertFalse(waiter.preparation_terminal(source)) + batch=dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=[dict(index=i) for i in range(499)]) + p=source/'batch-complete.json';p.write_text(json.dumps(batch));self.assertTrue(waiter.preparation_terminal(source)) + for change in [dict(status='STOPPED'),dict(stop_reason='GLOBAL_CPU_LIMIT'),dict(not_run_indices=[498]),dict(jobs=batch['jobs'][:-1]),dict(jobs=[dict(index=0)]*499)]: + p.write_text(json.dumps(dict(batch,**change))) + with self.assertRaises((RuntimeError,ValueError)):waiter.preparation_terminal(source) + def test_stopped_preparation_does_not_finalize_or_use_gpu(self): + with tempfile.TemporaryDirectory() as tmp: + source=Path(tmp)/'source';source.mkdir();(source/'batch-started.json').write_text('{}');(source/'contract.json').write_text('{}') + (source/'batch-complete.json').write_text('{"status":"STOPPED"}') + watch=Path(tmp)/'watch';root=Path(tmp)/'pilot' + with patch.object(waiter,'finalize') as finalize,patch.object(waiter,'pilot') as pilot: + with self.assertRaises(RuntimeError):waiter.run(source,root,watch,'synthetic') + finalize.assert_not_called();pilot.assert_not_called() + with self.assertRaises(FileExistsError):waiter.run(source,root,watch,'synthetic') + self.assertTrue((watch/'failure.json').exists());self.assertFalse(root.exists()) + + +if __name__=='__main__':unittest.main(verbosity=2)