From e527daf8f09dc86de7afd68d9eea1ce79e5afe56 Mon Sep 17 00:00:00 2001 From: Frank14f <1515444314@qq.com> Date: Sun, 15 Nov 2026 03:28:26 +0800 Subject: [PATCH] Accept hash-bound merged preparation terminals without retrying failed parts --- .../merged-continuation-receipt.json | 19 +++++ .../merged-continuation-tests.txt | 15 ++++ .../merged-publication-tests.txt | 7 ++ docs/s2-pilot-live-handoff-2026-11-15.md | 6 +- tools/prepare_s2_pilot_training.py | 10 ++- tools/run_s2_pilot_when_ready.py | 11 ++- tools/s2_pilot_terminal.py | 78 +++++++++++++++++++ tools/test_s2_pilot_waiter.py | 40 ++++++++++ 8 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-receipt.json create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-tests.txt create mode 100644 artifacts/s2-pilot-implementation-2026-11-15/merged-publication-tests.txt create mode 100644 tools/s2_pilot_terminal.py diff --git a/artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-receipt.json b/artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-receipt.json new file mode 100644 index 000000000..8b7b66fcd --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-receipt.json @@ -0,0 +1,19 @@ +{ + "status": "MERGED_CONTINUATION_GATES_PASS", + "unique_checks": 9, + "initial_run": "9/9 PASS, 0.148s; before atomic-publication refinement", + "final_targeted_run": "2/2 merged-manifest checks PASS after atomic create/no-overwrite refinement; other code unchanged", + "source_sha256": { + "tools/s2_pilot_terminal.py": "89dcdf837df09d343c44b60678f857391f24382b609f4446c9391dc73abfd4d9", + "tools/prepare_s2_pilot_training.py": "6eccaae3c08ef43207ce67f772e7db0671c74da9c9f85dfd49e2fe6790050423", + "tools/run_s2_pilot_when_ready.py": "03dca86c6aaa6a856a69836e2e913bf3f478ac1eb0d0d64adc083f4692b7d8cc", + "tools/test_s2_pilot_waiter.py": "9cb7096eedc0494e4bbe6faf015c8b16ca319d9d38b9cd6461724f77e9196aec" + }, + "logs_sha256": { + "artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-tests.txt": "88881465d9df52860961b2ee1f8c062734af340f8115ce26f5043f6cddd1ec87", + "artifacts/s2-pilot-implementation-2026-11-15/merged-publication-tests.txt": "fc6e1f2ba3ebfa846595e6153d5bfcf75ace6fe8380add1769f30a9a58099ae7" + }, + "real_continuation_queue_created": false, + "real_GPU_started": false, + "ruling": "1794684155.730259" +} diff --git a/artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-tests.txt b/artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-tests.txt new file mode 100644 index 000000000..889a23123 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-tests.txt @@ -0,0 +1,15 @@ +test_merge_rejects_retries_changed_specs_and_cumulative_overrun (test_s2_pilot_waiter.MergedTerminalTests) ... ok +test_waits_for_approved_merge_accepts_failed_parts_and_binds_both_sources (test_s2_pilot_waiter.MergedTerminalTests) ... ok +test_stopped_preparation_does_not_finalize_or_use_gpu (test_s2_pilot_waiter.WaiterTests) ... ok +test_terminal_gate_requires_full_roster_no_global_stop (test_s2_pilot_waiter.WaiterTests) ... ok +test_contract_records_two_node_rates_totals_and_cannot_overwrite (test_s2_pilot_orchestration.OrchestrationTests) ... ok +test_failed_probe_stops_before_training (test_s2_pilot_orchestration.OrchestrationTests) ... ok +test_partial_preparation_never_starts_pilot (test_s2_pilot_orchestration.OrchestrationTests) ... ok +test_probe_time_reduces_total_budget (test_s2_pilot_orchestration.OrchestrationTests) ... {"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "FAIL"}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 7190.000236072985, "probe_and_setup_wall_seconds": 7190.0, "test_labels_read": false, "test_predictions": false, "retries": 0} +ok +test_quantiles_only_learning_and_include_size_extremes (test_s2_pilot_orchestration.OrchestrationTests) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 0.148s + +OK diff --git a/artifacts/s2-pilot-implementation-2026-11-15/merged-publication-tests.txt b/artifacts/s2-pilot-implementation-2026-11-15/merged-publication-tests.txt new file mode 100644 index 000000000..5f5fcd232 --- /dev/null +++ b/artifacts/s2-pilot-implementation-2026-11-15/merged-publication-tests.txt @@ -0,0 +1,7 @@ +test_merge_rejects_retries_changed_specs_and_cumulative_overrun (test_s2_pilot_waiter.MergedTerminalTests) ... ok +test_waits_for_approved_merge_accepts_failed_parts_and_binds_both_sources (test_s2_pilot_waiter.MergedTerminalTests) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 0.161s + +OK diff --git a/docs/s2-pilot-live-handoff-2026-11-15.md b/docs/s2-pilot-live-handoff-2026-11-15.md index c1357d38d..d9e06c253 100644 --- a/docs/s2-pilot-live-handoff-2026-11-15.md +++ b/docs/s2-pilot-live-handoff-2026-11-15.md @@ -16,6 +16,10 @@ pilot仅通过run_s2_pilot.py启动:E256 20件学习几何合成目标计时pr 运行入口需要可见V100的主机环境;普通受限环境无法可靠读取GPU。用户已授权原计算范围与可控Git/Slack分享,不再要求重复授权。主分工保持:Claude整合/裁量,MeltAstra执行证据;用户休息勿扰。 -受控接续入口run_s2_pilot_when_ready.py另有2项合成门控检查PASS(.010s),不重复前26项验收。入口只等待现有队列,不启动准备;499个唯一终态、无全局停止/未尝试项、源码和原契约pin一致后才调用成功清单核验和既有GPU监督入口。等待不使用GPU,实际probe和四run的7200秒另从GPU入口计时。任何准备停止/不完整或pin变化都留失败记录并停止,不自动生成80h续队列。 +受控接续入口run_s2_pilot_when_ready.py另有2项合成门控检查PASS(.010s),不重复前26项验收。入口只等待现有队列,不启动准备;499个唯一终态、无全局停止/未尝试项、源码和原契约pin一致后才调用成功清单核验和既有GPU监督入口。等待不使用GPU,实际probe和四run的7200秒另从GPU入口计时。未经获准的准备停止/不完整或pin变化会留失败记录并停止,不自动生成80h续队列;已获准的60–80h首50续队列情形则等待合并终态清单。 此版本只固定接续实现;是否实际启动以artifacts/s2-pilot-continuation-2026-11-15/started.json为准。原准备session16682和现有batch-started.json仍是唯一准备队列;不能因为接续入口在等待而重复启动。 + +Claude 1794684155.730259已接受接续入口,并要求合法单件FAIL保持可剔除、获准续队列使用合并终态。新增s2_pilot_terminal.py绑定原队列和续队列各自contract/batch SHA,只接收原not_run集合且规格逐件相同,拒绝重复/重试、全局停止、hash变化与累计80h/100GiB超限。merged-terminal.json按原499索引保存来源ready目录,由完整临时文件原子创建且不可覆盖。finalizer据来源读各件回执;失败件仍列明剔除。原499队列正常完成时不生成续队列或合并清单。 + +相关9项检查PASS(0.148s);原子发布精化后定点重跑2项合并检查PASS,覆盖FAIL合法终态、等候获准合并、两来源hash、重复/规格变更/累计超限拒绝及不可覆盖。训练核心与GPU资源上限未改变;此前26项及接续入口初版2项记录原样保留。 diff --git a/tools/prepare_s2_pilot_training.py b/tools/prepare_s2_pilot_training.py index 7c331ad0f..8731c6921 100644 --- a/tools/prepare_s2_pilot_training.py +++ b/tools/prepare_s2_pilot_training.py @@ -6,6 +6,7 @@ import numpy as np from freeze_g1_components import sha from s2_e1_supervisor import save from s2_pilot_data import permitted_role +from s2_pilot_terminal import terminal_batch def quantile_probe(records,roles): @@ -15,7 +16,7 @@ def quantile_probe(records,roles): def finalize(source,root,gpu_uuid): - c=json.loads((source/'contract.json').read_text());batch=json.loads((source/'batch-complete.json').read_text()) + c=json.loads((source/'contract.json').read_text());batch,terminal_path=terminal_batch(source) assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None assert len(batch['jobs'])==len(c['cases'])==499 and not batch['not_run_indices'] assert sorted(r['index'] for r in batch['jobs'])==list(range(499)) @@ -25,7 +26,7 @@ def finalize(source,root,gpu_uuid): for job in sorted(batch['jobs'],key=lambda r:r['index']): spec=c['cases'][job['index']];case=spec['case'];role=permitted_role(roles,case) assert job['case']==case - dest=source/'ready'/case + dest=Path(job.get('ready_source_root',source))/'ready'/case if job['worker']['status']!='PASS': failed.append(dict(case=case,role=role,component_id=spec['component_id'],guard=job['worker'], failure=json.loads((dest/'failure.json').read_text()) if (dest/'failure.json').exists() else None)) @@ -46,11 +47,12 @@ def finalize(source,root,gpu_uuid): probe=quantile_probe(ready,roles) paths=['tools/prepare_s2_pilot_training.py','tools/run_s2_pilot.py','tools/run_s2_pilot_probe.py', 'tools/run_s2_pilot_training.py','tools/s2_pilot_train.py','tools/s2_pilot_gpu_guard.py','tools/s2_gpu_process_supervisor.py', - 'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/freeze_g1_components.py'] + 'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/s2_pilot_terminal.py','tools/freeze_g1_components.py'] paths+=sorted(str(p) for p in Path('tools/s2_models').glob('*.py')) code={p:sha(p) for p in paths} root.mkdir();save(root/'success-manifest.json',dict(ready_parts=ready,failed_parts=failed,successful=successful, - preparation_batch_sha256=sha(source/'batch-complete.json'),preparation_contract_sha256=sha(source/'contract.json'))) + preparation_batch_sha256=sha(terminal_path),preparation_terminal_manifest=str(terminal_path), + preparation_sources=batch.get('source_provenance',[]),preparation_contract_sha256=sha(source/'contract.json'))) plan=dict(fold=0,pool=100,seed=0,gpu_uuid=gpu_uuid,code_sha256=code,role_manifest=str(source/'role-manifest.json'), role_manifest_sha256=sha(source/'role-manifest.json'),success_manifest=str(root/'success-manifest.json'), success_manifest_sha256=sha(root/'success-manifest.json'),probe_cases=probe, diff --git a/tools/run_s2_pilot_when_ready.py b/tools/run_s2_pilot_when_ready.py index 90ecab5f0..b94927a71 100644 --- a/tools/run_s2_pilot_when_ready.py +++ b/tools/run_s2_pilot_when_ready.py @@ -13,12 +13,14 @@ from freeze_g1_components import sha from s2_e1_supervisor import save from prepare_s2_pilot_training import finalize from run_s2_pilot import run as pilot +from s2_pilot_terminal import terminal_batch,continuation_allowed def preparation_terminal(source): path=source/'batch-complete.json' if not path.exists():return False - batch=json.loads(path.read_text()) + batch,_=terminal_batch(source) + if not (source/'merged-terminal.json').exists() and continuation_allowed(source,batch):return False if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']: raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch') if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)): @@ -43,14 +45,17 @@ def run(source,root,watch,gpu_uuid): try: while not preparation_terminal(source): progress=json.loads((source/'progress.json').read_text()) - save(watch/'progress.json',dict(stage='WAITING_FOR_EXISTING_PREPARATION',completed=progress['completed'], + stage='WAITING_FOR_ACCEPTED_MERGED_CONTINUATION' if (source/'batch-complete.json').exists() else 'WAITING_FOR_EXISTING_PREPARATION' + save(watch/'progress.json',dict(stage=stage,completed=progress['completed'], waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True)) time.sleep(10) assert sha(source/'contract.json')==expected for p,h in frozen.items():assert sha(p)==h,p plan=finalize(source,root,gpu_uuid) + terminal,terminal_path=terminal_batch(source) save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED', - batch_sha256=sha(source/'batch-complete.json'),success_manifest_sha256=plan['success_manifest_sha256'], + batch_sha256=sha(terminal_path),terminal_manifest=str(terminal_path), + preparation_sources=terminal.get('source_provenance',[]),success_manifest_sha256=plan['success_manifest_sha256'], successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()), pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start)) save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root), diff --git a/tools/s2_pilot_terminal.py b/tools/s2_pilot_terminal.py new file mode 100644 index 000000000..68618a597 --- /dev/null +++ b/tools/s2_pilot_terminal.py @@ -0,0 +1,78 @@ +"""Validate original or merged preparation terminal evidence without rerunning it.""" +import argparse +import json +import os +from pathlib import Path +from freeze_g1_components import sha +from s2_e1_supervisor import disk_usage + + +def continuation_allowed(source,batch): + if batch['status']!='STOPPED' or batch.get('stop_reason')!='FIRST_50_PROJECTION_OVER_CAP':return False + checkpoint=json.loads((source/'first-50-checkpoint.json').read_text()) + return (60*3600=0 + assert job['worker']['status']!='GUARD_FAILED_ACCOUNTING_UNKNOWN' + jobs.append(dict(job,index=original_cases[case],source_index=job['index'],ready_source_root=str(root))) + assert seen==set(original_cases) and len(jobs)==499 + cpu=sum(r['worker']['cpu_seconds'] for r in jobs);used=sum(disk_usage(root) for root in roots) + assert cpu<=80*3600 and used<=100*2**30,'Merged budget exceeded' + return dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=sorted(jobs,key=lambda r:r['index']), + worker_cpu_seconds=cpu,guard_cpu_seconds=sum(b['guard_cpu_seconds'] for b in batches), + failed=sum(r['worker']['status']!='PASS' for r in jobs),disk_bytes=used, + source_provenance=proof,continuation_ruling='1794683049.057259',retry=False,replacement=False,test_labels_read=False) + + +def terminal_batch(source): + merged=source/'merged-terminal.json' + if not merged.exists(): + path=source/'batch-complete.json';return json.loads(path.read_text()),path + m=json.loads(merged.read_text());proof=m['source_provenance'] + batch=combine(source,Path(proof[1]['root']),proof) + assert batch['jobs']==m['jobs'] and batch['worker_cpu_seconds']==m['worker_cpu_seconds'] + return batch,merged + + +def merge(source,continuation): + proof=[dict(root=str(root),contract_sha256=sha(root/'contract.json'),batch_sha256=sha(root/'batch-complete.json')) for root in [source,continuation]] + batch=combine(source,continuation,proof) + pending=source/'merged-terminal.json.pending' + with pending.open('x') as f:json.dump(batch,f,indent=2);f.write('\n');f.flush();os.fsync(f.fileno()) + # The waiting entry sees either no merged manifest or the complete file. + # link creates the final name atomically and refuses an existing manifest. + os.link(pending,source/'merged-terminal.json');pending.unlink() + return batch + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('source',type=Path);p.add_argument('continuation',type=Path);a=p.parse_args();r=merge(a.source,a.continuation) + print(json.dumps(dict(status=r['status'],terminal=len(r['jobs']),failed=r['failed'],worker_cpu_seconds=r['worker_cpu_seconds']))) diff --git a/tools/test_s2_pilot_waiter.py b/tools/test_s2_pilot_waiter.py index 5ca2efa87..65233d746 100644 --- a/tools/test_s2_pilot_waiter.py +++ b/tools/test_s2_pilot_waiter.py @@ -4,6 +4,8 @@ import unittest from pathlib import Path from unittest.mock import patch import run_s2_pilot_when_ready as waiter +from s2_pilot_terminal import merge,terminal_batch +from freeze_g1_components import sha class WaiterTests(unittest.TestCase): @@ -27,4 +29,42 @@ class WaiterTests(unittest.TestCase): self.assertTrue((watch/'failure.json').exists());self.assertFalse(root.exists()) +class MergedTerminalTests(unittest.TestCase): + def fixture(self,root): + source=root/'original';later=root/'continuation';source.mkdir();later.mkdir() + cases=[dict(case=str(i),tet_nodes=i+1) for i in range(499)] + def job(i,case):return dict(index=i,case=str(case),worker=dict(status='STOPPED_OR_FAILED' if case==0 else 'PASS',cpu_seconds=1.)) + original=dict(cases=cases,role_manifest_sha256='synthetic-roles') + before=dict(status='STOPPED',stop_reason='FIRST_50_PROJECTION_OVER_CAP',not_run_indices=list(range(50,499)),jobs=[job(i,i) for i in range(50)],guard_cpu_seconds=1.) + (source/'contract.json').write_text(json.dumps(original));(source/'batch-complete.json').write_text(json.dumps(before)) + (source/'first-50-checkpoint.json').write_text(json.dumps(dict(projected_worker_cpu_seconds=70*3600,projected_disk_bytes=10*2**30))) + proof=dict(root=str(source),contract_sha256=sha(source/'contract.json'),batch_sha256=sha(source/'batch-complete.json')) + (later/'contract.json').write_text(json.dumps(dict(cases=cases[50:],role_manifest_sha256='synthetic-roles',continuation_of=proof))) + (later/'batch-complete.json').write_text(json.dumps(dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=[job(i,i+50) for i in range(449)],guard_cpu_seconds=1.))) + return source,later + def test_waits_for_approved_merge_accepts_failed_parts_and_binds_both_sources(self): + with tempfile.TemporaryDirectory() as tmp: + source,later=self.fixture(Path(tmp));self.assertFalse(waiter.preparation_terminal(source)) + batch=merge(source,later);self.assertEqual(batch['failed'],1);self.assertEqual(len(batch['jobs']),499) + self.assertEqual(batch['jobs'][49]['ready_source_root'],str(source));self.assertEqual(batch['jobs'][50]['ready_source_root'],str(later)) + self.assertEqual(batch['jobs'][50]['source_index'],0);self.assertEqual(batch['worker_cpu_seconds'],499.) + self.assertTrue(waiter.preparation_terminal(source)) + fixed=sha(source/'merged-terminal.json') + with self.assertRaises(FileExistsError):merge(source,later) + self.assertEqual(sha(source/'merged-terminal.json'),fixed) + (later/'contract.json').write_text('{}') + with self.assertRaises(AssertionError):terminal_batch(source) + def test_merge_rejects_retries_changed_specs_and_cumulative_overrun(self): + for change in ['retry','spec','cpu']: + with self.subTest(change=change),tempfile.TemporaryDirectory() as tmp: + source,later=self.fixture(Path(tmp)) + p=later/('batch-complete.json' if change=='cpu' else 'contract.json');data=json.loads(p.read_text()) + if change=='retry':data['cases'][0]['case']='0' + elif change=='spec':data['cases'][0]['tet_nodes']+=1 + else:data['jobs'][0]['worker']['cpu_seconds']=81*3600 + p.write_text(json.dumps(data)) + with self.assertRaises(AssertionError):merge(source,later) + self.assertFalse((source/'merged-terminal.json').exists()) + + if __name__=='__main__':unittest.main(verbosity=2)