Accept hash-bound merged preparation terminals without retrying failed parts

This commit is contained in:
Frank14f committed 2026-11-15 03:28:26 +08:00
1 parent f771dea144
commit e527daf8f0
8 files changed
+178 -8

No files matched your search

@@ -0,0 +1,19 @@
{
"status": "MERGED_CONTINUATION_GATES_PASS",
"unique_checks": 9,
"initial_run": "9/9 PASS, 0.148s; before atomic-publication refinement",
"final_targeted_run": "2/2 merged-manifest checks PASS after atomic create/no-overwrite refinement; other code unchanged",
"source_sha256": {
"tools/s2_pilot_terminal.py": "89dcdf837df09d343c44b60678f857391f24382b609f4446c9391dc73abfd4d9",
"tools/prepare_s2_pilot_training.py": "6eccaae3c08ef43207ce67f772e7db0671c74da9c9f85dfd49e2fe6790050423",
"tools/run_s2_pilot_when_ready.py": "03dca86c6aaa6a856a69836e2e913bf3f478ac1eb0d0d64adc083f4692b7d8cc",
"tools/test_s2_pilot_waiter.py": "9cb7096eedc0494e4bbe6faf015c8b16ca319d9d38b9cd6461724f77e9196aec"
},
"logs_sha256": {
"artifacts/s2-pilot-implementation-2026-11-15/merged-continuation-tests.txt": "88881465d9df52860961b2ee1f8c062734af340f8115ce26f5043f6cddd1ec87",
"artifacts/s2-pilot-implementation-2026-11-15/merged-publication-tests.txt": "fc6e1f2ba3ebfa846595e6153d5bfcf75ace6fe8380add1769f30a9a58099ae7"
},
"real_continuation_queue_created": false,
"real_GPU_started": false,
"ruling": "1794684155.730259"
}
@@ -0,0 +1,15 @@
test_merge_rejects_retries_changed_specs_and_cumulative_overrun (test_s2_pilot_waiter.MergedTerminalTests) ... ok
test_waits_for_approved_merge_accepts_failed_parts_and_binds_both_sources (test_s2_pilot_waiter.MergedTerminalTests) ... ok
test_stopped_preparation_does_not_finalize_or_use_gpu (test_s2_pilot_waiter.WaiterTests) ... ok
test_terminal_gate_requires_full_roster_no_global_stop (test_s2_pilot_waiter.WaiterTests) ... ok
test_contract_records_two_node_rates_totals_and_cannot_overwrite (test_s2_pilot_orchestration.OrchestrationTests) ... ok
test_failed_probe_stops_before_training (test_s2_pilot_orchestration.OrchestrationTests) ... ok
test_partial_preparation_never_starts_pilot (test_s2_pilot_orchestration.OrchestrationTests) ... ok
test_probe_time_reduces_total_budget (test_s2_pilot_orchestration.OrchestrationTests) ... {"status": "PILOT_INCOMPLETE", "reason": "PILOT_RUN_FAILED_NO_RETRY", "runs": [{"run": {"name": "N128", "arm": "N", "width": 128, "seed": 0}, "status": "FAIL"}], "selected_global_width": null, "selection_rule": "mean of N/E validation error at each width, ties smaller; all four runs required", "pilot_wall_seconds": 7190.000236072985, "probe_and_setup_wall_seconds": 7190.0, "test_labels_read": false, "test_predictions": false, "retries": 0}
ok
test_quantiles_only_learning_and_include_size_extremes (test_s2_pilot_orchestration.OrchestrationTests) ... ok
----------------------------------------------------------------------
Ran 9 tests in 0.148s
OK
@@ -0,0 +1,7 @@
test_merge_rejects_retries_changed_specs_and_cumulative_overrun (test_s2_pilot_waiter.MergedTerminalTests) ... ok
test_waits_for_approved_merge_accepts_failed_parts_and_binds_both_sources (test_s2_pilot_waiter.MergedTerminalTests) ... ok
----------------------------------------------------------------------
Ran 2 tests in 0.161s
OK
+5 -1
View File
@@ -16,6 +16,10 @@ pilot仅通过run_s2_pilot.py启动:E256 20件学习几何合成目标计时pr
运行入口需要可见V100的主机环境;普通受限环境无法可靠读取GPU。用户已授权原计算范围与可控Git/Slack分享,不再要求重复授权。主分工保持:Claude整合/裁量,MeltAstra执行证据;用户休息勿扰。
受控接续入口run_s2_pilot_when_ready.py另有2项合成门控检查PASS(.010s),不重复前26项验收。入口只等待现有队列,不启动准备;499个唯一终态、无全局停止/未尝试项、源码和原契约pin一致后才调用成功清单核验和既有GPU监督入口。等待不使用GPU,实际probe和四run的7200秒另从GPU入口计时。任何准备停止/不完整或pin变化都留失败记录并停止,不自动生成80h续队列。
受控接续入口run_s2_pilot_when_ready.py另有2项合成门控检查PASS(.010s),不重复前26项验收。入口只等待现有队列,不启动准备;499个唯一终态、无全局停止/未尝试项、源码和原契约pin一致后才调用成功清单核验和既有GPU监督入口。等待不使用GPU,实际probe和四run的7200秒另从GPU入口计时。未经获准的准备停止/不完整或pin变化会留失败记录并停止,不自动生成80h续队列;已获准的60–80h首50续队列情形则等待合并终态清单。
此版本只固定接续实现;是否实际启动以artifacts/s2-pilot-continuation-2026-11-15/started.json为准。原准备session16682和现有batch-started.json仍是唯一准备队列;不能因为接续入口在等待而重复启动。
Claude 1794684155.730259已接受接续入口,并要求合法单件FAIL保持可剔除、获准续队列使用合并终态。新增s2_pilot_terminal.py绑定原队列和续队列各自contract/batch SHA,只接收原not_run集合且规格逐件相同,拒绝重复/重试、全局停止、hash变化与累计80h/100GiB超限。merged-terminal.json按原499索引保存来源ready目录,由完整临时文件原子创建且不可覆盖。finalizer据来源读各件回执;失败件仍列明剔除。原499队列正常完成时不生成续队列或合并清单。
相关9项检查PASS(0.148s);原子发布精化后定点重跑2项合并检查PASS,覆盖FAIL合法终态、等候获准合并、两来源hash、重复/规格变更/累计超限拒绝及不可覆盖。训练核心与GPU资源上限未改变;此前26项及接续入口初版2项记录原样保留。
+6 -4
View File
@@ -6,6 +6,7 @@ import numpy as np
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_pilot_data import permitted_role
from s2_pilot_terminal import terminal_batch
def quantile_probe(records,roles):
@@ -15,7 +16,7 @@ def quantile_probe(records,roles):
def finalize(source,root,gpu_uuid):
c=json.loads((source/'contract.json').read_text());batch=json.loads((source/'batch-complete.json').read_text())
c=json.loads((source/'contract.json').read_text());batch,terminal_path=terminal_batch(source)
assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
assert len(batch['jobs'])==len(c['cases'])==499 and not batch['not_run_indices']
assert sorted(r['index'] for r in batch['jobs'])==list(range(499))
@@ -25,7 +26,7 @@ def finalize(source,root,gpu_uuid):
for job in sorted(batch['jobs'],key=lambda r:r['index']):
spec=c['cases'][job['index']];case=spec['case'];role=permitted_role(roles,case)
assert job['case']==case
dest=source/'ready'/case
dest=Path(job.get('ready_source_root',source))/'ready'/case
if job['worker']['status']!='PASS':
failed.append(dict(case=case,role=role,component_id=spec['component_id'],guard=job['worker'],
failure=json.loads((dest/'failure.json').read_text()) if (dest/'failure.json').exists() else None))
@@ -46,11 +47,12 @@ def finalize(source,root,gpu_uuid):
probe=quantile_probe(ready,roles)
paths=['tools/prepare_s2_pilot_training.py','tools/run_s2_pilot.py','tools/run_s2_pilot_probe.py',
'tools/run_s2_pilot_training.py','tools/s2_pilot_train.py','tools/s2_pilot_gpu_guard.py','tools/s2_gpu_process_supervisor.py',
'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/freeze_g1_components.py']
'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/s2_pilot_terminal.py','tools/freeze_g1_components.py']
paths+=sorted(str(p) for p in Path('tools/s2_models').glob('*.py'))
code={p:sha(p) for p in paths}
root.mkdir();save(root/'success-manifest.json',dict(ready_parts=ready,failed_parts=failed,successful=successful,
preparation_batch_sha256=sha(source/'batch-complete.json'),preparation_contract_sha256=sha(source/'contract.json')))
preparation_batch_sha256=sha(terminal_path),preparation_terminal_manifest=str(terminal_path),
preparation_sources=batch.get('source_provenance',[]),preparation_contract_sha256=sha(source/'contract.json')))
plan=dict(fold=0,pool=100,seed=0,gpu_uuid=gpu_uuid,code_sha256=code,role_manifest=str(source/'role-manifest.json'),
role_manifest_sha256=sha(source/'role-manifest.json'),success_manifest=str(root/'success-manifest.json'),
success_manifest_sha256=sha(root/'success-manifest.json'),probe_cases=probe,
+8 -3
View File
@@ -13,12 +13,14 @@ from freeze_g1_components import sha
from s2_e1_supervisor import save
from prepare_s2_pilot_training import finalize
from run_s2_pilot import run as pilot
from s2_pilot_terminal import terminal_batch,continuation_allowed
def preparation_terminal(source):
path=source/'batch-complete.json'
if not path.exists():return False
batch=json.loads(path.read_text())
batch,_=terminal_batch(source)
if not (source/'merged-terminal.json').exists() and continuation_allowed(source,batch):return False
if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']:
raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch')
if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)):
@@ -43,14 +45,17 @@ def run(source,root,watch,gpu_uuid):
try:
while not preparation_terminal(source):
progress=json.loads((source/'progress.json').read_text())
save(watch/'progress.json',dict(stage='WAITING_FOR_EXISTING_PREPARATION',completed=progress['completed'],
stage='WAITING_FOR_ACCEPTED_MERGED_CONTINUATION' if (source/'batch-complete.json').exists() else 'WAITING_FOR_EXISTING_PREPARATION'
save(watch/'progress.json',dict(stage=stage,completed=progress['completed'],
waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True))
time.sleep(10)
assert sha(source/'contract.json')==expected
for p,h in frozen.items():assert sha(p)==h,p
plan=finalize(source,root,gpu_uuid)
terminal,terminal_path=terminal_batch(source)
save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED',
batch_sha256=sha(source/'batch-complete.json'),success_manifest_sha256=plan['success_manifest_sha256'],
batch_sha256=sha(terminal_path),terminal_manifest=str(terminal_path),
preparation_sources=terminal.get('source_provenance',[]),success_manifest_sha256=plan['success_manifest_sha256'],
successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),
pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start))
save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),
+78
View File
@@ -0,0 +1,78 @@
"""Validate original or merged preparation terminal evidence without rerunning it."""
import argparse
import json
import os
from pathlib import Path
from freeze_g1_components import sha
from s2_e1_supervisor import disk_usage
def continuation_allowed(source,batch):
if batch['status']!='STOPPED' or batch.get('stop_reason')!='FIRST_50_PROJECTION_OVER_CAP':return False
checkpoint=json.loads((source/'first-50-checkpoint.json').read_text())
return (60*3600<checkpoint['projected_worker_cpu_seconds']<=80*3600
and checkpoint['projected_disk_bytes']<=100*2**30
and len(batch['jobs'])==50 and bool(batch['not_run_indices']))
def combine(source,continuation,proof):
roots=[source,continuation];contracts=[];batches=[]
assert len(proof)==2
for root,entry in zip(roots,proof):
assert str(root)==entry['root']
assert sha(root/'contract.json')==entry['contract_sha256']
assert sha(root/'batch-complete.json')==entry['batch_sha256']
contracts.append(json.loads((root/'contract.json').read_text()))
batches.append(json.loads((root/'batch-complete.json').read_text()))
original,later=contracts;before,after=batches
assert len(original['cases'])==499 and continuation_allowed(source,before)
assert after['status']=='ATTEMPTS_COMPLETE' and after['stop_reason'] is None and not after['not_run_indices']
assert later['continuation_of']==proof[0]
assert later['role_manifest_sha256']==original['role_manifest_sha256']
original_cases={s['case']:i for i,s in enumerate(original['cases'])};assert len(original_cases)==499
pending={original['cases'][i]['case'] for i in before['not_run_indices']}
assert {s['case'] for s in later['cases']}==pending and len(later['cases'])==len(pending)
assert sorted(r['index'] for r in after['jobs'])==list(range(len(later['cases'])))
for spec in later['cases']:
assert spec==original['cases'][original_cases[spec['case']]],'Continuation cannot replace or alter a part specification'
jobs=[];seen=set()
for root,c,b in zip(roots,contracts,batches):
for job in b['jobs']:
case=job['case'];assert c['cases'][job['index']]['case']==case
assert case not in seen,'An attempted part may not be retried by continuation'
seen.add(case);assert job['worker']['cpu_seconds']>=0
assert job['worker']['status']!='GUARD_FAILED_ACCOUNTING_UNKNOWN'
jobs.append(dict(job,index=original_cases[case],source_index=job['index'],ready_source_root=str(root)))
assert seen==set(original_cases) and len(jobs)==499
cpu=sum(r['worker']['cpu_seconds'] for r in jobs);used=sum(disk_usage(root) for root in roots)
assert cpu<=80*3600 and used<=100*2**30,'Merged budget exceeded'
return dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=sorted(jobs,key=lambda r:r['index']),
worker_cpu_seconds=cpu,guard_cpu_seconds=sum(b['guard_cpu_seconds'] for b in batches),
failed=sum(r['worker']['status']!='PASS' for r in jobs),disk_bytes=used,
source_provenance=proof,continuation_ruling='1794683049.057259',retry=False,replacement=False,test_labels_read=False)
def terminal_batch(source):
merged=source/'merged-terminal.json'
if not merged.exists():
path=source/'batch-complete.json';return json.loads(path.read_text()),path
m=json.loads(merged.read_text());proof=m['source_provenance']
batch=combine(source,Path(proof[1]['root']),proof)
assert batch['jobs']==m['jobs'] and batch['worker_cpu_seconds']==m['worker_cpu_seconds']
return batch,merged
def merge(source,continuation):
proof=[dict(root=str(root),contract_sha256=sha(root/'contract.json'),batch_sha256=sha(root/'batch-complete.json')) for root in [source,continuation]]
batch=combine(source,continuation,proof)
pending=source/'merged-terminal.json.pending'
with pending.open('x') as f:json.dump(batch,f,indent=2);f.write('\n');f.flush();os.fsync(f.fileno())
# The waiting entry sees either no merged manifest or the complete file.
# link creates the final name atomically and refuses an existing manifest.
os.link(pending,source/'merged-terminal.json');pending.unlink()
return batch
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('source',type=Path);p.add_argument('continuation',type=Path);a=p.parse_args();r=merge(a.source,a.continuation)
print(json.dumps(dict(status=r['status'],terminal=len(r['jobs']),failed=r['failed'],worker_cpu_seconds=r['worker_cpu_seconds'])))
+40
View File
@@ -4,6 +4,8 @@ import unittest
from pathlib import Path
from unittest.mock import patch
import run_s2_pilot_when_ready as waiter
from s2_pilot_terminal import merge,terminal_batch
from freeze_g1_components import sha
class WaiterTests(unittest.TestCase):
@@ -27,4 +29,42 @@ class WaiterTests(unittest.TestCase):
self.assertTrue((watch/'failure.json').exists());self.assertFalse(root.exists())
class MergedTerminalTests(unittest.TestCase):
def fixture(self,root):
source=root/'original';later=root/'continuation';source.mkdir();later.mkdir()
cases=[dict(case=str(i),tet_nodes=i+1) for i in range(499)]
def job(i,case):return dict(index=i,case=str(case),worker=dict(status='STOPPED_OR_FAILED' if case==0 else 'PASS',cpu_seconds=1.))
original=dict(cases=cases,role_manifest_sha256='synthetic-roles')
before=dict(status='STOPPED',stop_reason='FIRST_50_PROJECTION_OVER_CAP',not_run_indices=list(range(50,499)),jobs=[job(i,i) for i in range(50)],guard_cpu_seconds=1.)
(source/'contract.json').write_text(json.dumps(original));(source/'batch-complete.json').write_text(json.dumps(before))
(source/'first-50-checkpoint.json').write_text(json.dumps(dict(projected_worker_cpu_seconds=70*3600,projected_disk_bytes=10*2**30)))
proof=dict(root=str(source),contract_sha256=sha(source/'contract.json'),batch_sha256=sha(source/'batch-complete.json'))
(later/'contract.json').write_text(json.dumps(dict(cases=cases[50:],role_manifest_sha256='synthetic-roles',continuation_of=proof)))
(later/'batch-complete.json').write_text(json.dumps(dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=[job(i,i+50) for i in range(449)],guard_cpu_seconds=1.)))
return source,later
def test_waits_for_approved_merge_accepts_failed_parts_and_binds_both_sources(self):
with tempfile.TemporaryDirectory() as tmp:
source,later=self.fixture(Path(tmp));self.assertFalse(waiter.preparation_terminal(source))
batch=merge(source,later);self.assertEqual(batch['failed'],1);self.assertEqual(len(batch['jobs']),499)
self.assertEqual(batch['jobs'][49]['ready_source_root'],str(source));self.assertEqual(batch['jobs'][50]['ready_source_root'],str(later))
self.assertEqual(batch['jobs'][50]['source_index'],0);self.assertEqual(batch['worker_cpu_seconds'],499.)
self.assertTrue(waiter.preparation_terminal(source))
fixed=sha(source/'merged-terminal.json')
with self.assertRaises(FileExistsError):merge(source,later)
self.assertEqual(sha(source/'merged-terminal.json'),fixed)
(later/'contract.json').write_text('{}')
with self.assertRaises(AssertionError):terminal_batch(source)
def test_merge_rejects_retries_changed_specs_and_cumulative_overrun(self):
for change in ['retry','spec','cpu']:
with self.subTest(change=change),tempfile.TemporaryDirectory() as tmp:
source,later=self.fixture(Path(tmp))
p=later/('batch-complete.json' if change=='cpu' else 'contract.json');data=json.loads(p.read_text())
if change=='retry':data['cases'][0]['case']='0'
elif change=='spec':data['cases'][0]['tet_nodes']+=1
else:data['jobs'][0]['worker']['cpu_seconds']=81*3600
p.write_text(json.dumps(data))
with self.assertRaises(AssertionError):merge(source,later)
self.assertFalse((source/'merged-terminal.json').exists())
if __name__=='__main__':unittest.main(verbosity=2)