Accept hash-bound merged preparation terminals without retrying failed parts

This commit is contained in:
Frank14f committed 2026-11-15 03:28:26 +08:00
1 parent f771dea144
commit e527daf8f0
8 files changed
+178 -8

No files matched your search

+8 -3
View File
@@ -13,12 +13,14 @@ from freeze_g1_components import sha
from s2_e1_supervisor import save
from prepare_s2_pilot_training import finalize
from run_s2_pilot import run as pilot
from s2_pilot_terminal import terminal_batch,continuation_allowed
def preparation_terminal(source):
path=source/'batch-complete.json'
if not path.exists():return False
batch=json.loads(path.read_text())
batch,_=terminal_batch(source)
if not (source/'merged-terminal.json').exists() and continuation_allowed(source,batch):return False
if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']:
raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch')
if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)):
@@ -43,14 +45,17 @@ def run(source,root,watch,gpu_uuid):
try:
while not preparation_terminal(source):
progress=json.loads((source/'progress.json').read_text())
save(watch/'progress.json',dict(stage='WAITING_FOR_EXISTING_PREPARATION',completed=progress['completed'],
stage='WAITING_FOR_ACCEPTED_MERGED_CONTINUATION' if (source/'batch-complete.json').exists() else 'WAITING_FOR_EXISTING_PREPARATION'
save(watch/'progress.json',dict(stage=stage,completed=progress['completed'],
waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True))
time.sleep(10)
assert sha(source/'contract.json')==expected
for p,h in frozen.items():assert sha(p)==h,p
plan=finalize(source,root,gpu_uuid)
terminal,terminal_path=terminal_batch(source)
save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED',
batch_sha256=sha(source/'batch-complete.json'),success_manifest_sha256=plan['success_manifest_sha256'],
batch_sha256=sha(terminal_path),terminal_manifest=str(terminal_path),
preparation_sources=terminal.get('source_provenance',[]),success_manifest_sha256=plan['success_manifest_sha256'],
successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),
pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start))
save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),