Accept hash-bound merged preparation terminals without retrying failed parts
This commit is contained in:
1 parent
f771dea144
commit
e527daf8f0
8 files changed
+178
-8
No files matched your search
@@ -13,12 +13,14 @@ from freeze_g1_components import sha
|
||||
from s2_e1_supervisor import save
|
||||
from prepare_s2_pilot_training import finalize
|
||||
from run_s2_pilot import run as pilot
|
||||
from s2_pilot_terminal import terminal_batch,continuation_allowed
|
||||
|
||||
|
||||
def preparation_terminal(source):
|
||||
path=source/'batch-complete.json'
|
||||
if not path.exists():return False
|
||||
batch=json.loads(path.read_text())
|
||||
batch,_=terminal_batch(source)
|
||||
if not (source/'merged-terminal.json').exists() and continuation_allowed(source,batch):return False
|
||||
if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']:
|
||||
raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch')
|
||||
if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)):
|
||||
@@ -43,14 +45,17 @@ def run(source,root,watch,gpu_uuid):
|
||||
try:
|
||||
while not preparation_terminal(source):
|
||||
progress=json.loads((source/'progress.json').read_text())
|
||||
save(watch/'progress.json',dict(stage='WAITING_FOR_EXISTING_PREPARATION',completed=progress['completed'],
|
||||
stage='WAITING_FOR_ACCEPTED_MERGED_CONTINUATION' if (source/'batch-complete.json').exists() else 'WAITING_FOR_EXISTING_PREPARATION'
|
||||
save(watch/'progress.json',dict(stage=stage,completed=progress['completed'],
|
||||
waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True))
|
||||
time.sleep(10)
|
||||
assert sha(source/'contract.json')==expected
|
||||
for p,h in frozen.items():assert sha(p)==h,p
|
||||
plan=finalize(source,root,gpu_uuid)
|
||||
terminal,terminal_path=terminal_batch(source)
|
||||
save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED',
|
||||
batch_sha256=sha(source/'batch-complete.json'),success_manifest_sha256=plan['success_manifest_sha256'],
|
||||
batch_sha256=sha(terminal_path),terminal_manifest=str(terminal_path),
|
||||
preparation_sources=terminal.get('source_provenance',[]),success_manifest_sha256=plan['success_manifest_sha256'],
|
||||
successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),
|
||||
pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start))
|
||||
save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),
|
||||
|
||||
Reference in new issue
Block a user