Accept hash-bound merged preparation terminals without retrying failed parts
This commit is contained in:
1 parent
f771dea144
commit
e527daf8f0
8 files changed
+178
-8
No files matched your search
@@ -6,6 +6,7 @@ import numpy as np
|
||||
from freeze_g1_components import sha
|
||||
from s2_e1_supervisor import save
|
||||
from s2_pilot_data import permitted_role
|
||||
from s2_pilot_terminal import terminal_batch
|
||||
|
||||
|
||||
def quantile_probe(records,roles):
|
||||
@@ -15,7 +16,7 @@ def quantile_probe(records,roles):
|
||||
|
||||
|
||||
def finalize(source,root,gpu_uuid):
|
||||
c=json.loads((source/'contract.json').read_text());batch=json.loads((source/'batch-complete.json').read_text())
|
||||
c=json.loads((source/'contract.json').read_text());batch,terminal_path=terminal_batch(source)
|
||||
assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
|
||||
assert len(batch['jobs'])==len(c['cases'])==499 and not batch['not_run_indices']
|
||||
assert sorted(r['index'] for r in batch['jobs'])==list(range(499))
|
||||
@@ -25,7 +26,7 @@ def finalize(source,root,gpu_uuid):
|
||||
for job in sorted(batch['jobs'],key=lambda r:r['index']):
|
||||
spec=c['cases'][job['index']];case=spec['case'];role=permitted_role(roles,case)
|
||||
assert job['case']==case
|
||||
dest=source/'ready'/case
|
||||
dest=Path(job.get('ready_source_root',source))/'ready'/case
|
||||
if job['worker']['status']!='PASS':
|
||||
failed.append(dict(case=case,role=role,component_id=spec['component_id'],guard=job['worker'],
|
||||
failure=json.loads((dest/'failure.json').read_text()) if (dest/'failure.json').exists() else None))
|
||||
@@ -46,11 +47,12 @@ def finalize(source,root,gpu_uuid):
|
||||
probe=quantile_probe(ready,roles)
|
||||
paths=['tools/prepare_s2_pilot_training.py','tools/run_s2_pilot.py','tools/run_s2_pilot_probe.py',
|
||||
'tools/run_s2_pilot_training.py','tools/s2_pilot_train.py','tools/s2_pilot_gpu_guard.py','tools/s2_gpu_process_supervisor.py',
|
||||
'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/freeze_g1_components.py']
|
||||
'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/s2_pilot_terminal.py','tools/freeze_g1_components.py']
|
||||
paths+=sorted(str(p) for p in Path('tools/s2_models').glob('*.py'))
|
||||
code={p:sha(p) for p in paths}
|
||||
root.mkdir();save(root/'success-manifest.json',dict(ready_parts=ready,failed_parts=failed,successful=successful,
|
||||
preparation_batch_sha256=sha(source/'batch-complete.json'),preparation_contract_sha256=sha(source/'contract.json')))
|
||||
preparation_batch_sha256=sha(terminal_path),preparation_terminal_manifest=str(terminal_path),
|
||||
preparation_sources=batch.get('source_provenance',[]),preparation_contract_sha256=sha(source/'contract.json')))
|
||||
plan=dict(fold=0,pool=100,seed=0,gpu_uuid=gpu_uuid,code_sha256=code,role_manifest=str(source/'role-manifest.json'),
|
||||
role_manifest_sha256=sha(source/'role-manifest.json'),success_manifest=str(root/'success-manifest.json'),
|
||||
success_manifest_sha256=sha(root/'success-manifest.json'),probe_cases=probe,
|
||||
|
||||
Reference in new issue
Block a user