69 lines
5.2 KiB
Python
69 lines
5.2 KiB
Python
"""Finalize all 499 terminal preparation attempts and freeze one pilot plan."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import numpy as np
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from s2_pilot_data import permitted_role
|
|
|
|
|
|
def quantile_probe(records,roles):
|
|
learning=sorted((r for r in records if permitted_role(roles,r['case'])=='learning'),key=lambda r:(r['nodes'],r['case']))
|
|
if len(learning)<20:raise ValueError('Need 20 distinct successful learning parts')
|
|
return [learning[int(i)]['case'] for i in np.linspace(0,len(learning)-1,20,dtype=int)]
|
|
|
|
|
|
def finalize(source,root,gpu_uuid):
|
|
c=json.loads((source/'contract.json').read_text());batch=json.loads((source/'batch-complete.json').read_text())
|
|
assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
|
|
assert len(batch['jobs'])==len(c['cases'])==499 and not batch['not_run_indices']
|
|
assert sorted(r['index'] for r in batch['jobs'])==list(range(499))
|
|
assert not root.exists(),'No overwrite or retry of an existing pilot'
|
|
roles=json.loads((source/'role-manifest.json').read_text());assert sha(source/'role-manifest.json')==c['role_manifest_sha256']
|
|
ready=[];failed=[]
|
|
for job in sorted(batch['jobs'],key=lambda r:r['index']):
|
|
spec=c['cases'][job['index']];case=spec['case'];role=permitted_role(roles,case)
|
|
assert job['case']==case
|
|
dest=source/'ready'/case
|
|
if job['worker']['status']!='PASS':
|
|
failed.append(dict(case=case,role=role,component_id=spec['component_id'],guard=job['worker'],
|
|
failure=json.loads((dest/'failure.json').read_text()) if (dest/'failure.json').exists() else None))
|
|
continue
|
|
r=json.loads((dest/'receipt.json').read_text());assert r['status']=='READY' and r['role']==role and r['case']==case
|
|
prep=Path(r['preparation_root']);assert sha(prep/'result.json')==r['preparation_result_sha256']
|
|
pr=json.loads((prep/'result.json').read_text());assert pr['status']=='COMPLETE'
|
|
hashes={name:sha(prep/name) for name in ['base.npy','queries.npz']}
|
|
assert all(hashes[name]==pr['output_sha256'][name] for name in hashes)
|
|
label=dest/'labels.npz';hashes['labels.npz']=sha(label);assert hashes['labels.npz']==r['label_sha256']
|
|
assert r['nodes']==spec['tet_nodes'] and r['component_id']==spec['component_id'] and r['t_set']==spec['t_set']
|
|
ready.append(dict(case=case,role=role,component_id=r['component_id'],nodes=r['nodes'],t_set=r['t_set'],
|
|
preparation_root=str(prep),label_path=str(label),input_sha256=hashes,ready_receipt_sha256=sha(dest/'receipt.json')))
|
|
planned={role:sum(s['tet_nodes'] for s in c['cases'] if permitted_role(roles,s['case'])==role) for role in ['learning','validation']}
|
|
successful={role:dict(parts=sum(r['role']==role for r in ready),nodes=sum(r['nodes'] for r in ready if r['role']==role)) for role in planned}
|
|
assert planned==dict(learning=11863762,validation=3330152)
|
|
if not all(successful[k]['parts'] for k in successful):raise ValueError('Empty successful role')
|
|
probe=quantile_probe(ready,roles)
|
|
paths=['tools/prepare_s2_pilot_training.py','tools/run_s2_pilot.py','tools/run_s2_pilot_probe.py',
|
|
'tools/run_s2_pilot_training.py','tools/s2_pilot_train.py','tools/s2_pilot_gpu_guard.py','tools/s2_gpu_process_supervisor.py',
|
|
'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/freeze_g1_components.py']
|
|
paths+=sorted(str(p) for p in Path('tools/s2_models').glob('*.py'))
|
|
code={p:sha(p) for p in paths}
|
|
root.mkdir();save(root/'success-manifest.json',dict(ready_parts=ready,failed_parts=failed,successful=successful,
|
|
preparation_batch_sha256=sha(source/'batch-complete.json'),preparation_contract_sha256=sha(source/'contract.json')))
|
|
plan=dict(fold=0,pool=100,seed=0,gpu_uuid=gpu_uuid,code_sha256=code,role_manifest=str(source/'role-manifest.json'),
|
|
role_manifest_sha256=sha(source/'role-manifest.json'),success_manifest=str(root/'success-manifest.json'),
|
|
success_manifest_sha256=sha(root/'success-manifest.json'),probe_cases=probe,
|
|
probe_selection='20 equally spaced ranks in successful learning node-count order; includes min/max; tie by case',
|
|
learning_nodes=planned['learning'],validation_nodes=planned['validation'],successful=successful,
|
|
extrapolation='Train rate * all 387 planned learning nodes + FP32 inference rate * all 112 planned validation nodes; conservative if failed parts dropped',
|
|
runs=[dict(name=f'{arm}{width}-seed0',arm=arm,width=width,seed=0) for width in [128,256] for arm in ['N','E']],
|
|
limits=dict(total_gpu_wall_seconds=7200,probe_wall_seconds=300,run_wall_seconds=1800,host_rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=5*2**30),
|
|
authorizing_rulings=['1794680129.303719','1794681576.845789','1794681807.376669'],test_labels_read=False)
|
|
save(root/'pilot-plan.json',plan);return plan
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('source',type=Path);p.add_argument('root',type=Path);p.add_argument('gpu_uuid');a=p.parse_args()
|
|
plan=finalize(a.source,a.root,a.gpu_uuid);print(json.dumps(dict(successful=plan['successful'],probe_cases=plan['probe_cases'])))
|