Files
MeltMind/tools/prepare_s2_recovered_pilot_training.py

68 lines
5.7 KiB
Python

"""Finalize all 499 terminal preparation attempts and freeze one pilot plan."""
import argparse
import json
from pathlib import Path
import numpy as np
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_pilot_data import permitted_role
from s2_incident_recovery_manifest import load_manifest
def quantile_probe(records,roles):
learning=sorted((r for r in records if permitted_role(roles,r['case'])=='learning'),key=lambda r:(r['nodes'],r['case']))
if len(learning)<20:raise ValueError('Need 20 distinct successful learning parts')
return [learning[int(i)]['case'] for i in np.linspace(0,len(learning)-1,20,dtype=int)]
def finalize(source,root,gpu_uuid,terminal_path):
c=json.loads((source/'contract.json').read_text());batch=load_manifest(terminal_path)
assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
assert len(batch['jobs'])==len(c['cases'])==499 and not batch['not_run_indices']
assert sorted(r['index'] for r in batch['jobs'])==list(range(499))
assert not root.exists(),'No overwrite or retry of an existing pilot'
roles=json.loads((source/'role-manifest.json').read_text());assert sha(source/'role-manifest.json')==c['role_manifest_sha256']
ready=[];failed=[]
for job in sorted(batch['jobs'],key=lambda r:r['index']):
spec=c['cases'][job['index']];case=spec['case'];role=permitted_role(roles,case)
assert job['case']==case
dest=Path(job.get('ready_source_root',source))/'ready'/case
if job['worker']['status']!='PASS':
failed.append(dict(case=case,role=role,nodes=spec['tet_nodes'],component_id=spec['component_id'],guard=job['worker'],
failure=json.loads((dest/'failure.json').read_text()) if (dest/'failure.json').exists() else None))
continue
r=json.loads((dest/'receipt.json').read_text());assert r['status']=='READY' and r['role']==role and r['case']==case
prep=Path(r['preparation_root']);assert sha(prep/'result.json')==r['preparation_result_sha256']
pr=json.loads((prep/'result.json').read_text());assert pr['status']=='COMPLETE'
hashes={name:sha(prep/name) for name in ['base.npy','queries.npz']}
assert all(hashes[name]==pr['output_sha256'][name] for name in hashes)
label=dest/'labels.npz';hashes['labels.npz']=sha(label);assert hashes['labels.npz']==r['label_sha256']
assert r['nodes']==spec['tet_nodes'] and r['component_id']==spec['component_id'] and r['t_set']==spec['t_set']
ready.append(dict(case=case,role=role,component_id=r['component_id'],nodes=r['nodes'],t_set=r['t_set'],
preparation_root=str(prep),label_path=str(label),input_sha256=hashes,ready_receipt_sha256=sha(dest/'receipt.json')))
planned={role:sum(s['tet_nodes'] for s in c['cases'] if permitted_role(roles,s['case'])==role) for role in ['learning','validation']}
successful={role:dict(parts=sum(r['role']==role for r in ready),nodes=sum(r['nodes'] for r in ready if r['role']==role)) for role in planned}
assert planned==dict(learning=11863762,validation=3330152)
if not all(successful[k]['parts'] for k in successful):raise ValueError('Empty successful role')
probe=quantile_probe(ready,roles)
paths=['tools/prepare_s2_recovered_pilot_training.py','tools/run_s2_recovered_pilot.py','tools/run_s2_pilot_probe.py',
'tools/run_s2_pilot_training.py','tools/s2_pilot_train.py','tools/s2_recovered_pilot_gpu_guard.py','tools/s2_recovery_gpu_supervisor.py',
'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/s2_incident_recovery_manifest.py','tools/freeze_g1_components.py']
paths+=['tools/s2_recovery_monitoring.py','tools/prepare_s2_incident_recovery.py','tools/run_s2_recovery_chain.py']
paths+=sorted(str(p) for p in Path('tools/s2_models').glob('*.py'))
code={p:sha(p) for p in paths}
root.mkdir();save(root/'success-manifest.json',dict(ready_parts=ready,failed_parts=failed,successful=successful,
preparation_batch_sha256=sha(terminal_path),preparation_terminal_manifest=str(terminal_path),
preparation_sources=batch.get('source_provenance',[]),preparation_contract_sha256=sha(source/'contract.json')))
plan=dict(fold=0,pool=100,seed=0,gpu_uuid=gpu_uuid,code_sha256=code,role_manifest=str(source/'role-manifest.json'),
role_manifest_sha256=sha(source/'role-manifest.json'),success_manifest=str(root/'success-manifest.json'),
success_manifest_sha256=sha(root/'success-manifest.json'),probe_cases=probe,
probe_selection='20 equally spaced ranks in successful learning node-count order; includes min/max; tie by case',
learning_nodes=planned['learning'],validation_nodes=planned['validation'],successful=successful,
extrapolation='Train rate * all 387 planned learning nodes + FP32 inference rate * all 112 planned validation nodes; conservative if failed parts dropped',
runs=[dict(name=f'{arm}{width}-seed0',arm=arm,width=width,seed=0) for width in [128,256] for arm in ['N','E']],
limits=dict(total_gpu_wall_seconds=7200,probe_wall_seconds=300,run_wall_seconds=1800,host_rss_bytes=16*2**30,vram_bytes=15*2**30,disk_bytes=5*2**30),
coverage_boundary=dict(planned_max_learning_nodes=max(s['tet_nodes'] for s in c['cases'] if permitted_role(roles,s['case'])=='learning'),max_successful_learning_nodes=max(r['nodes'] for r in ready if r['role']=='learning'),failed_parts=failed,statement='VRAM probe covers only the maximum successful learning part included; not failed/larger unprobed parts'),
authorizing_rulings=['1794680129.303719','1794681576.845789','1794681807.376669','1794689807.792819'],test_labels_read=False)
save(root/'pilot-plan.json',plan);return plan