50 lines
3.2 KiB
Python
50 lines
3.2 KiB
Python
"""Freeze a new pilot after the approved, evidenced forward-precision diagnosis."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from freeze_g1_components import sha
|
|
from prepare_s2_recovered_pilot_training import finalize
|
|
from s2_e1_supervisor import save
|
|
|
|
|
|
def prepare(original,diagnostic,destination):
|
|
result=json.loads((diagnostic/'replay/result.json').read_text())
|
|
complete=json.loads((diagnostic/'complete.json').read_text())
|
|
guard=json.loads((diagnostic/'replay-guard-receipt.json').read_text())
|
|
assert complete['status']=='COMPLETE' and complete['total_wall_seconds']<=900
|
|
assert guard['status']=='PASS' and guard['result_sha256']==sha(diagnostic/'replay/result.json')
|
|
assert result['decision']=='FP32_FINITE_FP16_NONFINITE' and result['reproduced_original_failure']
|
|
assert result['fp32']['finite'] and not result['fp16']['finite']
|
|
assert all(r['matches_original_training_summary'] for r in result['prior_epoch_training_summaries'])
|
|
assert all(r['finite'] for r in result['parameters'].values())
|
|
assert result['validation_scoring'] is False and result['test_data_read'] is False
|
|
assert sha(diagnostic/'replay/failure-state.pt')==result['failure_state_sha256']
|
|
old=json.loads((original/'training-contract.json').read_text())
|
|
for path,digest in old['code_sha256'].items():assert sha(path)==digest,path
|
|
old_status=json.loads((original/'pilot-complete.json').read_text())
|
|
assert old_status['status']=='PILOT_INCOMPLETE' and old_status['selected_global_width'] is None
|
|
manifest=json.loads((original/'success-manifest.json').read_text())
|
|
source=Path(old['role_manifest']).parent
|
|
plan=finalize(source,destination,old['gpu_uuid'],Path(manifest['preparation_terminal_manifest']))
|
|
assert sha(destination/'success-manifest.json')==sha(original/'success-manifest.json')
|
|
additions=['tools/prepare_s2_precision_pilot.py','tools/run_s2_precision_pilot.py',
|
|
'tools/s2_precision_pilot_gpu_guard.py','tools/run_s2_pilot_probe_v2.py',
|
|
'tools/run_s2_pilot_training_v2.py','tools/run_s2_encoder_diagnostic.py']
|
|
additions+=sorted(str(p) for p in Path('tools/s2_models_v2').glob('*.py'))
|
|
plan['code_sha256'].update({p:sha(p) for p in additions})
|
|
plan['authorizing_rulings']+=['1794694544.819729','1794694646.032819']
|
|
plan['precision_variant']=dict(version='S2-pilot-v2-slice-FP32',
|
|
changed='Only shared N/E slice aggregation numerator, denominator and normalization in FP32 with autocast disabled',
|
|
original_pilot=str(original),original_contract_sha256=sha(original/'training-contract.json'),
|
|
original_status='PILOT_INCOMPLETE',diagnostic_root=str(diagnostic),
|
|
diagnostic_result_sha256=sha(diagnostic/'replay/result.json'),diagnostic_guard_sha256=sha(diagnostic/'replay-guard-receipt.json'),
|
|
from_scratch=True,original_weights_reused=False,preparation_repeated=False,
|
|
reprobe_max_nodes=187345,VRAM_limit_failure_no_batch_or_width_fallback=True)
|
|
save(destination/'pilot-plan.json',plan)
|
|
return plan
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('original',type=Path);p.add_argument('diagnostic',type=Path);p.add_argument('destination',type=Path);a=p.parse_args()
|
|
prepare(a.original,a.diagnostic,a.destination)
|