Files
MeltMind/tools/phase2_h2_s2_postfit_common.py
T

104 lines
8.9 KiB
Python

"""S2 postfit identities, all-four terminal barrier and unchanged paired statistics."""
import json,math,os
from pathlib import Path
import numpy as np
from phase2_h1_worker_v2 import checked,sha
from phase2_home_freeze_io import verify
from phase2_h2_s2_common import confirmed
from phase2_b13_e1b import pooled
def read(p):return json.loads(Path(p).read_text())
def roster(c,pilot):
rows=c['rows'];by={r['case']:r for r in rows}
if len(rows)!=552 or len(by)!=552 or len({r['component_id'] for r in rows})!=204:raise ValueError('Fixed552/204 roster')
alltest=[];component_folds={}
for f in pilot['folds']:
for case in f['test']:
row=by[case];pr=pilot['parts'][case];a=c['baseline_predictions'][case]
tests=[v['fold'] for v in row['assignments'] if v['role']=='test']
if tests!=[f['fold']] or a['fold']!=f['fold'] or row['component_id']!=a['component_id'] or row['component_id']!=pr['clock_group'] or row['v5_cache']['output_sha256']!=pr['output_sha256'] or row['nodes']!=pr['nodes'] or row['preparation']['t_set']!=pr['t_set']:raise ValueError('Original test/cache/clock/query baseline identity')
group=row['component_id']
if group in component_folds and component_folds[group]!=f['fold']:raise ValueError('Component spans test folds')
component_folds[group]=f['fold'];alltest.append(case)
if len(alltest)!=552 or len(set(alltest))!=552 or set(alltest)!=set(by):raise ValueError('Original five-fold partition')
return by
def config(root):
c=read(root/'contract.json');verify(root,'implementation')
if c['approval']!='1794925470.101149' or c['total_h2_gpu_seconds']!=144000 or c['prediction_gpu_seconds']!=2400 or c['postfit_cpu_seconds']!=3600 or c['new_fits']!=0 or c['setup_cpu_seconds']!=360:raise PermissionError('Fixed pooled postfit scope')
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
train=Path(c['training_root']);proof=verify(train,'implementation')
if proof['commit']!=c['training_implementation']:raise ValueError('Wrong frozen training implementation')
tc=read(train/'contract.json');pilot=read(c['pilot_contract'])
if np.__version__!=pilot['versions']['numpy']:raise ValueError('Original NumPy version required')
verify(Path(c['s1_root']),'predictions');verify(Path(c['s1_root']),'terminal')
if tc['prior_gpu_seconds']!=18858.21490871499 or tc['prediction_gpu_reserve']!=2400 or tc['postfit_cpu_cap']!=3600 or tc['epochs']!=200 or tc['seed']!=0:raise ValueError('Original training budget/recipe')
if [j['fold'] for j in tc['jobs']]!=[1,2,3,4] or c['training_jobs']!=tc['jobs']:raise ValueError('Original four jobs')
roster(c,pilot);return c,pilot,tc
def terminal_identity(tc,done,ledger,models):
if done.get('status')!='TRAIN_COMPLETE_POSTFIT_PENDING' or done.get('postfit_started') is not False:raise PermissionError('All-four training terminal required')
entries=done.get('terminal',[])
if len(entries)!=4 or {e['fold'] for e in entries}!={1,2,3,4} or any(e['status']!='PASS' or e['returncode']!=0 for e in entries):raise ValueError('Exactly four PASS terminals')
if ledger['entries']!=entries or len(models['models'])!=4 or {m['fold'] for m in models['models']}!={1,2,3,4}:raise ValueError('Complete terminal/model manifest')
if models['fold0_reused_by_hash']!=tc['fold0_reuse'] or models['new_fits']!=4 or models['test_targets_read'] is not False or models['postfit_started'] is not False:raise ValueError('Model manifest roles')
costs=[e['gpu_seconds_charged'] for e in entries]
if not all(math.isfinite(x) and x>=0 for x in costs):raise ValueError('Invalid settled GPU cost')
total=sum(costs)
if done['prior_gpu_once']!=tc['prior_gpu_seconds'] or ledger['prior_gpu_once']!=tc['prior_gpu_seconds'] or ledger['settled_gpu_seconds']!=total or done['new_gpu_seconds_charged']!=total or done['cumulative_h2_gpu_seconds']!=tc['prior_gpu_seconds']+total:raise ValueError('Prior and four new costs must be counted once')
if total>tc['train_gpu_cap'] or done['cumulative_h2_gpu_seconds']+2400>144000:raise ValueError('No original prediction reserve')
return done['cumulative_h2_gpu_seconds']
def terminal_inputs(c,pilot,tc):
import psutil
train=Path(c['training_root']);terminal=verify(train,'terminal');modelproof=verify(train,'models')
if (train/'global-stop.json').exists():raise PermissionError('Training stopped')
done=read(train/'complete.json');ledger=read(train/'terminal-ledger.json');manifest=read(train/'models-terminal.json');prior=terminal_identity(tc,done,ledger,manifest)
for p in psutil.process_iter(['pid','cmdline']):
args=p.info['cmdline'] or []
if str(train) in args and any(Path(x).name in ['phase2_h2_s2_queue.py','phase2_h2_s2_guard.py','phase2_h2_s2_worker.py'] for x in args):raise RuntimeError('Training process still alive')
entries={e['fold']:e for e in done['terminal']};out=[]
for m in sorted(manifest['models'],key=lambda m:m['fold']):
f=m['fold'];folder=train/'folds'/str(f);guard=read(checked(folder/'complete.json',entries[f]['complete_sha256']));resultpath=checked(folder/'worker/result.json',m['worker_result_sha256']);result=read(resultpath);p=pilot['folds'][f]
if guard['status']!='PASS' or guard['reasons'] or guard['monitor_errors'] or guard['worker_result_sha256']!=m['worker_result_sha256']:raise ValueError('Original guard/result identity')
if result['fold']!=f or result['completed_epochs']!=200 or result['optimizer_steps']!=200*len(p['learning']) or result['learning_nodes_per_epoch']!=p['learning_nodes'] or result['test_targets_read'] is not False or result['validation_used_for_selection'] is not False:raise ValueError('Final200/role identity')
expected=folder/'worker/last-epoch200.pt';stdpath=folder/'worker/standardization.json'
if Path(m['checkpoint']).resolve()!=expected.resolve() or Path(m['standardization']).resolve()!=stdpath.resolve() or m['checkpoint_sha256']!=result['checkpoint_sha256'] or m['standardization_sha256']!=result['standardization_sha256']:raise ValueError('Checkpoint/normalization path identity')
for path,digest in [(expected,m['checkpoint_sha256']),(stdpath,m['standardization_sha256'])]:
checked(path,digest)
if modelproof['files'].get(str(path.resolve()))!=digest:raise ValueError('Checkpoint/normalization omitted from model freeze')
stats=read(stdpath)
if stats['parts']!=p['learning'] or stats['nodes']!=p['learning_nodes'] or len(stats['mean'])!=19 or len(stats['scale'])!=19 or not np.isfinite(stats['mean']).all() or not np.isfinite(stats['scale']).all() or min(stats['scale'])<=0:raise ValueError('Original learning-only normalization')
out.append(m)
return dict(training_terminal_commit=terminal['commit'],training_models_commit=modelproof['commit'],training_complete_sha256=sha(train/'complete.json'),training_models_sha256=sha(train/'models-terminal.json'),training_contract_sha256=c['source_sha256'][str(train/'contract.json')],models=out,prior_gpu_seconds=prior,old_s1_gpu_once=tc['prior_gpu_seconds'],new_training_gpu_once=done['new_gpu_seconds_charged'])
def adopted_inputs(root,c,pilot,tc):
verify(root,'models');a=read(root/'models-adopted.json')
if a!=terminal_inputs(c,pilot,tc):raise ValueError('Adopted four frozen checkpoints changed')
return a
def checkpoint_identity(blob,c,fold,parts):
if fold not in (1,2,3,4) or blob['completed_epochs']!=200 or blob['optimizer_steps']!=200*parts or blob['seed']!=0 or blob['fold']!=fold or blob['role']!='last_epoch_only' or blob['contract_sha256']!=c['source_sha256'][str(Path(c['training_root'])/'contract.json')]:raise ValueError('Wrong fold/final checkpoint')
def comparisons(rows):
out={}
for reference in ['A','B13_E1b']:
raw=pooled([dict(component_id=r['component_id'],new=r['H2'],A=r[reference]) for r in rows]);out[reference]={}
for name,m in raw.items():out[reference][name]=dict(means={'H2':m['means']['new'],reference:m['means']['A']},components={'H2':m['components']['new'],reference:m['components']['A']},H2_minus_reference=m['new_minus_A'])
return out
def reuse_fold0_metric(prior,baseline,entry):
if entry['fold']!=0 or prior['fold']!=0 or prior['component_id']!=entry['component_id'] or prior['A']!=baseline['A']:raise ValueError('Fold0 frozen score identity')
return prior['new']
def decision(results,parts,groups):
field=results['A']['relative_l2'];paired=field['H2_minus_reference']
if parts!=552 or groups!=204 or field['components']!={'H2':204,'A':204} or paired['components']!=204:return 'PARK_INCOMPLETE_FIELD_SUPPORT'
return 'CONFIRMED' if confirmed(paired['high'],parts,groups) else 'NOT_CONFIRMED_FIXED_RECIPE'
def require_budget(prior,gpu_spent,cpu_spent,remaining_predictions):
if not all(math.isfinite(x) and x>=0 for x in [prior,gpu_spent,cpu_spent]) or remaining_predictions not in range(5):raise ValueError('Invalid resource ledger')
if gpu_spent+remaining_predictions*600>2400 or prior+gpu_spent+remaining_predictions*600>144000:raise RuntimeError('Prediction GPU reservation')
# All four60-second local CPU setup allowances are already in setup_cpu_seconds=360.
if cpu_spent+remaining_predictions*540+600+120>3600:raise RuntimeError('Postfit CPU reservation')