100 lines
8.0 KiB
Python
100 lines
8.0 KiB
Python
"""One reviewed item1 postfit; runtime checkpoint/prediction custody barriers."""
|
|
import json,math
|
|
from pathlib import Path
|
|
from phase2_h1_worker_v2 import checked,sha
|
|
from phase2_home_freeze_io import verify
|
|
MODEL_ARMS=('epoch25','epoch50','step50_U','step50_2U','step100_U','step100_2U','S1_epoch100_reuse')
|
|
DOMAIN={'learning':'own_learning','validation':'common_validation','test':'common_test'}
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
def config(root):
|
|
c=read(root/'contract.json');verify(root,'implementation')
|
|
if c['approval']!='1794950613.195499' or c['comparison_contract_sha256']!='332f2515911c99aadf2486086ddff82bd195d889faa8391296443e7aeacdfee9' or c['postfit_gpu_cap']!=7200 or c['postfit_cpu_cap']!=3600 or c['item_gpu_cap']!=120000:raise PermissionError('Frozen item1 postfit scope')
|
|
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h)
|
|
tc=read(c['training_contract']);pilot=read(tc['pilot_contract'])
|
|
verify(Path(c['training_root']),'implementation')
|
|
for p,h in tc['code_sha256'].items():checked(p,h)
|
|
if tc['new_fits']!=6 or tc['item_gpu_cap']!=120000 or tc['postfit_gpu_reserve']!=7200:raise ValueError('Training scope mismatch')
|
|
if [j['arm'] for j in c['jobs']]!=list(MODEL_ARMS):raise ValueError('Fixed seven endpoint identities')
|
|
return c,tc,pilot
|
|
|
|
def expected_checkpoint(job,meta,training_sha):
|
|
if meta.get('fold')!=0 or meta.get('seed')!=0 or meta.get('optimizer_steps')!=job['target_steps'] or meta.get('contract_sha256')!=training_sha:raise ValueError('Checkpoint fold/seed/step/source identity')
|
|
if job['id']<6:
|
|
if meta.get('pool')!=job['pool'] or meta.get('mode')!=job['mode'] or meta.get('role')!=job['tag']:raise ValueError('New checkpoint cell identity')
|
|
elif meta.get('completed_epochs')!=200 or meta.get('role')!='last_epoch_only':raise ValueError('S1 endpoint identity')
|
|
|
|
def terminal_identity(done,mm):
|
|
if done['status']!='TRAIN_COMPLETE_POSTFIT_PENDING' or len(done['entries'])!=6 or any(e['status']!='PASS' or e['returncode']!=0 for e in done['entries']):raise ValueError('All six terminal PASS required')
|
|
if {e['id'] for e in done['entries']}!=set(range(6)) or len(mm['models'])!=6 or {m['job'] for m in mm['models']}!=set(range(6)):raise ValueError('Six distinct model jobs')
|
|
if done['test_targets_read'] or mm['test_prediction_started'] or mm['new_fits']!=6 or mm['new_checkpoints']!=6:raise ValueError('Training custody identity')
|
|
costs=[e['gpu_seconds_charged'] for e in done['entries']];prior=done['training_gpu_seconds']
|
|
if any(not math.isfinite(x) or x<0 for x in costs+[prior]) or prior>112800 or abs(prior-sum(costs))>1e-8:raise ValueError('Settled training ledger')
|
|
return prior
|
|
|
|
def terminal_inputs(c,tc):
|
|
import psutil
|
|
train=Path(c['training_root'])
|
|
if not (train/'complete.json').exists():raise RuntimeError('WAIT_EXISTING_SIX_TRAINING_TERMINALS')
|
|
verify(train,'terminal');modelsproof=verify(train,'models');done=read(train/'complete.json');mm=read(train/'models-terminal.json')
|
|
if (train/'global-stop.json').exists():raise ValueError('Training global stop')
|
|
prior=terminal_identity(done,mm)
|
|
for p in psutil.process_iter(['pid','cmdline']):
|
|
if any(Path(a).name in {'phase3_data_sufficiency_v2_queue.py','phase3_data_sufficiency_v2_guard.py','phase3_data_sufficiency_v2_worker.py'} for a in p.info['cmdline'] or []):raise RuntimeError('Training processes still present')
|
|
by={m['job']:m for m in mm['models']};adopted=[]
|
|
for j in c['jobs']:
|
|
if j['id']<6:
|
|
m=by[j['id']];w=train/'jobs'/str(j['id'])/'worker';entry=next(e for e in done['entries'] if e['id']==j['id']);g=read(checked(w.parent/'complete.json',entry['guard_complete_sha256']));r=read(checked(w/'result.json',g['worker_result_sha256']))
|
|
if g['status']!='PASS' or r['status']!='COMPLETE' or r['job']!=j['id'] or r['pool']!=j['pool'] or r['mode']!=j['mode'] or r['optimizer_steps']!=j['target_steps'] or m['tag']!=j['tag'] or m['steps']!=j['target_steps'] or m['pool']!=j['pool'] or m['mode']!=j['mode'] or r['test_targets_read'] or r['test_prediction_started']:raise ValueError('Training worker/manifest identity')
|
|
if len(r['checkpoints'])!=1 or any(r['checkpoints'][0][k]!=m[k] for k in ['tag','steps','path','sha256','standardization','standardization_sha256']) or r['standardization_sha256']!=m['standardization_sha256']:raise ValueError('Worker checkpoint manifest mismatch')
|
|
if m['path']!=str(w/(j['tag']+'.pt')) or m['standardization']!=str(w/'standardization.json'):raise ValueError('Checkpoint path identity')
|
|
checked(m['path'],m['sha256']);stats=read(checked(m['standardization'],m['standardization_sha256']));record=dict(**j,checkpoint=m['path'],checkpoint_sha256=m['sha256'],standardization=m['standardization'],standardization_sha256=m['standardization_sha256'],training_contract_sha256=sha(train/'contract.json'))
|
|
else:
|
|
s=tc['S1_reuse'];checked(s['checkpoint'],s['checkpoint_sha256']);stats=read(checked(s['standardization'],s['standardization_sha256']));record=dict(**j,checkpoint=s['checkpoint'],checkpoint_sha256=s['checkpoint_sha256'],standardization=s['standardization'],standardization_sha256=s['standardization_sha256'],training_contract_sha256=c['S1_training_contract_sha256'])
|
|
if stats['parts']!=tc['pools'][str(j['pool'])]['learning']:raise ValueError('Pool standardization role mismatch')
|
|
adopted.append(record)
|
|
return dict(models=adopted,training_gpu_seconds_once=prior,training_terminal_sha256=sha(train/'complete.json'),training_models_sha256=sha(train/'models-terminal.json'),training_models_commit=modelsproof['commit'],new_fits=0,old_S1_GPU_not_recharged=True)
|
|
|
|
def adopted_inputs(root,c,tc):
|
|
verify(root,'models');a=read(root/'models-adopted.json')
|
|
if len(a['models'])!=7 or [m['arm'] for m in a['models']]!=list(MODEL_ARMS):raise ValueError('Seven adopted endpoints')
|
|
for m in a['models']:
|
|
checked(m['checkpoint'],m['checkpoint_sha256']);s=read(checked(m['standardization'],m['standardization_sha256']))
|
|
if s['parts']!=tc['pools'][str(m['pool'])]['learning']:raise ValueError('Adopted standardization')
|
|
return a
|
|
|
|
def expected_entries(c,tc):
|
|
out={}
|
|
for j in c['jobs']:
|
|
f=tc['pools'][str(j['pool'])]
|
|
for role,domain in DOMAIN.items():
|
|
for case in f[role]:out[(j['arm'],case)]=dict(pool=j['pool'],domain=domain)
|
|
for pool in [25,50,100]:
|
|
f=tc['pools'][str(pool)]
|
|
for role,domain in DOMAIN.items():
|
|
for case in f[role]:out[('c'+str(pool)+'F',case)]=dict(pool=pool,domain=domain)
|
|
return out
|
|
|
|
def manifest_identity(entries,c,tc,pilot):
|
|
expected=expected_entries(c,tc);keys=[(e['arm'],e['case']) for e in entries]
|
|
if len(set(keys))!=len(keys) or set(keys)!=set(expected):raise ValueError('Full model/clock/domain manifest required')
|
|
old={e['case']:e for e in read(tc['S1_reuse']['test_manifest'])['entries']}
|
|
for e in entries:
|
|
ex=expected[(e['arm'],e['case'])];row=c['rows'][e['case']]
|
|
if e['pool']!=ex['pool'] or e['domain']!=ex['domain'] or e['component_id']!=row['component_id'] or e['nodes']!=pilot['parts'][e['case']]['nodes']:raise ValueError('Prediction domain identity')
|
|
if e['arm']=='S1_epoch100_reuse' and e['domain']=='common_test':
|
|
o=old[e['case']]
|
|
if any(e[k]!=o[k] for k in ['path','sha256','nodes','query_nodes','negative_full_predictions','negative_query_predictions']) or e['origin']!='S1_HASH_REUSE':raise ValueError('Exact S1 test reuse')
|
|
else:
|
|
p=Path(c['postfit_root'])/'predictions'/e['arm']/(e['case']+'.npz')
|
|
if e['path']!=str(p) or e['origin']!='ITEM1_FROZEN_ENDPOINT_OR_CLOCK':raise ValueError('New prediction path/origin')
|
|
for name,n in [('negative_full_predictions',e['nodes']),('negative_query_predictions',e['query_nodes'])]:
|
|
if type(e[name]) is not int or not 0<=e[name]<=n:raise ValueError('Unclipped count range')
|
|
checked(e['path'],e['sha256'])
|
|
return expected
|
|
|
|
def require_budget(prior,gpu,cpu,remaining):
|
|
if any(not math.isfinite(v) or v<0 for v in [prior,gpu,cpu]):raise ValueError('Nonfinite budget ledger')
|
|
if remaining not in range(8):raise ValueError('Prediction job count')
|
|
if gpu+900*remaining>7200 or prior+gpu+900*remaining>120000:raise RuntimeError('GPU reserve exhausted')
|
|
if cpu+300*remaining+900+120>3600:raise RuntimeError('CPU reserve exhausted')
|