"""One reviewed item1 postfit; runtime checkpoint/prediction custody barriers.""" import json,math from pathlib import Path from phase2_h1_worker_v2 import checked,sha from phase2_home_freeze_io import verify MODEL_ARMS=('epoch25','epoch50','step50_U','step50_2U','step100_U','step100_2U','S1_epoch100_reuse') DOMAIN={'learning':'own_learning','validation':'common_validation','test':'common_test'} def read(p):return json.loads(Path(p).read_text()) def config(root): c=read(root/'contract.json');verify(root,'implementation') if c['approval']!='1794950613.195499' or c['comparison_contract_sha256']!='332f2515911c99aadf2486086ddff82bd195d889faa8391296443e7aeacdfee9' or c['postfit_gpu_cap']!=7200 or c['postfit_cpu_cap']!=3600 or c['item_gpu_cap']!=120000:raise PermissionError('Frozen item1 postfit scope') for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(p,h) tc=read(c['training_contract']);pilot=read(tc['pilot_contract']) verify(Path(c['training_root']),'implementation') for p,h in tc['code_sha256'].items():checked(p,h) if tc['new_fits']!=6 or tc['item_gpu_cap']!=120000 or tc['postfit_gpu_reserve']!=7200:raise ValueError('Training scope mismatch') if [j['arm'] for j in c['jobs']]!=list(MODEL_ARMS):raise ValueError('Fixed seven endpoint identities') return c,tc,pilot def expected_checkpoint(job,meta,training_sha): if meta.get('fold')!=0 or meta.get('seed')!=0 or meta.get('optimizer_steps')!=job['target_steps'] or meta.get('contract_sha256')!=training_sha:raise ValueError('Checkpoint fold/seed/step/source identity') if job['id']<6: if meta.get('pool')!=job['pool'] or meta.get('mode')!=job['mode'] or meta.get('role')!=job['tag']:raise ValueError('New checkpoint cell identity') elif meta.get('completed_epochs')!=200 or meta.get('role')!='last_epoch_only':raise ValueError('S1 endpoint identity') def terminal_identity(done,mm): if done['status']!='TRAIN_COMPLETE_POSTFIT_PENDING' or len(done['entries'])!=6 or any(e['status']!='PASS' or e['returncode']!=0 for e in done['entries']):raise ValueError('All six terminal PASS required') if {e['id'] for e in done['entries']}!=set(range(6)) or len(mm['models'])!=6 or {m['job'] for m in mm['models']}!=set(range(6)):raise ValueError('Six distinct model jobs') if done['test_targets_read'] or mm['test_prediction_started'] or mm['new_fits']!=6 or mm['new_checkpoints']!=6:raise ValueError('Training custody identity') costs=[e['gpu_seconds_charged'] for e in done['entries']];prior=done['training_gpu_seconds'] if any(not math.isfinite(x) or x<0 for x in costs+[prior]) or prior>112800 or abs(prior-sum(costs))>1e-8:raise ValueError('Settled training ledger') return prior def terminal_inputs(c,tc): import psutil train=Path(c['training_root']) if not (train/'complete.json').exists():raise RuntimeError('WAIT_EXISTING_SIX_TRAINING_TERMINALS') verify(train,'terminal');modelsproof=verify(train,'models');done=read(train/'complete.json');mm=read(train/'models-terminal.json') if (train/'global-stop.json').exists():raise ValueError('Training global stop') prior=terminal_identity(done,mm) for p in psutil.process_iter(['pid','cmdline']): if any(Path(a).name in {'phase3_data_sufficiency_v2_queue.py','phase3_data_sufficiency_v2_guard.py','phase3_data_sufficiency_v2_worker.py'} for a in p.info['cmdline'] or []):raise RuntimeError('Training processes still present') by={m['job']:m for m in mm['models']};adopted=[] for j in c['jobs']: if j['id']<6: m=by[j['id']];w=train/'jobs'/str(j['id'])/'worker';entry=next(e for e in done['entries'] if e['id']==j['id']);g=read(checked(w.parent/'complete.json',entry['guard_complete_sha256']));r=read(checked(w/'result.json',g['worker_result_sha256'])) if g['status']!='PASS' or r['status']!='COMPLETE' or r['job']!=j['id'] or r['pool']!=j['pool'] or r['mode']!=j['mode'] or r['optimizer_steps']!=j['target_steps'] or m['tag']!=j['tag'] or m['steps']!=j['target_steps'] or m['pool']!=j['pool'] or m['mode']!=j['mode'] or r['test_targets_read'] or r['test_prediction_started']:raise ValueError('Training worker/manifest identity') if len(r['checkpoints'])!=1 or any(r['checkpoints'][0][k]!=m[k] for k in ['tag','steps','path','sha256','standardization','standardization_sha256']) or r['standardization_sha256']!=m['standardization_sha256']:raise ValueError('Worker checkpoint manifest mismatch') if m['path']!=str(w/(j['tag']+'.pt')) or m['standardization']!=str(w/'standardization.json'):raise ValueError('Checkpoint path identity') checked(m['path'],m['sha256']);stats=read(checked(m['standardization'],m['standardization_sha256']));record=dict(**j,checkpoint=m['path'],checkpoint_sha256=m['sha256'],standardization=m['standardization'],standardization_sha256=m['standardization_sha256'],training_contract_sha256=sha(train/'contract.json')) else: s=tc['S1_reuse'];checked(s['checkpoint'],s['checkpoint_sha256']);stats=read(checked(s['standardization'],s['standardization_sha256']));record=dict(**j,checkpoint=s['checkpoint'],checkpoint_sha256=s['checkpoint_sha256'],standardization=s['standardization'],standardization_sha256=s['standardization_sha256'],training_contract_sha256=c['S1_training_contract_sha256']) if stats['parts']!=tc['pools'][str(j['pool'])]['learning']:raise ValueError('Pool standardization role mismatch') adopted.append(record) return dict(models=adopted,training_gpu_seconds_once=prior,training_terminal_sha256=sha(train/'complete.json'),training_models_sha256=sha(train/'models-terminal.json'),training_models_commit=modelsproof['commit'],new_fits=0,old_S1_GPU_not_recharged=True) def adopted_inputs(root,c,tc): verify(root,'models');a=read(root/'models-adopted.json') if len(a['models'])!=7 or [m['arm'] for m in a['models']]!=list(MODEL_ARMS):raise ValueError('Seven adopted endpoints') for m in a['models']: checked(m['checkpoint'],m['checkpoint_sha256']);s=read(checked(m['standardization'],m['standardization_sha256'])) if s['parts']!=tc['pools'][str(m['pool'])]['learning']:raise ValueError('Adopted standardization') return a def expected_entries(c,tc): out={} for j in c['jobs']: f=tc['pools'][str(j['pool'])] for role,domain in DOMAIN.items(): for case in f[role]:out[(j['arm'],case)]=dict(pool=j['pool'],domain=domain) for pool in [25,50,100]: f=tc['pools'][str(pool)] for role,domain in DOMAIN.items(): for case in f[role]:out[('c'+str(pool)+'F',case)]=dict(pool=pool,domain=domain) return out def manifest_identity(entries,c,tc,pilot): expected=expected_entries(c,tc);keys=[(e['arm'],e['case']) for e in entries] if len(set(keys))!=len(keys) or set(keys)!=set(expected):raise ValueError('Full model/clock/domain manifest required') old={e['case']:e for e in read(tc['S1_reuse']['test_manifest'])['entries']} for e in entries: ex=expected[(e['arm'],e['case'])];row=c['rows'][e['case']] if e['pool']!=ex['pool'] or e['domain']!=ex['domain'] or e['component_id']!=row['component_id'] or e['nodes']!=pilot['parts'][e['case']]['nodes']:raise ValueError('Prediction domain identity') if e['arm']=='S1_epoch100_reuse' and e['domain']=='common_test': o=old[e['case']] if any(e[k]!=o[k] for k in ['path','sha256','nodes','query_nodes','negative_full_predictions','negative_query_predictions']) or e['origin']!='S1_HASH_REUSE':raise ValueError('Exact S1 test reuse') else: p=Path(c['postfit_root'])/'predictions'/e['arm']/(e['case']+'.npz') if e['path']!=str(p) or e['origin']!='ITEM1_FROZEN_ENDPOINT_OR_CLOCK':raise ValueError('New prediction path/origin') for name,n in [('negative_full_predictions',e['nodes']),('negative_query_predictions',e['query_nodes'])]: if type(e[name]) is not int or not 0<=e[name]<=n:raise ValueError('Unclipped count range') checked(e['path'],e['sha256']) return expected def require_budget(prior,gpu,cpu,remaining): if any(not math.isfinite(v) or v<0 for v in [prior,gpu,cpu]):raise ValueError('Nonfinite budget ledger') if remaining not in range(8):raise ValueError('Prediction job count') if gpu+900*remaining>7200 or prior+gpu+900*remaining>120000:raise RuntimeError('GPU reserve exhausted') if cpu+300*remaining+900+120>3600:raise RuntimeError('CPU reserve exhausted')