"""Post-score determinism check approved in Claude1794874796; no refit/rescore.""" import argparse import base64 import fcntl import hashlib import os from pathlib import Path import subprocess import sys import time import numpy as np import psutil import r4bc_final as final import r4bc_postfit as post from r4bc_common import read,save,checked,sha,InputViolation from r4bc_guard import supervise from r4bc_interim import git_oid,tree_lookup,verify_git_file from s2_recovery_monitoring import disk_usage ROOT=final.BASE/'artifacts/r4bc-determinism-100-2026-11-17' ORIGINAL_GATE=post.training_gate def same_bits(a,b): return a.dtype.str==b.dtype.str and a.shape==b.shape and a.tobytes(order='C')==b.tobytes(order='C') def prepare(): start=time.process_time() if ROOT.exists():raise FileExistsError('Independent determinism root already exists') final.pins();post.git_gate=final.git_gate _,models=ORIGINAL_GATE(final.ROOT) if read(final.ROOT/'compare-complete.json')['status']!='COMPLETE' or read(final.ROOT/'INTERIM_100-exact-comparison.json')['status']!='EXACT_MATCH': raise InputViolation('Finish original final score/exact comparison first') models=[m for m in models if m['pool']==100] if len(models)!=35:raise InputViolation('Expected original35 full-pool final models') acceptance=sorted((final.BASE/'artifacts/r4bc-determinism-acceptance-2026-11-17').glob('attempt-*/worker.receipt.json')) if not acceptance or read(acceptance[-1])['status']!='PASS':raise InputViolation('Missing synthetic acceptance') source_paths=[final.ROOT/'compare-complete.json',final.ROOT/'INTERIM_100-exact-comparison.json', final.ROOT/'models-terminal.json',final.ROOT/'scores/summary.json',final.INTERIM/'predictions-freeze.json'] plan=dict(authorization='1794874796.184549',models=models,original_final_root=str(final.ROOT), prior_post_cpu_seconds=final.ledger(),synthetic_cpu_seconds=sum(read(p)['cpu_seconds'] for p in acceptance), metadata_reserve_cpu_seconds=30,source_sha256={str(p):sha(p) for p in source_paths}, code_sha256={p:sha(final.BASE/p) for p in ['tools/r4bc_determinism.py','tools/test_r4bc_determinism.py']}, shared_post_start_unix=read(final.ROOT/'adapter-freeze.json')['shared_post_start_unix'], post_cap=14400,total_cap=144000,worker_cpu=24,coordinator_cpu=25,retries=0,new_fits=0, no_new_score=True,no_new_decode=True,metadata_actual_cpu_seconds=time.process_time()-start) if plan['metadata_actual_cpu_seconds']>30:raise InputViolation('Metadata reserve exceeded') ROOT.mkdir();save(ROOT/'plan.json',plan) print('Prepared35-model post-score determinism check; not launched') def make_proof(commit): paths=[str((ROOT/'plan.json').relative_to(final.BASE))]+list(read(ROOT/'plan.json')['code_sha256']) proof=dict(commit=commit,objects={},files={}) def get(oid,kind): raw=subprocess.check_output(['git','cat-file',kind,oid],cwd=final.REPO) if git_oid(kind,raw)!=oid:raise InputViolation('Git object mismatch') proof['objects'][oid]=base64.b64encode(raw).decode();return raw top=get(commit,'commit').split(b'\n',1)[0].decode().removeprefix('tree ') for path in paths: oid=top for component in Path(path).parts:oid=tree_lookup(get(oid,'tree'),component) data=(final.BASE/path).read_bytes() if git_oid('blob',data)!=oid:raise InputViolation('Unfrozen determinism source') proof['files'][path]=hashlib.sha256(data).hexdigest();verify_git_file(proof,path) save(ROOT/'git-proof.json',proof) def pins(): plan=read(ROOT/'plan.json');proof=read(ROOT/'git-proof.json') for p in proof['files']:verify_git_file(proof,p) for p,h in plan['source_sha256'].items():checked(p,h) for p,h in plan['code_sha256'].items():checked(final.BASE/p,h) if final.ledger()!=plan['prior_post_cpu_seconds']:raise InputViolation('Original final ledger changed') final.pins();return plan def training_gate(root): if root!=ROOT:raise InputViolation('Determinism output root mismatch') pins();post.git_gate=final.git_gate return ORIGINAL_GATE(final.ROOT) def compare_arrays(): plan=pins();old=read(final.INTERIM/'predictions-freeze.json') expected={(e['case'],e['arm']):e for e in old['entries'] if e['arm'] in post.ARMS} seen=set();defects=[];index_hashes={} for m in plan['models']: dest=ROOT/'predictions'/f"cell-{m['cell']:02d}-{m['arm']}" if read(ROOT/'prediction-guards'/dest.name/'worker.receipt.json')['status']!='PASS':raise InputViolation('Determinism prediction failed') result=read(dest/'result.json') if result['model_sha256']!=m['model_sha256']:raise InputViolation('Determinism model changed') index=read(checked(dest/'index.json',result['index_sha256']));index_hashes[str(dest/'index.json')]=result['index_sha256'] for case,new in index.items(): key=(case,m['arm']) if key in seen or key not in expected:raise InputViolation('Determinism prediction roster mismatch') seen.add(key);prior=expected[key] with np.load(checked(new['path'],new['sha256']),allow_pickle=False) as lhs, np.load(checked(prior['path'],prior['sha256']),allow_pickle=False) as rhs: for field in ['raw_ids','counts','seconds']: oldfield=prior['array'] if field=='seconds' else field if not same_bits(lhs[field],rhs[oldfield]):defects.append(dict(case=case,arm=m['arm'],field=field)) if seen!=set(expected):raise InputViolation('Incomplete35 prediction domain') save(ROOT/'bitwise-comparison.json',dict(status='BITWISE_MATCH' if not defects else 'DEFECT', models=35,prediction_arrays=len(seen),fields=['raw_ids','counts','seconds'],defects=defects, comparison='dtype.str, shape and contiguous array bytes; no numeric tolerance', prediction_index_sha256=index_hashes,no_new_labels_fits_scores=True)) if defects:raise InputViolation('New100-percent predictions differ bitwise; preserve DEFECT') def worker(mode,cell=None,arm=None): os.sched_setaffinity(0,[24]);plan=pins() if mode=='predict': if (cell,arm) not in {(m['cell'],m['arm']) for m in plan['models']}:raise InputViolation('Only35 full-pool finals authorized') post.training_gate=training_gate;post.predict(ROOT,cell,arm) else:compare_arrays() def ledger(): p=read(ROOT/'plan.json');receipts=list((ROOT/'prediction-guards').glob('*/worker.receipt.json')) if (ROOT/'compare-guard/worker.receipt.json').exists():receipts.append(ROOT/'compare-guard/worker.receipt.json') return p['prior_post_cpu_seconds']+p['synthetic_cpu_seconds']+p['metadata_reserve_cpu_seconds']+sum(read(x)['cpu_seconds'] for x in receipts) def limits(plan,spent,mode,now): cpu=1800 if mode=='predict' else 600;reserve=600 if mode=='predict' else 0 if spent+cpu+reserve>14400:raise InputViolation('Shared post CPU cap') original=read(final.ROOT/'adapter-freeze.json') if original['fit_cpu_seconds']+original['setup_cpu_seconds']+spent+cpu+reserve>144000:raise InputViolation('Shared total CPU cap') wall=plan['shared_post_start_unix']+21600-now if wall<=0:raise InputViolation('Shared post wall cap') return dict(cpu_seconds=cpu,wall_seconds=min(3600 if mode=='predict' else 1800,wall),rss_bytes=8*2**30,disk_bytes=2*2**30) def run(): os.sched_setaffinity(0,[25]);plan=pins() with (ROOT/'execution.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (ROOT/'started.json').exists():raise FileExistsError('No determinism restart/retry') save(ROOT/'started.json',dict(pid=os.getpid(),unix=time.time(),authorization=plan['authorization'])) jobs=[('predict',m['cell'],m['arm']) for m in plan['models']]+[('compare',None,None)] for mode,cell,arm in jobs: cap=limits(plan,ledger(),mode,time.time()) if psutil.virtual_memory().available<10*2**30:raise InputViolation('Insufficient memory') if sum(disk_usage(p) for p in [final.ORIGINAL,final.INTERIM,final.ROOT,ROOT])>=100*2**30:raise InputViolation('Shared disk cap') command=[sys.executable,'tools/r4bc_determinism.py','worker','--mode',mode] if cell is not None:command+=['--cell',str(cell),'--arm',arm] dest=ROOT/'prediction-guards'/f'cell-{cell:02d}-{arm}' if mode=='predict' else ROOT/'compare-guard' save(ROOT/'progress.json',dict(mode=mode,cell=cell,arm=arm,post_cpu_seconds=ledger(),reserved_cpu=cap['cpu_seconds'])) result=supervise(command,dest,cap,global_stop=final.ORIGINAL/'global-stop.json') if result['status']!='PASS': save(ROOT/'complete.json',dict(status='STOPPED',mode=mode,cell=cell,arm=arm,receipt=result,post_cpu_seconds=ledger(),retry=False));return 1 save(ROOT/'complete.json',dict(status='COMPLETE_BITWISE_MATCH',post_cpu_seconds=ledger(),remaining_post_cpu_seconds=14400-ledger(), coordinator_cpu_seconds=time.process_time(),comparison_sha256=sha(ROOT/'bitwise-comparison.json'),new_fits_scores_decodes=0)) return 0 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('action',choices=['prepare','proof','worker','run']);p.add_argument('--commit') p.add_argument('--mode',choices=['predict','compare']);p.add_argument('--cell',type=int);p.add_argument('--arm');a=p.parse_args() try: if a.action=='prepare':prepare() elif a.action=='proof':make_proof(a.commit) elif a.action=='worker':worker(a.mode,a.cell,a.arm) else:raise SystemExit(run()) except Exception as exc: if a.action=='run':save(ROOT/'complete.json',dict(status='STOPPED',error=type(exc).__name__+': '+str(exc),post_cpu_seconds=ledger(),retry=False)) raise