Files
MeltMind/tools/r4bc_determinism.py
T

170 lines
9.7 KiB
Python

"""Post-score determinism check approved in Claude1794874796; no refit/rescore."""
import argparse
import base64
import fcntl
import hashlib
import os
from pathlib import Path
import subprocess
import sys
import time
import numpy as np
import psutil
import r4bc_final as final
import r4bc_postfit as post
from r4bc_common import read,save,checked,sha,InputViolation
from r4bc_guard import supervise
from r4bc_interim import git_oid,tree_lookup,verify_git_file
from s2_recovery_monitoring import disk_usage
ROOT=final.BASE/'artifacts/r4bc-determinism-100-2026-11-17'
ORIGINAL_GATE=post.training_gate
def same_bits(a,b):
return a.dtype.str==b.dtype.str and a.shape==b.shape and a.tobytes(order='C')==b.tobytes(order='C')
def prepare():
start=time.process_time()
if ROOT.exists():raise FileExistsError('Independent determinism root already exists')
final.pins();post.git_gate=final.git_gate
_,models=ORIGINAL_GATE(final.ROOT)
if read(final.ROOT/'compare-complete.json')['status']!='COMPLETE' or read(final.ROOT/'INTERIM_100-exact-comparison.json')['status']!='EXACT_MATCH':
raise InputViolation('Finish original final score/exact comparison first')
models=[m for m in models if m['pool']==100]
if len(models)!=35:raise InputViolation('Expected original35 full-pool final models')
acceptance=sorted((final.BASE/'artifacts/r4bc-determinism-acceptance-2026-11-17').glob('attempt-*/worker.receipt.json'))
if not acceptance or read(acceptance[-1])['status']!='PASS':raise InputViolation('Missing synthetic acceptance')
source_paths=[final.ROOT/'compare-complete.json',final.ROOT/'INTERIM_100-exact-comparison.json',
final.ROOT/'models-terminal.json',final.ROOT/'scores/summary.json',final.INTERIM/'predictions-freeze.json']
plan=dict(authorization='1794874796.184549',models=models,original_final_root=str(final.ROOT),
prior_post_cpu_seconds=final.ledger(),synthetic_cpu_seconds=sum(read(p)['cpu_seconds'] for p in acceptance),
metadata_reserve_cpu_seconds=30,source_sha256={str(p):sha(p) for p in source_paths},
code_sha256={p:sha(final.BASE/p) for p in ['tools/r4bc_determinism.py','tools/test_r4bc_determinism.py']},
shared_post_start_unix=read(final.ROOT/'adapter-freeze.json')['shared_post_start_unix'],
post_cap=14400,total_cap=144000,worker_cpu=24,coordinator_cpu=25,retries=0,new_fits=0,
no_new_score=True,no_new_decode=True,metadata_actual_cpu_seconds=time.process_time()-start)
if plan['metadata_actual_cpu_seconds']>30:raise InputViolation('Metadata reserve exceeded')
ROOT.mkdir();save(ROOT/'plan.json',plan)
print('Prepared35-model post-score determinism check; not launched')
def make_proof(commit):
paths=[str((ROOT/'plan.json').relative_to(final.BASE))]+list(read(ROOT/'plan.json')['code_sha256'])
proof=dict(commit=commit,objects={},files={})
def get(oid,kind):
raw=subprocess.check_output(['git','cat-file',kind,oid],cwd=final.REPO)
if git_oid(kind,raw)!=oid:raise InputViolation('Git object mismatch')
proof['objects'][oid]=base64.b64encode(raw).decode();return raw
top=get(commit,'commit').split(b'\n',1)[0].decode().removeprefix('tree ')
for path in paths:
oid=top
for component in Path(path).parts:oid=tree_lookup(get(oid,'tree'),component)
data=(final.BASE/path).read_bytes()
if git_oid('blob',data)!=oid:raise InputViolation('Unfrozen determinism source')
proof['files'][path]=hashlib.sha256(data).hexdigest();verify_git_file(proof,path)
save(ROOT/'git-proof.json',proof)
def pins():
plan=read(ROOT/'plan.json');proof=read(ROOT/'git-proof.json')
for p in proof['files']:verify_git_file(proof,p)
for p,h in plan['source_sha256'].items():checked(p,h)
for p,h in plan['code_sha256'].items():checked(final.BASE/p,h)
if final.ledger()!=plan['prior_post_cpu_seconds']:raise InputViolation('Original final ledger changed')
final.pins();return plan
def training_gate(root):
if root!=ROOT:raise InputViolation('Determinism output root mismatch')
pins();post.git_gate=final.git_gate
return ORIGINAL_GATE(final.ROOT)
def compare_arrays():
plan=pins();old=read(final.INTERIM/'predictions-freeze.json')
expected={(e['case'],e['arm']):e for e in old['entries'] if e['arm'] in post.ARMS}
seen=set();defects=[];index_hashes={}
for m in plan['models']:
dest=ROOT/'predictions'/f"cell-{m['cell']:02d}-{m['arm']}"
if read(ROOT/'prediction-guards'/dest.name/'worker.receipt.json')['status']!='PASS':raise InputViolation('Determinism prediction failed')
result=read(dest/'result.json')
if result['model_sha256']!=m['model_sha256']:raise InputViolation('Determinism model changed')
index=read(checked(dest/'index.json',result['index_sha256']));index_hashes[str(dest/'index.json')]=result['index_sha256']
for case,new in index.items():
key=(case,m['arm'])
if key in seen or key not in expected:raise InputViolation('Determinism prediction roster mismatch')
seen.add(key);prior=expected[key]
with np.load(checked(new['path'],new['sha256']),allow_pickle=False) as lhs, np.load(checked(prior['path'],prior['sha256']),allow_pickle=False) as rhs:
for field in ['raw_ids','counts','seconds']:
oldfield=prior['array'] if field=='seconds' else field
if not same_bits(lhs[field],rhs[oldfield]):defects.append(dict(case=case,arm=m['arm'],field=field))
if seen!=set(expected):raise InputViolation('Incomplete35 prediction domain')
save(ROOT/'bitwise-comparison.json',dict(status='BITWISE_MATCH' if not defects else 'DEFECT',
models=35,prediction_arrays=len(seen),fields=['raw_ids','counts','seconds'],defects=defects,
comparison='dtype.str, shape and contiguous array bytes; no numeric tolerance',
prediction_index_sha256=index_hashes,no_new_labels_fits_scores=True))
if defects:raise InputViolation('New100-percent predictions differ bitwise; preserve DEFECT')
def worker(mode,cell=None,arm=None):
os.sched_setaffinity(0,[24]);plan=pins()
if mode=='predict':
if (cell,arm) not in {(m['cell'],m['arm']) for m in plan['models']}:raise InputViolation('Only35 full-pool finals authorized')
post.training_gate=training_gate;post.predict(ROOT,cell,arm)
else:compare_arrays()
def ledger():
p=read(ROOT/'plan.json');receipts=list((ROOT/'prediction-guards').glob('*/worker.receipt.json'))
if (ROOT/'compare-guard/worker.receipt.json').exists():receipts.append(ROOT/'compare-guard/worker.receipt.json')
return p['prior_post_cpu_seconds']+p['synthetic_cpu_seconds']+p['metadata_reserve_cpu_seconds']+sum(read(x)['cpu_seconds'] for x in receipts)
def limits(plan,spent,mode,now):
cpu=1800 if mode=='predict' else 600;reserve=600 if mode=='predict' else 0
if spent+cpu+reserve>14400:raise InputViolation('Shared post CPU cap')
original=read(final.ROOT/'adapter-freeze.json')
if original['fit_cpu_seconds']+original['setup_cpu_seconds']+spent+cpu+reserve>144000:raise InputViolation('Shared total CPU cap')
wall=plan['shared_post_start_unix']+21600-now
if wall<=0:raise InputViolation('Shared post wall cap')
return dict(cpu_seconds=cpu,wall_seconds=min(3600 if mode=='predict' else 1800,wall),rss_bytes=8*2**30,disk_bytes=2*2**30)
def run():
os.sched_setaffinity(0,[25]);plan=pins()
with (ROOT/'execution.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (ROOT/'started.json').exists():raise FileExistsError('No determinism restart/retry')
save(ROOT/'started.json',dict(pid=os.getpid(),unix=time.time(),authorization=plan['authorization']))
jobs=[('predict',m['cell'],m['arm']) for m in plan['models']]+[('compare',None,None)]
for mode,cell,arm in jobs:
cap=limits(plan,ledger(),mode,time.time())
if psutil.virtual_memory().available<10*2**30:raise InputViolation('Insufficient memory')
if sum(disk_usage(p) for p in [final.ORIGINAL,final.INTERIM,final.ROOT,ROOT])>=100*2**30:raise InputViolation('Shared disk cap')
command=[sys.executable,'tools/r4bc_determinism.py','worker','--mode',mode]
if cell is not None:command+=['--cell',str(cell),'--arm',arm]
dest=ROOT/'prediction-guards'/f'cell-{cell:02d}-{arm}' if mode=='predict' else ROOT/'compare-guard'
save(ROOT/'progress.json',dict(mode=mode,cell=cell,arm=arm,post_cpu_seconds=ledger(),reserved_cpu=cap['cpu_seconds']))
result=supervise(command,dest,cap,global_stop=final.ORIGINAL/'global-stop.json')
if result['status']!='PASS':
save(ROOT/'complete.json',dict(status='STOPPED',mode=mode,cell=cell,arm=arm,receipt=result,post_cpu_seconds=ledger(),retry=False));return 1
save(ROOT/'complete.json',dict(status='COMPLETE_BITWISE_MATCH',post_cpu_seconds=ledger(),remaining_post_cpu_seconds=14400-ledger(),
coordinator_cpu_seconds=time.process_time(),comparison_sha256=sha(ROOT/'bitwise-comparison.json'),new_fits_scores_decodes=0))
return 0
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('action',choices=['prepare','proof','worker','run']);p.add_argument('--commit')
p.add_argument('--mode',choices=['predict','compare']);p.add_argument('--cell',type=int);p.add_argument('--arm');a=p.parse_args()
try:
if a.action=='prepare':prepare()
elif a.action=='proof':make_proof(a.commit)
elif a.action=='worker':worker(a.mode,a.cell,a.arm)
else:raise SystemExit(run())
except Exception as exc:
if a.action=='run':save(ROOT/'complete.json',dict(status='STOPPED',error=type(exc).__name__+': '+str(exc),post_cpu_seconds=ledger(),retry=False))
raise