170 lines
9.7 KiB
Python
170 lines
9.7 KiB
Python
"""Post-score determinism check approved in Claude1794874796; no refit/rescore."""
|
|
import argparse
|
|
import base64
|
|
import fcntl
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import numpy as np
|
|
import psutil
|
|
import r4bc_final as final
|
|
import r4bc_postfit as post
|
|
from r4bc_common import read,save,checked,sha,InputViolation
|
|
from r4bc_guard import supervise
|
|
from r4bc_interim import git_oid,tree_lookup,verify_git_file
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
ROOT=final.BASE/'artifacts/r4bc-determinism-100-2026-11-17'
|
|
ORIGINAL_GATE=post.training_gate
|
|
|
|
|
|
def same_bits(a,b):
|
|
return a.dtype.str==b.dtype.str and a.shape==b.shape and a.tobytes(order='C')==b.tobytes(order='C')
|
|
|
|
|
|
def prepare():
|
|
start=time.process_time()
|
|
if ROOT.exists():raise FileExistsError('Independent determinism root already exists')
|
|
final.pins();post.git_gate=final.git_gate
|
|
_,models=ORIGINAL_GATE(final.ROOT)
|
|
if read(final.ROOT/'compare-complete.json')['status']!='COMPLETE' or read(final.ROOT/'INTERIM_100-exact-comparison.json')['status']!='EXACT_MATCH':
|
|
raise InputViolation('Finish original final score/exact comparison first')
|
|
models=[m for m in models if m['pool']==100]
|
|
if len(models)!=35:raise InputViolation('Expected original35 full-pool final models')
|
|
acceptance=sorted((final.BASE/'artifacts/r4bc-determinism-acceptance-2026-11-17').glob('attempt-*/worker.receipt.json'))
|
|
if not acceptance or read(acceptance[-1])['status']!='PASS':raise InputViolation('Missing synthetic acceptance')
|
|
source_paths=[final.ROOT/'compare-complete.json',final.ROOT/'INTERIM_100-exact-comparison.json',
|
|
final.ROOT/'models-terminal.json',final.ROOT/'scores/summary.json',final.INTERIM/'predictions-freeze.json']
|
|
plan=dict(authorization='1794874796.184549',models=models,original_final_root=str(final.ROOT),
|
|
prior_post_cpu_seconds=final.ledger(),synthetic_cpu_seconds=sum(read(p)['cpu_seconds'] for p in acceptance),
|
|
metadata_reserve_cpu_seconds=30,source_sha256={str(p):sha(p) for p in source_paths},
|
|
code_sha256={p:sha(final.BASE/p) for p in ['tools/r4bc_determinism.py','tools/test_r4bc_determinism.py']},
|
|
shared_post_start_unix=read(final.ROOT/'adapter-freeze.json')['shared_post_start_unix'],
|
|
post_cap=14400,total_cap=144000,worker_cpu=24,coordinator_cpu=25,retries=0,new_fits=0,
|
|
no_new_score=True,no_new_decode=True,metadata_actual_cpu_seconds=time.process_time()-start)
|
|
if plan['metadata_actual_cpu_seconds']>30:raise InputViolation('Metadata reserve exceeded')
|
|
ROOT.mkdir();save(ROOT/'plan.json',plan)
|
|
print('Prepared35-model post-score determinism check; not launched')
|
|
|
|
|
|
def make_proof(commit):
|
|
paths=[str((ROOT/'plan.json').relative_to(final.BASE))]+list(read(ROOT/'plan.json')['code_sha256'])
|
|
proof=dict(commit=commit,objects={},files={})
|
|
def get(oid,kind):
|
|
raw=subprocess.check_output(['git','cat-file',kind,oid],cwd=final.REPO)
|
|
if git_oid(kind,raw)!=oid:raise InputViolation('Git object mismatch')
|
|
proof['objects'][oid]=base64.b64encode(raw).decode();return raw
|
|
top=get(commit,'commit').split(b'\n',1)[0].decode().removeprefix('tree ')
|
|
for path in paths:
|
|
oid=top
|
|
for component in Path(path).parts:oid=tree_lookup(get(oid,'tree'),component)
|
|
data=(final.BASE/path).read_bytes()
|
|
if git_oid('blob',data)!=oid:raise InputViolation('Unfrozen determinism source')
|
|
proof['files'][path]=hashlib.sha256(data).hexdigest();verify_git_file(proof,path)
|
|
save(ROOT/'git-proof.json',proof)
|
|
|
|
|
|
def pins():
|
|
plan=read(ROOT/'plan.json');proof=read(ROOT/'git-proof.json')
|
|
for p in proof['files']:verify_git_file(proof,p)
|
|
for p,h in plan['source_sha256'].items():checked(p,h)
|
|
for p,h in plan['code_sha256'].items():checked(final.BASE/p,h)
|
|
if final.ledger()!=plan['prior_post_cpu_seconds']:raise InputViolation('Original final ledger changed')
|
|
final.pins();return plan
|
|
|
|
|
|
def training_gate(root):
|
|
if root!=ROOT:raise InputViolation('Determinism output root mismatch')
|
|
pins();post.git_gate=final.git_gate
|
|
return ORIGINAL_GATE(final.ROOT)
|
|
|
|
|
|
def compare_arrays():
|
|
plan=pins();old=read(final.INTERIM/'predictions-freeze.json')
|
|
expected={(e['case'],e['arm']):e for e in old['entries'] if e['arm'] in post.ARMS}
|
|
seen=set();defects=[];index_hashes={}
|
|
for m in plan['models']:
|
|
dest=ROOT/'predictions'/f"cell-{m['cell']:02d}-{m['arm']}"
|
|
if read(ROOT/'prediction-guards'/dest.name/'worker.receipt.json')['status']!='PASS':raise InputViolation('Determinism prediction failed')
|
|
result=read(dest/'result.json')
|
|
if result['model_sha256']!=m['model_sha256']:raise InputViolation('Determinism model changed')
|
|
index=read(checked(dest/'index.json',result['index_sha256']));index_hashes[str(dest/'index.json')]=result['index_sha256']
|
|
for case,new in index.items():
|
|
key=(case,m['arm'])
|
|
if key in seen or key not in expected:raise InputViolation('Determinism prediction roster mismatch')
|
|
seen.add(key);prior=expected[key]
|
|
with np.load(checked(new['path'],new['sha256']),allow_pickle=False) as lhs, np.load(checked(prior['path'],prior['sha256']),allow_pickle=False) as rhs:
|
|
for field in ['raw_ids','counts','seconds']:
|
|
oldfield=prior['array'] if field=='seconds' else field
|
|
if not same_bits(lhs[field],rhs[oldfield]):defects.append(dict(case=case,arm=m['arm'],field=field))
|
|
if seen!=set(expected):raise InputViolation('Incomplete35 prediction domain')
|
|
save(ROOT/'bitwise-comparison.json',dict(status='BITWISE_MATCH' if not defects else 'DEFECT',
|
|
models=35,prediction_arrays=len(seen),fields=['raw_ids','counts','seconds'],defects=defects,
|
|
comparison='dtype.str, shape and contiguous array bytes; no numeric tolerance',
|
|
prediction_index_sha256=index_hashes,no_new_labels_fits_scores=True))
|
|
if defects:raise InputViolation('New100-percent predictions differ bitwise; preserve DEFECT')
|
|
|
|
|
|
def worker(mode,cell=None,arm=None):
|
|
os.sched_setaffinity(0,[24]);plan=pins()
|
|
if mode=='predict':
|
|
if (cell,arm) not in {(m['cell'],m['arm']) for m in plan['models']}:raise InputViolation('Only35 full-pool finals authorized')
|
|
post.training_gate=training_gate;post.predict(ROOT,cell,arm)
|
|
else:compare_arrays()
|
|
|
|
|
|
def ledger():
|
|
p=read(ROOT/'plan.json');receipts=list((ROOT/'prediction-guards').glob('*/worker.receipt.json'))
|
|
if (ROOT/'compare-guard/worker.receipt.json').exists():receipts.append(ROOT/'compare-guard/worker.receipt.json')
|
|
return p['prior_post_cpu_seconds']+p['synthetic_cpu_seconds']+p['metadata_reserve_cpu_seconds']+sum(read(x)['cpu_seconds'] for x in receipts)
|
|
|
|
|
|
def limits(plan,spent,mode,now):
|
|
cpu=1800 if mode=='predict' else 600;reserve=600 if mode=='predict' else 0
|
|
if spent+cpu+reserve>14400:raise InputViolation('Shared post CPU cap')
|
|
original=read(final.ROOT/'adapter-freeze.json')
|
|
if original['fit_cpu_seconds']+original['setup_cpu_seconds']+spent+cpu+reserve>144000:raise InputViolation('Shared total CPU cap')
|
|
wall=plan['shared_post_start_unix']+21600-now
|
|
if wall<=0:raise InputViolation('Shared post wall cap')
|
|
return dict(cpu_seconds=cpu,wall_seconds=min(3600 if mode=='predict' else 1800,wall),rss_bytes=8*2**30,disk_bytes=2*2**30)
|
|
|
|
|
|
def run():
|
|
os.sched_setaffinity(0,[25]);plan=pins()
|
|
with (ROOT/'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (ROOT/'started.json').exists():raise FileExistsError('No determinism restart/retry')
|
|
save(ROOT/'started.json',dict(pid=os.getpid(),unix=time.time(),authorization=plan['authorization']))
|
|
jobs=[('predict',m['cell'],m['arm']) for m in plan['models']]+[('compare',None,None)]
|
|
for mode,cell,arm in jobs:
|
|
cap=limits(plan,ledger(),mode,time.time())
|
|
if psutil.virtual_memory().available<10*2**30:raise InputViolation('Insufficient memory')
|
|
if sum(disk_usage(p) for p in [final.ORIGINAL,final.INTERIM,final.ROOT,ROOT])>=100*2**30:raise InputViolation('Shared disk cap')
|
|
command=[sys.executable,'tools/r4bc_determinism.py','worker','--mode',mode]
|
|
if cell is not None:command+=['--cell',str(cell),'--arm',arm]
|
|
dest=ROOT/'prediction-guards'/f'cell-{cell:02d}-{arm}' if mode=='predict' else ROOT/'compare-guard'
|
|
save(ROOT/'progress.json',dict(mode=mode,cell=cell,arm=arm,post_cpu_seconds=ledger(),reserved_cpu=cap['cpu_seconds']))
|
|
result=supervise(command,dest,cap,global_stop=final.ORIGINAL/'global-stop.json')
|
|
if result['status']!='PASS':
|
|
save(ROOT/'complete.json',dict(status='STOPPED',mode=mode,cell=cell,arm=arm,receipt=result,post_cpu_seconds=ledger(),retry=False));return 1
|
|
save(ROOT/'complete.json',dict(status='COMPLETE_BITWISE_MATCH',post_cpu_seconds=ledger(),remaining_post_cpu_seconds=14400-ledger(),
|
|
coordinator_cpu_seconds=time.process_time(),comparison_sha256=sha(ROOT/'bitwise-comparison.json'),new_fits_scores_decodes=0))
|
|
return 0
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('action',choices=['prepare','proof','worker','run']);p.add_argument('--commit')
|
|
p.add_argument('--mode',choices=['predict','compare']);p.add_argument('--cell',type=int);p.add_argument('--arm');a=p.parse_args()
|
|
try:
|
|
if a.action=='prepare':prepare()
|
|
elif a.action=='proof':make_proof(a.commit)
|
|
elif a.action=='worker':worker(a.mode,a.cell,a.arm)
|
|
else:raise SystemExit(run())
|
|
except Exception as exc:
|
|
if a.action=='run':save(ROOT/'complete.json',dict(status='STOPPED',error=type(exc).__name__+': '+str(exc),post_cpu_seconds=ledger(),retry=False))
|
|
raise
|