"""Final R4bc orchestration; original prediction/scoring functions stay immutable.""" import argparse import base64 import fcntl import hashlib import os from pathlib import Path import subprocess import sys import time import psutil import r4bc_postfit as post from r4bc_common import contract, read, save, checked, sha, InputViolation from r4bc_guard import supervise from r4bc_interim import git_oid, tree_lookup, verify_git_file from s2_recovery_monitoring import disk_usage BASE=Path('/home/frank14f/optane/MeltMind') ORIGINAL=BASE/'artifacts/r4bc-execution-2026-11-16' INTERIM=BASE/'artifacts/r4bc-interim-100-2026-11-17' ROOT=BASE/'artifacts/r4bc-final-postfit-2026-11-17' REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8') POST_CAP=14400 def terminal_check(c, done, models): if (done['status']!='FIT_QUEUE_TERMINAL' or done['global_stop'] or done['cut'] or not done['accounting_complete'] or done['monitor_errors'] or done['never_attempted'] or done['postfit_state']!='READY_FOR_GIT_CHECKPOINT_FREEZE'): raise InputViolation('Original full queue not normally terminal') if len(done['records'])!=1005 or {r['index'] for r in done['records']}!=set(range(1005)): raise InputViolation('Incomplete or duplicate attempts') bad=[r for r in done['records'] if r['worker']['status']!='PASS'] if len(bad)!=1 or bad[0]['index']!=90 or bad[0]['worker']['stop_reason']!='process_count_limit': raise InputViolation('Terminal failure set differs from accepted original job90') if any(r['global_failure'] for r in done['records']):raise InputViolation('Global worker failure') expected={(j['cell'],j['arm']) for j in c['jobs'] if j['kind']=='FINAL'} if len(models)!=105 or {(m['cell'],m['arm']) for m in models}!=expected or any(m['status']!='PASS' for m in models): raise InputViolation('Final model roster differs from actual accepted terminal') def match_interim_models(models, old): current={(m['cell'],m['arm']):m for m in models if m['pool']==100} prior={(m['cell'],m['arm']):m for m in old} if len(current)!=35 or current.keys()!=prior.keys():raise InputViolation('Interim model roster changed') for k in current: if any(current[k][f]!=prior[k][f] for f in ['pool','fold','status','model_path','model_sha256']): raise InputViolation('Interim checkpoint changed: '+str(k)) def prepare(): start=time.process_time() if ROOT.exists():raise FileExistsError('Final postfit root already exists; no duplicate preparation') c=contract(ORIGINAL);done=read(ORIGINAL/'batch-complete.json') manifest=read(checked(ORIGINAL/'models-terminal.json',done['model_manifest_sha256'])) terminal_check(c,done,manifest['models']) match_interim_models(manifest['models'],read(INTERIM/'models-terminal.json')['models']) if sha(ORIGINAL/'contract.json')!=sha(INTERIM/'contract.json'):raise InputViolation('Interim contract differs') originals={};model_files={} for r in done['records']: guard=ORIGINAL/f"jobs/job-{r['index']:04d}/guard-result.json" raw=read(guard) if {k:v for k,v in r.items() if k!='attempted'}!=raw:raise InputViolation('Guard/terminal receipt mismatch') originals[str(guard)]=sha(guard) if r['worker']['status']=='PASS': result=read(checked(r['result_path'],r['result_sha256'])) originals[r['result_path']]=r['result_sha256'] checked(result['model_path'],result['model_sha256']);model_files[result['model_path']]=result['model_sha256'] for m in manifest['models']:checked(m['model_path'],m['model_sha256']) for name in ['contract.json','batch-complete.json','models-terminal.json']: originals[str(ORIGINAL/name)]=sha(ORIGINAL/name) prior=read(INTERIM/'score-complete.json') if prior['status']!='COMPLETE' or prior['post_cpu_seconds']!=346.870387: raise InputViolation('Interim ledger changed') for name in ['contract.json','models-terminal.json','predictions-freeze.json','score-complete.json','predict-started.json']: originals[str(INTERIM/name)]=sha(INTERIM/name) oldpred=read(INTERIM/'predictions-freeze.json') if len(oldpred['entries'])!=4968 or any(e['pool']!=100 for e in oldpred['entries']): raise InputViolation('Interim prediction manifest changed') for e in oldpred['entries']:checked(e['path'],e['sha256']) tests=list((BASE/'artifacts/r4bc-final-acceptance-2026-11-17').glob('attempt-*/worker.receipt.json')) if not tests or read(sorted(tests)[-1])['status']!='PASS':raise InputViolation('Missing final adapter synthetic acceptance') test_cpu=sum(read(p)['cpu_seconds'] for p in tests) if time.process_time()-start>60:raise InputViolation('Metadata/checkpoint audit reserve exceeded') ROOT.mkdir() for name in ['contract.json','batch-complete.json','models-terminal.json']: (ROOT/name).write_bytes((ORIGINAL/name).read_bytes()) save(ROOT/'terminal-input-audit.json',dict(status='PASS',original_files=originals, checkpoints=model_files,original_failed_job=90,model_count=105,all_checkpoint_count=len(model_files), actual_cpu_seconds=time.process_time()-start,labels_read=False,new_fits=0)) save(ROOT/'adapter-freeze.json',dict(code_sha256={p:sha(BASE/p) for p in ['tools/r4bc_final.py','tools/test_r4bc_final.py','tools/r4bc_interim.py','tools/r4bc_interim_compare.py']}, prior_post_cpu_seconds=prior['post_cpu_seconds'],synthetic_cpu_seconds=test_cpu, metadata_reserve_cpu_seconds=60,post_cap_seconds=14400, shared_post_start_unix=read(INTERIM/'predict-started.json')['unix'],post_wall_seconds=21600, fit_cpu_seconds=done['fit_cpu_seconds'],setup_cpu_seconds=done['setup_cpu_seconds'], worker_cpu=24,coordinator_cpu=25,old100_predictions_reused=True,retries=0,new_fits=0, terminal_audit_sha256=sha(ROOT/'terminal-input-audit.json'))) print('Prepared final postfit; no prediction or score launched') def pins(): f=read(ROOT/'adapter-freeze.json') for p,h in f['code_sha256'].items():checked(BASE/p,h) checked(ROOT/'terminal-input-audit.json',f['terminal_audit_sha256']) for p,h in read(ROOT/'terminal-input-audit.json')['original_files'].items():checked(p,h) return f def make_proof(mode, commit): names=['contract.json','batch-complete.json','models-terminal.json','adapter-freeze.json','terminal-input-audit.json'] if mode=='scoring':names+=['predictions-freeze.json'] paths=[str((ROOT/n).relative_to(BASE)) for n in names] paths+=list(read(ROOT/'adapter-freeze.json')['code_sha256']) if mode=='training': paths+=[str(Path(m['model_path']).relative_to(BASE)) for m in read(ROOT/'models-terminal.json')['models']] proof=dict(commit=commit,objects={},files={}) def get(oid,kind): data=subprocess.check_output(['git','cat-file',kind,oid],cwd=REPO) if git_oid(kind,data)!=oid:raise InputViolation('Git object mismatch') proof['objects'][oid]=base64.b64encode(data).decode();return data oid=get(commit,'commit').split(b'\n',1)[0].decode().removeprefix('tree ') for path in paths: current=oid for component in Path(path).parts:current=tree_lookup(get(current,'tree'),component) data=(BASE/path).read_bytes() if git_oid('blob',data)!=current:raise InputViolation('Not Git-frozen: '+path) proof['files'][path]=hashlib.sha256(data).hexdigest();verify_git_file(proof,path) save(ROOT/(mode+'-git-proof.json'),proof) def git_gate(root, name, required): proof=read(root/('scoring-git-proof.json' if name.startswith('scoring') else 'training-git-proof.json')) for path in proof['files']:verify_git_file(proof,path) for rel in required: if str((root/rel).relative_to(BASE)) not in proof['files']:raise InputViolation('Missing Git-frozen required manifest') return proof def worker(mode,cell=None,arm=None): os.sched_setaffinity(0,[24]);pins();post.git_gate=git_gate if mode=='predict': m=next(m for m in read(ROOT/'models-terminal.json')['models'] if m['cell']==cell and m['arm']==arm) if m['pool']==100:raise InputViolation('Do not regenerate frozen interim100 predictions') post.predict(ROOT,cell,arm) elif mode=='score':post.score(ROOT) elif mode=='compare': from r4bc_interim_compare import compare if compare(INTERIM,ROOT,ROOT/'INTERIM_100-exact-comparison.json'): raise InputViolation('Final/interim comparison DEFECT; preserve originals') def freeze_predictions(): post.git_gate=git_gate;c,models=post.training_gate(ROOT);entries=[];missing=[] interim=read(INTERIM/'predictions-freeze.json') entries.extend(interim['entries']) for m in models: if m['pool']==100:continue dest=ROOT/'predictions'/f"cell-{m['cell']:02d}-{m['arm']}" guard=ROOT/'prediction-guards'/dest.name/'worker.receipt.json' receipt=read(guard) if receipt['status']!='PASS': missing.append(dict(cell=m['cell'],arm=m['arm'],reason=receipt['stop_reason'] or 'PREDICTION_FAILED'));continue result=read(dest/'result.json') if result['model_sha256']!=m['model_sha256']:raise InputViolation('Prediction model mismatch') index=read(checked(dest/'index.json',result['index_sha256'])) if set(index)!={r['case'] for r in post.role_rows(c,m['fold'],m['pool'],'test')}:raise InputViolation('Incomplete prediction domain') for case,v in index.items(): checked(v['path'],v['sha256']);entries.append(dict(pool=m['pool'],fold=m['fold'],arm=m['arm'],case=case,**v)) for row in c['parts']: fold=next(a['fold'] for a in row['assignments'] if a['role']=='test') for pool in [50,25]: h=row['historical_predictions'][str(pool)];checked(h['path'],h['sha256']) entries.append(dict(pool=pool,fold=fold,arm='H0',case=row['case'],array='seconds',**h)) e=row['r4a'];checked(e['predictions'],e['predictions_sha256']) entries.append(dict(pool=pool,fold=fold,arm='EIK',case=row['case'],array='F',path=e['predictions'],sha256=e['predictions_sha256'])) for e in entries:checked(e['path'],e['sha256']) keys=[(e['pool'],e['case'],e['arm']) for e in entries] if len(keys)!=len(set(keys)):raise InputViolation('Duplicate prediction entry') save(ROOT/'predictions-freeze.json',dict(contract_sha256=sha(ROOT/'contract.json'), models_terminal_sha256=sha(ROOT/'models-terminal.json'),entries=entries,missing=missing, interim_reused_manifest_sha256=sha(INTERIM/'predictions-freeze.json'), target_values_read=False,new_decodes=0,common_parts=552,common_components=204)) def ledger(): f=read(ROOT/'adapter-freeze.json') receipts=list((ROOT/'prediction-guards').glob('*/worker.receipt.json')) for name in ['score-guard','compare-guard']: p=ROOT/name/'worker.receipt.json' if p.exists():receipts.append(p) return f['prior_post_cpu_seconds']+f['synthetic_cpu_seconds']+f['metadata_reserve_cpu_seconds']+sum(read(p)['cpu_seconds'] for p in receipts) def budget(f,spent,cpu,wall,mode,now): remaining_wall=f['shared_post_start_unix']+21600-now reserve_after=7800 if mode=='predict' else (600 if mode=='score' else 0) if spent+cpu+reserve_after>POST_CAP:raise InputViolation('Shared post CPU cap/reserve') if f['fit_cpu_seconds']+f['setup_cpu_seconds']+spent+cpu+reserve_after>144000: raise InputViolation('Shared total CPU cap') if remaining_wall<=0:raise InputViolation('Shared post wall cap') return min(wall,remaining_wall) def run(mode): os.sched_setaffinity(0,[25]);f=pins();post.git_gate=git_gate with (ROOT/'postfit.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) started=ROOT/(mode+'-started.json') if started.exists() or (ROOT/'stop.json').exists():raise FileExistsError('No restart/retry after attempted stage') _,models=post.training_gate(ROOT) if mode in ['score','compare'] and not (ROOT/('predict-complete.json' if mode=='score' else 'score-complete.json')).exists(): raise InputViolation('Previous stage not complete') if mode=='score':git_gate(ROOT,'scoring-git-freeze.json',['contract.json','predictions-freeze.json']) save(started,dict(pid=os.getpid(),unix=time.time(),stage=mode,shared_post_start=f['shared_post_start_unix'],no_new_fits=True)) jobs=[(m['cell'],m['arm']) for m in models if m['pool']!=100] if mode=='predict' else [(None,None)] for cell,arm in jobs: cpu=1800 if mode=='predict' else (7200 if mode=='score' else 600) wall=budget(f,ledger(),cpu,3600 if mode=='predict' else (10800 if mode=='score' else 1800),mode,time.time()) if (ORIGINAL/'global-stop.json').exists():raise InputViolation('Original global stop') if psutil.virtual_memory().available<10*2**30:raise InputViolation('Insufficient available memory') if sum(disk_usage(r) for r in [ORIGINAL,INTERIM,ROOT])>=100*2**30:raise InputViolation('Shared disk cap') dest=ROOT/'prediction-guards'/f'cell-{cell:02d}-{arm}' if mode=='predict' else ROOT/(mode+'-guard') command=[sys.executable,'tools/r4bc_final.py','worker','--mode',mode] if cell is not None:command+=['--cell',str(cell),'--arm',arm] save(ROOT/'progress.json',dict(stage=mode,cell=cell,arm=arm,post_cpu_seconds=ledger(),reserved_cpu=cpu, post_cap=14400,remaining_shared_wall=wall,new_fits=0)) receipt=supervise(command,dest,dict(cpu_seconds=cpu,wall_seconds=wall,rss_bytes=8*2**30,disk_bytes=2*2**30),global_stop=ORIGINAL/'global-stop.json') if receipt['status']!='PASS': # Preserve failures, do not automatically retry any postfit stage. save(ROOT/'stop.json',dict(mode=mode,cell=cell,arm=arm,receipt=receipt,post_cpu_seconds=ledger())) return 1 if mode=='predict':freeze_predictions() save(ROOT/(mode+'-complete.json'),dict(status='COMPLETE',mode=mode,post_cpu_seconds=ledger(),remaining_post_cpu_seconds=POST_CAP-ledger(), coordinator_cpu_seconds=time.process_time(),next={'predict':'GIT_FREEZE_ALL_PREDICTIONS','score':'EXACT_INTERIM100_COMPARE','compare':'FIXED_RESULT_DELIVERY'}[mode])) return 0 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('action',choices=['prepare','proof','worker','run']) p.add_argument('--mode',choices=['training','scoring','predict','score','compare']);p.add_argument('--commit');p.add_argument('--cell',type=int);p.add_argument('--arm');a=p.parse_args() try: if a.action=='prepare':prepare() elif a.action=='proof':make_proof(a.mode,a.commit) elif a.action=='worker':worker(a.mode,a.cell,a.arm) else:raise SystemExit(run(a.mode)) except Exception as exc: if a.action=='run':save(ROOT/'stop.json',dict(mode=a.mode,error=type(exc).__name__+': '+str(exc),post_cpu_seconds=ledger())) raise