Files
MeltMind/tools/r4bc_final.py

251 lines
15 KiB
Python

"""Final R4bc orchestration; original prediction/scoring functions stay immutable."""
import argparse
import base64
import fcntl
import hashlib
import os
from pathlib import Path
import subprocess
import sys
import time
import psutil
import r4bc_postfit as post
from r4bc_common import contract, read, save, checked, sha, InputViolation
from r4bc_guard import supervise
from r4bc_interim import git_oid, tree_lookup, verify_git_file
from s2_recovery_monitoring import disk_usage
BASE=Path('/home/frank14f/optane/MeltMind')
ORIGINAL=BASE/'artifacts/r4bc-execution-2026-11-16'
INTERIM=BASE/'artifacts/r4bc-interim-100-2026-11-17'
ROOT=BASE/'artifacts/r4bc-final-postfit-2026-11-17'
REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
POST_CAP=14400
def terminal_check(c, done, models):
if (done['status']!='FIT_QUEUE_TERMINAL' or done['global_stop'] or done['cut']
or not done['accounting_complete'] or done['monitor_errors'] or done['never_attempted']
or done['postfit_state']!='READY_FOR_GIT_CHECKPOINT_FREEZE'):
raise InputViolation('Original full queue not normally terminal')
if len(done['records'])!=1005 or {r['index'] for r in done['records']}!=set(range(1005)):
raise InputViolation('Incomplete or duplicate attempts')
bad=[r for r in done['records'] if r['worker']['status']!='PASS']
if len(bad)!=1 or bad[0]['index']!=90 or bad[0]['worker']['stop_reason']!='process_count_limit':
raise InputViolation('Terminal failure set differs from accepted original job90')
if any(r['global_failure'] for r in done['records']):raise InputViolation('Global worker failure')
expected={(j['cell'],j['arm']) for j in c['jobs'] if j['kind']=='FINAL'}
if len(models)!=105 or {(m['cell'],m['arm']) for m in models}!=expected or any(m['status']!='PASS' for m in models):
raise InputViolation('Final model roster differs from actual accepted terminal')
def match_interim_models(models, old):
current={(m['cell'],m['arm']):m for m in models if m['pool']==100}
prior={(m['cell'],m['arm']):m for m in old}
if len(current)!=35 or current.keys()!=prior.keys():raise InputViolation('Interim model roster changed')
for k in current:
if any(current[k][f]!=prior[k][f] for f in ['pool','fold','status','model_path','model_sha256']):
raise InputViolation('Interim checkpoint changed: '+str(k))
def prepare():
start=time.process_time()
if ROOT.exists():raise FileExistsError('Final postfit root already exists; no duplicate preparation')
c=contract(ORIGINAL);done=read(ORIGINAL/'batch-complete.json')
manifest=read(checked(ORIGINAL/'models-terminal.json',done['model_manifest_sha256']))
terminal_check(c,done,manifest['models'])
match_interim_models(manifest['models'],read(INTERIM/'models-terminal.json')['models'])
if sha(ORIGINAL/'contract.json')!=sha(INTERIM/'contract.json'):raise InputViolation('Interim contract differs')
originals={};model_files={}
for r in done['records']:
guard=ORIGINAL/f"jobs/job-{r['index']:04d}/guard-result.json"
raw=read(guard)
if {k:v for k,v in r.items() if k!='attempted'}!=raw:raise InputViolation('Guard/terminal receipt mismatch')
originals[str(guard)]=sha(guard)
if r['worker']['status']=='PASS':
result=read(checked(r['result_path'],r['result_sha256']))
originals[r['result_path']]=r['result_sha256']
checked(result['model_path'],result['model_sha256']);model_files[result['model_path']]=result['model_sha256']
for m in manifest['models']:checked(m['model_path'],m['model_sha256'])
for name in ['contract.json','batch-complete.json','models-terminal.json']:
originals[str(ORIGINAL/name)]=sha(ORIGINAL/name)
prior=read(INTERIM/'score-complete.json')
if prior['status']!='COMPLETE' or prior['post_cpu_seconds']!=346.870387:
raise InputViolation('Interim ledger changed')
for name in ['contract.json','models-terminal.json','predictions-freeze.json','score-complete.json','predict-started.json']:
originals[str(INTERIM/name)]=sha(INTERIM/name)
oldpred=read(INTERIM/'predictions-freeze.json')
if len(oldpred['entries'])!=4968 or any(e['pool']!=100 for e in oldpred['entries']):
raise InputViolation('Interim prediction manifest changed')
for e in oldpred['entries']:checked(e['path'],e['sha256'])
tests=list((BASE/'artifacts/r4bc-final-acceptance-2026-11-17').glob('attempt-*/worker.receipt.json'))
if not tests or read(sorted(tests)[-1])['status']!='PASS':raise InputViolation('Missing final adapter synthetic acceptance')
test_cpu=sum(read(p)['cpu_seconds'] for p in tests)
if time.process_time()-start>60:raise InputViolation('Metadata/checkpoint audit reserve exceeded')
ROOT.mkdir()
for name in ['contract.json','batch-complete.json','models-terminal.json']:
(ROOT/name).write_bytes((ORIGINAL/name).read_bytes())
save(ROOT/'terminal-input-audit.json',dict(status='PASS',original_files=originals,
checkpoints=model_files,original_failed_job=90,model_count=105,all_checkpoint_count=len(model_files),
actual_cpu_seconds=time.process_time()-start,labels_read=False,new_fits=0))
save(ROOT/'adapter-freeze.json',dict(code_sha256={p:sha(BASE/p) for p in
['tools/r4bc_final.py','tools/test_r4bc_final.py','tools/r4bc_interim.py','tools/r4bc_interim_compare.py']},
prior_post_cpu_seconds=prior['post_cpu_seconds'],synthetic_cpu_seconds=test_cpu,
metadata_reserve_cpu_seconds=60,post_cap_seconds=14400,
shared_post_start_unix=read(INTERIM/'predict-started.json')['unix'],post_wall_seconds=21600,
fit_cpu_seconds=done['fit_cpu_seconds'],setup_cpu_seconds=done['setup_cpu_seconds'],
worker_cpu=24,coordinator_cpu=25,old100_predictions_reused=True,retries=0,new_fits=0,
terminal_audit_sha256=sha(ROOT/'terminal-input-audit.json')))
print('Prepared final postfit; no prediction or score launched')
def pins():
f=read(ROOT/'adapter-freeze.json')
for p,h in f['code_sha256'].items():checked(BASE/p,h)
checked(ROOT/'terminal-input-audit.json',f['terminal_audit_sha256'])
for p,h in read(ROOT/'terminal-input-audit.json')['original_files'].items():checked(p,h)
return f
def make_proof(mode, commit):
names=['contract.json','batch-complete.json','models-terminal.json','adapter-freeze.json','terminal-input-audit.json']
if mode=='scoring':names+=['predictions-freeze.json']
paths=[str((ROOT/n).relative_to(BASE)) for n in names]
paths+=list(read(ROOT/'adapter-freeze.json')['code_sha256'])
if mode=='training':
paths+=[str(Path(m['model_path']).relative_to(BASE)) for m in read(ROOT/'models-terminal.json')['models']]
proof=dict(commit=commit,objects={},files={})
def get(oid,kind):
data=subprocess.check_output(['git','cat-file',kind,oid],cwd=REPO)
if git_oid(kind,data)!=oid:raise InputViolation('Git object mismatch')
proof['objects'][oid]=base64.b64encode(data).decode();return data
oid=get(commit,'commit').split(b'\n',1)[0].decode().removeprefix('tree ')
for path in paths:
current=oid
for component in Path(path).parts:current=tree_lookup(get(current,'tree'),component)
data=(BASE/path).read_bytes()
if git_oid('blob',data)!=current:raise InputViolation('Not Git-frozen: '+path)
proof['files'][path]=hashlib.sha256(data).hexdigest();verify_git_file(proof,path)
save(ROOT/(mode+'-git-proof.json'),proof)
def git_gate(root, name, required):
proof=read(root/('scoring-git-proof.json' if name.startswith('scoring') else 'training-git-proof.json'))
for path in proof['files']:verify_git_file(proof,path)
for rel in required:
if str((root/rel).relative_to(BASE)) not in proof['files']:raise InputViolation('Missing Git-frozen required manifest')
return proof
def worker(mode,cell=None,arm=None):
os.sched_setaffinity(0,[24]);pins();post.git_gate=git_gate
if mode=='predict':
m=next(m for m in read(ROOT/'models-terminal.json')['models'] if m['cell']==cell and m['arm']==arm)
if m['pool']==100:raise InputViolation('Do not regenerate frozen interim100 predictions')
post.predict(ROOT,cell,arm)
elif mode=='score':post.score(ROOT)
elif mode=='compare':
from r4bc_interim_compare import compare
if compare(INTERIM,ROOT,ROOT/'INTERIM_100-exact-comparison.json'):
raise InputViolation('Final/interim comparison DEFECT; preserve originals')
def freeze_predictions():
post.git_gate=git_gate;c,models=post.training_gate(ROOT);entries=[];missing=[]
interim=read(INTERIM/'predictions-freeze.json')
entries.extend(interim['entries'])
for m in models:
if m['pool']==100:continue
dest=ROOT/'predictions'/f"cell-{m['cell']:02d}-{m['arm']}"
guard=ROOT/'prediction-guards'/dest.name/'worker.receipt.json'
receipt=read(guard)
if receipt['status']!='PASS':
missing.append(dict(cell=m['cell'],arm=m['arm'],reason=receipt['stop_reason'] or 'PREDICTION_FAILED'));continue
result=read(dest/'result.json')
if result['model_sha256']!=m['model_sha256']:raise InputViolation('Prediction model mismatch')
index=read(checked(dest/'index.json',result['index_sha256']))
if set(index)!={r['case'] for r in post.role_rows(c,m['fold'],m['pool'],'test')}:raise InputViolation('Incomplete prediction domain')
for case,v in index.items():
checked(v['path'],v['sha256']);entries.append(dict(pool=m['pool'],fold=m['fold'],arm=m['arm'],case=case,**v))
for row in c['parts']:
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
for pool in [50,25]:
h=row['historical_predictions'][str(pool)];checked(h['path'],h['sha256'])
entries.append(dict(pool=pool,fold=fold,arm='H0',case=row['case'],array='seconds',**h))
e=row['r4a'];checked(e['predictions'],e['predictions_sha256'])
entries.append(dict(pool=pool,fold=fold,arm='EIK',case=row['case'],array='F',path=e['predictions'],sha256=e['predictions_sha256']))
for e in entries:checked(e['path'],e['sha256'])
keys=[(e['pool'],e['case'],e['arm']) for e in entries]
if len(keys)!=len(set(keys)):raise InputViolation('Duplicate prediction entry')
save(ROOT/'predictions-freeze.json',dict(contract_sha256=sha(ROOT/'contract.json'),
models_terminal_sha256=sha(ROOT/'models-terminal.json'),entries=entries,missing=missing,
interim_reused_manifest_sha256=sha(INTERIM/'predictions-freeze.json'),
target_values_read=False,new_decodes=0,common_parts=552,common_components=204))
def ledger():
f=read(ROOT/'adapter-freeze.json')
receipts=list((ROOT/'prediction-guards').glob('*/worker.receipt.json'))
for name in ['score-guard','compare-guard']:
p=ROOT/name/'worker.receipt.json'
if p.exists():receipts.append(p)
return f['prior_post_cpu_seconds']+f['synthetic_cpu_seconds']+f['metadata_reserve_cpu_seconds']+sum(read(p)['cpu_seconds'] for p in receipts)
def budget(f,spent,cpu,wall,mode,now):
remaining_wall=f['shared_post_start_unix']+21600-now
reserve_after=7800 if mode=='predict' else (600 if mode=='score' else 0)
if spent+cpu+reserve_after>POST_CAP:raise InputViolation('Shared post CPU cap/reserve')
if f['fit_cpu_seconds']+f['setup_cpu_seconds']+spent+cpu+reserve_after>144000:
raise InputViolation('Shared total CPU cap')
if remaining_wall<=0:raise InputViolation('Shared post wall cap')
return min(wall,remaining_wall)
def run(mode):
os.sched_setaffinity(0,[25]);f=pins();post.git_gate=git_gate
with (ROOT/'postfit.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
started=ROOT/(mode+'-started.json')
if started.exists() or (ROOT/'stop.json').exists():raise FileExistsError('No restart/retry after attempted stage')
_,models=post.training_gate(ROOT)
if mode in ['score','compare'] and not (ROOT/('predict-complete.json' if mode=='score' else 'score-complete.json')).exists():
raise InputViolation('Previous stage not complete')
if mode=='score':git_gate(ROOT,'scoring-git-freeze.json',['contract.json','predictions-freeze.json'])
save(started,dict(pid=os.getpid(),unix=time.time(),stage=mode,shared_post_start=f['shared_post_start_unix'],no_new_fits=True))
jobs=[(m['cell'],m['arm']) for m in models if m['pool']!=100] if mode=='predict' else [(None,None)]
for cell,arm in jobs:
cpu=1800 if mode=='predict' else (7200 if mode=='score' else 600)
wall=budget(f,ledger(),cpu,3600 if mode=='predict' else (10800 if mode=='score' else 1800),mode,time.time())
if (ORIGINAL/'global-stop.json').exists():raise InputViolation('Original global stop')
if psutil.virtual_memory().available<10*2**30:raise InputViolation('Insufficient available memory')
if sum(disk_usage(r) for r in [ORIGINAL,INTERIM,ROOT])>=100*2**30:raise InputViolation('Shared disk cap')
dest=ROOT/'prediction-guards'/f'cell-{cell:02d}-{arm}' if mode=='predict' else ROOT/(mode+'-guard')
command=[sys.executable,'tools/r4bc_final.py','worker','--mode',mode]
if cell is not None:command+=['--cell',str(cell),'--arm',arm]
save(ROOT/'progress.json',dict(stage=mode,cell=cell,arm=arm,post_cpu_seconds=ledger(),reserved_cpu=cpu,
post_cap=14400,remaining_shared_wall=wall,new_fits=0))
receipt=supervise(command,dest,dict(cpu_seconds=cpu,wall_seconds=wall,rss_bytes=8*2**30,disk_bytes=2*2**30),global_stop=ORIGINAL/'global-stop.json')
if receipt['status']!='PASS':
# Preserve failures, do not automatically retry any postfit stage.
save(ROOT/'stop.json',dict(mode=mode,cell=cell,arm=arm,receipt=receipt,post_cpu_seconds=ledger()))
return 1
if mode=='predict':freeze_predictions()
save(ROOT/(mode+'-complete.json'),dict(status='COMPLETE',mode=mode,post_cpu_seconds=ledger(),remaining_post_cpu_seconds=POST_CAP-ledger(),
coordinator_cpu_seconds=time.process_time(),next={'predict':'GIT_FREEZE_ALL_PREDICTIONS','score':'EXACT_INTERIM100_COMPARE','compare':'FIXED_RESULT_DELIVERY'}[mode]))
return 0
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('action',choices=['prepare','proof','worker','run'])
p.add_argument('--mode',choices=['training','scoring','predict','score','compare']);p.add_argument('--commit');p.add_argument('--cell',type=int);p.add_argument('--arm');a=p.parse_args()
try:
if a.action=='prepare':prepare()
elif a.action=='proof':make_proof(a.mode,a.commit)
elif a.action=='worker':worker(a.mode,a.cell,a.arm)
else:raise SystemExit(run(a.mode))
except Exception as exc:
if a.action=='run':save(ROOT/'stop.json',dict(mode=a.mode,error=type(exc).__name__+': '+str(exc),post_cpu_seconds=ledger()))
raise