251 lines
15 KiB
Python
251 lines
15 KiB
Python
"""Final R4bc orchestration; original prediction/scoring functions stay immutable."""
|
|
import argparse
|
|
import base64
|
|
import fcntl
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import psutil
|
|
import r4bc_postfit as post
|
|
from r4bc_common import contract, read, save, checked, sha, InputViolation
|
|
from r4bc_guard import supervise
|
|
from r4bc_interim import git_oid, tree_lookup, verify_git_file
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
BASE=Path('/home/frank14f/optane/MeltMind')
|
|
ORIGINAL=BASE/'artifacts/r4bc-execution-2026-11-16'
|
|
INTERIM=BASE/'artifacts/r4bc-interim-100-2026-11-17'
|
|
ROOT=BASE/'artifacts/r4bc-final-postfit-2026-11-17'
|
|
REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
|
|
POST_CAP=14400
|
|
|
|
|
|
def terminal_check(c, done, models):
|
|
if (done['status']!='FIT_QUEUE_TERMINAL' or done['global_stop'] or done['cut']
|
|
or not done['accounting_complete'] or done['monitor_errors'] or done['never_attempted']
|
|
or done['postfit_state']!='READY_FOR_GIT_CHECKPOINT_FREEZE'):
|
|
raise InputViolation('Original full queue not normally terminal')
|
|
if len(done['records'])!=1005 or {r['index'] for r in done['records']}!=set(range(1005)):
|
|
raise InputViolation('Incomplete or duplicate attempts')
|
|
bad=[r for r in done['records'] if r['worker']['status']!='PASS']
|
|
if len(bad)!=1 or bad[0]['index']!=90 or bad[0]['worker']['stop_reason']!='process_count_limit':
|
|
raise InputViolation('Terminal failure set differs from accepted original job90')
|
|
if any(r['global_failure'] for r in done['records']):raise InputViolation('Global worker failure')
|
|
expected={(j['cell'],j['arm']) for j in c['jobs'] if j['kind']=='FINAL'}
|
|
if len(models)!=105 or {(m['cell'],m['arm']) for m in models}!=expected or any(m['status']!='PASS' for m in models):
|
|
raise InputViolation('Final model roster differs from actual accepted terminal')
|
|
|
|
|
|
def match_interim_models(models, old):
|
|
current={(m['cell'],m['arm']):m for m in models if m['pool']==100}
|
|
prior={(m['cell'],m['arm']):m for m in old}
|
|
if len(current)!=35 or current.keys()!=prior.keys():raise InputViolation('Interim model roster changed')
|
|
for k in current:
|
|
if any(current[k][f]!=prior[k][f] for f in ['pool','fold','status','model_path','model_sha256']):
|
|
raise InputViolation('Interim checkpoint changed: '+str(k))
|
|
|
|
|
|
def prepare():
|
|
start=time.process_time()
|
|
if ROOT.exists():raise FileExistsError('Final postfit root already exists; no duplicate preparation')
|
|
c=contract(ORIGINAL);done=read(ORIGINAL/'batch-complete.json')
|
|
manifest=read(checked(ORIGINAL/'models-terminal.json',done['model_manifest_sha256']))
|
|
terminal_check(c,done,manifest['models'])
|
|
match_interim_models(manifest['models'],read(INTERIM/'models-terminal.json')['models'])
|
|
if sha(ORIGINAL/'contract.json')!=sha(INTERIM/'contract.json'):raise InputViolation('Interim contract differs')
|
|
originals={};model_files={}
|
|
for r in done['records']:
|
|
guard=ORIGINAL/f"jobs/job-{r['index']:04d}/guard-result.json"
|
|
raw=read(guard)
|
|
if {k:v for k,v in r.items() if k!='attempted'}!=raw:raise InputViolation('Guard/terminal receipt mismatch')
|
|
originals[str(guard)]=sha(guard)
|
|
if r['worker']['status']=='PASS':
|
|
result=read(checked(r['result_path'],r['result_sha256']))
|
|
originals[r['result_path']]=r['result_sha256']
|
|
checked(result['model_path'],result['model_sha256']);model_files[result['model_path']]=result['model_sha256']
|
|
for m in manifest['models']:checked(m['model_path'],m['model_sha256'])
|
|
for name in ['contract.json','batch-complete.json','models-terminal.json']:
|
|
originals[str(ORIGINAL/name)]=sha(ORIGINAL/name)
|
|
prior=read(INTERIM/'score-complete.json')
|
|
if prior['status']!='COMPLETE' or prior['post_cpu_seconds']!=346.870387:
|
|
raise InputViolation('Interim ledger changed')
|
|
for name in ['contract.json','models-terminal.json','predictions-freeze.json','score-complete.json','predict-started.json']:
|
|
originals[str(INTERIM/name)]=sha(INTERIM/name)
|
|
oldpred=read(INTERIM/'predictions-freeze.json')
|
|
if len(oldpred['entries'])!=4968 or any(e['pool']!=100 for e in oldpred['entries']):
|
|
raise InputViolation('Interim prediction manifest changed')
|
|
for e in oldpred['entries']:checked(e['path'],e['sha256'])
|
|
tests=list((BASE/'artifacts/r4bc-final-acceptance-2026-11-17').glob('attempt-*/worker.receipt.json'))
|
|
if not tests or read(sorted(tests)[-1])['status']!='PASS':raise InputViolation('Missing final adapter synthetic acceptance')
|
|
test_cpu=sum(read(p)['cpu_seconds'] for p in tests)
|
|
if time.process_time()-start>60:raise InputViolation('Metadata/checkpoint audit reserve exceeded')
|
|
ROOT.mkdir()
|
|
for name in ['contract.json','batch-complete.json','models-terminal.json']:
|
|
(ROOT/name).write_bytes((ORIGINAL/name).read_bytes())
|
|
save(ROOT/'terminal-input-audit.json',dict(status='PASS',original_files=originals,
|
|
checkpoints=model_files,original_failed_job=90,model_count=105,all_checkpoint_count=len(model_files),
|
|
actual_cpu_seconds=time.process_time()-start,labels_read=False,new_fits=0))
|
|
save(ROOT/'adapter-freeze.json',dict(code_sha256={p:sha(BASE/p) for p in
|
|
['tools/r4bc_final.py','tools/test_r4bc_final.py','tools/r4bc_interim.py','tools/r4bc_interim_compare.py']},
|
|
prior_post_cpu_seconds=prior['post_cpu_seconds'],synthetic_cpu_seconds=test_cpu,
|
|
metadata_reserve_cpu_seconds=60,post_cap_seconds=14400,
|
|
shared_post_start_unix=read(INTERIM/'predict-started.json')['unix'],post_wall_seconds=21600,
|
|
fit_cpu_seconds=done['fit_cpu_seconds'],setup_cpu_seconds=done['setup_cpu_seconds'],
|
|
worker_cpu=24,coordinator_cpu=25,old100_predictions_reused=True,retries=0,new_fits=0,
|
|
terminal_audit_sha256=sha(ROOT/'terminal-input-audit.json')))
|
|
print('Prepared final postfit; no prediction or score launched')
|
|
|
|
|
|
def pins():
|
|
f=read(ROOT/'adapter-freeze.json')
|
|
for p,h in f['code_sha256'].items():checked(BASE/p,h)
|
|
checked(ROOT/'terminal-input-audit.json',f['terminal_audit_sha256'])
|
|
for p,h in read(ROOT/'terminal-input-audit.json')['original_files'].items():checked(p,h)
|
|
return f
|
|
|
|
|
|
def make_proof(mode, commit):
|
|
names=['contract.json','batch-complete.json','models-terminal.json','adapter-freeze.json','terminal-input-audit.json']
|
|
if mode=='scoring':names+=['predictions-freeze.json']
|
|
paths=[str((ROOT/n).relative_to(BASE)) for n in names]
|
|
paths+=list(read(ROOT/'adapter-freeze.json')['code_sha256'])
|
|
if mode=='training':
|
|
paths+=[str(Path(m['model_path']).relative_to(BASE)) for m in read(ROOT/'models-terminal.json')['models']]
|
|
proof=dict(commit=commit,objects={},files={})
|
|
def get(oid,kind):
|
|
data=subprocess.check_output(['git','cat-file',kind,oid],cwd=REPO)
|
|
if git_oid(kind,data)!=oid:raise InputViolation('Git object mismatch')
|
|
proof['objects'][oid]=base64.b64encode(data).decode();return data
|
|
oid=get(commit,'commit').split(b'\n',1)[0].decode().removeprefix('tree ')
|
|
for path in paths:
|
|
current=oid
|
|
for component in Path(path).parts:current=tree_lookup(get(current,'tree'),component)
|
|
data=(BASE/path).read_bytes()
|
|
if git_oid('blob',data)!=current:raise InputViolation('Not Git-frozen: '+path)
|
|
proof['files'][path]=hashlib.sha256(data).hexdigest();verify_git_file(proof,path)
|
|
save(ROOT/(mode+'-git-proof.json'),proof)
|
|
|
|
|
|
def git_gate(root, name, required):
|
|
proof=read(root/('scoring-git-proof.json' if name.startswith('scoring') else 'training-git-proof.json'))
|
|
for path in proof['files']:verify_git_file(proof,path)
|
|
for rel in required:
|
|
if str((root/rel).relative_to(BASE)) not in proof['files']:raise InputViolation('Missing Git-frozen required manifest')
|
|
return proof
|
|
|
|
|
|
def worker(mode,cell=None,arm=None):
|
|
os.sched_setaffinity(0,[24]);pins();post.git_gate=git_gate
|
|
if mode=='predict':
|
|
m=next(m for m in read(ROOT/'models-terminal.json')['models'] if m['cell']==cell and m['arm']==arm)
|
|
if m['pool']==100:raise InputViolation('Do not regenerate frozen interim100 predictions')
|
|
post.predict(ROOT,cell,arm)
|
|
elif mode=='score':post.score(ROOT)
|
|
elif mode=='compare':
|
|
from r4bc_interim_compare import compare
|
|
if compare(INTERIM,ROOT,ROOT/'INTERIM_100-exact-comparison.json'):
|
|
raise InputViolation('Final/interim comparison DEFECT; preserve originals')
|
|
|
|
|
|
def freeze_predictions():
|
|
post.git_gate=git_gate;c,models=post.training_gate(ROOT);entries=[];missing=[]
|
|
interim=read(INTERIM/'predictions-freeze.json')
|
|
entries.extend(interim['entries'])
|
|
for m in models:
|
|
if m['pool']==100:continue
|
|
dest=ROOT/'predictions'/f"cell-{m['cell']:02d}-{m['arm']}"
|
|
guard=ROOT/'prediction-guards'/dest.name/'worker.receipt.json'
|
|
receipt=read(guard)
|
|
if receipt['status']!='PASS':
|
|
missing.append(dict(cell=m['cell'],arm=m['arm'],reason=receipt['stop_reason'] or 'PREDICTION_FAILED'));continue
|
|
result=read(dest/'result.json')
|
|
if result['model_sha256']!=m['model_sha256']:raise InputViolation('Prediction model mismatch')
|
|
index=read(checked(dest/'index.json',result['index_sha256']))
|
|
if set(index)!={r['case'] for r in post.role_rows(c,m['fold'],m['pool'],'test')}:raise InputViolation('Incomplete prediction domain')
|
|
for case,v in index.items():
|
|
checked(v['path'],v['sha256']);entries.append(dict(pool=m['pool'],fold=m['fold'],arm=m['arm'],case=case,**v))
|
|
for row in c['parts']:
|
|
fold=next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
|
for pool in [50,25]:
|
|
h=row['historical_predictions'][str(pool)];checked(h['path'],h['sha256'])
|
|
entries.append(dict(pool=pool,fold=fold,arm='H0',case=row['case'],array='seconds',**h))
|
|
e=row['r4a'];checked(e['predictions'],e['predictions_sha256'])
|
|
entries.append(dict(pool=pool,fold=fold,arm='EIK',case=row['case'],array='F',path=e['predictions'],sha256=e['predictions_sha256']))
|
|
for e in entries:checked(e['path'],e['sha256'])
|
|
keys=[(e['pool'],e['case'],e['arm']) for e in entries]
|
|
if len(keys)!=len(set(keys)):raise InputViolation('Duplicate prediction entry')
|
|
save(ROOT/'predictions-freeze.json',dict(contract_sha256=sha(ROOT/'contract.json'),
|
|
models_terminal_sha256=sha(ROOT/'models-terminal.json'),entries=entries,missing=missing,
|
|
interim_reused_manifest_sha256=sha(INTERIM/'predictions-freeze.json'),
|
|
target_values_read=False,new_decodes=0,common_parts=552,common_components=204))
|
|
|
|
|
|
def ledger():
|
|
f=read(ROOT/'adapter-freeze.json')
|
|
receipts=list((ROOT/'prediction-guards').glob('*/worker.receipt.json'))
|
|
for name in ['score-guard','compare-guard']:
|
|
p=ROOT/name/'worker.receipt.json'
|
|
if p.exists():receipts.append(p)
|
|
return f['prior_post_cpu_seconds']+f['synthetic_cpu_seconds']+f['metadata_reserve_cpu_seconds']+sum(read(p)['cpu_seconds'] for p in receipts)
|
|
|
|
|
|
def budget(f,spent,cpu,wall,mode,now):
|
|
remaining_wall=f['shared_post_start_unix']+21600-now
|
|
reserve_after=7800 if mode=='predict' else (600 if mode=='score' else 0)
|
|
if spent+cpu+reserve_after>POST_CAP:raise InputViolation('Shared post CPU cap/reserve')
|
|
if f['fit_cpu_seconds']+f['setup_cpu_seconds']+spent+cpu+reserve_after>144000:
|
|
raise InputViolation('Shared total CPU cap')
|
|
if remaining_wall<=0:raise InputViolation('Shared post wall cap')
|
|
return min(wall,remaining_wall)
|
|
|
|
|
|
def run(mode):
|
|
os.sched_setaffinity(0,[25]);f=pins();post.git_gate=git_gate
|
|
with (ROOT/'postfit.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
started=ROOT/(mode+'-started.json')
|
|
if started.exists() or (ROOT/'stop.json').exists():raise FileExistsError('No restart/retry after attempted stage')
|
|
_,models=post.training_gate(ROOT)
|
|
if mode in ['score','compare'] and not (ROOT/('predict-complete.json' if mode=='score' else 'score-complete.json')).exists():
|
|
raise InputViolation('Previous stage not complete')
|
|
if mode=='score':git_gate(ROOT,'scoring-git-freeze.json',['contract.json','predictions-freeze.json'])
|
|
save(started,dict(pid=os.getpid(),unix=time.time(),stage=mode,shared_post_start=f['shared_post_start_unix'],no_new_fits=True))
|
|
jobs=[(m['cell'],m['arm']) for m in models if m['pool']!=100] if mode=='predict' else [(None,None)]
|
|
for cell,arm in jobs:
|
|
cpu=1800 if mode=='predict' else (7200 if mode=='score' else 600)
|
|
wall=budget(f,ledger(),cpu,3600 if mode=='predict' else (10800 if mode=='score' else 1800),mode,time.time())
|
|
if (ORIGINAL/'global-stop.json').exists():raise InputViolation('Original global stop')
|
|
if psutil.virtual_memory().available<10*2**30:raise InputViolation('Insufficient available memory')
|
|
if sum(disk_usage(r) for r in [ORIGINAL,INTERIM,ROOT])>=100*2**30:raise InputViolation('Shared disk cap')
|
|
dest=ROOT/'prediction-guards'/f'cell-{cell:02d}-{arm}' if mode=='predict' else ROOT/(mode+'-guard')
|
|
command=[sys.executable,'tools/r4bc_final.py','worker','--mode',mode]
|
|
if cell is not None:command+=['--cell',str(cell),'--arm',arm]
|
|
save(ROOT/'progress.json',dict(stage=mode,cell=cell,arm=arm,post_cpu_seconds=ledger(),reserved_cpu=cpu,
|
|
post_cap=14400,remaining_shared_wall=wall,new_fits=0))
|
|
receipt=supervise(command,dest,dict(cpu_seconds=cpu,wall_seconds=wall,rss_bytes=8*2**30,disk_bytes=2*2**30),global_stop=ORIGINAL/'global-stop.json')
|
|
if receipt['status']!='PASS':
|
|
# Preserve failures, do not automatically retry any postfit stage.
|
|
save(ROOT/'stop.json',dict(mode=mode,cell=cell,arm=arm,receipt=receipt,post_cpu_seconds=ledger()))
|
|
return 1
|
|
if mode=='predict':freeze_predictions()
|
|
save(ROOT/(mode+'-complete.json'),dict(status='COMPLETE',mode=mode,post_cpu_seconds=ledger(),remaining_post_cpu_seconds=POST_CAP-ledger(),
|
|
coordinator_cpu_seconds=time.process_time(),next={'predict':'GIT_FREEZE_ALL_PREDICTIONS','score':'EXACT_INTERIM100_COMPARE','compare':'FIXED_RESULT_DELIVERY'}[mode]))
|
|
return 0
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('action',choices=['prepare','proof','worker','run'])
|
|
p.add_argument('--mode',choices=['training','scoring','predict','score','compare']);p.add_argument('--commit');p.add_argument('--cell',type=int);p.add_argument('--arm');a=p.parse_args()
|
|
try:
|
|
if a.action=='prepare':prepare()
|
|
elif a.action=='proof':make_proof(a.mode,a.commit)
|
|
elif a.action=='worker':worker(a.mode,a.cell,a.arm)
|
|
else:raise SystemExit(run(a.mode))
|
|
except Exception as exc:
|
|
if a.action=='run':save(ROOT/'stop.json',dict(mode=a.mode,error=type(exc).__name__+': '+str(exc),post_cpu_seconds=ledger()))
|
|
raise
|