"""INTERIM_100 orchestration only; frozen prediction/scoring functions unchanged.""" import argparse import base64 import collections import hashlib import json import os from pathlib import Path import subprocess import sys import time import r4bc_postfit as post from r4bc_common import contract, read, save, checked, sha, ARMS from r4bc_guard import supervise BASE = Path('/home/frank14f/optane/MeltMind') ORIGINAL = BASE / 'artifacts/r4bc-execution-2026-11-16' ROOT = BASE / 'artifacts/r4bc-interim-100-2026-11-17' REPO = Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8') POST_CAP = 14400 INTERIM_CAP = 7200 # retain at least 7200 CPU-s for the final postfit stage def git_oid(kind, data): return hashlib.sha1(kind.encode() + b' ' + str(len(data)).encode() + b'\0' + data).hexdigest() def tree_lookup(raw, name): offset = 0 while offset < len(raw): end = raw.index(b'\0', offset) entry = raw[offset:end].split(b' ', 1)[1].decode() oid = raw[end + 1:end + 21].hex() if entry == name: return oid offset = end + 21 raise ValueError('Missing Git tree entry: ' + name) def verify_git_file(proof, path): """Verify commit/tree/blob linkage without child processes inside CPU guard.""" def obj(oid, kind): data = base64.b64decode(proof['objects'][oid]) if git_oid(kind, data) != oid: raise ValueError('Git object identity mismatch') return data commit = obj(proof['commit'], 'commit') oid = commit.split(b'\n', 1)[0].decode().removeprefix('tree ') parts = Path(path).parts for component in parts: oid = tree_lookup(obj(oid, 'tree'), component) data = (BASE / path).read_bytes() if git_oid('blob', data) != oid or hashlib.sha256(data).hexdigest() != proof['files'][path]: raise ValueError('Git-frozen file mismatch: ' + path) def make_proof(mode, commit): names = ['contract.json', 'models-terminal.json', 'interim-freeze.json', 'adapter-freeze.json'] if mode == 'scoring': names += ['predictions-freeze.json'] paths = [str((ROOT / n).relative_to(BASE)) for n in names] paths += list(read(ROOT / 'adapter-freeze.json')['code_sha256']) if mode == 'training': paths += [str(Path(m['model_path']).relative_to(BASE)) for m in read(ROOT / 'models-terminal.json')['models']] proof = {'commit': commit, 'objects': {}, 'files': {}} def get(oid, kind): raw = subprocess.check_output(['git', 'cat-file', kind, oid], cwd=REPO) assert git_oid(kind, raw) == oid proof['objects'][oid] = base64.b64encode(raw).decode() return raw root_oid = get(commit, 'commit').split(b'\n', 1)[0].decode().removeprefix('tree ') for path in paths: oid = root_oid for component in Path(path).parts: oid = tree_lookup(get(oid, 'tree'), component) data = (BASE / path).read_bytes() assert git_oid('blob', data) == oid, path proof['files'][path] = hashlib.sha256(data).hexdigest() verify_git_file(proof, path) save(ROOT / (mode + '-git-proof.json'), proof) def git_gate(root, name, required): mode = 'scoring' if name.startswith('scoring') else 'training' proof = read(root / (mode + '-git-proof.json')) for path in proof['files']: verify_git_file(proof, path) for rel in required: if rel == 'batch-complete.json': raise ValueError('Never manufacture whole-run terminal evidence') if str((root / rel).relative_to(BASE)) not in proof['files']: raise ValueError('Required manifest absent from Git proof') return proof def training_gate(root): c = contract(root) if sha(root / 'contract.json') != sha(ORIGINAL / 'contract.json'): raise ValueError('Original contract changed') git_gate(root, 'training-git-freeze.json', ['contract.json', 'models-terminal.json', 'interim-freeze.json']) frozen = read(root / 'interim-freeze.json') if frozen['scope'] != 'INTERIM_100' or frozen['terminal_attempts'] != 335: raise ValueError('Incomplete 100-percent terminal scope') models = read(root / 'models-terminal.json')['models'] expected = {(f * 3 + 2, a) for f in range(5) for a in ARMS} if len(models) != 35 or {(m['cell'], m['arm']) for m in models} != expected: raise ValueError('Incomplete 35-model roster') if any(m['pool'] != 100 or m['status'] != 'PASS' for m in models): raise ValueError('Invalid interim model') for path, digest in frozen['original_files'].items(): checked(path, digest) if (ORIGINAL / 'global-stop.json').exists(): raise ValueError('Original queue global stop') return c, models def prepare(): ROOT.mkdir(exist_ok=False) c = contract(ORIGINAL) records = [] original_files = {} models = [] for j in c['jobs']: if j['pool'] != 100: continue receipt_path = ORIGINAL / f"jobs/job-{j['index']:04d}/guard-result.json" r = read(receipt_path) assert r['job'] == j and not r['global_failure'] records.append(r) original_files[str(receipt_path)] = sha(receipt_path) if j['kind'] == 'FINAL': assert r['worker']['status'] == 'PASS' result = read(checked(r['result_path'], r['result_sha256'])) original_files[r['result_path']] = r['result_sha256'] original_files[result['model_path']] = result['model_sha256'] checked(result['model_path'], result['model_sha256']) models.append(dict(cell=j['cell'], fold=j['fold'], pool=100, arm=j['arm'], status='PASS', model_path=result['model_path'], model_sha256=result['model_sha256'], reason=None)) assert len(records) == 335 and len(models) == 35 (ROOT / 'contract.json').write_bytes((ORIGINAL / 'contract.json').read_bytes()) save(ROOT / 'models-terminal.json', dict(scope='INTERIM_100', models=models)) save(ROOT / 'interim-freeze.json', dict(scope='INTERIM_100', authorization='1794858745.402629', terminal_attempts=335, terminal_statuses=dict(collections.Counter(r['worker']['status'] for r in records)), original_files=original_files, contract_sha256=sha(ROOT / 'contract.json'), models_sha256=sha(ROOT / 'models-terminal.json'), bootstrap_seed=2026111408, whole_queue_terminal=False, original_queue_unchanged=True, final_exact_reproduction_required=True)) save(ROOT / 'adapter-freeze.json', dict(scope='INTERIM_100', code_sha256={ 'tools/r4bc_interim.py': sha(BASE / 'tools/r4bc_interim.py')}, frozen_computation_commit='d272e608987840e3c272eb3a862919123b805bc6', post_cap_seconds=POST_CAP, interim_cap_seconds=INTERIM_CAP, final_reserved_seconds=7200, worker_cpu=0, coordinator_cpu=1, no_new_fit=True, no_new_decode=True, gate_adapter_only=True, original_score_empty_pools_not_results=[50,25])) def worker(mode, cell=None, arm=None): os.sched_setaffinity(0, [0]) post.training_gate = training_gate post.git_gate = git_gate if mode == 'predict': post.predict(ROOT, cell, arm) elif mode == 'score': post.score(ROOT) # Original function; only pool100 prediction entries exist. raw = read(ROOT / 'scores/summary.json') rows = read(ROOT / 'scores/part-scores.json') assert {r['pool'] for r in rows} == {100} save(ROOT / 'INTERIM_100.json', dict(status='INTERIM_100', pool100=raw['pools']['100'], raw_summary_sha256=sha(ROOT / 'scores/summary.json'), part_scores_sha256=sha(ROOT / 'scores/part-scores.json'), roster_parts=552, roster_components=204, final_exact_comparison_required=True, excluded_empty_pool_summaries=[50,25], new_fits=0, new_decodes=0)) def freeze(): c, models = training_gate(ROOT) entries = [] for m in models: dest = ROOT / 'predictions' / f"cell-{m['cell']:02d}-{m['arm']}" guard = ROOT / 'prediction-guards' / dest.name / 'worker.receipt.json' assert read(guard)['status'] == 'PASS' result = read(dest / 'result.json') index = read(checked(dest / 'index.json', result['index_sha256'])) assert set(index) == {r['case'] for r in post.role_rows(c, m['fold'], 100, 'test')} for case, value in index.items(): checked(value['path'], value['sha256']) entries.append(dict(pool=100, fold=m['fold'], arm=m['arm'], case=case, **value)) for row in c['parts']: fold = next(a['fold'] for a in row['assignments'] if a['role'] == 'test') h = row['historical_predictions']['100']; checked(h['path'], h['sha256']) entries.append(dict(pool=100, fold=fold, arm='H0', case=row['case'], array='seconds', **h)) e = row['r4a']; checked(e['predictions'], e['predictions_sha256']) entries.append(dict(pool=100, fold=fold, arm='EIK', case=row['case'], array='F', path=e['predictions'], sha256=e['predictions_sha256'])) assert len(entries) == 552 * 9 save(ROOT / 'predictions-freeze.json', dict(scope='INTERIM_100', contract_sha256=sha(ROOT / 'contract.json'), models_terminal_sha256=sha(ROOT / 'models-terminal.json'), entries=entries, missing=[], target_values_read=False, new_decodes=0, common_parts=552, common_components=204)) def ledger(): receipts = list((ROOT / 'prediction-guards').glob('*/worker.receipt.json')) if (ROOT / 'score-guard/worker.receipt.json').exists(): receipts.append(ROOT / 'score-guard/worker.receipt.json') return 60 + sum(read(p)['cpu_seconds'] for p in receipts) # engineering/metadata reserve def run(mode): import fcntl os.sched_setaffinity(0, [1]) with (ROOT / (mode + '.lock')).open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) started = ROOT / (mode + '-started.json') if started.exists(): raise ValueError('No restart or retry') save(started, dict(pid=os.getpid(), unix=time.time(), scope='INTERIM_100')) _, models = training_gate(ROOT) jobs = [(m['cell'], m['arm']) for m in models] if mode == 'predict' else [(None, None)] for cell, arm in jobs: import psutil from s2_recovery_monitoring import disk_usage if ledger() + 1800 > INTERIM_CAP: raise ValueError('Interim cap/final postfit reserve exhausted') if (ORIGINAL / 'global-stop.json').exists(): raise ValueError('Original global stop') elapsed = time.time() - read(ROOT / 'predict-started.json')['unix'] if elapsed >= 21600: raise ValueError('Shared postfit wall limit') live = read(ORIGINAL / 'progress.json') if live['global_stop'] or live['monitor_errors']: raise ValueError('Original queue global/monitor failure') if psutil.virtual_memory().available < 10*2**30 or live['total_rss_bytes'] + 8*2**30 > 36*2**30: raise ValueError('Insufficient shared memory headroom') if disk_usage(ROOT) + disk_usage(ORIGINAL) >= 100*2**30: raise ValueError('Shared disk cap') dest = ROOT / 'prediction-guards' / f'cell-{cell:02d}-{arm}' if mode == 'predict' else ROOT / 'score-guard' command = [sys.executable, 'tools/r4bc_interim.py', 'worker', '--mode', mode] if cell is not None: command += ['--cell', str(cell), '--arm', arm] save(ROOT / 'interim-progress.json', dict(stage=mode, cell=cell, arm=arm, post_cpu_settled=ledger(), reserved_active_cpu=1800, post_cap=POST_CAP, interim_cap=INTERIM_CAP, final_reserve=7200)) receipt = supervise(command, dest, dict(cpu_seconds=1800, wall_seconds=min(3600,21600-elapsed), rss_bytes=8*2**30, disk_bytes=2*2**30), global_stop=ORIGINAL / 'global-stop.json') if receipt['status'] != 'PASS': save(ROOT / 'interim-stop.json', dict(mode=mode, cell=cell, arm=arm, receipt=receipt, post_cpu=ledger())) return 1 if mode == 'predict': freeze() save(ROOT / (mode + '-complete.json'), dict(status='COMPLETE', scope='INTERIM_100', post_cpu_seconds=ledger(), remaining_post_cpu_seconds=POST_CAP-ledger(), coordinator_cpu_seconds=time.process_time(), next='GIT_FREEZE_PREDICTIONS' if mode == 'predict' else 'FIXED_RESULT_DELIVERY')) return 0 if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('action', choices=['prepare','proof','worker','run']) parser.add_argument('--mode', choices=['training','scoring','predict','score']) parser.add_argument('--commit'); parser.add_argument('--cell', type=int); parser.add_argument('--arm') args = parser.parse_args() if args.action == 'prepare': prepare() elif args.action == 'proof': make_proof(args.mode, args.commit) elif args.action == 'worker': worker(args.mode, args.cell, args.arm) else: raise SystemExit(run(args.mode))