Files
MeltMind/tools/r4bc_interim.py

262 lines
13 KiB
Python

"""INTERIM_100 orchestration only; frozen prediction/scoring functions unchanged."""
import argparse
import base64
import collections
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import r4bc_postfit as post
from r4bc_common import contract, read, save, checked, sha, ARMS
from r4bc_guard import supervise
BASE = Path('/home/frank14f/optane/MeltMind')
ORIGINAL = BASE / 'artifacts/r4bc-execution-2026-11-16'
ROOT = BASE / 'artifacts/r4bc-interim-100-2026-11-17'
REPO = Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
POST_CAP = 14400
INTERIM_CAP = 7200 # retain at least 7200 CPU-s for the final postfit stage
def git_oid(kind, data):
return hashlib.sha1(kind.encode() + b' ' + str(len(data)).encode() + b'\0' + data).hexdigest()
def tree_lookup(raw, name):
offset = 0
while offset < len(raw):
end = raw.index(b'\0', offset)
entry = raw[offset:end].split(b' ', 1)[1].decode()
oid = raw[end + 1:end + 21].hex()
if entry == name:
return oid
offset = end + 21
raise ValueError('Missing Git tree entry: ' + name)
def verify_git_file(proof, path):
"""Verify commit/tree/blob linkage without child processes inside CPU guard."""
def obj(oid, kind):
data = base64.b64decode(proof['objects'][oid])
if git_oid(kind, data) != oid:
raise ValueError('Git object identity mismatch')
return data
commit = obj(proof['commit'], 'commit')
oid = commit.split(b'\n', 1)[0].decode().removeprefix('tree ')
parts = Path(path).parts
for component in parts:
oid = tree_lookup(obj(oid, 'tree'), component)
data = (BASE / path).read_bytes()
if git_oid('blob', data) != oid or hashlib.sha256(data).hexdigest() != proof['files'][path]:
raise ValueError('Git-frozen file mismatch: ' + path)
def make_proof(mode, commit):
names = ['contract.json', 'models-terminal.json', 'interim-freeze.json', 'adapter-freeze.json']
if mode == 'scoring':
names += ['predictions-freeze.json']
paths = [str((ROOT / n).relative_to(BASE)) for n in names]
paths += list(read(ROOT / 'adapter-freeze.json')['code_sha256'])
if mode == 'training':
paths += [str(Path(m['model_path']).relative_to(BASE)) for m in read(ROOT / 'models-terminal.json')['models']]
proof = {'commit': commit, 'objects': {}, 'files': {}}
def get(oid, kind):
raw = subprocess.check_output(['git', 'cat-file', kind, oid], cwd=REPO)
assert git_oid(kind, raw) == oid
proof['objects'][oid] = base64.b64encode(raw).decode()
return raw
root_oid = get(commit, 'commit').split(b'\n', 1)[0].decode().removeprefix('tree ')
for path in paths:
oid = root_oid
for component in Path(path).parts:
oid = tree_lookup(get(oid, 'tree'), component)
data = (BASE / path).read_bytes()
assert git_oid('blob', data) == oid, path
proof['files'][path] = hashlib.sha256(data).hexdigest()
verify_git_file(proof, path)
save(ROOT / (mode + '-git-proof.json'), proof)
def git_gate(root, name, required):
mode = 'scoring' if name.startswith('scoring') else 'training'
proof = read(root / (mode + '-git-proof.json'))
for path in proof['files']:
verify_git_file(proof, path)
for rel in required:
if rel == 'batch-complete.json':
raise ValueError('Never manufacture whole-run terminal evidence')
if str((root / rel).relative_to(BASE)) not in proof['files']:
raise ValueError('Required manifest absent from Git proof')
return proof
def training_gate(root):
c = contract(root)
if sha(root / 'contract.json') != sha(ORIGINAL / 'contract.json'):
raise ValueError('Original contract changed')
git_gate(root, 'training-git-freeze.json', ['contract.json', 'models-terminal.json', 'interim-freeze.json'])
frozen = read(root / 'interim-freeze.json')
if frozen['scope'] != 'INTERIM_100' or frozen['terminal_attempts'] != 335:
raise ValueError('Incomplete 100-percent terminal scope')
models = read(root / 'models-terminal.json')['models']
expected = {(f * 3 + 2, a) for f in range(5) for a in ARMS}
if len(models) != 35 or {(m['cell'], m['arm']) for m in models} != expected:
raise ValueError('Incomplete 35-model roster')
if any(m['pool'] != 100 or m['status'] != 'PASS' for m in models):
raise ValueError('Invalid interim model')
for path, digest in frozen['original_files'].items():
checked(path, digest)
if (ORIGINAL / 'global-stop.json').exists():
raise ValueError('Original queue global stop')
return c, models
def prepare():
ROOT.mkdir(exist_ok=False)
c = contract(ORIGINAL)
records = []
original_files = {}
models = []
for j in c['jobs']:
if j['pool'] != 100:
continue
receipt_path = ORIGINAL / f"jobs/job-{j['index']:04d}/guard-result.json"
r = read(receipt_path)
assert r['job'] == j and not r['global_failure']
records.append(r)
original_files[str(receipt_path)] = sha(receipt_path)
if j['kind'] == 'FINAL':
assert r['worker']['status'] == 'PASS'
result = read(checked(r['result_path'], r['result_sha256']))
original_files[r['result_path']] = r['result_sha256']
original_files[result['model_path']] = result['model_sha256']
checked(result['model_path'], result['model_sha256'])
models.append(dict(cell=j['cell'], fold=j['fold'], pool=100, arm=j['arm'], status='PASS',
model_path=result['model_path'], model_sha256=result['model_sha256'], reason=None))
assert len(records) == 335 and len(models) == 35
(ROOT / 'contract.json').write_bytes((ORIGINAL / 'contract.json').read_bytes())
save(ROOT / 'models-terminal.json', dict(scope='INTERIM_100', models=models))
save(ROOT / 'interim-freeze.json', dict(scope='INTERIM_100', authorization='1794858745.402629',
terminal_attempts=335, terminal_statuses=dict(collections.Counter(r['worker']['status'] for r in records)),
original_files=original_files, contract_sha256=sha(ROOT / 'contract.json'),
models_sha256=sha(ROOT / 'models-terminal.json'), bootstrap_seed=2026111408,
whole_queue_terminal=False, original_queue_unchanged=True, final_exact_reproduction_required=True))
save(ROOT / 'adapter-freeze.json', dict(scope='INTERIM_100', code_sha256={
'tools/r4bc_interim.py': sha(BASE / 'tools/r4bc_interim.py')},
frozen_computation_commit='d272e608987840e3c272eb3a862919123b805bc6',
post_cap_seconds=POST_CAP, interim_cap_seconds=INTERIM_CAP, final_reserved_seconds=7200,
worker_cpu=0, coordinator_cpu=1, no_new_fit=True, no_new_decode=True,
gate_adapter_only=True, original_score_empty_pools_not_results=[50,25]))
def worker(mode, cell=None, arm=None):
os.sched_setaffinity(0, [0])
post.training_gate = training_gate
post.git_gate = git_gate
if mode == 'predict':
post.predict(ROOT, cell, arm)
elif mode == 'score':
post.score(ROOT) # Original function; only pool100 prediction entries exist.
raw = read(ROOT / 'scores/summary.json')
rows = read(ROOT / 'scores/part-scores.json')
assert {r['pool'] for r in rows} == {100}
save(ROOT / 'INTERIM_100.json', dict(status='INTERIM_100', pool100=raw['pools']['100'],
raw_summary_sha256=sha(ROOT / 'scores/summary.json'), part_scores_sha256=sha(ROOT / 'scores/part-scores.json'),
roster_parts=552, roster_components=204, final_exact_comparison_required=True,
excluded_empty_pool_summaries=[50,25], new_fits=0, new_decodes=0))
def freeze():
c, models = training_gate(ROOT)
entries = []
for m in models:
dest = ROOT / 'predictions' / f"cell-{m['cell']:02d}-{m['arm']}"
guard = ROOT / 'prediction-guards' / dest.name / 'worker.receipt.json'
assert read(guard)['status'] == 'PASS'
result = read(dest / 'result.json')
index = read(checked(dest / 'index.json', result['index_sha256']))
assert set(index) == {r['case'] for r in post.role_rows(c, m['fold'], 100, 'test')}
for case, value in index.items():
checked(value['path'], value['sha256'])
entries.append(dict(pool=100, fold=m['fold'], arm=m['arm'], case=case, **value))
for row in c['parts']:
fold = next(a['fold'] for a in row['assignments'] if a['role'] == 'test')
h = row['historical_predictions']['100']; checked(h['path'], h['sha256'])
entries.append(dict(pool=100, fold=fold, arm='H0', case=row['case'], array='seconds', **h))
e = row['r4a']; checked(e['predictions'], e['predictions_sha256'])
entries.append(dict(pool=100, fold=fold, arm='EIK', case=row['case'], array='F', path=e['predictions'], sha256=e['predictions_sha256']))
assert len(entries) == 552 * 9
save(ROOT / 'predictions-freeze.json', dict(scope='INTERIM_100', contract_sha256=sha(ROOT / 'contract.json'),
models_terminal_sha256=sha(ROOT / 'models-terminal.json'), entries=entries, missing=[],
target_values_read=False, new_decodes=0, common_parts=552, common_components=204))
def ledger():
receipts = list((ROOT / 'prediction-guards').glob('*/worker.receipt.json'))
if (ROOT / 'score-guard/worker.receipt.json').exists():
receipts.append(ROOT / 'score-guard/worker.receipt.json')
return 60 + sum(read(p)['cpu_seconds'] for p in receipts) # engineering/metadata reserve
def run(mode):
import fcntl
os.sched_setaffinity(0, [1])
with (ROOT / (mode + '.lock')).open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
started = ROOT / (mode + '-started.json')
if started.exists():
raise ValueError('No restart or retry')
save(started, dict(pid=os.getpid(), unix=time.time(), scope='INTERIM_100'))
_, models = training_gate(ROOT)
jobs = [(m['cell'], m['arm']) for m in models] if mode == 'predict' else [(None, None)]
for cell, arm in jobs:
import psutil
from s2_recovery_monitoring import disk_usage
if ledger() + 1800 > INTERIM_CAP:
raise ValueError('Interim cap/final postfit reserve exhausted')
if (ORIGINAL / 'global-stop.json').exists():
raise ValueError('Original global stop')
elapsed = time.time() - read(ROOT / 'predict-started.json')['unix']
if elapsed >= 21600:
raise ValueError('Shared postfit wall limit')
live = read(ORIGINAL / 'progress.json')
if live['global_stop'] or live['monitor_errors']:
raise ValueError('Original queue global/monitor failure')
if psutil.virtual_memory().available < 10*2**30 or live['total_rss_bytes'] + 8*2**30 > 36*2**30:
raise ValueError('Insufficient shared memory headroom')
if disk_usage(ROOT) + disk_usage(ORIGINAL) >= 100*2**30:
raise ValueError('Shared disk cap')
dest = ROOT / 'prediction-guards' / f'cell-{cell:02d}-{arm}' if mode == 'predict' else ROOT / 'score-guard'
command = [sys.executable, 'tools/r4bc_interim.py', 'worker', '--mode', mode]
if cell is not None:
command += ['--cell', str(cell), '--arm', arm]
save(ROOT / 'interim-progress.json', dict(stage=mode, cell=cell, arm=arm, post_cpu_settled=ledger(),
reserved_active_cpu=1800, post_cap=POST_CAP, interim_cap=INTERIM_CAP, final_reserve=7200))
receipt = supervise(command, dest, dict(cpu_seconds=1800, wall_seconds=min(3600,21600-elapsed),
rss_bytes=8*2**30, disk_bytes=2*2**30), global_stop=ORIGINAL / 'global-stop.json')
if receipt['status'] != 'PASS':
save(ROOT / 'interim-stop.json', dict(mode=mode, cell=cell, arm=arm, receipt=receipt, post_cpu=ledger()))
return 1
if mode == 'predict':
freeze()
save(ROOT / (mode + '-complete.json'), dict(status='COMPLETE', scope='INTERIM_100',
post_cpu_seconds=ledger(), remaining_post_cpu_seconds=POST_CAP-ledger(), coordinator_cpu_seconds=time.process_time(),
next='GIT_FREEZE_PREDICTIONS' if mode == 'predict' else 'FIXED_RESULT_DELIVERY'))
return 0
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('action', choices=['prepare','proof','worker','run'])
parser.add_argument('--mode', choices=['training','scoring','predict','score'])
parser.add_argument('--commit'); parser.add_argument('--cell', type=int); parser.add_argument('--arm')
args = parser.parse_args()
if args.action == 'prepare': prepare()
elif args.action == 'proof': make_proof(args.mode, args.commit)
elif args.action == 'worker': worker(args.mode, args.cell, args.arm)
else: raise SystemExit(run(args.mode))