"""Score only hash-frozen new predictions against the existing S2 custody cache.""" import argparse from pathlib import Path import time import numpy as np from freeze_g1_components import sha from s2_custody_common import checked, read_json, prediction_inputs from s2_recovery_supervisor import save from s3a_data import ARMS, cached_truth from s3a_statistics import part_metrics, reports def review_gate(plan): c = read_json(plan/'contract.json') for path, digest in {**c['code_sha256'], **c['source_sha256']}.items(): checked(path, digest) approval = read_json(plan/'main-review.json') if (approval.get('approved') is not True or not approval.get('slack_ts') or not approval.get('git_commit') or approval.get('contract_sha256') != sha(plan/'contract.json')): raise PermissionError('S3-A main review missing or mismatched') return c def expected_predictions(root, c, terminals, roster): by_cell = {r['cell']: r for r in terminals} if len(terminals) != 30 or len(by_cell) != 30 or set(by_cell) != set(range(30)): raise ValueError('Exactly 30 unique new final terminal cells required') expected = {} for cell in c['cells']: terminal = by_cell[cell['index']] if (terminal['arm'], terminal['fold'], terminal['pool']) != (cell['arm'], cell['fold'], cell['pool']): raise ValueError('Final cell identity mismatch') if terminal['status'] == 'NA': if not terminal.get('reason'): raise ValueError('NA requires original reason') continue if terminal['status'] != 'PASS': raise ValueError('Nonterminal final cell') checked(terminal['model_path'], terminal['model_sha256']) for row in roster: if row['included'] and row['assignments'][cell['fold']]['role'] == 'test': path = root/'predictions'/f"cell-{cell['index']:02d}"/(row['case']+'.npz') expected[str(path)] = (cell, row) return expected def prediction_gate(plan, root, c): proof = read_json(root/'predictions-complete.json') if (proof['status'] != 'ALL_S3A_PREDICTIONS_FROZEN' or not proof.get('git_commit') or proof['contract_sha256'] != sha(plan/'contract.json')): raise ValueError('New predictions must be frozen against approved contract') frozen = read_json(checked(root/'terminal-freeze.json', proof['terminal_freeze_sha256'])) if (frozen['status'] != 'ALL_S3A_FITS_TERMINAL' or frozen['contract_sha256'] != sha(plan/'contract.json') or sorted(frozen['hpo_terminal_indices']) != list(range(360))): raise ValueError('All 360 HPO and 30 final terminal records required before test predictions') checked(frozen['batch_receipt'], frozen['batch_receipt_sha256']) roster = read_json(checked(c['roster'], c['roster_sha256']))['parts'] expected = expected_predictions(root, c, frozen['final_fits'], roster) index = read_json(checked(root/'prediction-index.json', proof['prediction_index_sha256'])) if set(index) != set(expected): raise ValueError('Missing or extra new predictions; cannot silently drop part') for p, h in index.items(): checked(p, h) return roster, expected, index, frozen def baseline_rows(c, roster): rows = [r for r in read_json(checked(c['baseline_scores'], c['baseline_scores_sha256'])) if r['arm'] == 'R2'] expected = {(r['case'], p) for r in roster if r['included'] for p in (25, 50, 100)} if len(rows) != len(expected) or {(r['case'], r['pool']) for r in rows} != expected: raise ValueError('Original R2 part scores incomplete or duplicated') if any(r['seed'] is not None for r in rows): raise ValueError('R2 is single fit, not three copies') index = read_json(checked(c['baseline_prediction_index'], c['baseline_prediction_index_sha256'])) paths = {p: h for p, h in index.items() if Path(p).parent.name.endswith('-R2')} actual = {(Path(p).stem, int(Path(p).parent.name.split('-pool')[1].split('-')[0])) for p in paths} if len(paths) != len(expected) or actual != expected: raise ValueError('Original R2 prediction manifest differs') for p, h in paths.items(): checked(p, h) for model in c['baseline_fits']: checked(model['model_path'], model['model_sha256']) return rows def run(plan, root): c = review_gate(plan) roster, expected, index, frozen = prediction_gate(plan, root, c) dest = root/'scores' dest.mkdir(exist_ok=False) start = time.monotonic() cpu = time.process_time() save(dest/'started.json', dict(contract_sha256=sha(plan/'contract.json'), unix=time.time(), predictions_complete_sha256=sha(root/'predictions-complete.json'))) scores = baseline_rows(c, roster) entries = read_json(checked(c['label_index'], c['label_index_sha256'])) labels = {r['case']: r for r in entries} members = {case for f in read_json(checked(c['exposure_membership'], c['exposure_membership_sha256'])) for case in f['parts']} # Load one part at a time, use identical IDs for all new arms/pools, retain finite negative predictions. by_case = {} for p, (cell, row) in expected.items(): by_case.setdefault(row['case'], []).append((p, cell, row)) for case, items in sorted(by_case.items()): d = prediction_inputs(items[0][2]) truth = cached_truth(labels[case], d) for path, cell, row in items: with np.load(checked(path, index[path]), allow_pickle=False) as z: if not np.array_equal(z['raw_ids'], d['raw_ids']) or not np.array_equal(z['counts'], d['counts']): raise ValueError('Prediction/query IDs or multiplicity mismatch') metrics = part_metrics(z['seconds'], truth, d['counts'], d['near_wall'], d['raw_ids'], d['coordinates_over_Rg'], case.split('_')[0]) scores.append(dict(case=case, component_id=row['component_id'], fold=cell['fold'], pool=cell['pool'], arm=cell['arm'], seed=None, **metrics)) tables, contrasts, components = reports(scores, roster, members) save(dest/'part-scores.json', scores) save(dest/'tables.json', tables) save(dest/'contrasts.json', contrasts) save(dest/'component-scores.json', components) save(dest/'learning-pool-counts.json', read_json(c['roster'])['fold_pool_counts']) save(dest/'result.json', dict(status='S3A_SCORED', new_prediction_rows=len(expected), reused_R2_score_rows=len(scores)-len(expected), primary=next(r for r in contrasts if r['primary']), labels_newly_decoded=0, original_R2_refits=0, cpu_seconds=time.process_time()-cpu, wall_seconds=time.monotonic()-start, final_NA=[r for r in frozen['final_fits'] if r['status']=='NA'], interpretation=c['interpretation'], output_sha256={p.name: sha(p) for p in dest.glob('*.json')})) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('plan', type=Path) parser.add_argument('root', type=Path) args = parser.parse_args() run(args.plan, args.root)