"""Require final full scoring to reproduce INTERIM_100 exactly; no fitting/scoring.""" import argparse import json from pathlib import Path import numpy as np from r4bc_common import read, checked, sha, save def compare(interim, final, output): defects = [] if sha(interim / 'contract.json') != sha(final / 'contract.json'): defects.append('contract bytes differ') for root in (interim, final): for path, digest in read(root / 'contract.json')['code_sha256'].items(): checked(path, digest) def models(root): return {(m['cell'], m['arm']): m for m in read(root / 'models-terminal.json')['models'] if m['pool'] == 100} a, b = models(interim), models(final) if set(a) != set(b): defects.append('100-percent model roster differs') for key in a.keys() & b.keys(): if a[key]['status'] != b[key]['status'] or a[key]['model_sha256'] != b[key]['model_sha256']: defects.append('model differs: ' + str(key)) if b[key]['status'] == 'PASS': checked(b[key]['model_path'], b[key]['model_sha256']) def entries(root): return {(e['case'], e['arm']): e for e in read(root / 'predictions-freeze.json')['entries'] if e['pool'] == 100} ae, be = entries(interim), entries(final) if set(ae) != set(be): defects.append('100-percent prediction roster differs') for key in ae.keys() & be.keys(): x, y = ae[key], be[key] with np.load(checked(x['path'], x['sha256']), allow_pickle=False) as left, np.load(checked(y['path'], y['sha256']), allow_pickle=False) as right: for field in ['raw_ids', 'counts']: if not np.array_equal(left[field], right[field]): defects.append(str(key) + '/' + field) if not np.array_equal(left[x['array']], right[y['array']]): defects.append(str(key) + '/predictions') def scores(root): return sorted((r for r in read(root / 'scores/part-scores.json') if r['pool'] == 100), key=lambda r:(r['case'],r['arm'])) if scores(interim) != scores(final): defects.append('100-percent part scores differ') if read(interim / 'scores/summary.json')['pools']['100'] != read(final / 'scores/summary.json')['pools']['100']: defects.append('100-percent summary/contrasts/bootstrap differ') save(output, dict(status='EXACT_MATCH' if not defects else 'DEFECT', defects=defects, interim_root=str(interim), final_root=str(final), checked_models=len(a), checked_predictions=len(ae), numeric_tolerance=0, no_new_fits_predictions_scores_decodes=True)) return 1 if defects else 0 if __name__ == '__main__': p=argparse.ArgumentParser();p.add_argument('interim',type=Path);p.add_argument('final',type=Path);p.add_argument('output',type=Path) a=p.parse_args() try: status=compare(a.interim,a.final,a.output) except Exception as exc: save(a.output,dict(status='DEFECT',defects=[type(exc).__name__+': '+str(exc)], interim_root=str(a.interim),final_root=str(a.final),numeric_tolerance=0)) status=1 raise SystemExit(status)