Files
MeltMind/tools/r4bc_interim_compare.py
T

55 lines
3.0 KiB
Python

"""Require final full scoring to reproduce INTERIM_100 exactly; no fitting/scoring."""
import argparse
import json
from pathlib import Path
import numpy as np
from r4bc_common import read, checked, sha, save
def compare(interim, final, output):
defects = []
if sha(interim / 'contract.json') != sha(final / 'contract.json'):
defects.append('contract bytes differ')
for root in (interim, final):
for path, digest in read(root / 'contract.json')['code_sha256'].items():
checked(path, digest)
def models(root):
return {(m['cell'], m['arm']): m for m in read(root / 'models-terminal.json')['models'] if m['pool'] == 100}
a, b = models(interim), models(final)
if set(a) != set(b): defects.append('100-percent model roster differs')
for key in a.keys() & b.keys():
if a[key]['status'] != b[key]['status'] or a[key]['model_sha256'] != b[key]['model_sha256']:
defects.append('model differs: ' + str(key))
if b[key]['status'] == 'PASS': checked(b[key]['model_path'], b[key]['model_sha256'])
def entries(root):
return {(e['case'], e['arm']): e for e in read(root / 'predictions-freeze.json')['entries'] if e['pool'] == 100}
ae, be = entries(interim), entries(final)
if set(ae) != set(be): defects.append('100-percent prediction roster differs')
for key in ae.keys() & be.keys():
x, y = ae[key], be[key]
with np.load(checked(x['path'], x['sha256']), allow_pickle=False) as left, np.load(checked(y['path'], y['sha256']), allow_pickle=False) as right:
for field in ['raw_ids', 'counts']:
if not np.array_equal(left[field], right[field]): defects.append(str(key) + '/' + field)
if not np.array_equal(left[x['array']], right[y['array']]): defects.append(str(key) + '/predictions')
def scores(root):
return sorted((r for r in read(root / 'scores/part-scores.json') if r['pool'] == 100), key=lambda r:(r['case'],r['arm']))
if scores(interim) != scores(final): defects.append('100-percent part scores differ')
if read(interim / 'scores/summary.json')['pools']['100'] != read(final / 'scores/summary.json')['pools']['100']:
defects.append('100-percent summary/contrasts/bootstrap differ')
save(output, dict(status='EXACT_MATCH' if not defects else 'DEFECT', defects=defects,
interim_root=str(interim), final_root=str(final), checked_models=len(a), checked_predictions=len(ae),
numeric_tolerance=0, no_new_fits_predictions_scores_decodes=True))
return 1 if defects else 0
if __name__ == '__main__':
p=argparse.ArgumentParser();p.add_argument('interim',type=Path);p.add_argument('final',type=Path);p.add_argument('output',type=Path)
a=p.parse_args()
try:
status=compare(a.interim,a.final,a.output)
except Exception as exc:
save(a.output,dict(status='DEFECT',defects=[type(exc).__name__+': '+str(exc)],
interim_root=str(a.interim),final_root=str(a.final),numeric_tolerance=0))
status=1
raise SystemExit(status)