55 lines
3.0 KiB
Python
55 lines
3.0 KiB
Python
"""Require final full scoring to reproduce INTERIM_100 exactly; no fitting/scoring."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import numpy as np
|
|
from r4bc_common import read, checked, sha, save
|
|
|
|
|
|
def compare(interim, final, output):
|
|
defects = []
|
|
if sha(interim / 'contract.json') != sha(final / 'contract.json'):
|
|
defects.append('contract bytes differ')
|
|
for root in (interim, final):
|
|
for path, digest in read(root / 'contract.json')['code_sha256'].items():
|
|
checked(path, digest)
|
|
def models(root):
|
|
return {(m['cell'], m['arm']): m for m in read(root / 'models-terminal.json')['models'] if m['pool'] == 100}
|
|
a, b = models(interim), models(final)
|
|
if set(a) != set(b): defects.append('100-percent model roster differs')
|
|
for key in a.keys() & b.keys():
|
|
if a[key]['status'] != b[key]['status'] or a[key]['model_sha256'] != b[key]['model_sha256']:
|
|
defects.append('model differs: ' + str(key))
|
|
if b[key]['status'] == 'PASS': checked(b[key]['model_path'], b[key]['model_sha256'])
|
|
def entries(root):
|
|
return {(e['case'], e['arm']): e for e in read(root / 'predictions-freeze.json')['entries'] if e['pool'] == 100}
|
|
ae, be = entries(interim), entries(final)
|
|
if set(ae) != set(be): defects.append('100-percent prediction roster differs')
|
|
for key in ae.keys() & be.keys():
|
|
x, y = ae[key], be[key]
|
|
with np.load(checked(x['path'], x['sha256']), allow_pickle=False) as left, np.load(checked(y['path'], y['sha256']), allow_pickle=False) as right:
|
|
for field in ['raw_ids', 'counts']:
|
|
if not np.array_equal(left[field], right[field]): defects.append(str(key) + '/' + field)
|
|
if not np.array_equal(left[x['array']], right[y['array']]): defects.append(str(key) + '/predictions')
|
|
def scores(root):
|
|
return sorted((r for r in read(root / 'scores/part-scores.json') if r['pool'] == 100), key=lambda r:(r['case'],r['arm']))
|
|
if scores(interim) != scores(final): defects.append('100-percent part scores differ')
|
|
if read(interim / 'scores/summary.json')['pools']['100'] != read(final / 'scores/summary.json')['pools']['100']:
|
|
defects.append('100-percent summary/contrasts/bootstrap differ')
|
|
save(output, dict(status='EXACT_MATCH' if not defects else 'DEFECT', defects=defects,
|
|
interim_root=str(interim), final_root=str(final), checked_models=len(a), checked_predictions=len(ae),
|
|
numeric_tolerance=0, no_new_fits_predictions_scores_decodes=True))
|
|
return 1 if defects else 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('interim',type=Path);p.add_argument('final',type=Path);p.add_argument('output',type=Path)
|
|
a=p.parse_args()
|
|
try:
|
|
status=compare(a.interim,a.final,a.output)
|
|
except Exception as exc:
|
|
save(a.output,dict(status='DEFECT',defects=[type(exc).__name__+': '+str(exc)],
|
|
interim_root=str(a.interim),final_root=str(a.final),numeric_tolerance=0))
|
|
status=1
|
|
raise SystemExit(status)
|