133 lines
7.0 KiB
Python
133 lines
7.0 KiB
Python
"""Score only hash-frozen new predictions against the existing S2 custody cache."""
|
|
import argparse
|
|
from pathlib import Path
|
|
import time
|
|
import numpy as np
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import checked, read_json, prediction_inputs
|
|
from s2_recovery_supervisor import save
|
|
from s3a_data import ARMS, cached_truth
|
|
from s3a_statistics import part_metrics, reports
|
|
|
|
|
|
def review_gate(plan):
|
|
c = read_json(plan/'contract.json')
|
|
for path, digest in {**c['code_sha256'], **c['source_sha256']}.items():
|
|
checked(path, digest)
|
|
approval = read_json(plan/'main-review.json')
|
|
if (approval.get('approved') is not True or not approval.get('slack_ts') or not approval.get('git_commit')
|
|
or approval.get('contract_sha256') != sha(plan/'contract.json')):
|
|
raise PermissionError('S3-A main review missing or mismatched')
|
|
return c
|
|
|
|
|
|
def expected_predictions(root, c, terminals, roster):
|
|
by_cell = {r['cell']: r for r in terminals}
|
|
if len(terminals) != 30 or len(by_cell) != 30 or set(by_cell) != set(range(30)):
|
|
raise ValueError('Exactly 30 unique new final terminal cells required')
|
|
expected = {}
|
|
for cell in c['cells']:
|
|
terminal = by_cell[cell['index']]
|
|
if (terminal['arm'], terminal['fold'], terminal['pool']) != (cell['arm'], cell['fold'], cell['pool']):
|
|
raise ValueError('Final cell identity mismatch')
|
|
if terminal['status'] == 'NA':
|
|
if not terminal.get('reason'):
|
|
raise ValueError('NA requires original reason')
|
|
continue
|
|
if terminal['status'] != 'PASS':
|
|
raise ValueError('Nonterminal final cell')
|
|
checked(terminal['model_path'], terminal['model_sha256'])
|
|
for row in roster:
|
|
if row['included'] and row['assignments'][cell['fold']]['role'] == 'test':
|
|
path = root/'predictions'/f"cell-{cell['index']:02d}"/(row['case']+'.npz')
|
|
expected[str(path)] = (cell, row)
|
|
return expected
|
|
|
|
|
|
def prediction_gate(plan, root, c):
|
|
proof = read_json(root/'predictions-complete.json')
|
|
if (proof['status'] != 'ALL_S3A_PREDICTIONS_FROZEN' or not proof.get('git_commit')
|
|
or proof['contract_sha256'] != sha(plan/'contract.json')):
|
|
raise ValueError('New predictions must be frozen against approved contract')
|
|
frozen = read_json(checked(root/'terminal-freeze.json', proof['terminal_freeze_sha256']))
|
|
if (frozen['status'] != 'ALL_S3A_FITS_TERMINAL' or frozen['contract_sha256'] != sha(plan/'contract.json')
|
|
or sorted(frozen['hpo_terminal_indices']) != list(range(360))):
|
|
raise ValueError('All 360 HPO and 30 final terminal records required before test predictions')
|
|
checked(frozen['batch_receipt'], frozen['batch_receipt_sha256'])
|
|
roster = read_json(checked(c['roster'], c['roster_sha256']))['parts']
|
|
expected = expected_predictions(root, c, frozen['final_fits'], roster)
|
|
index = read_json(checked(root/'prediction-index.json', proof['prediction_index_sha256']))
|
|
if set(index) != set(expected):
|
|
raise ValueError('Missing or extra new predictions; cannot silently drop part')
|
|
for p, h in index.items():
|
|
checked(p, h)
|
|
return roster, expected, index, frozen
|
|
|
|
|
|
def baseline_rows(c, roster):
|
|
rows = [r for r in read_json(checked(c['baseline_scores'], c['baseline_scores_sha256'])) if r['arm'] == 'R2']
|
|
expected = {(r['case'], p) for r in roster if r['included'] for p in (25, 50, 100)}
|
|
if len(rows) != len(expected) or {(r['case'], r['pool']) for r in rows} != expected:
|
|
raise ValueError('Original R2 part scores incomplete or duplicated')
|
|
if any(r['seed'] is not None for r in rows):
|
|
raise ValueError('R2 is single fit, not three copies')
|
|
index = read_json(checked(c['baseline_prediction_index'], c['baseline_prediction_index_sha256']))
|
|
paths = {p: h for p, h in index.items() if Path(p).parent.name.endswith('-R2')}
|
|
actual = {(Path(p).stem, int(Path(p).parent.name.split('-pool')[1].split('-')[0])) for p in paths}
|
|
if len(paths) != len(expected) or actual != expected:
|
|
raise ValueError('Original R2 prediction manifest differs')
|
|
for p, h in paths.items():
|
|
checked(p, h)
|
|
for model in c['baseline_fits']:
|
|
checked(model['model_path'], model['model_sha256'])
|
|
return rows
|
|
|
|
|
|
def run(plan, root):
|
|
c = review_gate(plan)
|
|
roster, expected, index, frozen = prediction_gate(plan, root, c)
|
|
dest = root/'scores'
|
|
dest.mkdir(exist_ok=False)
|
|
start = time.monotonic()
|
|
cpu = time.process_time()
|
|
save(dest/'started.json', dict(contract_sha256=sha(plan/'contract.json'), unix=time.time(),
|
|
predictions_complete_sha256=sha(root/'predictions-complete.json')))
|
|
scores = baseline_rows(c, roster)
|
|
entries = read_json(checked(c['label_index'], c['label_index_sha256']))
|
|
labels = {r['case']: r for r in entries}
|
|
members = {case for f in read_json(checked(c['exposure_membership'], c['exposure_membership_sha256'])) for case in f['parts']}
|
|
# Load one part at a time, use identical IDs for all new arms/pools, retain finite negative predictions.
|
|
by_case = {}
|
|
for p, (cell, row) in expected.items():
|
|
by_case.setdefault(row['case'], []).append((p, cell, row))
|
|
for case, items in sorted(by_case.items()):
|
|
d = prediction_inputs(items[0][2])
|
|
truth = cached_truth(labels[case], d)
|
|
for path, cell, row in items:
|
|
with np.load(checked(path, index[path]), allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'], d['raw_ids']) or not np.array_equal(z['counts'], d['counts']):
|
|
raise ValueError('Prediction/query IDs or multiplicity mismatch')
|
|
metrics = part_metrics(z['seconds'], truth, d['counts'], d['near_wall'],
|
|
d['raw_ids'], d['coordinates_over_Rg'], case.split('_')[0])
|
|
scores.append(dict(case=case, component_id=row['component_id'], fold=cell['fold'],
|
|
pool=cell['pool'], arm=cell['arm'], seed=None, **metrics))
|
|
tables, contrasts, components = reports(scores, roster, members)
|
|
save(dest/'part-scores.json', scores)
|
|
save(dest/'tables.json', tables)
|
|
save(dest/'contrasts.json', contrasts)
|
|
save(dest/'component-scores.json', components)
|
|
save(dest/'learning-pool-counts.json', read_json(c['roster'])['fold_pool_counts'])
|
|
save(dest/'result.json', dict(status='S3A_SCORED', new_prediction_rows=len(expected),
|
|
reused_R2_score_rows=len(scores)-len(expected), primary=next(r for r in contrasts if r['primary']),
|
|
labels_newly_decoded=0, original_R2_refits=0, cpu_seconds=time.process_time()-cpu,
|
|
wall_seconds=time.monotonic()-start, final_NA=[r for r in frozen['final_fits'] if r['status']=='NA'],
|
|
interpretation=c['interpretation'], output_sha256={p.name: sha(p) for p in dest.glob('*.json')}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('plan', type=Path)
|
|
parser.add_argument('root', type=Path)
|
|
args = parser.parse_args()
|
|
run(args.plan, args.root)
|