Files
MeltMind/tools/run_s3a_score.py

133 lines
7.0 KiB
Python

"""Score only hash-frozen new predictions against the existing S2 custody cache."""
import argparse
from pathlib import Path
import time
import numpy as np
from freeze_g1_components import sha
from s2_custody_common import checked, read_json, prediction_inputs
from s2_recovery_supervisor import save
from s3a_data import ARMS, cached_truth
from s3a_statistics import part_metrics, reports
def review_gate(plan):
c = read_json(plan/'contract.json')
for path, digest in {**c['code_sha256'], **c['source_sha256']}.items():
checked(path, digest)
approval = read_json(plan/'main-review.json')
if (approval.get('approved') is not True or not approval.get('slack_ts') or not approval.get('git_commit')
or approval.get('contract_sha256') != sha(plan/'contract.json')):
raise PermissionError('S3-A main review missing or mismatched')
return c
def expected_predictions(root, c, terminals, roster):
by_cell = {r['cell']: r for r in terminals}
if len(terminals) != 30 or len(by_cell) != 30 or set(by_cell) != set(range(30)):
raise ValueError('Exactly 30 unique new final terminal cells required')
expected = {}
for cell in c['cells']:
terminal = by_cell[cell['index']]
if (terminal['arm'], terminal['fold'], terminal['pool']) != (cell['arm'], cell['fold'], cell['pool']):
raise ValueError('Final cell identity mismatch')
if terminal['status'] == 'NA':
if not terminal.get('reason'):
raise ValueError('NA requires original reason')
continue
if terminal['status'] != 'PASS':
raise ValueError('Nonterminal final cell')
checked(terminal['model_path'], terminal['model_sha256'])
for row in roster:
if row['included'] and row['assignments'][cell['fold']]['role'] == 'test':
path = root/'predictions'/f"cell-{cell['index']:02d}"/(row['case']+'.npz')
expected[str(path)] = (cell, row)
return expected
def prediction_gate(plan, root, c):
proof = read_json(root/'predictions-complete.json')
if (proof['status'] != 'ALL_S3A_PREDICTIONS_FROZEN' or not proof.get('git_commit')
or proof['contract_sha256'] != sha(plan/'contract.json')):
raise ValueError('New predictions must be frozen against approved contract')
frozen = read_json(checked(root/'terminal-freeze.json', proof['terminal_freeze_sha256']))
if (frozen['status'] != 'ALL_S3A_FITS_TERMINAL' or frozen['contract_sha256'] != sha(plan/'contract.json')
or sorted(frozen['hpo_terminal_indices']) != list(range(360))):
raise ValueError('All 360 HPO and 30 final terminal records required before test predictions')
checked(frozen['batch_receipt'], frozen['batch_receipt_sha256'])
roster = read_json(checked(c['roster'], c['roster_sha256']))['parts']
expected = expected_predictions(root, c, frozen['final_fits'], roster)
index = read_json(checked(root/'prediction-index.json', proof['prediction_index_sha256']))
if set(index) != set(expected):
raise ValueError('Missing or extra new predictions; cannot silently drop part')
for p, h in index.items():
checked(p, h)
return roster, expected, index, frozen
def baseline_rows(c, roster):
rows = [r for r in read_json(checked(c['baseline_scores'], c['baseline_scores_sha256'])) if r['arm'] == 'R2']
expected = {(r['case'], p) for r in roster if r['included'] for p in (25, 50, 100)}
if len(rows) != len(expected) or {(r['case'], r['pool']) for r in rows} != expected:
raise ValueError('Original R2 part scores incomplete or duplicated')
if any(r['seed'] is not None for r in rows):
raise ValueError('R2 is single fit, not three copies')
index = read_json(checked(c['baseline_prediction_index'], c['baseline_prediction_index_sha256']))
paths = {p: h for p, h in index.items() if Path(p).parent.name.endswith('-R2')}
actual = {(Path(p).stem, int(Path(p).parent.name.split('-pool')[1].split('-')[0])) for p in paths}
if len(paths) != len(expected) or actual != expected:
raise ValueError('Original R2 prediction manifest differs')
for p, h in paths.items():
checked(p, h)
for model in c['baseline_fits']:
checked(model['model_path'], model['model_sha256'])
return rows
def run(plan, root):
c = review_gate(plan)
roster, expected, index, frozen = prediction_gate(plan, root, c)
dest = root/'scores'
dest.mkdir(exist_ok=False)
start = time.monotonic()
cpu = time.process_time()
save(dest/'started.json', dict(contract_sha256=sha(plan/'contract.json'), unix=time.time(),
predictions_complete_sha256=sha(root/'predictions-complete.json')))
scores = baseline_rows(c, roster)
entries = read_json(checked(c['label_index'], c['label_index_sha256']))
labels = {r['case']: r for r in entries}
members = {case for f in read_json(checked(c['exposure_membership'], c['exposure_membership_sha256'])) for case in f['parts']}
# Load one part at a time, use identical IDs for all new arms/pools, retain finite negative predictions.
by_case = {}
for p, (cell, row) in expected.items():
by_case.setdefault(row['case'], []).append((p, cell, row))
for case, items in sorted(by_case.items()):
d = prediction_inputs(items[0][2])
truth = cached_truth(labels[case], d)
for path, cell, row in items:
with np.load(checked(path, index[path]), allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'], d['raw_ids']) or not np.array_equal(z['counts'], d['counts']):
raise ValueError('Prediction/query IDs or multiplicity mismatch')
metrics = part_metrics(z['seconds'], truth, d['counts'], d['near_wall'],
d['raw_ids'], d['coordinates_over_Rg'], case.split('_')[0])
scores.append(dict(case=case, component_id=row['component_id'], fold=cell['fold'],
pool=cell['pool'], arm=cell['arm'], seed=None, **metrics))
tables, contrasts, components = reports(scores, roster, members)
save(dest/'part-scores.json', scores)
save(dest/'tables.json', tables)
save(dest/'contrasts.json', contrasts)
save(dest/'component-scores.json', components)
save(dest/'learning-pool-counts.json', read_json(c['roster'])['fold_pool_counts'])
save(dest/'result.json', dict(status='S3A_SCORED', new_prediction_rows=len(expected),
reused_R2_score_rows=len(scores)-len(expected), primary=next(r for r in contrasts if r['primary']),
labels_newly_decoded=0, original_R2_refits=0, cpu_seconds=time.process_time()-cpu,
wall_seconds=time.monotonic()-start, final_NA=[r for r in frozen['final_fits'] if r['status']=='NA'],
interpretation=c['interpretation'], output_sha256={p.name: sha(p) for p in dest.glob('*.json')}))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('plan', type=Path)
parser.add_argument('root', type=Path)
args = parser.parse_args()
run(args.plan, args.root)