Files
MeltMind/tools/evaluate_s1_shortfall.py

56 lines
3.2 KiB
Python

"""Frozen v1.2 LOPO shortfall gate only; no scoring or training."""
import csv
import hashlib
import json
from pathlib import Path
def read(p):return json.loads(Path(p).read_text())
def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest()
def decision(nrefs):
if not nrefs or any(n is None for n in nrefs):
return dict(comparable=False,reason='missing_training_result',below_500=None,below_1000=None)
assert all(isinstance(n,int) and n>=0 for n in nrefs)
below500=sum(n<500 for n in nrefs);below1000=sum(n<1000 for n in nrefs)
# Strictly greater than 20 percent, expressed exactly with integer arithmetic.
fail=below500>0 or 5*below1000>len(nrefs)
return dict(comparable=not fail,reason='shortfall' if fail else 'PASS',below_500=below500,below_1000=below1000)
def main():
assert decision([500,1000,1000,1000,1000])['comparable']
assert not decision([499,1000,1000,1000,1000])['comparable']
assert decision([999,1000,1000,1000,1000])['comparable']
assert not decision([999,999,1000,1000,1000])['comparable']
assert not decision([None,2000])['comparable']
out=Path('artifacts/s1-sampled-o2-2026-11-14')
release=read(out/'review_release.json');assert release['process_hold_cleared']
c=read(out/'execution_contract.json');assert read(out/'verification.json')['status']=='PASS'
assert len(read(out/'panel.completion.json')['records'])==24
foldpath=Path('artifacts/s1-sampled-draw-2026-11-14/fold_table.csv')
folds=list(csv.DictReader(foldpath.open()));rows=[];inputs={}
for f in folds:
test=int(f['test_panel_index']);training=list(map(int,f['train_panel_indices'].split(';')))
assert training==[i for i in range(24) if i!=test]
nrefs=[]
for i in training:
s=c['panel'][i];p=out/'panel'/(s['job']+'.supervisor.json');sup=read(p);inputs[str(p)]=sha(p)
nrefs.append(sup['n_ref'] if sup['status']=='PASS' else None)
d=decision(nrefs)
rows.append(dict(fold_index=int(f['fold_index']),test_panel_index=test,test_case=f['test_case'],training_parts=len(training),training_min_n_ref=min(nrefs) if all(n is not None for n in nrefs) else None,**d))
with (out/'shortfall.csv').open('x') as f:
w=csv.DictWriter(f,fieldnames=list(rows[0]),lineterminator='\n');w.writeheader();w.writerows(rows)
result=dict(status='COMPLETE',folds=rows,comparable_folds=sum(r['comparable'] for r in rows),
authority='https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794640949342329',
rule='Any training n_ref<500 OR fraction(training n_ref<1000)>0.20 is not comparable; equality endpoints do not trigger.',
boundary_tests='PASS: 499/500, 999/1000, exactly20percent/above20percent, missing result',
fold_table_sha256=sha(foldpath),supervisor_sha256=inputs,code_sha256=sha(__file__),
process_release_sha256=sha(out/'review_release.json'),S1_scoring=False,training=False)
with (out/'shortfall.json').open('x') as f:json.dump(result,f,indent=2);f.write('\n')
print(json.dumps(dict(folds=len(rows),comparable_folds=result['comparable_folds'],shortfall_failures=sum(not r['comparable'] for r in rows),S1_scoring=False),indent=2))
if __name__=='__main__':main()