89 lines
5.3 KiB
Python
89 lines
5.3 KiB
Python
"""Verify completed fits and freeze evidence before any held-out inference."""
|
|
import json
|
|
from pathlib import Path
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
from freeze_g1_components import sha
|
|
from s2_recovery_supervisor import save
|
|
|
|
ROOT = Path('artifacts/s2-terminal-freeze-2026-11-15')
|
|
NEURAL = Path('artifacts/s2-main-matrix-2026-11-15')
|
|
R2 = Path('artifacts/s2-r2-main-2026-11-15')
|
|
|
|
|
|
def main():
|
|
if ROOT.exists():
|
|
raise FileExistsError('Immutable freeze already exists')
|
|
n = json.loads((NEURAL/'matrix-complete.json').read_text())
|
|
r = json.loads((R2/'batch-complete.json').read_text())
|
|
nc = json.loads((NEURAL/'training-contract.json').read_text())
|
|
rc = json.loads((R2/'contract.json').read_text())
|
|
assert n['status']=='ALL_NEURAL_TERMINAL' and r['status']=='ALL_R2_TERMINAL'
|
|
assert n['stop_reason'] is None and r['stop_reason'] is None
|
|
assert len(n['runs'])==90 and len(r['final_fits'])==15 and len(r['jobs'])==195
|
|
assert not r['not_run_indices']
|
|
assert all(not x['test_predictions'] and not x['test_scoring'] for x in [n,r])
|
|
assert nc['roster_sha256']==rc['roster_sha256']==sha(nc['roster'])
|
|
for c in [nc,rc]:
|
|
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():
|
|
assert sha(p)==h,p
|
|
models=[]; access_count=0
|
|
for spec,t in zip(nc['runs'],n['runs']):
|
|
assert t['index']==nc['runs'].index(spec) and spec['name']==t['name']
|
|
assert sha(t['guard_receipt'])==t['guard_sha256']
|
|
base=NEURAL/'cells'/spec['name']/spec['name']
|
|
access=json.loads((base/'target-access.json').read_text())
|
|
assert access['outer_fold_test_labels_read'] is False
|
|
roles=next(x['roles'] for x in nc['cells'] if x['fold']==spec['fold'] and x['pool']==spec['pool'])
|
|
assert all(x['case'] not in roles['sealed_test_parts'] for x in access['access'])
|
|
access_count+=1
|
|
if t['status']=='PASS':
|
|
model=base/'best.pt';assert sha(model)==t['checkpoint_sha256']
|
|
result=json.loads((base/'result.json').read_text())
|
|
assert result['status']=='COMPLETE' and not result['test_labels_read']
|
|
models.append(dict(kind='neural',**spec,path=str(model),sha256=sha(model),
|
|
standardizer_fingerprint=t['standardizer_fingerprint']))
|
|
for t in r['final_fits']:
|
|
if t['status']=='PASS':
|
|
assert sha(t['model_path'])==t['model_sha256']
|
|
models.append(dict(kind='R2',name=f"fold{t['fold']}-pool{t['pool']}-R2",arm='R2',
|
|
fold=t['fold'],pool=t['pool'],seed=None,path=t['model_path'],sha256=t['model_sha256']))
|
|
for p in R2.glob('jobs/*/output/access.json'):
|
|
a=json.loads(p.read_text()); roles=next(x['roles'] for x in nc['cells'] if x['fold']==a['fold'] and x['pool']==a['pool'])
|
|
assert not a['outer_fold_test_labels_read']
|
|
assert all(x['case'] not in roles['sealed_test_parts'] for x in a['cases'])
|
|
access_count+=1
|
|
files=set()
|
|
for root in [NEURAL,R2]:
|
|
files.update(root.glob('*.json'))
|
|
for folder in [NEURAL/'cells',NEURAL/'blocks',R2/'jobs',R2/'selections']:
|
|
files.update(p for p in folder.rglob('*') if p.is_file())
|
|
files.add(Path(nc['roster']))
|
|
evidence={str(p):dict(sha256=sha(p),bytes=p.stat().st_size) for p in sorted(files)}
|
|
ROOT.mkdir()
|
|
save(ROOT/'evidence-manifest.json',evidence)
|
|
save(ROOT/'models.json',models)
|
|
save(ROOT/'terminal-audit.json',dict(status='FROZEN_BEFORE_TEST_PREDICTIONS',
|
|
timestamp=datetime.now(timezone.utc).isoformat(),neural_counts=dict(Counter(t['status'] for t in n['runs'])),
|
|
neural_NA=[t for t in n['runs'] if t['status']!='PASS'],
|
|
pairs_valid=sum(t['pair_valid'] for t in n['blocks']),pairs_total=45,
|
|
primary_pairs_valid=sum(t['pair_valid'] for t in n['blocks'] if nc['blocks'][t['block']]['pool']==100),
|
|
r2_final_counts=dict(Counter(t['status'] for t in r['final_fits'])),
|
|
r2_attempt_counts=dict(Counter(t['worker']['status'] for t in r['jobs'])),
|
|
neural_gpu_seconds=n['charged_gpu_seconds'],r2_worker_cpu_seconds=r['worker_cpu_seconds'],
|
|
r2_guard_cpu_seconds=r['guard_cpu_seconds'],r2_coordinator_cpu_seconds=r['coordinator_cpu_seconds'],
|
|
test_role_access_records_checked=access_count,test_role_access_violations=0,
|
|
test_seal_scope='Each fold held out from its own fits; parts may be non-test in other folds. No untouched-corpus claim.',
|
|
models=104,evidence_files=len(evidence),evidence_bytes=sum(x['bytes'] for x in evidence.values()),
|
|
roster=nc['roster'],roster_sha256=nc['roster_sha256'],
|
|
neural_contract_sha256=sha(NEURAL/'training-contract.json'),r2_contract_sha256=sha(R2/'contract.json'),
|
|
evidence_manifest_sha256=sha(ROOT/'evidence-manifest.json'),models_sha256=sha(ROOT/'models.json'),
|
|
test_predictions=False,test_scoring=False,retries=False,
|
|
limitations=['AMP skipped case/step not saved','R2 job118 second process identity not saved',
|
|
'50pct fold1 seed0 N/E pair invalid; component contrasts require all parts and all3 matched seeds',
|
|
'617 included/7 globally excluded; query domain only; width128 feasibility, width effect not tested']))
|
|
print((ROOT/'terminal-audit.json').read_text())
|
|
|
|
|
|
if __name__=='__main__': main()
|