Files
MeltMind/tools/track_b_gino_greif_custody.py
T

211 lines
10 KiB
Python

"""Drive-backed Greif safekeeping and one guarded custody read per test case."""
from collections import defaultdict
import hashlib
import io
import json
import os
from pathlib import Path
import resource
import time
from zipfile import ZIP_STORED, ZipFile
import numpy as np
from phase2_h2_data import cached
from s2_custody_common import prediction_inputs
from s2_custody_statistics import part_metrics
import track_b_gino_greif_entry as entry
from track_b_gino_greif_drive_acl import prove as prove_drive_acl
from track_b_gino_greif_prepare import PILOT, QUERY, PINS
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
METRICS=('relative_l2','last_location','kendall_tau_b')
def frozen(c):
root=entry.ROOT
terminal=read(root/'complete.json')
if (terminal['status'],terminal['contract_sha256'],terminal['test_targets_read']) != (
'PREDICTIONS_FROZEN_NO_TARGETS',sha(root/'contract.json'),False) or not 0<terminal['charged_gpu_seconds']<=c['budgets']['gpu_cap']:
raise PermissionError('A capped completed prediction terminal must precede custody')
models=[]
predictions=[]
for fold,seed in entry.JOBS:
models.extend(entry.stage_terminal(fold,seed,'fit')['files'])
predictions.extend(entry.stage_terminal(fold,seed,'predict')['files'])
if models!=read(root/'models-frozen.json') or predictions!=read(root/'predictions-frozen.json'):
raise ValueError('Final sealed indexes differ from accepted terminals')
if len(models)!=4 or len(predictions)!=424 or len({r['path'] for r in models+predictions})!=428:
raise ValueError('Four final weights and 424 forecasts are required')
for item in models+predictions:
checked(item['path'],item['sha256'])
if not Path(item['path']).resolve().is_relative_to(root):
raise ValueError('Artifact path leaves the frozen run root')
return models,predictions
def pack(c):
root=entry.ROOT
models,predictions=frozen(c)
dest=root/'gino-greif-models-and-predictions.zip'
with ZipFile(dest,'x',compression=ZIP_STORED) as archive:
for item in models+predictions:
archive.write(item['path'],Path(item['path']).relative_to(root))
for name in ('models-frozen.json','predictions-frozen.json','contract.json'):
archive.write(root/name,name)
with ZipFile(dest,'r') as archive:
for item in models+predictions:
key=str(Path(item['path']).relative_to(root))
with archive.open(key) as stream:
actual=hashlib.sha256()
for data in iter(lambda:stream.read(1024**2),b''):
actual.update(data)
if actual.hexdigest()!=item['sha256']:
raise ValueError('Archived bytes differ from the frozen file index')
save(root/'drive-upload-index.json',dict(status='READY_FOR_PRIVATE_PROJECT_DRIVE',path=str(dest),
bytes=dest.stat().st_size,sha256=sha(dest),contract_sha256=sha(root/'contract.json'),
model_index_sha256=sha(root/'models-frozen.json'),prediction_index_sha256=sha(root/'predictions-frozen.json')))
return 0
def verify_drive(auth):
root=entry.ROOT
bundle=read(root/'drive-upload-index.json')
path=checked(bundle['path'],bundle['sha256'])
if (path!=root/'gino-greif-models-and-predictions.zip'
or path.stat().st_size!=bundle['bytes']
or bundle['contract_sha256']!=sha(root/'contract.json')
or bundle['model_index_sha256']!=sha(root/'models-frozen.json')
or bundle['prediction_index_sha256']!=sha(root/'predictions-frozen.json')):
raise ValueError('Local archive no longer matches the fixed models and forecasts')
receipt=read(root/'drive-readback.json')
# Transfer bookkeeping binds the file ID to the expected archive. The
# actual ACL and remote file identity come from the live provider.
if (receipt.get('sha256Checksum'),receipt.get('size'),receipt.get('parent_id')) != (
bundle['sha256'],str(bundle['bytes']),auth['private_drive_folder_id']):
raise PermissionError('Private Drive readback SHA, size and folder are required before scoring')
if receipt.get('contract_sha256')!=bundle['contract_sha256']:
raise PermissionError('Drive transfer does not match the reviewed contract')
return prove_drive_acl(receipt.get('file_id'),auth['private_drive_folder_id'],bundle['sha256'],bundle['bytes'])
def bootstrap(rows, metric, fixed_support=None):
values=defaultdict(list)
unavailable=[]
for row in rows:
scores=[row[arm][seed][metric]['value'] for arm in ('gino','h2') for seed in (0,1)]
if any(v is None or not np.isfinite(v) for v in scores):
unavailable.append(row['case'])
continue
values[row['family']].append(((scores[0]+scores[1])/2,(scores[2]+scores[3])/2))
absent=set(unavailable)
rejected={r['family'] for r in rows if r['case'] in absent}
family=np.array([np.mean(values[k],axis=0) for k in sorted(set(values)-rejected)])
support=dict(parts=len(rows),components=len(family),undefined_cases=unavailable)
if not len(family) or (fixed_support is not None and (len(rows),len(family))!=fixed_support):
return dict(**support,decision='INCOMPLETE_PAIRED_SUPPORT',ci95=None)
delta=family[:,0]-family[:,1]
rng=np.random.default_rng(2026111408)
draw=rng.integers(len(family),size=(2000,len(family)))
low,high=map(float,np.percentile(delta[draw].mean(axis=1),[2.5,97.5]))
return dict(**support,gino=float(family[:,0].mean()),h2=float(family[:,1].mean()),delta=float(delta.mean()),
ci95=[low,high],replicates=2000,seed=2026111408)
def summary(rows, compare):
if sorted((r['fold'],r['case'],r['family']) for r in rows)!=sorted(
(r['fold'],r['case'],r['family']) for r in compare['roster']):
raise ValueError('Exact two-fold paired roster required')
primary=bootstrap(rows,'relative_l2',fixed_support=(212,78))
if primary['ci95'] is not None:
low,high=primary['ci95']
primary['decision']='GINO_BETTER' if high<0 else 'H2_BETTER' if low>0 else 'COMPARABLE'
hard=[r for r in rows if r['case'] in compare['hard34']]
return dict(field=primary,secondary={metric:bootstrap(rows,metric) for metric in METRICS[1:]},
hard_subset=dict(frozen_total=34,two_fold_cases=[r['case'] for r in hard],
descriptive={metric:bootstrap(hard,metric) for metric in METRICS}))
def target_array(pilot,case):
item=pilot['parts'][case]
file=BASE/item['output']/'seconds.npy'
blob=file.read_bytes()
if hashlib.sha256(blob).hexdigest()!=item['output_sha256']['seconds.npy']:
raise ValueError('Pinned held-out bytes differ')
return np.load(io.BytesIO(blob),allow_pickle=False)
def one_case(pilot, meta, row, forecasts, reference, log):
case=row['case']
fold=row['fold']
q=prediction_inputs(meta)
raw=cached(pilot,case,'raw_ids')
ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):
raise ValueError('Frozen PPS coordinates differ')
new=[]
for seed in (0,1):
f=forecasts[fold,seed,case]
if f['family']!=row['family']:
raise ValueError('Predicted family differs')
with np.load(checked(f['path'],f['sha256']),allow_pickle=False) as z:
seconds=z['seconds']
if (seconds.dtype!=np.float64 or seconds.shape!=raw.shape or not np.isfinite(seconds).all()
or not np.array_equal(raw,z['raw_ids'])
or any(not np.array_equal(q[name],z['query_raw_ids' if name=='raw_ids' else name])
for name in ('counts','near_wall','coordinates_over_Rg','raw_ids'))):
raise ValueError('Frozen full node or PPS support differs')
new.append(seconds[ix])
old=reference[case]
if (old['fold'],old['component_id'])!=(fold,row['family']):
raise ValueError('H2 score pairing differs')
log.write(json.dumps(dict(case=case,fold=fold,role='test',sha256=pilot['parts'][case]['output_sha256']['seconds.npy']))+'\n')
log.flush()
truth=target_array(pilot,case)
if truth.shape!=raw.shape:
raise ValueError('Original held-out node support differs')
gino=[part_metrics(p,truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],
int(case.split('_')[0])) for p in new]
return dict(row,gino=gino,h2=[old['seed0'],old['seed1']])
def score(c):
auth,_=entry.authorize(c)
if os.environ.get('CUDA_VISIBLE_DEVICES')!='':
raise PermissionError('Custody scoring must run with CUDA hidden')
ceiling=resource.getrlimit(resource.RLIMIT_CPU)[1]
limit=c['budgets']['score_cpu_seconds']
if ceiling!=resource.RLIM_INFINITY:
limit=min(limit,ceiling)
resource.setrlimit(resource.RLIMIT_CPU,(limit,limit))
_,predictions=frozen(c)
drive=verify_drive(auth)
compare=read(checked(c['comparison_contract'],c['comparison_contract_sha256']))
rosters={(r['fold'],r['case']) for r in compare['roster']}
new={(r['fold'],r['seed'],r['case']):r for r in predictions}
if set(new)!={(fold,seed,case) for fold,case in rosters for seed in (0,1)}:
raise ValueError('Frozen four-set test support differs')
prior=read(checked(compare['h2_score']['path'],compare['h2_score']['sha256']))
reference={r['case']:r for r in prior['rows']}
if prior['status']!='COMPLETE' or len(reference)!=552 or not {r['case'] for r in compare['roster']}<=set(reference):
raise ValueError('H2 frozen score support differs')
pilot=read(checked(PILOT,PINS[str(PILOT)]))
query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']}
out=entry.ROOT/'score'
out.mkdir(exist_ok=False)
evidence=out/'drive-acl-evidence.json'
save(evidence,dict(drive,observed_before_target_reads=True))
save(out/'custody-start.json',dict(unix=time.time(),drive_file_id=drive['file_id'],
drive_acl_evidence_sha256=sha(evidence), manifest_sha256=sha(entry.ROOT/'predictions-frozen.json'),
before_test_target_reads=True))
results=[]
with (out/'heldout-target-access.jsonl').open('x') as log:
for row in compare['roster']:
results.append(one_case(pilot,query[row['case']],row,new,reference,log))
save(out/'result.json',dict(status='COMPLETE',summary=summary(results,compare),rows=results,
unique_test_target_reads=len(results),no_new_decode=True,drive_file_id=drive['file_id'],
drive_acl_evidence_sha256=sha(evidence)))
return 0