206 lines
10 KiB
Python
206 lines
10 KiB
Python
"""Drive-backed Greif safekeeping and one guarded custody read per test case."""
|
|
|
|
from collections import defaultdict
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import time
|
|
from zipfile import ZIP_STORED, ZipFile
|
|
|
|
import numpy as np
|
|
|
|
from phase2_h2_data import cached
|
|
from s2_custody_common import prediction_inputs
|
|
from s2_custody_statistics import part_metrics
|
|
import track_b_gino_greif_entry as entry
|
|
from track_b_gino_greif_prepare import PILOT, QUERY, PINS
|
|
from track_b_gino_shapenet_entry import BASE, checked, read, save, sha
|
|
|
|
|
|
METRICS=('relative_l2','last_location','kendall_tau_b')
|
|
|
|
|
|
def frozen(c):
|
|
root=entry.ROOT
|
|
terminal=read(root/'complete.json')
|
|
if (terminal['status'],terminal['contract_sha256'],terminal['test_targets_read']) != (
|
|
'PREDICTIONS_FROZEN_NO_TARGETS',sha(root/'contract.json'),False) or not 0<terminal['charged_gpu_seconds']<=c['budgets']['gpu_cap']:
|
|
raise PermissionError('A capped completed prediction terminal must precede custody')
|
|
models=[]
|
|
predictions=[]
|
|
for fold,seed in entry.JOBS:
|
|
models.extend(entry.stage_terminal(fold,seed,'fit')['files'])
|
|
predictions.extend(entry.stage_terminal(fold,seed,'predict')['files'])
|
|
if models!=read(root/'models-frozen.json') or predictions!=read(root/'predictions-frozen.json'):
|
|
raise ValueError('Final sealed indexes differ from accepted terminals')
|
|
if len(models)!=4 or len(predictions)!=424 or len({r['path'] for r in models+predictions})!=428:
|
|
raise ValueError('Four final weights and 424 forecasts are required')
|
|
for item in models+predictions:
|
|
checked(item['path'],item['sha256'])
|
|
if not Path(item['path']).resolve().is_relative_to(root):
|
|
raise ValueError('Artifact path leaves the frozen run root')
|
|
return models,predictions
|
|
|
|
|
|
def pack(c):
|
|
root=entry.ROOT
|
|
models,predictions=frozen(c)
|
|
dest=root/'gino-greif-models-and-predictions.zip'
|
|
with ZipFile(dest,'x',compression=ZIP_STORED) as archive:
|
|
for item in models+predictions:
|
|
archive.write(item['path'],Path(item['path']).relative_to(root))
|
|
for name in ('models-frozen.json','predictions-frozen.json','contract.json'):
|
|
archive.write(root/name,name)
|
|
with ZipFile(dest,'r') as archive:
|
|
for item in models+predictions:
|
|
key=str(Path(item['path']).relative_to(root))
|
|
with archive.open(key) as stream:
|
|
actual=hashlib.sha256()
|
|
for data in iter(lambda:stream.read(1024**2),b''):
|
|
actual.update(data)
|
|
if actual.hexdigest()!=item['sha256']:
|
|
raise ValueError('Archived bytes differ from the frozen file index')
|
|
save(root/'drive-upload-index.json',dict(status='READY_FOR_PRIVATE_PROJECT_DRIVE',path=str(dest),
|
|
bytes=dest.stat().st_size,sha256=sha(dest),contract_sha256=sha(root/'contract.json'),
|
|
model_index_sha256=sha(root/'models-frozen.json'),prediction_index_sha256=sha(root/'predictions-frozen.json')))
|
|
return 0
|
|
|
|
|
|
def verify_drive(auth):
|
|
root=entry.ROOT
|
|
bundle=read(root/'drive-upload-index.json')
|
|
path=checked(bundle['path'],bundle['sha256'])
|
|
if (path!=root/'gino-greif-models-and-predictions.zip'
|
|
or path.stat().st_size!=bundle['bytes']
|
|
or bundle['contract_sha256']!=sha(root/'contract.json')
|
|
or bundle['model_index_sha256']!=sha(root/'models-frozen.json')
|
|
or bundle['prediction_index_sha256']!=sha(root/'predictions-frozen.json')):
|
|
raise ValueError('Local archive no longer matches the fixed models and forecasts')
|
|
receipt=read(root/'drive-readback.json')
|
|
# Drive file metadata, recorded by the authorized connector from the private project folder.
|
|
if (receipt.get('sha256Checksum'),receipt.get('size'),receipt.get('parent_id')) != (
|
|
bundle['sha256'],str(bundle['bytes']),auth['private_drive_folder_id']):
|
|
raise PermissionError('Private Drive readback SHA, size and folder are required before scoring')
|
|
if (not receipt.get('file_id') or receipt.get('project_private') is not True
|
|
or receipt.get('contract_sha256')!=bundle['contract_sha256']):
|
|
raise PermissionError('Private Drive identity required before opening targets')
|
|
return receipt
|
|
|
|
|
|
def bootstrap(rows, metric, fixed_support=None):
|
|
values=defaultdict(list)
|
|
unavailable=[]
|
|
for row in rows:
|
|
scores=[row[arm][seed][metric]['value'] for arm in ('gino','h2') for seed in (0,1)]
|
|
if any(v is None or not np.isfinite(v) for v in scores):
|
|
unavailable.append(row['case'])
|
|
continue
|
|
values[row['family']].append(((scores[0]+scores[1])/2,(scores[2]+scores[3])/2))
|
|
absent=set(unavailable)
|
|
rejected={r['family'] for r in rows if r['case'] in absent}
|
|
family=np.array([np.mean(values[k],axis=0) for k in sorted(set(values)-rejected)])
|
|
support=dict(parts=len(rows),components=len(family),undefined_cases=unavailable)
|
|
if not len(family) or (fixed_support is not None and (len(rows),len(family))!=fixed_support):
|
|
return dict(**support,decision='INCOMPLETE_PAIRED_SUPPORT',ci95=None)
|
|
delta=family[:,0]-family[:,1]
|
|
rng=np.random.default_rng(2026111408)
|
|
draw=rng.integers(len(family),size=(2000,len(family)))
|
|
low,high=map(float,np.percentile(delta[draw].mean(axis=1),[2.5,97.5]))
|
|
return dict(**support,gino=float(family[:,0].mean()),h2=float(family[:,1].mean()),delta=float(delta.mean()),
|
|
ci95=[low,high],replicates=2000,seed=2026111408)
|
|
|
|
|
|
def summary(rows, compare):
|
|
if sorted((r['fold'],r['case'],r['family']) for r in rows)!=sorted(
|
|
(r['fold'],r['case'],r['family']) for r in compare['roster']):
|
|
raise ValueError('Exact two-fold paired roster required')
|
|
primary=bootstrap(rows,'relative_l2',fixed_support=(212,78))
|
|
if primary['ci95'] is not None:
|
|
low,high=primary['ci95']
|
|
primary['decision']='GINO_BETTER' if high<0 else 'H2_BETTER' if low>0 else 'COMPARABLE'
|
|
hard=[r for r in rows if r['case'] in compare['hard34']]
|
|
return dict(field=primary,secondary={metric:bootstrap(rows,metric) for metric in METRICS[1:]},
|
|
hard_subset=dict(frozen_total=34,two_fold_cases=[r['case'] for r in hard],
|
|
descriptive={metric:bootstrap(hard,metric) for metric in METRICS}))
|
|
|
|
|
|
def target_array(pilot,case):
|
|
item=pilot['parts'][case]
|
|
file=BASE/item['output']/'seconds.npy'
|
|
blob=file.read_bytes()
|
|
if hashlib.sha256(blob).hexdigest()!=item['output_sha256']['seconds.npy']:
|
|
raise ValueError('Pinned held-out bytes differ')
|
|
return np.load(io.BytesIO(blob),allow_pickle=False)
|
|
|
|
|
|
def one_case(pilot, meta, row, forecasts, reference, log):
|
|
case=row['case']
|
|
fold=row['fold']
|
|
q=prediction_inputs(meta)
|
|
raw=cached(pilot,case,'raw_ids')
|
|
ix=np.searchsorted(raw,q['raw_ids'])
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):
|
|
raise ValueError('Frozen PPS coordinates differ')
|
|
new=[]
|
|
for seed in (0,1):
|
|
f=forecasts[fold,seed,case]
|
|
if f['family']!=row['family']:
|
|
raise ValueError('Predicted family differs')
|
|
with np.load(checked(f['path'],f['sha256']),allow_pickle=False) as z:
|
|
seconds=z['seconds']
|
|
if (seconds.dtype!=np.float64 or seconds.shape!=raw.shape or not np.isfinite(seconds).all()
|
|
or not np.array_equal(raw,z['raw_ids'])
|
|
or any(not np.array_equal(q[name],z['query_raw_ids' if name=='raw_ids' else name])
|
|
for name in ('counts','near_wall','coordinates_over_Rg','raw_ids'))):
|
|
raise ValueError('Frozen full node or PPS support differs')
|
|
new.append(seconds[ix])
|
|
old=reference[case]
|
|
if (old['fold'],old['component_id'])!=(fold,row['family']):
|
|
raise ValueError('H2 score pairing differs')
|
|
log.write(json.dumps(dict(case=case,fold=fold,role='test',sha256=pilot['parts'][case]['output_sha256']['seconds.npy']))+'\n')
|
|
log.flush()
|
|
truth=target_array(pilot,case)
|
|
if truth.shape!=raw.shape:
|
|
raise ValueError('Original held-out node support differs')
|
|
gino=[part_metrics(p,truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],
|
|
int(case.split('_')[0])) for p in new]
|
|
return dict(row,gino=gino,h2=[old['seed0'],old['seed1']])
|
|
|
|
|
|
def score(c):
|
|
auth,_=entry.authorize(c)
|
|
if os.environ.get('CUDA_VISIBLE_DEVICES')!='':
|
|
raise PermissionError('Custody scoring must run with CUDA hidden')
|
|
ceiling=resource.getrlimit(resource.RLIMIT_CPU)[1]
|
|
limit=c['budgets']['score_cpu_seconds']
|
|
if ceiling!=resource.RLIM_INFINITY:
|
|
limit=min(limit,ceiling)
|
|
resource.setrlimit(resource.RLIMIT_CPU,(limit,limit))
|
|
_,predictions=frozen(c)
|
|
drive=verify_drive(auth)
|
|
compare=read(checked(c['comparison_contract'],c['comparison_contract_sha256']))
|
|
rosters={(r['fold'],r['case']) for r in compare['roster']}
|
|
new={(r['fold'],r['seed'],r['case']):r for r in predictions}
|
|
if set(new)!={(fold,seed,case) for fold,case in rosters for seed in (0,1)}:
|
|
raise ValueError('Frozen four-set test support differs')
|
|
prior=read(checked(compare['h2_score']['path'],compare['h2_score']['sha256']))
|
|
reference={r['case']:r for r in prior['rows']}
|
|
if prior['status']!='COMPLETE' or len(reference)!=552 or not {r['case'] for r in compare['roster']}<=set(reference):
|
|
raise ValueError('H2 frozen score support differs')
|
|
pilot=read(checked(PILOT,PINS[str(PILOT)]))
|
|
query={r['case']:r for r in read(checked(QUERY,PINS[str(QUERY)]))['rows'] if r['included']}
|
|
out=entry.ROOT/'score'
|
|
out.mkdir(exist_ok=False)
|
|
save(out/'custody-start.json',dict(unix=time.time(),drive_file_id=drive['file_id'],
|
|
manifest_sha256=sha(entry.ROOT/'predictions-frozen.json'),before_test_target_reads=True))
|
|
results=[]
|
|
with (out/'heldout-target-access.jsonl').open('x') as log:
|
|
for row in compare['roster']:
|
|
results.append(one_case(pilot,query[row['case']],row,new,reference,log))
|
|
save(out/'result.json',dict(status='COMPLETE',summary=summary(results,compare),rows=results,
|
|
unique_test_target_reads=len(results),no_new_decode=True,drive_file_id=drive['file_id']))
|
|
return 0
|