83 lines
5.0 KiB
Python
83 lines
5.0 KiB
Python
"""One held-out fold, frozen models only; no target reader imported."""
|
|
import argparse
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
import traceback
|
|
import numpy as np
|
|
from s2_custody_common import contract,checked,test_rows,prediction_inputs,expected_models,prediction_path
|
|
from s2_recovery_supervisor import save
|
|
from freeze_g1_components import sha
|
|
from s2_models.features import R2_COLUMNS
|
|
from s2_models.objectives import b0_seconds
|
|
|
|
|
|
def load_neural(m,device):
|
|
import torch
|
|
from s2_models.normalization import BaseStandardizer
|
|
from s2_models_v2.network import QueryModel
|
|
checkpoint=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=True)
|
|
assert (checkpoint['arm'],checkpoint['width'],checkpoint['seed'])==(m['arm'],m['width'],m['seed'])
|
|
d=checkpoint['standardizer']
|
|
stats=BaseStandardizer(d['fold'],d['pool'],tuple(d['learning_part_ids']),d['node_count'],tuple(d['mean']),tuple(d['std']))
|
|
assert stats.fingerprint==m['standardizer_fingerprint']==checkpoint['standardizer_fingerprint']
|
|
assert stats.fold==str(m['fold']) and stats.pool==str(m['pool'])
|
|
model=QueryModel(m['arm'],m['width'],standardizer=stats)
|
|
model.load_state_dict(checkpoint['state_dict'],strict=True)
|
|
return model.to(device).eval()
|
|
|
|
|
|
def run(root,fold):
|
|
import torch,joblib,sklearn
|
|
from threadpoolctl import threadpool_limits
|
|
root=Path(root);c=contract(root);dest=root/'folds'/str(fold);dest.mkdir(parents=True,exist_ok=False)
|
|
start=time.monotonic();cpu=time.process_time()
|
|
save(dest/'started.json',dict(pid=os.getpid(),fold=fold,unix=time.time(),contract_sha256=sha(root/'contract.json')))
|
|
try:
|
|
assert c['execution_approved'] and c['limits']['retries']==0
|
|
assert torch.cuda.is_available() and torch.cuda.device_count()==1 and 'V100' in torch.cuda.get_device_name()
|
|
assert sklearn.__version__=='1.7.2'
|
|
torch.set_num_threads(1);torch.backends.cuda.matmul.allow_tf32=False;torch.cuda.reset_peak_memory_stats()
|
|
models=expected_models(c,fold);loaded={};device=torch.device('cuda:0')
|
|
with threadpool_limits(1):
|
|
for m in models:
|
|
if m['kind']=='neural': loaded[m['name']]=load_neural(m,device)
|
|
elif m['kind']=='R2': loaded[m['name']]=joblib.load(checked(m['path'],m['sha256']))
|
|
index={};rows=test_rows(c,fold)
|
|
for i,row in enumerate(rows):
|
|
d=prediction_inputs(row)
|
|
tensors={k:torch.as_tensor(d[k],device=device,dtype=torch.long if k=='ids' else torch.float32) for k in ['base','ids','gaps','edge']}
|
|
for m in models:
|
|
if m['kind']=='neural':
|
|
with torch.inference_mode(),torch.autocast('cuda',enabled=False):
|
|
y=loaded[m['name']](tensors['base'],tensors['ids'],tensors['gaps'],tensors['edge'] if m['arm']=='E' else None).cpu().numpy().astype(np.float64)
|
|
elif m['kind']=='R2':y=loaded[m['name']].predict(d['x'])
|
|
else:
|
|
assert R2_COLUMNS[0]=='Vf'
|
|
y=d['x'][:,R2_COLUMNS.index('Vf')].copy()
|
|
seconds=b0_seconds(y,d['t_set']) if m['kind']=='B0' else y*d['t_set']
|
|
if seconds.shape!=d['ids'].shape or not np.isfinite(seconds).all():raise ValueError('Invalid prediction '+m['name'])
|
|
p=prediction_path(root,m,row['case']);p.parent.mkdir(parents=True,exist_ok=True)
|
|
if p.exists():raise FileExistsError('No prediction overwrite/retry')
|
|
np.savez_compressed(p,seconds=seconds,raw_ids=d['raw_ids'],counts=d['counts'])
|
|
index[str(p)]=sha(p)
|
|
del tensors,d
|
|
torch.cuda.synchronize()
|
|
save(dest/'progress.json',dict(parts_done=i+1,parts_total=len(rows),wall_seconds=time.monotonic()-start,
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
|
|
save(dest/'index.json',index)
|
|
save(dest/'result.json',dict(status='COMPLETE',fold=fold,parts=len(rows),models=len(models),predictions=len(index),
|
|
labels_read=False,model_fit=False,index_sha256=sha(dest/'index.json'),
|
|
wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
|
|
return 0
|
|
except Exception as exc:
|
|
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated() if torch.cuda.is_initialized() else 0,
|
|
peak_reserved_bytes=torch.cuda.max_memory_reserved() if torch.cuda.is_initialized() else 0))
|
|
raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('fold',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.fold))
|