Files
MeltMind/tools/run_s2_custody_predict.py
T

83 lines
5.0 KiB
Python

"""One held-out fold, frozen models only; no target reader imported."""
import argparse
import os
from pathlib import Path
import time
import traceback
import numpy as np
from s2_custody_common import contract,checked,test_rows,prediction_inputs,expected_models,prediction_path
from s2_recovery_supervisor import save
from freeze_g1_components import sha
from s2_models.features import R2_COLUMNS
from s2_models.objectives import b0_seconds
def load_neural(m,device):
import torch
from s2_models.normalization import BaseStandardizer
from s2_models_v2.network import QueryModel
checkpoint=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=True)
assert (checkpoint['arm'],checkpoint['width'],checkpoint['seed'])==(m['arm'],m['width'],m['seed'])
d=checkpoint['standardizer']
stats=BaseStandardizer(d['fold'],d['pool'],tuple(d['learning_part_ids']),d['node_count'],tuple(d['mean']),tuple(d['std']))
assert stats.fingerprint==m['standardizer_fingerprint']==checkpoint['standardizer_fingerprint']
assert stats.fold==str(m['fold']) and stats.pool==str(m['pool'])
model=QueryModel(m['arm'],m['width'],standardizer=stats)
model.load_state_dict(checkpoint['state_dict'],strict=True)
return model.to(device).eval()
def run(root,fold):
import torch,joblib,sklearn
from threadpoolctl import threadpool_limits
root=Path(root);c=contract(root);dest=root/'folds'/str(fold);dest.mkdir(parents=True,exist_ok=False)
start=time.monotonic();cpu=time.process_time()
save(dest/'started.json',dict(pid=os.getpid(),fold=fold,unix=time.time(),contract_sha256=sha(root/'contract.json')))
try:
assert c['execution_approved'] and c['limits']['retries']==0
assert torch.cuda.is_available() and torch.cuda.device_count()==1 and 'V100' in torch.cuda.get_device_name()
assert sklearn.__version__=='1.7.2'
torch.set_num_threads(1);torch.backends.cuda.matmul.allow_tf32=False;torch.cuda.reset_peak_memory_stats()
models=expected_models(c,fold);loaded={};device=torch.device('cuda:0')
with threadpool_limits(1):
for m in models:
if m['kind']=='neural': loaded[m['name']]=load_neural(m,device)
elif m['kind']=='R2': loaded[m['name']]=joblib.load(checked(m['path'],m['sha256']))
index={};rows=test_rows(c,fold)
for i,row in enumerate(rows):
d=prediction_inputs(row)
tensors={k:torch.as_tensor(d[k],device=device,dtype=torch.long if k=='ids' else torch.float32) for k in ['base','ids','gaps','edge']}
for m in models:
if m['kind']=='neural':
with torch.inference_mode(),torch.autocast('cuda',enabled=False):
y=loaded[m['name']](tensors['base'],tensors['ids'],tensors['gaps'],tensors['edge'] if m['arm']=='E' else None).cpu().numpy().astype(np.float64)
elif m['kind']=='R2':y=loaded[m['name']].predict(d['x'])
else:
assert R2_COLUMNS[0]=='Vf'
y=d['x'][:,R2_COLUMNS.index('Vf')].copy()
seconds=b0_seconds(y,d['t_set']) if m['kind']=='B0' else y*d['t_set']
if seconds.shape!=d['ids'].shape or not np.isfinite(seconds).all():raise ValueError('Invalid prediction '+m['name'])
p=prediction_path(root,m,row['case']);p.parent.mkdir(parents=True,exist_ok=True)
if p.exists():raise FileExistsError('No prediction overwrite/retry')
np.savez_compressed(p,seconds=seconds,raw_ids=d['raw_ids'],counts=d['counts'])
index[str(p)]=sha(p)
del tensors,d
torch.cuda.synchronize()
save(dest/'progress.json',dict(parts_done=i+1,parts_total=len(rows),wall_seconds=time.monotonic()-start,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
save(dest/'index.json',index)
save(dest/'result.json',dict(status='COMPLETE',fold=fold,parts=len(rows),models=len(models),predictions=len(index),
labels_read=False,model_fit=False,index_sha256=sha(dest/'index.json'),
wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
return 0
except Exception as exc:
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
peak_allocated_bytes=torch.cuda.max_memory_allocated() if torch.cuda.is_initialized() else 0,
peak_reserved_bytes=torch.cuda.max_memory_reserved() if torch.cuda.is_initialized() else 0))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('fold',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.fold))