"""One held-out fold, frozen models only; no target reader imported.""" import argparse import os from pathlib import Path import time import traceback import numpy as np from s2_custody_common import contract,checked,test_rows,prediction_inputs,expected_models,prediction_path from s2_recovery_supervisor import save from freeze_g1_components import sha from s2_models.features import R2_COLUMNS from s2_models.objectives import b0_seconds def load_neural(m,device): import torch from s2_models.normalization import BaseStandardizer from s2_models_v2.network import QueryModel checkpoint=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=True) assert (checkpoint['arm'],checkpoint['width'],checkpoint['seed'])==(m['arm'],m['width'],m['seed']) d=checkpoint['standardizer'] stats=BaseStandardizer(d['fold'],d['pool'],tuple(d['learning_part_ids']),d['node_count'],tuple(d['mean']),tuple(d['std'])) assert stats.fingerprint==m['standardizer_fingerprint']==checkpoint['standardizer_fingerprint'] assert stats.fold==str(m['fold']) and stats.pool==str(m['pool']) model=QueryModel(m['arm'],m['width'],standardizer=stats) model.load_state_dict(checkpoint['state_dict'],strict=True) return model.to(device).eval() def run(root,fold): import torch,joblib,sklearn from threadpoolctl import threadpool_limits root=Path(root);c=contract(root);dest=root/'folds'/str(fold);dest.mkdir(parents=True,exist_ok=False) start=time.monotonic();cpu=time.process_time() save(dest/'started.json',dict(pid=os.getpid(),fold=fold,unix=time.time(),contract_sha256=sha(root/'contract.json'))) try: assert c['execution_approved'] and c['limits']['retries']==0 assert torch.cuda.is_available() and torch.cuda.device_count()==1 and 'V100' in torch.cuda.get_device_name() assert sklearn.__version__=='1.7.2' torch.set_num_threads(1);torch.backends.cuda.matmul.allow_tf32=False;torch.cuda.reset_peak_memory_stats() models=expected_models(c,fold);loaded={};device=torch.device('cuda:0') with threadpool_limits(1): for m in models: if m['kind']=='neural': loaded[m['name']]=load_neural(m,device) elif m['kind']=='R2': loaded[m['name']]=joblib.load(checked(m['path'],m['sha256'])) index={};rows=test_rows(c,fold) for i,row in enumerate(rows): d=prediction_inputs(row) tensors={k:torch.as_tensor(d[k],device=device,dtype=torch.long if k=='ids' else torch.float32) for k in ['base','ids','gaps','edge']} for m in models: if m['kind']=='neural': with torch.inference_mode(),torch.autocast('cuda',enabled=False): y=loaded[m['name']](tensors['base'],tensors['ids'],tensors['gaps'],tensors['edge'] if m['arm']=='E' else None).cpu().numpy().astype(np.float64) elif m['kind']=='R2':y=loaded[m['name']].predict(d['x']) else: assert R2_COLUMNS[0]=='Vf' y=d['x'][:,R2_COLUMNS.index('Vf')].copy() seconds=b0_seconds(y,d['t_set']) if m['kind']=='B0' else y*d['t_set'] if seconds.shape!=d['ids'].shape or not np.isfinite(seconds).all():raise ValueError('Invalid prediction '+m['name']) p=prediction_path(root,m,row['case']);p.parent.mkdir(parents=True,exist_ok=True) if p.exists():raise FileExistsError('No prediction overwrite/retry') np.savez_compressed(p,seconds=seconds,raw_ids=d['raw_ids'],counts=d['counts']) index[str(p)]=sha(p) del tensors,d torch.cuda.synchronize() save(dest/'progress.json',dict(parts_done=i+1,parts_total=len(rows),wall_seconds=time.monotonic()-start, peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())) save(dest/'index.json',index) save(dest/'result.json',dict(status='COMPLETE',fold=fold,parts=len(rows),models=len(models),predictions=len(index), labels_read=False,model_fit=False,index_sha256=sha(dest/'index.json'), wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu, peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())) return 0 except Exception as exc: save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(), peak_allocated_bytes=torch.cuda.max_memory_allocated() if torch.cuda.is_initialized() else 0, peak_reserved_bytes=torch.cuda.max_memory_reserved() if torch.cuda.is_initialized() else 0)) raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('fold',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.fold))