"""Predictions frozen before test-target scoring, all successful tet nodes.""" import argparse import json from pathlib import Path import time import numpy as np from gre24_prepare_v2 import sha from gre24_features_v2 import per_geometry_metrics from gre24_models import fnn_model from gre24_fit import contract from s2_recovery_supervisor import save def predict(root): c,prep,splits=contract(root) models=json.loads((root/'models-freeze.json').read_text()) dest=root/'predictions';dest.mkdir(exist_ok=False) start=time.process_time();rows=[] for split in splits: available={'naive':None} for name in ['GBRT','FNN']: r=models['fits'][f"{split['index']}/{name}"] if r['status']!='PASS':continue if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze') if name=='GBRT': from xgboost import XGBRegressor m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint']) else: import torch torch.set_num_threads(4);m=fnn_model() m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval() available[name]=m for case in split['test']: if case not in prep['successful']: rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={})) continue r=prep['successful'][case];source=Path(r['output']);outputs={} for name in ['features.npy','Vf.npy','raw_ids.npy']: if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed') x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False) # No seconds/target file is opened in this phase. for name,m in available.items(): if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False) elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32)) else: chunks=[] with torch.no_grad(): for a in range(0,len(x),65536): chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy()) pred=np.concatenate(chunks) if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction') folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True) path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds']) outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True) rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs, raw_ids_sha256=r['output_sha256']['raw_ids.npy'])) save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start)) save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows, worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds')) def score(root): c,prep,splits=contract(root);frozen=root/'predictions-freeze.json' if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring') freeze=json.loads(frozen.read_text());rows=[];start=time.process_time() for r in freeze['rows']: if r['status']!='PREDICTED': rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue p=prep['successful'][r['case']];source=Path(p['output']) if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed') y=np.load(source/'seconds.npy',allow_pickle=False);scores={} for name,value in r['predictions'].items(): if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed') pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred) rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores)) summary={} for name in ['naive','GBRT','FNN']: summary[name]={} for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']: per=[] for split in splits: a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores'] and r['scores'][name][metric] is not None] per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a), missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None, median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None)) # Do not average only whichever model splits happened to finish. avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']} summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg, full_requested_coverage=all(r['missing']==0 for r in per)) save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen))) save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'], status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start, node_domain='All retained tetrahedral nodes including walls, not S2 queries; auxiliary nodes separately reported', outside_tet_node_counts={case:r.get('outside_tet_nodes',0) for case,r in prep['successful'].items() if r.get('outside_tet_nodes',0)}, scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True, limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements', 't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices', 'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit'])) if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']) a=p.parse_args();(predict if a.phase=='predict' else score)(a.root)