Files
MeltMind/tools/gre24_score.py

105 lines
6.5 KiB
Python

"""Predictions frozen before test-target scoring, all successful tet nodes."""
import argparse
import json
from pathlib import Path
import time
import numpy as np
from gre24_prepare_v2 import sha
from gre24_features_v2 import per_geometry_metrics
from gre24_models import fnn_model
from gre24_fit import contract
from s2_recovery_supervisor import save
def predict(root):
c,prep,splits=contract(root)
models=json.loads((root/'models-freeze.json').read_text())
dest=root/'predictions';dest.mkdir(exist_ok=False)
start=time.process_time();rows=[]
for split in splits:
available={'naive':None}
for name in ['GBRT','FNN']:
r=models['fits'][f"{split['index']}/{name}"]
if r['status']!='PASS':continue
if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze')
if name=='GBRT':
from xgboost import XGBRegressor
m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint'])
else:
import torch
torch.set_num_threads(4);m=fnn_model()
m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval()
available[name]=m
for case in split['test']:
if case not in prep['successful']:
rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={}))
continue
r=prep['successful'][case];source=Path(r['output']);outputs={}
for name in ['features.npy','Vf.npy','raw_ids.npy']:
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed')
x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
# No seconds/target file is opened in this phase.
for name,m in available.items():
if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False)
elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32))
else:
chunks=[]
with torch.no_grad():
for a in range(0,len(x),65536):
chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy())
pred=np.concatenate(chunks)
if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction')
folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True)
path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds'])
outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True)
rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs,
raw_ids_sha256=r['output_sha256']['raw_ids.npy']))
save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start))
save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows,
worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds'))
def score(root):
c,prep,splits=contract(root);frozen=root/'predictions-freeze.json'
if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring')
freeze=json.loads(frozen.read_text());rows=[];start=time.process_time()
for r in freeze['rows']:
if r['status']!='PREDICTED':
rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue
p=prep['successful'][r['case']];source=Path(p['output'])
if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed')
y=np.load(source/'seconds.npy',allow_pickle=False);scores={}
for name,value in r['predictions'].items():
if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed')
pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred)
rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores))
summary={}
for name in ['naive','GBRT','FNN']:
summary[name]={}
for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']:
per=[]
for split in splits:
a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores']
and r['scores'][name][metric] is not None]
per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a),
missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None,
median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None))
# Do not average only whichever model splits happened to finish.
avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']}
summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg,
full_requested_coverage=all(r['missing']==0 for r in per))
save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen)))
save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'],
status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start,
node_domain='All retained tetrahedral nodes including walls, not S2 queries; auxiliary nodes separately reported',
outside_tet_node_counts={case:r.get('outside_tet_nodes',0) for case,r in prep['successful'].items() if r.get('outside_tet_nodes',0)},
scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True,
limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements',
't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices',
'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit']))
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score'])
a=p.parse_args();(predict if a.phase=='predict' else score)(a.root)