105 lines
6.5 KiB
Python
105 lines
6.5 KiB
Python
"""Predictions frozen before test-target scoring, all successful tet nodes."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import numpy as np
|
|
from gre24_prepare_v2 import sha
|
|
from gre24_features_v2 import per_geometry_metrics
|
|
from gre24_models import fnn_model
|
|
from gre24_fit import contract
|
|
from s2_recovery_supervisor import save
|
|
|
|
|
|
def predict(root):
|
|
c,prep,splits=contract(root)
|
|
models=json.loads((root/'models-freeze.json').read_text())
|
|
dest=root/'predictions';dest.mkdir(exist_ok=False)
|
|
start=time.process_time();rows=[]
|
|
for split in splits:
|
|
available={'naive':None}
|
|
for name in ['GBRT','FNN']:
|
|
r=models['fits'][f"{split['index']}/{name}"]
|
|
if r['status']!='PASS':continue
|
|
if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze')
|
|
if name=='GBRT':
|
|
from xgboost import XGBRegressor
|
|
m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint'])
|
|
else:
|
|
import torch
|
|
torch.set_num_threads(4);m=fnn_model()
|
|
m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval()
|
|
available[name]=m
|
|
for case in split['test']:
|
|
if case not in prep['successful']:
|
|
rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={}))
|
|
continue
|
|
r=prep['successful'][case];source=Path(r['output']);outputs={}
|
|
for name in ['features.npy','Vf.npy','raw_ids.npy']:
|
|
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed')
|
|
x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
|
|
# No seconds/target file is opened in this phase.
|
|
for name,m in available.items():
|
|
if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False)
|
|
elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32))
|
|
else:
|
|
chunks=[]
|
|
with torch.no_grad():
|
|
for a in range(0,len(x),65536):
|
|
chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy())
|
|
pred=np.concatenate(chunks)
|
|
if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction')
|
|
folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True)
|
|
path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds'])
|
|
outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True)
|
|
rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs,
|
|
raw_ids_sha256=r['output_sha256']['raw_ids.npy']))
|
|
save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start))
|
|
save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows,
|
|
worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds'))
|
|
|
|
|
|
def score(root):
|
|
c,prep,splits=contract(root);frozen=root/'predictions-freeze.json'
|
|
if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring')
|
|
freeze=json.loads(frozen.read_text());rows=[];start=time.process_time()
|
|
for r in freeze['rows']:
|
|
if r['status']!='PREDICTED':
|
|
rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue
|
|
p=prep['successful'][r['case']];source=Path(p['output'])
|
|
if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed')
|
|
y=np.load(source/'seconds.npy',allow_pickle=False);scores={}
|
|
for name,value in r['predictions'].items():
|
|
if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed')
|
|
pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred)
|
|
rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores))
|
|
summary={}
|
|
for name in ['naive','GBRT','FNN']:
|
|
summary[name]={}
|
|
for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']:
|
|
per=[]
|
|
for split in splits:
|
|
a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores']
|
|
and r['scores'][name][metric] is not None]
|
|
per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a),
|
|
missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None,
|
|
median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None))
|
|
# Do not average only whichever model splits happened to finish.
|
|
avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']}
|
|
summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg,
|
|
full_requested_coverage=all(r['missing']==0 for r in per))
|
|
save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen)))
|
|
save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'],
|
|
status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start,
|
|
node_domain='All retained tetrahedral nodes including walls, not S2 queries; auxiliary nodes separately reported',
|
|
outside_tet_node_counts={case:r.get('outside_tet_nodes',0) for case,r in prep['successful'].items() if r.get('outside_tet_nodes',0)},
|
|
scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True,
|
|
limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements',
|
|
't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices',
|
|
'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit']))
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score'])
|
|
a=p.parse_args();(predict if a.phase=='predict' else score)(a.root)
|