104 lines
7.5 KiB
Python
104 lines
7.5 KiB
Python
"""Four fixed test split predictions then independently gated paired full-node scoring."""
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
import numpy as np
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h1_worker_v2 import checked, sha, save
|
|
from phase2_home_freeze_io import verify
|
|
from track_a_postfit_common import checkpoint_identity, prediction_identity
|
|
from track_a_postfit_rebind_common import config, read, adopted, selected_training, SPLITS
|
|
from track_a_postfit_rebind_statistics import summarize
|
|
from phase2_residual_math import decode_seconds
|
|
|
|
|
|
def predict(root,index):
|
|
import torch
|
|
from phase2_h2_model import H2Geo
|
|
from phase2_h2_data import to_device,arguments
|
|
from phase3_item1_postfit_data import load_evaluation_part
|
|
c,tc,entry=config(root);validate_review(root);a=adopted(root,c)
|
|
if index not in SPLITS:raise ValueError('Split outside approved four-of-five support')
|
|
j=tc['jobs'][index];m=next(m for m in a['models'] if m['job']==index)
|
|
out=root/'jobs'/str(index)/'prediction-records';out.mkdir(exist_ok=False)
|
|
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity')
|
|
if torch.__version__!=tc['versions']['torch'] or np.__version__!=tc['versions']['numpy']:raise ValueError('Library identity')
|
|
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
|
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
|
|
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
|
parts=0;entries=[];start=time.monotonic();cpu=time.process_time()
|
|
try:
|
|
blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False)
|
|
checkpoint_identity(blob,j,m['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
|
|
stats=read(checked(m['standardization'],m['standardization_sha256']));view=dict(tc,folds=[j])
|
|
with torch.no_grad():
|
|
for case in j['test']:
|
|
data=to_device(load_evaluation_part(view,case,stats,'test'),'cuda')
|
|
residual=model(*arguments(data),checkpoint_blocks=False)
|
|
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite test residual')
|
|
seconds=decode_seconds(residual.cpu().numpy(),data['F'],j['c'],data['t_set']);torch.cuda.synchronize()
|
|
if seconds.shape!=(tc['parts'][case]['nodes'],) or not np.isfinite(seconds).all():raise ValueError('Full-node prediction invalid')
|
|
p=root/'predictions'/str(index)/(case+'.npz');p.parent.mkdir(parents=True,exist_ok=True)
|
|
if p.exists():raise FileExistsError('No prediction overwrite/retry')
|
|
np.savez_compressed(p,seconds=seconds,raw_ids=data['raw_ids'])
|
|
entries.append(dict(split=index,case=case,path=str(p),sha256=sha(p),nodes=len(seconds),
|
|
raw_ids_sha256=tc['parts'][case]['output_sha256']['raw_ids.npy'],negative_predictions=int(np.sum(seconds<0))))
|
|
parts+=1;save(out/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',parts=parts,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
|
|
del data,residual,seconds
|
|
save(out/'manifest.json',dict(entries=entries,test_targets_read=False))
|
|
save(out/'result.json',dict(status='COMPLETE',parts=parts,checkpoint_sha256=m['sha256'],test_targets_read=False,**peaks()))
|
|
except BaseException as exc:
|
|
save(out/'failure.json',dict(status='FAILED',parts=parts,reason=type(exc).__name__+': '+str(exc),**peaks()));raise
|
|
|
|
|
|
def score(root):
|
|
import json
|
|
from phase2_h2_data import cached
|
|
from gre24_features_v2 import per_geometry_metrics
|
|
c,tc,entry=config(root);validate_review(root);adopted(root,c);verify(root,'predictions')
|
|
selected=selected_training(tc)
|
|
entries=read(root/'predictions-manifest.json')['entries'];prediction_identity(root,c,selected,entries)
|
|
out=root/'score';out.mkdir(exist_ok=False);bycase={}
|
|
save(out/'review-consumed.json',dict(review_sha256=sha(root/'review.json'),prediction_manifest_sha256=sha(root/'predictions-manifest.json'),before_target_values=True))
|
|
for e in entries:bycase.setdefault(e['case'],[]).append(e)
|
|
expected=set().union(*(set(j['test']) for j in selected['jobs']))
|
|
if set(bycase)!=expected or len(expected)!=c['expected_unique_target_reads']:raise ValueError('Frozen unique test domain')
|
|
rows=[];count=0
|
|
for case in sorted(bycase):
|
|
raw=cached(tc,case,'raw_ids')
|
|
with (out/'target-access.jsonl').open('a') as log:
|
|
log.write(json.dumps(dict(case=case,role='test',source_sha256=tc['parts'][case]['output_sha256']['seconds.npy'],unix=time.time()))+'\n')
|
|
truth=cached(tc,case,'seconds');count+=1
|
|
if truth.shape!=raw.shape:raise ValueError('Target native node domain')
|
|
for e in bycase[case]:
|
|
ref=c['A_predictions'][str(e['split'])][case]
|
|
A=np.load(checked(ref['path'],ref['sha256']),allow_pickle=False)
|
|
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
|
|
if not np.array_equal(z['raw_ids'],raw):raise ValueError('H2 native raw-ID ordering')
|
|
H2=z['seconds'].copy()
|
|
am=per_geometry_metrics(truth,A);hm=per_geometry_metrics(truth,H2)
|
|
if am['nodes']!=e['nodes'] or hm['nodes']!=e['nodes'] or int(np.sum(H2<0))!=e['negative_predictions']:raise ValueError('Paired full-node coverage')
|
|
av=am['rrmse_maximum'];hv=hm['rrmse_maximum']
|
|
rows.append(dict(split=e['split'],case=case,nodes=e['nodes'],A=av,H2=hv,
|
|
delta_H2_minus_A=hv-av if av is not None and hv is not None else None,
|
|
truth_maximum=am['denominators']['maximum'],undefined=av is None or hv is None,
|
|
A_rmse=am['rmse'],H2_rmse=hm['rmse'],A_negative=int(np.sum(A<0)),H2_negative=e['negative_predictions']))
|
|
del truth
|
|
save(out/'progress.json',dict(unique_test_target_reads=count,paired_rows=len(rows)))
|
|
if len(rows)!=c['expected_pair_visits'] or count!=c['expected_unique_target_reads']:raise ValueError('Complete paired scoring required')
|
|
save(out/'result.json',dict(status='COMPLETE_FOUR_OF_FIVE',rows=rows,summary=summarize(rows,selected['jobs']),
|
|
evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],
|
|
unique_test_target_reads=count,paired_visits=len(rows),original_failed=tc['original_failed'],
|
|
A_refit=False,H2_refit=False,clipped=False,new_decodes=0,comparison_contract_sha256=c['comparison_contract_sha256'],
|
|
review_sha256=sha(root/'review.json'),limitations=['Four of five fixed splits; split 3 has no completed training checkpoint after the budget stop',
|
|
'Protocol-level re-creation; original author split IDs unavailable',
|
|
'Random-part splits can share geometry families','Original14 preparation failures are retained, not resampled',
|
|
'RMSE/max(full truth) is the frozen documented proxy; paper numbers are context only']))
|
|
|
|
|
|
if __name__=='__main__':
|
|
if sys.argv[1]=='predict':predict(Path(sys.argv[2]),int(sys.argv[3]))
|
|
elif sys.argv[1]=='score' and len(sys.argv)==3:score(Path(sys.argv[2]))
|
|
else:raise ValueError('predict root index or score root')
|