Files
MeltMind/tools/track_a_postfit_worker.py
T

99 lines
7.1 KiB
Python

"""Five fixed test predictions then independently gated paired full-node scoring."""
import os
from pathlib import Path
import sys
import time
import numpy as np
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, sha, save
from phase2_home_freeze_io import verify
from track_a_postfit_common import config, read, adopted, checkpoint_identity, prediction_identity
from track_a_postfit_statistics import summarize
from phase2_residual_math import decode_seconds
def predict(root,index):
import torch
from phase2_h2_model import H2Geo
from phase2_h2_data import to_device,arguments
from phase3_item1_postfit_data import load_evaluation_part
c,tc,entry=config(root);validate_review(root);a=adopted(root,tc)
j=tc['jobs'][index];m=next(m for m in a['models'] if m['job']==index)
out=root/'jobs'/str(index)/'prediction-records';out.mkdir(exist_ok=False)
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity')
if torch.__version__!=tc['versions']['torch'] or np.__version__!=tc['versions']['numpy']:raise ValueError('Library identity')
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
parts=0;entries=[];start=time.monotonic();cpu=time.process_time()
try:
blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False)
checkpoint_identity(blob,j,a['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
stats=read(checked(m['standardization'],m['standardization_sha256']));view=dict(tc,folds=[j])
with torch.no_grad():
for case in j['test']:
data=to_device(load_evaluation_part(view,case,stats,'test'),'cuda')
residual=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite test residual')
seconds=decode_seconds(residual.cpu().numpy(),data['F'],j['c'],data['t_set']);torch.cuda.synchronize()
if seconds.shape!=(tc['parts'][case]['nodes'],) or not np.isfinite(seconds).all():raise ValueError('Full-node prediction invalid')
p=root/'predictions'/str(index)/(case+'.npz');p.parent.mkdir(parents=True,exist_ok=True)
if p.exists():raise FileExistsError('No prediction overwrite/retry')
np.savez_compressed(p,seconds=seconds,raw_ids=data['raw_ids'])
entries.append(dict(split=index,case=case,path=str(p),sha256=sha(p),nodes=len(seconds),
raw_ids_sha256=tc['parts'][case]['output_sha256']['raw_ids.npy'],negative_predictions=int(np.sum(seconds<0))))
parts+=1;save(out/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',parts=parts,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
del data,residual,seconds
save(out/'manifest.json',dict(entries=entries,test_targets_read=False))
save(out/'result.json',dict(status='COMPLETE',parts=parts,checkpoint_sha256=m['sha256'],test_targets_read=False,**peaks()))
except BaseException as exc:
save(out/'failure.json',dict(status='FAILED',parts=parts,reason=type(exc).__name__+': '+str(exc),**peaks()));raise
def score(root):
import json
from phase2_h2_data import cached
from gre24_features_v2 import per_geometry_metrics
c,tc,entry=config(root);validate_review(root);adopted(root,tc);verify(root,'predictions')
entries=read(root/'predictions-manifest.json')['entries'];prediction_identity(root,c,tc,entries)
out=root/'score';out.mkdir(exist_ok=False);bycase={}
save(out/'review-consumed.json',dict(review_sha256=sha(root/'review.json'),prediction_manifest_sha256=sha(root/'predictions-manifest.json'),before_target_values=True))
for e in entries:bycase.setdefault(e['case'],[]).append(e)
expected=set().union(*(set(j['test']) for j in tc['jobs']))
if set(bycase)!=expected or len(expected)!=c['expected_unique_target_reads']:raise ValueError('Frozen unique test domain')
rows=[];count=0
for case in sorted(bycase):
raw=cached(tc,case,'raw_ids')
with (out/'target-access.jsonl').open('a') as log:
log.write(json.dumps(dict(case=case,role='test',source_sha256=tc['parts'][case]['output_sha256']['seconds.npy'],unix=time.time()))+'\n')
truth=cached(tc,case,'seconds');count+=1
if truth.shape!=raw.shape:raise ValueError('Target native node domain')
for e in bycase[case]:
ref=c['A_predictions'][str(e['split'])][case]
A=np.load(checked(ref['path'],ref['sha256']),allow_pickle=False)
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],raw):raise ValueError('H2 native raw-ID ordering')
H2=z['seconds'].copy()
am=per_geometry_metrics(truth,A);hm=per_geometry_metrics(truth,H2)
if am['nodes']!=e['nodes'] or hm['nodes']!=e['nodes'] or int(np.sum(H2<0))!=e['negative_predictions']:raise ValueError('Paired full-node coverage')
av=am['rrmse_maximum'];hv=hm['rrmse_maximum']
rows.append(dict(split=e['split'],case=case,nodes=e['nodes'],A=av,H2=hv,
delta_H2_minus_A=hv-av if av is not None and hv is not None else None,
truth_maximum=am['denominators']['maximum'],undefined=av is None or hv is None,
A_rmse=am['rmse'],H2_rmse=hm['rmse'],A_negative=int(np.sum(A<0)),H2_negative=e['negative_predictions']))
del truth
save(out/'progress.json',dict(unique_test_target_reads=count,paired_rows=len(rows)))
if len(rows)!=c['expected_pair_visits'] or count!=c['expected_unique_target_reads']:raise ValueError('Complete paired scoring required')
save(out/'result.json',dict(status='COMPLETE',rows=rows,summary=summarize(rows,tc['jobs']),
unique_test_target_reads=count,paired_visits=len(rows),original_failed=tc['original_failed'],
A_refit=False,H2_refit=False,clipped=False,new_decodes=0,comparison_contract_sha256=c['comparison_contract_sha256'],
review_sha256=sha(root/'review.json'),limitations=['Protocol-level re-creation; original author split IDs unavailable',
'Random-part splits can share geometry families','Original14 preparation failures are retained, not resampled',
'RMSE/max(full truth) is the frozen documented proxy; paper numbers are context only']))
if __name__=='__main__':
if sys.argv[1]=='predict':predict(Path(sys.argv[2]),int(sys.argv[3]))
elif sys.argv[1]=='score' and len(sys.argv)==3:score(Path(sys.argv[2]))
else:raise ValueError('predict root index or score root')