"""Predict seed1 without targets; score once only after all552 predictions freeze.""" import os, sys, time from pathlib import Path import numpy as np from phase2_h1_worker_v2 import save, sha, checked from phase2_home_freeze_io import verify from phase2_gate_outcome_review import validate_review from phase3_seed1_postfit_common import config, adopted, checkpoint_identity, prediction_identity, read from phase3_seed1_postfit_statistics import summary from phase3_seed1_v2_common import fold_view from phase2_h2_s1_postfit import query_index from phase2_residual_math import decode_seconds from phase2_b1213_features import cached from s2_custody_statistics import part_metrics from s2_custody_common import prediction_inputs def paired_query(raw, q, seed0): original={k:seed0['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']} ix=query_index(raw,q,original) if (not np.array_equal(seed0['raw_ids'],raw) or not np.array_equal(seed0['pred'],seed0['seconds'][ix]) or not np.isfinite(seed0['seconds']).all()): raise ValueError('Seed0 full/query identity') return ix def predict(root, fold): c,tc,pilot=config(root); validate_review(root); models=adopted(root,c,tc) view=fold_view(pilot,fold); f=view['folds'][0]; job=tc['jobs'][fold] m=next(m for m in models['models'] if m['job']==fold); stats=read(m['standardization']) import torch from phase2_h2_model import H2Geo from phase2_h2_data import to_device, arguments from phase2_h2_s1_test_data import load_test_part if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True': raise ValueError('Selected GPU/allocator identity') os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1) torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False dest=root/'jobs'/str(fold)/'prediction-records';dest.mkdir(exist_ok=False) torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats() start=time.monotonic();cpu=time.process_time();entries=[];by={r['case']:r for r in c['rows']} def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved())) try: blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False) checkpoint_identity(blob,job,c['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob save(dest/'environment.json',dict(fold=fold,seed=1,torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,checkpoint_sha256=m['sha256'])) with torch.no_grad(): for case in f['test']: row=by[case];data=to_device(load_test_part(view,case,stats),'cuda');residual=model(*arguments(data),checkpoint_blocks=False) if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite residual') seconds=decode_seconds(residual.cpu().numpy(),data['F'],f['c'],data['t_set']);torch.cuda.synchronize() q=prediction_inputs(row);old=c['seed0_predictions'][case] with np.load(checked(old['path'],old['sha256']),allow_pickle=False) as z: ix=paired_query(data['raw_ids'],q,z) if len(seconds)!=row['nodes'] or not np.isfinite(seconds).all() or len(ix)!=old['query_nodes']:raise ValueError('Prediction coverage') path=dest/(case+'.npz') np.savez_compressed(path,seconds=seconds,raw_ids=data['raw_ids'],pred=seconds[ix],query_raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall']) entries.append(dict(case=case,component_id=row['component_id'],fold=fold,path=str(path),sha256=sha(path),nodes=row['nodes'],query_nodes=len(ix),seed0_sha256=old['sha256'],raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'],negative_full_predictions=int(np.sum(seconds<0)),negative_query_predictions=int(np.sum(seconds[ix]<0)))) save(dest/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',fold=fold,parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks())) del data,residual,seconds save(dest/'manifest.json',dict(entries=entries,test_targets_read=False,new_fits=0,full_nodes=sum(e['nodes'] for e in entries))) save(dest/'result.json',dict(status='COMPLETE',fold=fold,parts=len(entries),test_targets_read=False,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks())) except BaseException as exc: save(dest/'failure.json',dict(status='FAILED',fold=fold,reason=type(exc).__name__+': '+str(exc),completed_parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()));raise def score(root): c,tc,pilot=config(root);validate_review(root);adopted(root,c,tc);verify(root,'predictions') entries=read(root/'predictions-manifest.json')['entries'];by=prediction_identity(root,c,tc,entries) dest=root/'score';dest.mkdir(exist_ok=False) save(dest/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'),before_old_score_or_target_values=True)) frozen=read(checked(c['seed0_score'],c['source_sha256'][c['seed0_score']])) olds=frozen['rows'];oldrows={r['case']:r for r in olds} if frozen['status']!='COMPLETE' or len(olds)!=552 or len(oldrows)!=552 or set(oldrows)!=set(by):raise ValueError('Seed0 score support') rows=[];access=[] for e in entries: case=e['case'];row=by[case];old=oldrows[case];base=c['seed0_predictions'][case] if old['fold']!=e['fold'] or old['component_id']!=e['component_id']:raise ValueError('Seed0 score pairing') access.append(dict(case=case,fold=e['fold'],role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy'])) save(dest/'target-access.json',access) truth=cached(row,'seconds');raw=cached(row,'raw_ids') with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z, np.load(checked(base['path'],base['sha256']),allow_pickle=False) as original: q={k:z['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']} ix=paired_query(raw,q,original) if (not np.array_equal(z['raw_ids'],raw) or not np.array_equal(z['pred'],z['seconds'][ix]) or not np.isfinite(z['seconds']).all() or len(z['seconds'])!=e['nodes'] or len(ix)!=e['query_nodes'] or int(np.sum(z['seconds']<0))!=e['negative_full_predictions'] or int(np.sum(z['pred']<0))!=e['negative_query_predictions']):raise ValueError('Seed1 full/query/count identity') metric=part_metrics(z['pred'],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0])) rows.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],seed1=metric,seed0=old['H2'])) save(dest/'progress.json',dict(parts=len(rows),unique_test_target_reads=len(access))) if len(access)!=552 or len({x['case'] for x in access})!=552:raise ValueError('Exactly552 unique existing target-cache reads') save(dest/'result.json',dict(status='COMPLETE',**summary(rows),rows=rows,seed0_metrics_reused=552, seed0_score_sha256=c['source_sha256'][c['seed0_score']],seed0_predictions_reused=552, unique_test_target_reads=552,new_decodes=0,new_fits=0,review_sha256=sha(root/'review.json'), negative_predictions={arm:dict(full_nodes=sum(e['negative_full_predictions'] for e in es),query_nodes=sum(e['negative_query_predictions'] for e in es),clipped=False) for arm,es in [('seed1',entries),('seed0',list(c['seed0_predictions'].values()))]}, full_nodes=sum(e['nodes'] for e in entries),eligible_unique_query_nodes=sum(e['query_nodes'] for e in entries), later_stages_started=False,seed2_started=False,ensemble_started=False)) if __name__=='__main__': action=sys.argv[1];root=Path(sys.argv[2]) if action=='predict' and len(sys.argv)==4:predict(root,int(sys.argv[3])) elif action=='score' and len(sys.argv)==3:score(root) else:raise ValueError('Use predict root fold or score root')