Files
MeltMind/tools/phase3_seed1_postfit_worker.py
T

105 lines
8.4 KiB
Python

"""Predict seed1 without targets; score once only after all552 predictions freeze."""
import os, sys, time
from pathlib import Path
import numpy as np
from phase2_h1_worker_v2 import save, sha, checked
from phase2_home_freeze_io import verify
from phase2_gate_outcome_review import validate_review
from phase3_seed1_postfit_common import config, adopted, checkpoint_identity, prediction_identity, read
from phase3_seed1_postfit_statistics import summary
from phase3_seed1_v2_common import fold_view
from phase2_h2_s1_postfit import query_index
from phase2_residual_math import decode_seconds
from phase2_b1213_features import cached
from s2_custody_statistics import part_metrics
from s2_custody_common import prediction_inputs
def paired_query(raw, q, seed0):
original={k:seed0['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']}
ix=query_index(raw,q,original)
if (not np.array_equal(seed0['raw_ids'],raw) or not np.array_equal(seed0['pred'],seed0['seconds'][ix])
or not np.isfinite(seed0['seconds']).all()): raise ValueError('Seed0 full/query identity')
return ix
def predict(root, fold):
c,tc,pilot=config(root); validate_review(root); models=adopted(root,c,tc)
view=fold_view(pilot,fold); f=view['folds'][0]; job=tc['jobs'][fold]
m=next(m for m in models['models'] if m['job']==fold); stats=read(m['standardization'])
import torch
from phase2_h2_model import H2Geo
from phase2_h2_data import to_device, arguments
from phase2_h2_s1_test_data import load_test_part
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
raise ValueError('Selected GPU/allocator identity')
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
dest=root/'jobs'/str(fold)/'prediction-records';dest.mkdir(exist_ok=False)
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
start=time.monotonic();cpu=time.process_time();entries=[];by={r['case']:r for r in c['rows']}
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
try:
blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False)
checkpoint_identity(blob,job,c['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
save(dest/'environment.json',dict(fold=fold,seed=1,torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,checkpoint_sha256=m['sha256']))
with torch.no_grad():
for case in f['test']:
row=by[case];data=to_device(load_test_part(view,case,stats),'cuda');residual=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite residual')
seconds=decode_seconds(residual.cpu().numpy(),data['F'],f['c'],data['t_set']);torch.cuda.synchronize()
q=prediction_inputs(row);old=c['seed0_predictions'][case]
with np.load(checked(old['path'],old['sha256']),allow_pickle=False) as z:
ix=paired_query(data['raw_ids'],q,z)
if len(seconds)!=row['nodes'] or not np.isfinite(seconds).all() or len(ix)!=old['query_nodes']:raise ValueError('Prediction coverage')
path=dest/(case+'.npz')
np.savez_compressed(path,seconds=seconds,raw_ids=data['raw_ids'],pred=seconds[ix],query_raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
entries.append(dict(case=case,component_id=row['component_id'],fold=fold,path=str(path),sha256=sha(path),nodes=row['nodes'],query_nodes=len(ix),seed0_sha256=old['sha256'],raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'],negative_full_predictions=int(np.sum(seconds<0)),negative_query_predictions=int(np.sum(seconds[ix]<0))))
save(dest/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',fold=fold,parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
del data,residual,seconds
save(dest/'manifest.json',dict(entries=entries,test_targets_read=False,new_fits=0,full_nodes=sum(e['nodes'] for e in entries)))
save(dest/'result.json',dict(status='COMPLETE',fold=fold,parts=len(entries),test_targets_read=False,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
save(dest/'failure.json',dict(status='FAILED',fold=fold,reason=type(exc).__name__+': '+str(exc),completed_parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()));raise
def score(root):
c,tc,pilot=config(root);validate_review(root);adopted(root,c,tc);verify(root,'predictions')
entries=read(root/'predictions-manifest.json')['entries'];by=prediction_identity(root,c,tc,entries)
dest=root/'score';dest.mkdir(exist_ok=False)
save(dest/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'),before_old_score_or_target_values=True))
frozen=read(checked(c['seed0_score'],c['source_sha256'][c['seed0_score']]))
olds=frozen['rows'];oldrows={r['case']:r for r in olds}
if frozen['status']!='COMPLETE' or len(olds)!=552 or len(oldrows)!=552 or set(oldrows)!=set(by):raise ValueError('Seed0 score support')
rows=[];access=[]
for e in entries:
case=e['case'];row=by[case];old=oldrows[case];base=c['seed0_predictions'][case]
if old['fold']!=e['fold'] or old['component_id']!=e['component_id']:raise ValueError('Seed0 score pairing')
access.append(dict(case=case,fold=e['fold'],role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy']))
save(dest/'target-access.json',access)
truth=cached(row,'seconds');raw=cached(row,'raw_ids')
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z, np.load(checked(base['path'],base['sha256']),allow_pickle=False) as original:
q={k:z['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']}
ix=paired_query(raw,q,original)
if (not np.array_equal(z['raw_ids'],raw) or not np.array_equal(z['pred'],z['seconds'][ix])
or not np.isfinite(z['seconds']).all() or len(z['seconds'])!=e['nodes'] or len(ix)!=e['query_nodes']
or int(np.sum(z['seconds']<0))!=e['negative_full_predictions']
or int(np.sum(z['pred']<0))!=e['negative_query_predictions']):raise ValueError('Seed1 full/query/count identity')
metric=part_metrics(z['pred'],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
rows.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],seed1=metric,seed0=old['H2']))
save(dest/'progress.json',dict(parts=len(rows),unique_test_target_reads=len(access)))
if len(access)!=552 or len({x['case'] for x in access})!=552:raise ValueError('Exactly552 unique existing target-cache reads')
save(dest/'result.json',dict(status='COMPLETE',**summary(rows),rows=rows,seed0_metrics_reused=552,
seed0_score_sha256=c['source_sha256'][c['seed0_score']],seed0_predictions_reused=552,
unique_test_target_reads=552,new_decodes=0,new_fits=0,review_sha256=sha(root/'review.json'),
negative_predictions={arm:dict(full_nodes=sum(e['negative_full_predictions'] for e in es),query_nodes=sum(e['negative_query_predictions'] for e in es),clipped=False) for arm,es in [('seed1',entries),('seed0',list(c['seed0_predictions'].values()))]},
full_nodes=sum(e['nodes'] for e in entries),eligible_unique_query_nodes=sum(e['query_nodes'] for e in entries),
later_stages_started=False,seed2_started=False,ensemble_started=False))
if __name__=='__main__':
action=sys.argv[1];root=Path(sys.argv[2])
if action=='predict' and len(sys.argv)==4:predict(root,int(sys.argv[3]))
elif action=='score' and len(sys.argv)==3:score(root)
else:raise ValueError('Use predict root fold or score root')