Files
MeltMind/tools/phase3_item1_postfit_worker.py
T

89 lines
9.0 KiB
Python

"""Target-free endpoint prediction, then separately gated single-read scoring."""
import json,os,sys,time
from pathlib import Path
import numpy as np
from phase2_h1_worker_v2 import save,sha,checked
from phase2_home_freeze_io import verify
from phase2_gate_outcome_review import validate_review
from phase3_item1_postfit_common import config,read,adopted_inputs,expected_checkpoint,manifest_identity,DOMAIN
from phase3_item1_postfit_statistics import summarize
from phase2_residual_math import decode_seconds
from s2_custody_common import prediction_inputs
from s2_custody_statistics import part_metrics
def query_indices(raw,q):
ix=np.searchsorted(raw,q['raw_ids'])
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Fixed query/raw ID mapping')
return ix
def save_prediction(root,arm,case,seconds,raw,q):
ix=query_indices(raw,q);seconds=np.asarray(seconds,np.float64)
if seconds.shape!=raw.shape or not np.isfinite(seconds).all():raise ValueError('Full-node prediction validity')
p=root/'predictions'/arm/(case+'.npz');p.parent.mkdir(parents=True,exist_ok=True)
if p.exists():raise FileExistsError('No prediction overwrite/retry')
np.savez_compressed(p,seconds=seconds,raw_ids=raw,pred=seconds[ix],query_raw_ids=q['raw_ids'],counts=q['counts'],near_wall=q['near_wall'],coordinates_over_Rg=q['coordinates_over_Rg'])
return dict(path=str(p),sha256=sha(p),nodes=len(raw),query_nodes=len(ix),negative_full_predictions=int(np.sum(seconds<0)),negative_query_predictions=int(np.sum(seconds[ix]<0)),origin='ITEM1_FROZEN_ENDPOINT_OR_CLOCK')
def predict(root,index):
import torch
from phase2_h2_model import H2Geo
from phase2_h2_data import to_device,arguments,cached
from phase3_item1_postfit_data import load_evaluation_part
c,tc,pilot=config(root);validate_review(root);adopted=adopted_inputs(root,c,tc);m=adopted['models'][index];job=c['jobs'][index];f=tc['pools'][str(m['pool'])];view=dict(pilot,folds=[f]);out=root/'jobs'/str(index)/'prediction-records';out.mkdir(exist_ok=False);stats=read(m['standardization']);entries=[];parts=0;cpu=time.process_time();start=time.monotonic()
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity')
if torch.__version__!=pilot['versions']['torch'] or np.__version__!=pilot['versions']['numpy']:raise ValueError('Library identity')
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1);torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
try:
blob=torch.load(checked(m['checkpoint'],m['checkpoint_sha256']),map_location='cpu',weights_only=False);expected_checkpoint(job,blob,m['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=c['gpu_uuid'],checkpoint_sha256=m['checkpoint_sha256'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),dtype='FP32',autocast=False))
old={e['case']:e for e in read(tc['S1_reuse']['test_manifest'])['entries']};emit_clock=index in [0,1,6]
with torch.no_grad():
for role,domain in DOMAIN.items():
for case in f[role]:
row=c['rows'][case];q=prediction_inputs(row);identity=dict(case=case,component_id=row['component_id'],pool=m['pool'],domain=domain)
if index==6 and role=='test':
e=old[case];checked(e['path'],e['sha256']);entries.append(dict(**identity,arm=m['arm'],**{k:e[k] for k in ['path','sha256','nodes','query_nodes','negative_full_predictions','negative_query_predictions']},origin='S1_HASH_REUSE'))
else:
data=to_device(load_evaluation_part(view,case,stats,role),'cuda');residual=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite residual')
seconds=decode_seconds(residual.cpu().numpy(),data['F'],f['c'],data['t_set']);torch.cuda.synchronize();entries.append(dict(**identity,arm=m['arm'],**save_prediction(root,m['arm'],case,seconds,data['raw_ids'],q)));del residual,data,seconds
if emit_clock:
raw=cached(view,case,'raw_ids');F=cached(view,case,'Vf');seconds=np.asarray(F,np.float64)*f['c']*pilot['parts'][case]['t_set'];arm='c'+str(m['pool'])+'F';entries.append(dict(**identity,arm=arm,**save_prediction(root,arm,case,seconds,raw,q)))
parts+=1;save(out/'progress.json',dict(stage='PREDICTION_NO_TARGETS',job=index,parts=parts,entries=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
save(out/'manifest.json',dict(entries=entries,targets_read=False));save(out/'result.json',dict(status='COMPLETE',parts=parts,entries=len(entries),targets_read=False,new_fits=0,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
save(out/'failure.json',dict(status='FAILED',parts=parts,reason=type(exc).__name__+': '+str(exc),**peaks()));raise
def score(root):
from phase2_h2_data import cached
c,tc,pilot=config(root);validate_review(root);adopted_inputs(root,c,tc);verify(root,'predictions');entries=read(root/'predictions-manifest.json')['entries'];manifest_identity(entries,c,tc,pilot);out=root/'score';out.mkdir(exist_ok=False)
save(out/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'),model_manifest_sha256=sha(root/'models-adopted.json'),prediction_manifest_sha256=sha(root/'predictions-manifest.json'),before_target_values=True))
old=read(checked(tc['S1_reuse']['test_score'],tc['S1_reuse']['test_score_sha256']));oldrows={r['case']:r for r in old['rows']};f0=tc['pools']['100'];roles={case:role for role in DOMAIN for case in f0[role]};bycase={}
for e in entries:bycase.setdefault(e['case'],[]).append(e)
if set(bycase)!=set(roles) or len(bycase)!=552:raise ValueError('Complete common552 evaluation support')
rows=[];access=[]
for case in sorted(bycase):
role=roles[case];q=prediction_inputs(c['rows'][case]);raw=cached(pilot,case,'raw_ids');ix=query_indices(raw,q);record=dict(case=case,role=role,target_sha256=pilot['parts'][case]['output_sha256']['seconds.npy'],unix=time.time())
with (out/'target-access.jsonl').open('a') as log:log.write(json.dumps(record)+'\n')
access.append(record);truth=cached(pilot,case,'seconds')
for e in bycase[case]:
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
if not np.array_equal(z['raw_ids'],raw) or not np.array_equal(z['query_raw_ids'],q['raw_ids']) or not np.array_equal(z['counts'],q['counts']) or not np.array_equal(z['near_wall'],q['near_wall']) or not np.array_equal(z['coordinates_over_Rg'],q['coordinates_over_Rg']) or not np.array_equal(z['pred'],z['seconds'][ix]):raise ValueError('Prediction/full/query identity')
if len(z['seconds'])!=e['nodes'] or len(ix)!=e['query_nodes'] or int(np.sum(z['seconds']<0))!=e['negative_full_predictions'] or int(np.sum(z['pred']<0))!=e['negative_query_predictions']:raise ValueError('Negative/coverage metadata mismatch')
if e['origin']=='S1_HASH_REUSE':
oldrow=oldrows[case]
if oldrow['component_id']!=e['component_id'] or oldrow['fold']!=0:raise ValueError('S1 stored metric identity')
metrics=oldrow['new']
else:metrics=part_metrics(z['pred'],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
rows.append(dict(case=case,component_id=e['component_id'],arm=e['arm'],pool=e['pool'],domain=e['domain'],metrics=metrics,origin=e['origin']))
del truth;save(out/'progress.json',dict(unique_target_reads=len(access),rows=len(rows)))
if {r:sum(a['role']==r for a in access) for r in DOMAIN}!={'learning':320,'validation':111,'test':121}:raise ValueError('Exactly one read per frozen target/domain')
comparison=read(c['comparison_contract']);result=summarize(rows,comparison['component_counts'],comparison['bootstrap']['replicates'],comparison['bootstrap']['seed'])
negatives={arm:dict(full_nodes=sum(e['negative_full_predictions'] for e in entries if e['arm']==arm),query_nodes=sum(e['negative_query_predictions'] for e in entries if e['arm']==arm)) for arm in sorted({e['arm'] for e in entries})}
save(out/'result.json',dict(status='COMPLETE',summary=result,rows=rows,unique_target_reads=len(access),test_target_reads=121,S1_metric_rows_reused=121,new_fits=0,new_decodes=0,negative_predictions=negatives,clipped=False,model_predictions_full_nodes=True,metrics='Original fixed PPS query readout; multiplicity once, query argmax last/Rg, original tau pairing',comparison_contract_sha256=c['comparison_contract_sha256'],review_sha256=sha(root/'review.json'),future_phase3_started=False))
if __name__=='__main__':
if sys.argv[1]=='predict':predict(Path(sys.argv[2]),int(sys.argv[3]))
elif sys.argv[1]=='score' and len(sys.argv)==3:score(Path(sys.argv[2]))
else:raise ValueError('predict root index or score root')