"""Label-free candidate inference and one custody pass after all predictions freeze.""" import json import os from pathlib import Path import sys import time import numpy as np from phase2_b1213_features import cached from phase2_h1_worker_v2 import checked, save, sha from phase2_h2_data import cached as original_cache from phase2_h2_s1_test_data import load_test_part from phase2_home_freeze_io import verify from phase2_residual_math import decode_seconds from phase3_followup_postfit_common import ( BASELINE, SEED, checkpoint_identity, config, frozen_file, prediction_identity, read, ) from phase3_followup_postfit_statistics import comparisons, decision, tail_comparisons from phase3_item_b_data import encode, read_context from phase3_seed1_postfit_statistics import summary as seed_summary from phase3_seed1_postfit_worker import paired_query from phase3_seed1_v2_common import fold_view from s2_custody_common import prediction_inputs from s2_custody_statistics import part_metrics from s2_tail_diagnostic_math import peak_diagnostic, tail_error, tail_membership, weights def predict(root, fold): import torch from phase2_h2_data import arguments, to_device from phase2_h2_model import H2Geo from phase3_item_b_model import H2Context c, tc, pilot, by = config(root) if fold not in range(5): raise ValueError('Original five prediction folds only') verify(root, 'models') if read(root/'models-adopted.json') != c['training']: raise ValueError('Adopted candidate models differ from binding') view = fold_view(pilot, fold) job = tc['jobs'][fold] m = c['training']['models'][fold] stats = read(checked(m['standardization'], m['standardization_sha256'])) if (os.environ.get('CUDA_VISIBLE_DEVICES') != c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF') != 'expandable_segments:True'): raise ValueError('Wrong assigned device or allocator') os.sched_setaffinity(0, c['worker_cpus']) torch.set_num_threads(1) torch.set_num_interop_threads(1) torch.backends.cuda.matmul.allow_tf32 = False torch.backends.cudnn.allow_tf32 = False dest = root/'jobs'/str(fold)/'prediction-records' dest.mkdir(exist_ok=False) torch.cuda.set_device(0) torch.cuda.init() torch.cuda.reset_peak_memory_stats() start = time.monotonic() cpu = time.process_time() entries = [] def record(): return dict(fold=fold, parts=len(entries), wall_seconds=time.monotonic()-start, cpu_seconds=time.process_time()-cpu, test_targets_read=False, peak_allocated_bytes=torch.cuda.max_memory_allocated(), peak_reserved_bytes=torch.cuda.max_memory_reserved()) try: blob = torch.load(checked(m['path'], m['sha256']), map_location='cpu', weights_only=False) checkpoint_identity(blob, job, c) model = (H2Geo() if c['item']=='c' else H2Context()).cuda() model.load_state_dict(blob['model'], strict=True) model.eval() del blob save(dest/'environment.json', dict(fold=fold, checkpoint_sha256=m['sha256'], model_dtype='FP32', gpu_uuid=c['gpu_uuid'], torch=torch.__version__, allocator=str(torch.cuda.memory._snapshot().get('allocator_settings',{})))) with torch.no_grad(): for case in job['test']: row = by[case] data = load_test_part(view, case, stats) if c['item'] == 'b': extra, valid = read_context(view, tc, case) context = encode(extra, valid, stats['context']) data['x'] = torch.cat([data['x'], torch.from_numpy(context)], dim=1) data = to_device(data, 'cuda') residual = model(*arguments(data), checkpoint_blocks=False) if not bool(torch.isfinite(residual).all()): raise ValueError('Nonfinite candidate prediction') seconds = decode_seconds(residual.cpu().numpy(), data['F'], job['c'], data['t_set']) torch.cuda.synchronize() q = prediction_inputs(row) base = c['seed0_predictions'][case] with np.load(checked(base['path'],base['sha256']), allow_pickle=False) as old: ix = paired_query(data['raw_ids'], q, old) if len(seconds) != row['nodes'] or len(ix) != base['query_nodes']: raise ValueError('Prediction support mismatch') path = dest/f'{case}.npz' np.savez_compressed(path, seconds=seconds, raw_ids=data['raw_ids'], pred=seconds[ix], query_raw_ids=q['raw_ids'], counts=q['counts'], coordinates_over_Rg=q['coordinates_over_Rg'], near_wall=q['near_wall']) entries.append(dict(case=case, component_id=row['component_id'], fold=fold, path=str(path), sha256=sha(path), nodes=row['nodes'], query_nodes=len(ix), seed0_sha256=base['sha256'], raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'], negative_full_predictions=int(np.sum(seconds<0)), negative_query_predictions=int(np.sum(seconds[ix]<0)))) save(dest/'progress.json', dict(stage='TEST_PREDICTION_NO_TARGETS', **record())) del data, residual, seconds save(dest/'manifest.json', dict(entries=entries, test_targets_read=False)) save(dest/'result.json', dict(status='COMPLETE', **record())) except BaseException as exc: save(dest/'failure.json', dict(status='FAILED', reason=f'{type(exc).__name__}: {exc}', **record())) raise def same_rows(rows, by, entries): expected = {(e['case'], e['component_id'], e['fold']) for e in entries} observed = {(r['case'], r['component_id'], r['fold']) for r in rows} if len(rows) != 552 or observed != expected or {r['case'] for r in rows} != set(by): raise ValueError('Reference score roster/pairing differs') return {r['case']:r for r in rows} def seed_reference(by, entries, old_rows, regional): if not (SEED/'post-terminal-cost.json').exists(): return None, {}, None terminal = verify(SEED, 'terminal') pred_proof = verify(SEED, 'predictions') result_path = SEED/'score/result.json' manifest = SEED/'predictions-manifest.json' done_path = SEED/'complete.json' done = read(frozen_file(terminal, done_path, sha(done_path))) cost = read(SEED/'post-terminal-cost.json') if done['status'] != 'COMPLETE' or cost['within_cap'] is not True: return None, {}, None score = read(frozen_file(terminal, result_path, sha(result_path))) if score['status'] != 'COMPLETE' or not score['review_sha256'].startswith('7b4da089'): raise ValueError('Wrong seed1 result or review') srows = same_rows(score['rows'], by, entries) if any(srows[k]['seed0'] != r['H2'] for k,r in old_rows.items()): raise ValueError('Seed1 did not use the same frozen seed0 metrics') pooled = seed_summary(score['rows'])['pooled'] if pooled != score['pooled']: raise ValueError('Seed paired summaries differ from frozen result') manifest_entries = read(frozen_file(pred_proof, manifest, sha(manifest)))['entries'] s_preds = same_rows(manifest_entries, by, entries) if regional else {} if regional: for e in entries: s = s_preds[e['case']] if any(s[k] != e[k] for k in ('seed0_sha256','raw_ids_sha256','nodes','query_nodes')): raise ValueError('Seed full-domain prediction identity differs') checked(s['path'], s['sha256']) return pooled, s_preds, dict(terminal_commit=terminal['commit'], score_sha256=sha(result_path), prediction_manifest_sha256=sha(manifest)) def full_prediction(z, raw, ix, e): seconds = z['seconds'] if (seconds.shape != raw.shape or not np.array_equal(z['raw_ids'],raw) or not np.array_equal(z['pred'], seconds[ix]) or not np.isfinite(seconds).all() or int(np.sum(seconds<0)) != e['negative_full_predictions'] or int(np.sum(seconds[ix]<0)) != e['negative_query_predictions']): raise ValueError('Nonfinite or inconsistent frozen full/query prediction') return seconds def tail_diagnostics(pilot, case, raw, q, ix, truth, predictions): geom = pilot['geometry'][case] with np.load(checked(geom['path'], geom['sha256']), allow_pickle=False) as z: points, mass, rg = z['points'], z['mass'], float(z['rg']) if not np.array_equal(z['raw_ids'],raw) or not np.array_equal(mass,original_cache(pilot,case,'mass')): raise ValueError('Frozen geometry IDs or lumped masses differ') truth = np.asarray(truth, np.float64) w = weights(mass) a, td = tail_membership(truth,w) arms = {} for arm, field in predictions.items(): p, pd = tail_membership(field,w) arms[arm] = dict(prediction=pd, overlap=tail_error(a,p,w), full=peak_diagnostic(truth,field,points,raw,rg,w=w,true_tail=a), query=peak_diagnostic(truth[ix],field[ix],points[ix],q['raw_ids'],rg)) return dict(truth=td, arms=arms, Rg=rg) def score(root): c, tc, pilot, by = config(root) verify(root,'models') if read(root/'models-adopted.json') != c['training']: raise ValueError('Adopted model evidence differs') proof = verify(root,'predictions') manifest = root/'predictions-manifest.json' entries = read(frozen_file(proof,manifest,sha(manifest)))['entries'] prediction_identity(root,c,tc,by,entries) os.sched_setaffinity(0,c['worker_cpus']) item = c['item'] baseline_path = BASELINE/'score/result.json' baseline = read(checked(baseline_path,c['source_sha256'][str(baseline_path)])) if baseline['status'] != 'COMPLETE': raise ValueError('Original seed0 score incomplete') old = same_rows(baseline['rows'],by,entries) seed, seed_preds, seed_evidence = seed_reference(by,entries,old,item=='c') dest = root/'score' dest.mkdir(exist_ok=False) save(dest/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'), predictions_commit=proof['commit'],before_target_values=True,seed_reference=seed_evidence)) rows = [] for e in entries: case = e['case'] row = by[case] raw = cached(row,'raw_ids') base = c['seed0_predictions'][case] r = dict(case=case, component_id=e['component_id'], fold=e['fold'], seed0=old[case]['H2']) with np.load(e['path'],allow_pickle=False) as z, np.load(base['path'],allow_pickle=False) as original: q = {k:z['query_raw_ids' if k=='raw_ids' else k] for k in ('raw_ids','counts','coordinates_over_Rg','near_wall')} ix = paired_query(raw,q,original) candidate = full_prediction(z,raw,ix,e) predicted = {item:candidate, 'seed0':full_prediction(original,raw,ix,base)} if seed_preds: s = seed_preds[case] with np.load(s['path'],allow_pickle=False) as replica: if not np.array_equal(paired_query(raw,q,replica),ix): raise ValueError('Seed1 query differs') predicted['seed1'] = full_prediction(replica,raw,ix,s) # Fail closed before opening the one existing full-node target cache. with (dest/'target-access.jsonl').open('a') as receipt: receipt.write(json.dumps(dict(case=case,fold=e['fold'],role='test', target_sha256=row['v5_cache']['output_sha256']['seconds.npy'], unix=time.time()))+'\n') truth = cached(row,'seconds') if truth.shape != raw.shape or not np.isfinite(truth).all(): raise ValueError('Wrong full-domain target') r[item] = part_metrics(candidate[ix],truth[ix],q['counts'],q['near_wall'], q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0])) if item == 'c': scale = pilot['parts'][case]['t_set'] if not np.isfinite(scale) or scale <= 0: raise ValueError('Original positive time scale required') r['tail'] = tail_diagnostics(pilot,case,raw,q,ix,np.asarray(truth,np.float64)/scale, {a:np.asarray(p,np.float64)/scale for a,p in predicted.items()}) else: r['context_valid'] = tc['context'][case]['status']=='GEOMETRY_COMPLETE' rows.append(r) save(dest/'progress.json',dict(parts=len(rows),unique_test_target_reads=len(rows))) metrics = comparisons(rows,item) result = dict(status='COMPLETE',item=item,parts=552,components=204,rows=rows,pooled=metrics, per_fold={str(f):comparisons([r for r in rows if r['fold']==f],item) for f in range(5)}, decision=decision(item,metrics,seed),same_endpoint_seed_reference=seed,seed_evidence=seed_evidence, unique_test_target_reads=552,new_decodes=0,new_fits=0,negative_clipping=False, negative_predictions={arm:dict(full_nodes=sum(e['negative_full_predictions'] for e in es), query_nodes=sum(e['negative_query_predictions'] for e in es)) for arm,es in [(item,entries),('seed0',list(c['seed0_predictions'].values()))]}) if item == 'c': result['tail'] = tail_comparisons(rows,item,bool(seed_preds)) else: result['context_strata'] = {} for valid in (True,False): subset = [r for r in rows if r['context_valid'] is valid] result['context_strata'][str(valid)] = dict(parts=len(subset), components=len({r['component_id'] for r in subset}),metrics=comparisons(subset,item)) result['uncached_partition_runtime'] = 'NOT_MEASURED' save(dest/'result.json',result) if __name__=='__main__': mode, root = sys.argv[1], Path(sys.argv[2]) if mode=='predict' and len(sys.argv)==4: predict(root,int(sys.argv[3])) elif mode=='score' and len(sys.argv)==3: score(root) else: raise ValueError('Use predict root fold or score root')