271 lines
14 KiB
Python
271 lines
14 KiB
Python
"""Label-free candidate inference and one custody pass after all predictions freeze."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
|
|
import numpy as np
|
|
|
|
from phase2_b1213_features import cached
|
|
from phase2_h1_worker_v2 import checked, save, sha
|
|
from phase2_h2_data import cached as original_cache
|
|
from phase2_h2_s1_test_data import load_test_part
|
|
from phase2_home_freeze_io import verify
|
|
from phase2_residual_math import decode_seconds
|
|
from phase3_followup_postfit_common import (
|
|
BASELINE, SEED, checkpoint_identity, frozen_file, prediction_identity, read,
|
|
)
|
|
from phase3_followup_postfit_recovery_common import config
|
|
from phase3_followup_postfit_statistics import comparisons, decision, tail_comparisons
|
|
from phase3_item_b_data import encode, read_context
|
|
from phase3_seed1_postfit_statistics import summary as seed_summary
|
|
from phase3_seed1_postfit_worker import paired_query
|
|
from phase3_seed1_v2_common import fold_view
|
|
from s2_custody_common import prediction_inputs
|
|
from s2_custody_statistics import part_metrics
|
|
from s2_tail_diagnostic_math import peak_diagnostic, tail_error, tail_membership, weights
|
|
|
|
|
|
def predict(root, fold):
|
|
import torch
|
|
from phase2_h2_data import arguments, to_device
|
|
from phase2_h2_model import H2Geo
|
|
from phase3_item_b_model import H2Context
|
|
|
|
c, tc, pilot, by = config(root)
|
|
if fold not in range(5):
|
|
raise ValueError('Original five prediction folds only')
|
|
verify(root, 'models')
|
|
if read(root/'models-adopted.json') != c['training']:
|
|
raise ValueError('Adopted candidate models differ from binding')
|
|
view = fold_view(pilot, fold)
|
|
job = tc['jobs'][fold]
|
|
m = c['training']['models'][fold]
|
|
stats = read(checked(m['standardization'], m['standardization_sha256']))
|
|
if (os.environ.get('CUDA_VISIBLE_DEVICES') != c['gpu_uuid']
|
|
or os.environ.get('PYTORCH_CUDA_ALLOC_CONF') != 'expandable_segments:True'):
|
|
raise ValueError('Wrong assigned device or allocator')
|
|
os.sched_setaffinity(0, c['worker_cpus'])
|
|
torch.set_num_threads(1)
|
|
torch.set_num_interop_threads(1)
|
|
torch.backends.cuda.matmul.allow_tf32 = False
|
|
torch.backends.cudnn.allow_tf32 = False
|
|
dest = root/'jobs'/str(fold)/'prediction-records'
|
|
dest.mkdir(exist_ok=False)
|
|
torch.cuda.set_device(0)
|
|
torch.cuda.init()
|
|
torch.cuda.reset_peak_memory_stats()
|
|
start = time.monotonic()
|
|
cpu = time.process_time()
|
|
entries = []
|
|
|
|
def record():
|
|
return dict(fold=fold, parts=len(entries), wall_seconds=time.monotonic()-start,
|
|
cpu_seconds=time.process_time()-cpu, test_targets_read=False,
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated(),
|
|
peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
|
|
|
try:
|
|
blob = torch.load(checked(m['path'], m['sha256']), map_location='cpu', weights_only=False)
|
|
checkpoint_identity(blob, job, c)
|
|
model = (H2Geo() if c['item']=='c' else H2Context()).cuda()
|
|
model.load_state_dict(blob['model'], strict=True)
|
|
model.eval()
|
|
del blob
|
|
save(dest/'environment.json', dict(fold=fold, checkpoint_sha256=m['sha256'], model_dtype='FP32',
|
|
gpu_uuid=c['gpu_uuid'], torch=torch.__version__, allocator=str(torch.cuda.memory._snapshot().get('allocator_settings',{}))))
|
|
with torch.no_grad():
|
|
for case in job['test']:
|
|
row = by[case]
|
|
data = load_test_part(view, case, stats)
|
|
if c['item'] == 'b':
|
|
extra, valid = read_context(view, tc, case)
|
|
context = encode(extra, valid, stats['context'])
|
|
data['x'] = torch.cat([data['x'], torch.from_numpy(context)], dim=1)
|
|
data = to_device(data, 'cuda')
|
|
residual = model(*arguments(data), checkpoint_blocks=False)
|
|
if not bool(torch.isfinite(residual).all()):
|
|
raise ValueError('Nonfinite candidate prediction')
|
|
seconds = decode_seconds(residual.cpu().numpy(), data['F'], job['c'], data['t_set'])
|
|
torch.cuda.synchronize()
|
|
q = prediction_inputs(row)
|
|
base = c['seed0_predictions'][case]
|
|
with np.load(checked(base['path'],base['sha256']), allow_pickle=False) as old:
|
|
ix = paired_query(data['raw_ids'], q, old)
|
|
if len(seconds) != row['nodes'] or len(ix) != base['query_nodes']:
|
|
raise ValueError('Prediction support mismatch')
|
|
path = dest/f'{case}.npz'
|
|
np.savez_compressed(path, seconds=seconds, raw_ids=data['raw_ids'], pred=seconds[ix],
|
|
query_raw_ids=q['raw_ids'], counts=q['counts'],
|
|
coordinates_over_Rg=q['coordinates_over_Rg'], near_wall=q['near_wall'])
|
|
entries.append(dict(case=case, component_id=row['component_id'], fold=fold, path=str(path),
|
|
sha256=sha(path), nodes=row['nodes'], query_nodes=len(ix), seed0_sha256=base['sha256'],
|
|
raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'],
|
|
negative_full_predictions=int(np.sum(seconds<0)), negative_query_predictions=int(np.sum(seconds[ix]<0))))
|
|
save(dest/'progress.json', dict(stage='TEST_PREDICTION_NO_TARGETS', **record()))
|
|
del data, residual, seconds
|
|
save(dest/'manifest.json', dict(entries=entries, test_targets_read=False))
|
|
save(dest/'result.json', dict(status='COMPLETE', **record()))
|
|
except BaseException as exc:
|
|
save(dest/'failure.json', dict(status='FAILED', reason=f'{type(exc).__name__}: {exc}', **record()))
|
|
raise
|
|
|
|
|
|
def same_rows(rows, by, entries):
|
|
expected = {(e['case'], e['component_id'], e['fold']) for e in entries}
|
|
observed = {(r['case'], r['component_id'], r['fold']) for r in rows}
|
|
if len(rows) != 552 or observed != expected or {r['case'] for r in rows} != set(by):
|
|
raise ValueError('Reference score roster/pairing differs')
|
|
return {r['case']:r for r in rows}
|
|
|
|
|
|
def seed_reference(by, entries, old_rows, regional):
|
|
if not (SEED/'post-terminal-cost.json').exists():
|
|
return None, {}, None
|
|
terminal = verify(SEED, 'terminal')
|
|
pred_proof = verify(SEED, 'predictions')
|
|
result_path = SEED/'score/result.json'
|
|
manifest = SEED/'predictions-manifest.json'
|
|
done_path = SEED/'complete.json'
|
|
done = read(frozen_file(terminal, done_path, sha(done_path)))
|
|
cost = read(SEED/'post-terminal-cost.json')
|
|
if done['status'] != 'COMPLETE' or cost['within_cap'] is not True:
|
|
return None, {}, None
|
|
score = read(frozen_file(terminal, result_path, sha(result_path)))
|
|
if score['status'] != 'COMPLETE' or not score['review_sha256'].startswith('7b4da089'):
|
|
raise ValueError('Wrong seed1 result or review')
|
|
srows = same_rows(score['rows'], by, entries)
|
|
if any(srows[k]['seed0'] != r['H2'] for k,r in old_rows.items()):
|
|
raise ValueError('Seed1 did not use the same frozen seed0 metrics')
|
|
pooled = seed_summary(score['rows'])['pooled']
|
|
if pooled != score['pooled']:
|
|
raise ValueError('Seed paired summaries differ from frozen result')
|
|
manifest_entries = read(frozen_file(pred_proof, manifest, sha(manifest)))['entries']
|
|
s_preds = same_rows(manifest_entries, by, entries) if regional else {}
|
|
if regional:
|
|
for e in entries:
|
|
s = s_preds[e['case']]
|
|
if any(s[k] != e[k] for k in ('seed0_sha256','raw_ids_sha256','nodes','query_nodes')):
|
|
raise ValueError('Seed full-domain prediction identity differs')
|
|
checked(s['path'], s['sha256'])
|
|
return pooled, s_preds, dict(terminal_commit=terminal['commit'], score_sha256=sha(result_path),
|
|
prediction_manifest_sha256=sha(manifest))
|
|
|
|
|
|
def full_prediction(z, raw, ix, e):
|
|
seconds = z['seconds']
|
|
if (seconds.shape != raw.shape or not np.array_equal(z['raw_ids'],raw)
|
|
or not np.array_equal(z['pred'], seconds[ix]) or not np.isfinite(seconds).all()
|
|
or int(np.sum(seconds<0)) != e['negative_full_predictions']
|
|
or int(np.sum(seconds[ix]<0)) != e['negative_query_predictions']):
|
|
raise ValueError('Nonfinite or inconsistent frozen full/query prediction')
|
|
return seconds
|
|
|
|
|
|
def tail_diagnostics(pilot, case, raw, q, ix, truth, predictions):
|
|
geom = pilot['geometry'][case]
|
|
with np.load(checked(geom['path'], geom['sha256']), allow_pickle=False) as z:
|
|
points, mass, rg = z['points'], z['mass'], float(z['rg'])
|
|
if not np.array_equal(z['raw_ids'],raw) or not np.array_equal(mass,original_cache(pilot,case,'mass')):
|
|
raise ValueError('Frozen geometry IDs or lumped masses differ')
|
|
truth = np.asarray(truth, np.float64)
|
|
w = weights(mass)
|
|
a, td = tail_membership(truth,w)
|
|
arms = {}
|
|
for arm, field in predictions.items():
|
|
p, pd = tail_membership(field,w)
|
|
arms[arm] = dict(prediction=pd, overlap=tail_error(a,p,w),
|
|
full=peak_diagnostic(truth,field,points,raw,rg,w=w,true_tail=a),
|
|
query=peak_diagnostic(truth[ix],field[ix],points[ix],q['raw_ids'],rg))
|
|
return dict(truth=td, arms=arms, Rg=rg)
|
|
|
|
|
|
def score(root):
|
|
c, tc, pilot, by = config(root)
|
|
verify(root,'models')
|
|
if read(root/'models-adopted.json') != c['training']:
|
|
raise ValueError('Adopted model evidence differs')
|
|
proof = verify(root,'predictions')
|
|
manifest = root/'predictions-manifest.json'
|
|
entries = read(frozen_file(proof,manifest,sha(manifest)))['entries']
|
|
prediction_identity(root,c,tc,by,entries)
|
|
os.sched_setaffinity(0,c['worker_cpus'])
|
|
item = c['item']
|
|
baseline_path = BASELINE/'score/result.json'
|
|
baseline = read(checked(baseline_path,c['source_sha256'][str(baseline_path)]))
|
|
if baseline['status'] != 'COMPLETE':
|
|
raise ValueError('Original seed0 score incomplete')
|
|
old = same_rows(baseline['rows'],by,entries)
|
|
seed, seed_preds, seed_evidence = seed_reference(by,entries,old,item=='c')
|
|
dest = root/'score'
|
|
dest.mkdir(exist_ok=False)
|
|
save(dest/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'),
|
|
predictions_commit=proof['commit'],before_target_values=True,seed_reference=seed_evidence))
|
|
rows = []
|
|
for e in entries:
|
|
case = e['case']
|
|
row = by[case]
|
|
raw = cached(row,'raw_ids')
|
|
base = c['seed0_predictions'][case]
|
|
r = dict(case=case, component_id=e['component_id'], fold=e['fold'], seed0=old[case]['H2'])
|
|
with np.load(e['path'],allow_pickle=False) as z, np.load(base['path'],allow_pickle=False) as original:
|
|
q = {k:z['query_raw_ids' if k=='raw_ids' else k] for k in
|
|
('raw_ids','counts','coordinates_over_Rg','near_wall')}
|
|
ix = paired_query(raw,q,original)
|
|
candidate = full_prediction(z,raw,ix,e)
|
|
predicted = {item:candidate, 'seed0':full_prediction(original,raw,ix,base)}
|
|
if seed_preds:
|
|
s = seed_preds[case]
|
|
with np.load(s['path'],allow_pickle=False) as replica:
|
|
if not np.array_equal(paired_query(raw,q,replica),ix):
|
|
raise ValueError('Seed1 query differs')
|
|
predicted['seed1'] = full_prediction(replica,raw,ix,s)
|
|
# Fail closed before opening the one existing full-node target cache.
|
|
with (dest/'target-access.jsonl').open('a') as receipt:
|
|
receipt.write(json.dumps(dict(case=case,fold=e['fold'],role='test',
|
|
target_sha256=row['v5_cache']['output_sha256']['seconds.npy'], unix=time.time()))+'\n')
|
|
truth = cached(row,'seconds')
|
|
if truth.shape != raw.shape or not np.isfinite(truth).all():
|
|
raise ValueError('Wrong full-domain target')
|
|
r[item] = part_metrics(candidate[ix],truth[ix],q['counts'],q['near_wall'],
|
|
q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
|
|
if item == 'c':
|
|
scale = pilot['parts'][case]['t_set']
|
|
if not np.isfinite(scale) or scale <= 0:
|
|
raise ValueError('Original positive time scale required')
|
|
r['tail'] = tail_diagnostics(pilot,case,raw,q,ix,np.asarray(truth,np.float64)/scale,
|
|
{a:np.asarray(p,np.float64)/scale for a,p in predicted.items()})
|
|
else:
|
|
r['context_valid'] = tc['context'][case]['status']=='GEOMETRY_COMPLETE'
|
|
rows.append(r)
|
|
save(dest/'progress.json',dict(parts=len(rows),unique_test_target_reads=len(rows)))
|
|
metrics = comparisons(rows,item)
|
|
result = dict(status='COMPLETE',item=item,parts=552,components=204,rows=rows,pooled=metrics,
|
|
per_fold={str(f):comparisons([r for r in rows if r['fold']==f],item) for f in range(5)},
|
|
decision=decision(item,metrics,seed),same_endpoint_seed_reference=seed,seed_evidence=seed_evidence,
|
|
unique_test_target_reads=552,new_decodes=0,new_fits=0,negative_clipping=False,
|
|
negative_predictions={arm:dict(full_nodes=sum(e['negative_full_predictions'] for e in es),
|
|
query_nodes=sum(e['negative_query_predictions'] for e in es)) for arm,es in
|
|
[(item,entries),('seed0',list(c['seed0_predictions'].values()))]})
|
|
if item == 'c':
|
|
result['tail'] = tail_comparisons(rows,item,bool(seed_preds))
|
|
else:
|
|
result['context_strata'] = {}
|
|
for valid in (True,False):
|
|
subset = [r for r in rows if r['context_valid'] is valid]
|
|
result['context_strata'][str(valid)] = dict(parts=len(subset),
|
|
components=len({r['component_id'] for r in subset}),metrics=comparisons(subset,item))
|
|
result['uncached_partition_runtime'] = 'NOT_MEASURED'
|
|
save(dest/'result.json',result)
|
|
|
|
|
|
if __name__=='__main__':
|
|
mode, root = sys.argv[1], Path(sys.argv[2])
|
|
if mode=='predict' and len(sys.argv)==4:
|
|
predict(root,int(sys.argv[3]))
|
|
elif mode=='score' and len(sys.argv)==3:
|
|
score(root)
|
|
else:
|
|
raise ValueError('Use predict root fold or score root')
|