Files
MeltMind/tools/phase3_followup_postfit_recovery_worker.py
T

271 lines
14 KiB
Python

"""Label-free candidate inference and one custody pass after all predictions freeze."""
import json
import os
from pathlib import Path
import sys
import time
import numpy as np
from phase2_b1213_features import cached
from phase2_h1_worker_v2 import checked, save, sha
from phase2_h2_data import cached as original_cache
from phase2_h2_s1_test_data import load_test_part
from phase2_home_freeze_io import verify
from phase2_residual_math import decode_seconds
from phase3_followup_postfit_common import (
BASELINE, SEED, checkpoint_identity, frozen_file, prediction_identity, read,
)
from phase3_followup_postfit_recovery_common import config
from phase3_followup_postfit_statistics import comparisons, decision, tail_comparisons
from phase3_item_b_data import encode, read_context
from phase3_seed1_postfit_statistics import summary as seed_summary
from phase3_seed1_postfit_worker import paired_query
from phase3_seed1_v2_common import fold_view
from s2_custody_common import prediction_inputs
from s2_custody_statistics import part_metrics
from s2_tail_diagnostic_math import peak_diagnostic, tail_error, tail_membership, weights
def predict(root, fold):
import torch
from phase2_h2_data import arguments, to_device
from phase2_h2_model import H2Geo
from phase3_item_b_model import H2Context
c, tc, pilot, by = config(root)
if fold not in range(5):
raise ValueError('Original five prediction folds only')
verify(root, 'models')
if read(root/'models-adopted.json') != c['training']:
raise ValueError('Adopted candidate models differ from binding')
view = fold_view(pilot, fold)
job = tc['jobs'][fold]
m = c['training']['models'][fold]
stats = read(checked(m['standardization'], m['standardization_sha256']))
if (os.environ.get('CUDA_VISIBLE_DEVICES') != c['gpu_uuid']
or os.environ.get('PYTORCH_CUDA_ALLOC_CONF') != 'expandable_segments:True'):
raise ValueError('Wrong assigned device or allocator')
os.sched_setaffinity(0, c['worker_cpus'])
torch.set_num_threads(1)
torch.set_num_interop_threads(1)
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
dest = root/'jobs'/str(fold)/'prediction-records'
dest.mkdir(exist_ok=False)
torch.cuda.set_device(0)
torch.cuda.init()
torch.cuda.reset_peak_memory_stats()
start = time.monotonic()
cpu = time.process_time()
entries = []
def record():
return dict(fold=fold, parts=len(entries), wall_seconds=time.monotonic()-start,
cpu_seconds=time.process_time()-cpu, test_targets_read=False,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),
peak_reserved_bytes=torch.cuda.max_memory_reserved())
try:
blob = torch.load(checked(m['path'], m['sha256']), map_location='cpu', weights_only=False)
checkpoint_identity(blob, job, c)
model = (H2Geo() if c['item']=='c' else H2Context()).cuda()
model.load_state_dict(blob['model'], strict=True)
model.eval()
del blob
save(dest/'environment.json', dict(fold=fold, checkpoint_sha256=m['sha256'], model_dtype='FP32',
gpu_uuid=c['gpu_uuid'], torch=torch.__version__, allocator=str(torch.cuda.memory._snapshot().get('allocator_settings',{}))))
with torch.no_grad():
for case in job['test']:
row = by[case]
data = load_test_part(view, case, stats)
if c['item'] == 'b':
extra, valid = read_context(view, tc, case)
context = encode(extra, valid, stats['context'])
data['x'] = torch.cat([data['x'], torch.from_numpy(context)], dim=1)
data = to_device(data, 'cuda')
residual = model(*arguments(data), checkpoint_blocks=False)
if not bool(torch.isfinite(residual).all()):
raise ValueError('Nonfinite candidate prediction')
seconds = decode_seconds(residual.cpu().numpy(), data['F'], job['c'], data['t_set'])
torch.cuda.synchronize()
q = prediction_inputs(row)
base = c['seed0_predictions'][case]
with np.load(checked(base['path'],base['sha256']), allow_pickle=False) as old:
ix = paired_query(data['raw_ids'], q, old)
if len(seconds) != row['nodes'] or len(ix) != base['query_nodes']:
raise ValueError('Prediction support mismatch')
path = dest/f'{case}.npz'
np.savez_compressed(path, seconds=seconds, raw_ids=data['raw_ids'], pred=seconds[ix],
query_raw_ids=q['raw_ids'], counts=q['counts'],
coordinates_over_Rg=q['coordinates_over_Rg'], near_wall=q['near_wall'])
entries.append(dict(case=case, component_id=row['component_id'], fold=fold, path=str(path),
sha256=sha(path), nodes=row['nodes'], query_nodes=len(ix), seed0_sha256=base['sha256'],
raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'],
negative_full_predictions=int(np.sum(seconds<0)), negative_query_predictions=int(np.sum(seconds[ix]<0))))
save(dest/'progress.json', dict(stage='TEST_PREDICTION_NO_TARGETS', **record()))
del data, residual, seconds
save(dest/'manifest.json', dict(entries=entries, test_targets_read=False))
save(dest/'result.json', dict(status='COMPLETE', **record()))
except BaseException as exc:
save(dest/'failure.json', dict(status='FAILED', reason=f'{type(exc).__name__}: {exc}', **record()))
raise
def same_rows(rows, by, entries):
expected = {(e['case'], e['component_id'], e['fold']) for e in entries}
observed = {(r['case'], r['component_id'], r['fold']) for r in rows}
if len(rows) != 552 or observed != expected or {r['case'] for r in rows} != set(by):
raise ValueError('Reference score roster/pairing differs')
return {r['case']:r for r in rows}
def seed_reference(by, entries, old_rows, regional):
if not (SEED/'post-terminal-cost.json').exists():
return None, {}, None
terminal = verify(SEED, 'terminal')
pred_proof = verify(SEED, 'predictions')
result_path = SEED/'score/result.json'
manifest = SEED/'predictions-manifest.json'
done_path = SEED/'complete.json'
done = read(frozen_file(terminal, done_path, sha(done_path)))
cost = read(SEED/'post-terminal-cost.json')
if done['status'] != 'COMPLETE' or cost['within_cap'] is not True:
return None, {}, None
score = read(frozen_file(terminal, result_path, sha(result_path)))
if score['status'] != 'COMPLETE' or not score['review_sha256'].startswith('7b4da089'):
raise ValueError('Wrong seed1 result or review')
srows = same_rows(score['rows'], by, entries)
if any(srows[k]['seed0'] != r['H2'] for k,r in old_rows.items()):
raise ValueError('Seed1 did not use the same frozen seed0 metrics')
pooled = seed_summary(score['rows'])['pooled']
if pooled != score['pooled']:
raise ValueError('Seed paired summaries differ from frozen result')
manifest_entries = read(frozen_file(pred_proof, manifest, sha(manifest)))['entries']
s_preds = same_rows(manifest_entries, by, entries) if regional else {}
if regional:
for e in entries:
s = s_preds[e['case']]
if any(s[k] != e[k] for k in ('seed0_sha256','raw_ids_sha256','nodes','query_nodes')):
raise ValueError('Seed full-domain prediction identity differs')
checked(s['path'], s['sha256'])
return pooled, s_preds, dict(terminal_commit=terminal['commit'], score_sha256=sha(result_path),
prediction_manifest_sha256=sha(manifest))
def full_prediction(z, raw, ix, e):
seconds = z['seconds']
if (seconds.shape != raw.shape or not np.array_equal(z['raw_ids'],raw)
or not np.array_equal(z['pred'], seconds[ix]) or not np.isfinite(seconds).all()
or int(np.sum(seconds<0)) != e['negative_full_predictions']
or int(np.sum(seconds[ix]<0)) != e['negative_query_predictions']):
raise ValueError('Nonfinite or inconsistent frozen full/query prediction')
return seconds
def tail_diagnostics(pilot, case, raw, q, ix, truth, predictions):
geom = pilot['geometry'][case]
with np.load(checked(geom['path'], geom['sha256']), allow_pickle=False) as z:
points, mass, rg = z['points'], z['mass'], float(z['rg'])
if not np.array_equal(z['raw_ids'],raw) or not np.array_equal(mass,original_cache(pilot,case,'mass')):
raise ValueError('Frozen geometry IDs or lumped masses differ')
truth = np.asarray(truth, np.float64)
w = weights(mass)
a, td = tail_membership(truth,w)
arms = {}
for arm, field in predictions.items():
p, pd = tail_membership(field,w)
arms[arm] = dict(prediction=pd, overlap=tail_error(a,p,w),
full=peak_diagnostic(truth,field,points,raw,rg,w=w,true_tail=a),
query=peak_diagnostic(truth[ix],field[ix],points[ix],q['raw_ids'],rg))
return dict(truth=td, arms=arms, Rg=rg)
def score(root):
c, tc, pilot, by = config(root)
verify(root,'models')
if read(root/'models-adopted.json') != c['training']:
raise ValueError('Adopted model evidence differs')
proof = verify(root,'predictions')
manifest = root/'predictions-manifest.json'
entries = read(frozen_file(proof,manifest,sha(manifest)))['entries']
prediction_identity(root,c,tc,by,entries)
os.sched_setaffinity(0,c['worker_cpus'])
item = c['item']
baseline_path = BASELINE/'score/result.json'
baseline = read(checked(baseline_path,c['source_sha256'][str(baseline_path)]))
if baseline['status'] != 'COMPLETE':
raise ValueError('Original seed0 score incomplete')
old = same_rows(baseline['rows'],by,entries)
seed, seed_preds, seed_evidence = seed_reference(by,entries,old,item=='c')
dest = root/'score'
dest.mkdir(exist_ok=False)
save(dest/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'),
predictions_commit=proof['commit'],before_target_values=True,seed_reference=seed_evidence))
rows = []
for e in entries:
case = e['case']
row = by[case]
raw = cached(row,'raw_ids')
base = c['seed0_predictions'][case]
r = dict(case=case, component_id=e['component_id'], fold=e['fold'], seed0=old[case]['H2'])
with np.load(e['path'],allow_pickle=False) as z, np.load(base['path'],allow_pickle=False) as original:
q = {k:z['query_raw_ids' if k=='raw_ids' else k] for k in
('raw_ids','counts','coordinates_over_Rg','near_wall')}
ix = paired_query(raw,q,original)
candidate = full_prediction(z,raw,ix,e)
predicted = {item:candidate, 'seed0':full_prediction(original,raw,ix,base)}
if seed_preds:
s = seed_preds[case]
with np.load(s['path'],allow_pickle=False) as replica:
if not np.array_equal(paired_query(raw,q,replica),ix):
raise ValueError('Seed1 query differs')
predicted['seed1'] = full_prediction(replica,raw,ix,s)
# Fail closed before opening the one existing full-node target cache.
with (dest/'target-access.jsonl').open('a') as receipt:
receipt.write(json.dumps(dict(case=case,fold=e['fold'],role='test',
target_sha256=row['v5_cache']['output_sha256']['seconds.npy'], unix=time.time()))+'\n')
truth = cached(row,'seconds')
if truth.shape != raw.shape or not np.isfinite(truth).all():
raise ValueError('Wrong full-domain target')
r[item] = part_metrics(candidate[ix],truth[ix],q['counts'],q['near_wall'],
q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
if item == 'c':
scale = pilot['parts'][case]['t_set']
if not np.isfinite(scale) or scale <= 0:
raise ValueError('Original positive time scale required')
r['tail'] = tail_diagnostics(pilot,case,raw,q,ix,np.asarray(truth,np.float64)/scale,
{a:np.asarray(p,np.float64)/scale for a,p in predicted.items()})
else:
r['context_valid'] = tc['context'][case]['status']=='GEOMETRY_COMPLETE'
rows.append(r)
save(dest/'progress.json',dict(parts=len(rows),unique_test_target_reads=len(rows)))
metrics = comparisons(rows,item)
result = dict(status='COMPLETE',item=item,parts=552,components=204,rows=rows,pooled=metrics,
per_fold={str(f):comparisons([r for r in rows if r['fold']==f],item) for f in range(5)},
decision=decision(item,metrics,seed),same_endpoint_seed_reference=seed,seed_evidence=seed_evidence,
unique_test_target_reads=552,new_decodes=0,new_fits=0,negative_clipping=False,
negative_predictions={arm:dict(full_nodes=sum(e['negative_full_predictions'] for e in es),
query_nodes=sum(e['negative_query_predictions'] for e in es)) for arm,es in
[(item,entries),('seed0',list(c['seed0_predictions'].values()))]})
if item == 'c':
result['tail'] = tail_comparisons(rows,item,bool(seed_preds))
else:
result['context_strata'] = {}
for valid in (True,False):
subset = [r for r in rows if r['context_valid'] is valid]
result['context_strata'][str(valid)] = dict(parts=len(subset),
components=len({r['component_id'] for r in subset}),metrics=comparisons(subset,item))
result['uncached_partition_runtime'] = 'NOT_MEASURED'
save(dest/'result.json',result)
if __name__=='__main__':
mode, root = sys.argv[1], Path(sys.argv[2])
if mode=='predict' and len(sys.argv)==4:
predict(root,int(sys.argv[3]))
elif mode=='score' and len(sys.argv)==3:
score(root)
else:
raise ValueError('Use predict root fold or score root')