Files
MeltMind/tools/r4bc_fit.py
T

123 lines
7.6 KiB
Python

"""One R4bc HPO/final job: cached targets scoped to learning/validation only."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
import joblib
import numpy as np
import sklearn
from sklearn.ensemble import HistGradientBoostingRegressor
from threadpoolctl import threadpool_limits, threadpool_info
from r4bc_common import (ARMS, HIST, InputViolation, contract, read, checked, sha, save,
spec, role_rows, require_role, features, targets, hierarchy_weights,
histgbrt_grid, relative_l2, selection_path, job_dir, fingerprint)
from gre24_models import xgb_model
def assemble(c, job, output, access):
rows = role_rows(c, job['fold'], job['pool'], 'learning'); cfg = spec(job['arm'])
full = cfg['support'] == 'all'
n = sum(r['nodes'] if full else r['preparation']['eligible_unique'] for r in rows)
x = np.empty((n, cfg['columns']), dtype=cfg['dtype']); y = np.empty(n, dtype=cfg['dtype'])
cursor = 0; counts = []; groups = []; identities = []
for row in rows:
require_role(row, job['fold'], 'learning', job['pool'])
save(output/'next-access.json', dict(case=row['case'], role='learning', fold=job['fold'], full=full))
xx, q = features(row, job['arm'], full=full)
yy, _ = targets(row, job, 'learning', full=full, access=access)
if len(xx) != len(yy): raise InputViolation('Feature/target rows differ')
end = cursor+len(yy); x[cursor:end] = xx; y[cursor:end] = yy; cursor = end
counts.append(q['counts']); groups.append(row['component_id'])
identities.append(dict(case=row['case'], nodes=len(yy), query_raw_ids=fingerprint(q['raw_ids']),
counts=fingerprint(q['counts'])))
save(output/'access.json', access)
if cursor != n or not n: raise InputViolation('Empty/miscounted learning rows')
w = hierarchy_weights(counts, groups, xgb=job['arm']=='Q') if cfg['weights']=='hierarchy' else np.ones(n, dtype=np.float64)
if len(w) != n: raise InputViolation('Weights/support mismatch')
save(output/'training-fingerprints.json', dict(x=fingerprint(x), y=fingerprint(y), weights=fingerprint(w),
order=identities, weight_sum=float(w.sum()), spec=cfg))
return x, y, w, rows
def validate(model, c, job, output, access):
rows = role_rows(c, job['fold'], job['pool'], 'validation'); groups = {}; parts = []
for row in rows:
require_role(row, job['fold'], 'validation', job['pool'])
save(output/'next-access.json', dict(case=row['case'], role='validation', fold=job['fold'], full=False))
x, q = features(row, job['arm'], full=False)
_, truth = targets(row, job, 'validation', full=False, access=access)
pred = np.asarray(model.predict(x), dtype=np.float64)*row['preparation']['t_set']
if not np.isfinite(pred).all(): raise ValueError('Nonfinite validation prediction')
error = relative_l2(pred, truth, q['counts'])
if error is None or not np.isfinite(error): raise ValueError('Undefined validation error')
groups.setdefault(row['component_id'], []).append(error)
parts.append(dict(case=row['case'], error=error)); save(output/'access.json', access)
if not groups: raise InputViolation('No validation components')
save(output/'validation-parts.json', parts)
return float(np.mean([np.mean(v) for v in groups.values()]))
def run(root, index):
start = time.monotonic(); cpu = time.process_time(); c = contract(root); job = c['jobs'][index]
output = job_dir(root, index)/'output'; output.mkdir(exist_ok=False); access = []
save(output/'started.json', dict(pid=os.getpid(), unix=time.time(), job=job, contract_sha256=sha(root/'contract.json')))
try:
if sklearn.__version__ != '1.7.2' or np.__version__ != '1.26.4':
raise InputViolation('Frozen library version differs')
config_index = job['config_index']; selected = None
if job['arm'] in HIST and job['kind']=='FINAL':
path = selection_path(root, job); selected = read(path)
if selected['status'] != 'SELECTED': raise InputViolation('Final without completed selection')
for trial in selected['trials']: checked(trial['receipt_path'], trial['receipt_sha256'])
config_index = selected['config_index']; save(output/'selection.json', dict(path=str(path), sha256=sha(path), **selected))
with threadpool_limits(1):
pools = threadpool_info()
if any(p['num_threads']!=1 for p in pools): raise InputViolation('Thread limit violated')
save(output/'progress.json', dict(stage='LOAD_LEARNING', job=job))
x, y, weights, rows = assemble(c, job, output, access)
loaded_cpu = time.process_time()-cpu
if job['arm'] in HIST:
params = histgbrt_grid()[config_index]; model = HistGradientBoostingRegressor(**params)
else:
import xgboost
if xgboost.__version__ != '2.1.4': raise InputViolation('XGBoost version differs')
model = xgb_model(0, 1); params = {k:('NaN library missing marker' if isinstance(v,float) and np.isnan(v) else v) for k,v in model.get_params().items()}
save(output/'progress.json', dict(stage='FIT', job=job, rows=len(y), load_cpu_seconds=loaded_cpu))
before = time.process_time()
model.fit(x, y, sample_weight=weights)
fitted_cpu = time.process_time()-before
if job['arm'] in HIST and model.n_iter_ != params['max_iter']: raise InputViolation('Unplanned early stop')
learning_nodes = len(y); del x, y, weights
score = None
if job['kind']=='HPO':
save(output/'progress.json', dict(stage='VALIDATION', job=job, fit_cpu_seconds=fitted_cpu))
score = validate(model, c, job, output, access)
path = output/'model.joblib'
joblib.dump(model, path, compress=3)
if job['arm'] not in HIST: (output/'xgboost-config.json').write_text(model.get_booster().save_config())
result = dict(status='COMPLETE', job=job, config_index=config_index, params=params,
validation_score=score, model_path=str(path), model_sha256=sha(path),
load_cpu_seconds=loaded_cpu, fit_cpu_seconds=fitted_cpu,
worker_cpu_seconds=time.process_time()-cpu, wall_seconds=time.monotonic()-start,
learning_nodes=learning_nodes, learning_parts=len(rows),
learning_components=len({r['component_id'] for r in rows}),
access_sha256=sha(output/'access.json'), threadpools=pools,
cpu_affinity=sorted(os.sched_getaffinity(0)), test_labels_read=False,
new_decodes=0, retry=False, final_learning_only=job['kind']=='FINAL')
save(output/'result.json', result); save(output/'progress.json', dict(stage='COMPLETE', job=job))
return 0
except Exception as exc:
save(output/'access.json', access)
save(output/'failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc(),
global_stop=isinstance(exc, InputViolation), cpu_seconds=time.process_time()-cpu,
wall_seconds=time.monotonic()-start, retry=False))
raise
if __name__ == '__main__':
parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path); parser.add_argument('index', type=int)
args = parser.parse_args(); raise SystemExit(run(args.root, args.index))