123 lines
7.6 KiB
Python
123 lines
7.6 KiB
Python
"""One R4bc HPO/final job: cached targets scoped to learning/validation only."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
import traceback
|
|
|
|
import joblib
|
|
import numpy as np
|
|
import sklearn
|
|
from sklearn.ensemble import HistGradientBoostingRegressor
|
|
from threadpoolctl import threadpool_limits, threadpool_info
|
|
from r4bc_common import (ARMS, HIST, InputViolation, contract, read, checked, sha, save,
|
|
spec, role_rows, require_role, features, targets, hierarchy_weights,
|
|
histgbrt_grid, relative_l2, selection_path, job_dir, fingerprint)
|
|
from gre24_models import xgb_model
|
|
|
|
|
|
def assemble(c, job, output, access):
|
|
rows = role_rows(c, job['fold'], job['pool'], 'learning'); cfg = spec(job['arm'])
|
|
full = cfg['support'] == 'all'
|
|
n = sum(r['nodes'] if full else r['preparation']['eligible_unique'] for r in rows)
|
|
x = np.empty((n, cfg['columns']), dtype=cfg['dtype']); y = np.empty(n, dtype=cfg['dtype'])
|
|
cursor = 0; counts = []; groups = []; identities = []
|
|
for row in rows:
|
|
require_role(row, job['fold'], 'learning', job['pool'])
|
|
save(output/'next-access.json', dict(case=row['case'], role='learning', fold=job['fold'], full=full))
|
|
xx, q = features(row, job['arm'], full=full)
|
|
yy, _ = targets(row, job, 'learning', full=full, access=access)
|
|
if len(xx) != len(yy): raise InputViolation('Feature/target rows differ')
|
|
end = cursor+len(yy); x[cursor:end] = xx; y[cursor:end] = yy; cursor = end
|
|
counts.append(q['counts']); groups.append(row['component_id'])
|
|
identities.append(dict(case=row['case'], nodes=len(yy), query_raw_ids=fingerprint(q['raw_ids']),
|
|
counts=fingerprint(q['counts'])))
|
|
save(output/'access.json', access)
|
|
if cursor != n or not n: raise InputViolation('Empty/miscounted learning rows')
|
|
w = hierarchy_weights(counts, groups, xgb=job['arm']=='Q') if cfg['weights']=='hierarchy' else np.ones(n, dtype=np.float64)
|
|
if len(w) != n: raise InputViolation('Weights/support mismatch')
|
|
save(output/'training-fingerprints.json', dict(x=fingerprint(x), y=fingerprint(y), weights=fingerprint(w),
|
|
order=identities, weight_sum=float(w.sum()), spec=cfg))
|
|
return x, y, w, rows
|
|
|
|
|
|
def validate(model, c, job, output, access):
|
|
rows = role_rows(c, job['fold'], job['pool'], 'validation'); groups = {}; parts = []
|
|
for row in rows:
|
|
require_role(row, job['fold'], 'validation', job['pool'])
|
|
save(output/'next-access.json', dict(case=row['case'], role='validation', fold=job['fold'], full=False))
|
|
x, q = features(row, job['arm'], full=False)
|
|
_, truth = targets(row, job, 'validation', full=False, access=access)
|
|
pred = np.asarray(model.predict(x), dtype=np.float64)*row['preparation']['t_set']
|
|
if not np.isfinite(pred).all(): raise ValueError('Nonfinite validation prediction')
|
|
error = relative_l2(pred, truth, q['counts'])
|
|
if error is None or not np.isfinite(error): raise ValueError('Undefined validation error')
|
|
groups.setdefault(row['component_id'], []).append(error)
|
|
parts.append(dict(case=row['case'], error=error)); save(output/'access.json', access)
|
|
if not groups: raise InputViolation('No validation components')
|
|
save(output/'validation-parts.json', parts)
|
|
return float(np.mean([np.mean(v) for v in groups.values()]))
|
|
|
|
|
|
def run(root, index):
|
|
start = time.monotonic(); cpu = time.process_time(); c = contract(root); job = c['jobs'][index]
|
|
output = job_dir(root, index)/'output'; output.mkdir(exist_ok=False); access = []
|
|
save(output/'started.json', dict(pid=os.getpid(), unix=time.time(), job=job, contract_sha256=sha(root/'contract.json')))
|
|
try:
|
|
if sklearn.__version__ != '1.7.2' or np.__version__ != '1.26.4':
|
|
raise InputViolation('Frozen library version differs')
|
|
config_index = job['config_index']; selected = None
|
|
if job['arm'] in HIST and job['kind']=='FINAL':
|
|
path = selection_path(root, job); selected = read(path)
|
|
if selected['status'] != 'SELECTED': raise InputViolation('Final without completed selection')
|
|
for trial in selected['trials']: checked(trial['receipt_path'], trial['receipt_sha256'])
|
|
config_index = selected['config_index']; save(output/'selection.json', dict(path=str(path), sha256=sha(path), **selected))
|
|
with threadpool_limits(1):
|
|
pools = threadpool_info()
|
|
if any(p['num_threads']!=1 for p in pools): raise InputViolation('Thread limit violated')
|
|
save(output/'progress.json', dict(stage='LOAD_LEARNING', job=job))
|
|
x, y, weights, rows = assemble(c, job, output, access)
|
|
loaded_cpu = time.process_time()-cpu
|
|
if job['arm'] in HIST:
|
|
params = histgbrt_grid()[config_index]; model = HistGradientBoostingRegressor(**params)
|
|
else:
|
|
import xgboost
|
|
if xgboost.__version__ != '2.1.4': raise InputViolation('XGBoost version differs')
|
|
model = xgb_model(0, 1); params = {k:('NaN library missing marker' if isinstance(v,float) and np.isnan(v) else v) for k,v in model.get_params().items()}
|
|
save(output/'progress.json', dict(stage='FIT', job=job, rows=len(y), load_cpu_seconds=loaded_cpu))
|
|
before = time.process_time()
|
|
model.fit(x, y, sample_weight=weights)
|
|
fitted_cpu = time.process_time()-before
|
|
if job['arm'] in HIST and model.n_iter_ != params['max_iter']: raise InputViolation('Unplanned early stop')
|
|
learning_nodes = len(y); del x, y, weights
|
|
score = None
|
|
if job['kind']=='HPO':
|
|
save(output/'progress.json', dict(stage='VALIDATION', job=job, fit_cpu_seconds=fitted_cpu))
|
|
score = validate(model, c, job, output, access)
|
|
path = output/'model.joblib'
|
|
joblib.dump(model, path, compress=3)
|
|
if job['arm'] not in HIST: (output/'xgboost-config.json').write_text(model.get_booster().save_config())
|
|
result = dict(status='COMPLETE', job=job, config_index=config_index, params=params,
|
|
validation_score=score, model_path=str(path), model_sha256=sha(path),
|
|
load_cpu_seconds=loaded_cpu, fit_cpu_seconds=fitted_cpu,
|
|
worker_cpu_seconds=time.process_time()-cpu, wall_seconds=time.monotonic()-start,
|
|
learning_nodes=learning_nodes, learning_parts=len(rows),
|
|
learning_components=len({r['component_id'] for r in rows}),
|
|
access_sha256=sha(output/'access.json'), threadpools=pools,
|
|
cpu_affinity=sorted(os.sched_getaffinity(0)), test_labels_read=False,
|
|
new_decodes=0, retry=False, final_learning_only=job['kind']=='FINAL')
|
|
save(output/'result.json', result); save(output/'progress.json', dict(stage='COMPLETE', job=job))
|
|
return 0
|
|
except Exception as exc:
|
|
save(output/'access.json', access)
|
|
save(output/'failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc(),
|
|
global_stop=isinstance(exc, InputViolation), cpu_seconds=time.process_time()-cpu,
|
|
wall_seconds=time.monotonic()-start, retry=False))
|
|
raise
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path); parser.add_argument('index', type=int)
|
|
args = parser.parse_args(); raise SystemExit(run(args.root, args.index))
|