"""One R4bc HPO/final job: cached targets scoped to learning/validation only.""" import argparse import json import os from pathlib import Path import time import traceback import joblib import numpy as np import sklearn from sklearn.ensemble import HistGradientBoostingRegressor from threadpoolctl import threadpool_limits, threadpool_info from r4bc_common import (ARMS, HIST, InputViolation, contract, read, checked, sha, save, spec, role_rows, require_role, features, targets, hierarchy_weights, histgbrt_grid, relative_l2, selection_path, job_dir, fingerprint) from gre24_models import xgb_model def assemble(c, job, output, access): rows = role_rows(c, job['fold'], job['pool'], 'learning'); cfg = spec(job['arm']) full = cfg['support'] == 'all' n = sum(r['nodes'] if full else r['preparation']['eligible_unique'] for r in rows) x = np.empty((n, cfg['columns']), dtype=cfg['dtype']); y = np.empty(n, dtype=cfg['dtype']) cursor = 0; counts = []; groups = []; identities = [] for row in rows: require_role(row, job['fold'], 'learning', job['pool']) save(output/'next-access.json', dict(case=row['case'], role='learning', fold=job['fold'], full=full)) xx, q = features(row, job['arm'], full=full) yy, _ = targets(row, job, 'learning', full=full, access=access) if len(xx) != len(yy): raise InputViolation('Feature/target rows differ') end = cursor+len(yy); x[cursor:end] = xx; y[cursor:end] = yy; cursor = end counts.append(q['counts']); groups.append(row['component_id']) identities.append(dict(case=row['case'], nodes=len(yy), query_raw_ids=fingerprint(q['raw_ids']), counts=fingerprint(q['counts']))) save(output/'access.json', access) if cursor != n or not n: raise InputViolation('Empty/miscounted learning rows') w = hierarchy_weights(counts, groups, xgb=job['arm']=='Q') if cfg['weights']=='hierarchy' else np.ones(n, dtype=np.float64) if len(w) != n: raise InputViolation('Weights/support mismatch') save(output/'training-fingerprints.json', dict(x=fingerprint(x), y=fingerprint(y), weights=fingerprint(w), order=identities, weight_sum=float(w.sum()), spec=cfg)) return x, y, w, rows def validate(model, c, job, output, access): rows = role_rows(c, job['fold'], job['pool'], 'validation'); groups = {}; parts = [] for row in rows: require_role(row, job['fold'], 'validation', job['pool']) save(output/'next-access.json', dict(case=row['case'], role='validation', fold=job['fold'], full=False)) x, q = features(row, job['arm'], full=False) _, truth = targets(row, job, 'validation', full=False, access=access) pred = np.asarray(model.predict(x), dtype=np.float64)*row['preparation']['t_set'] if not np.isfinite(pred).all(): raise ValueError('Nonfinite validation prediction') error = relative_l2(pred, truth, q['counts']) if error is None or not np.isfinite(error): raise ValueError('Undefined validation error') groups.setdefault(row['component_id'], []).append(error) parts.append(dict(case=row['case'], error=error)); save(output/'access.json', access) if not groups: raise InputViolation('No validation components') save(output/'validation-parts.json', parts) return float(np.mean([np.mean(v) for v in groups.values()])) def run(root, index): start = time.monotonic(); cpu = time.process_time(); c = contract(root); job = c['jobs'][index] output = job_dir(root, index)/'output'; output.mkdir(exist_ok=False); access = [] save(output/'started.json', dict(pid=os.getpid(), unix=time.time(), job=job, contract_sha256=sha(root/'contract.json'))) try: if sklearn.__version__ != '1.7.2' or np.__version__ != '1.26.4': raise InputViolation('Frozen library version differs') config_index = job['config_index']; selected = None if job['arm'] in HIST and job['kind']=='FINAL': path = selection_path(root, job); selected = read(path) if selected['status'] != 'SELECTED': raise InputViolation('Final without completed selection') for trial in selected['trials']: checked(trial['receipt_path'], trial['receipt_sha256']) config_index = selected['config_index']; save(output/'selection.json', dict(path=str(path), sha256=sha(path), **selected)) with threadpool_limits(1): pools = threadpool_info() if any(p['num_threads']!=1 for p in pools): raise InputViolation('Thread limit violated') save(output/'progress.json', dict(stage='LOAD_LEARNING', job=job)) x, y, weights, rows = assemble(c, job, output, access) loaded_cpu = time.process_time()-cpu if job['arm'] in HIST: params = histgbrt_grid()[config_index]; model = HistGradientBoostingRegressor(**params) else: import xgboost if xgboost.__version__ != '2.1.4': raise InputViolation('XGBoost version differs') model = xgb_model(0, 1); params = {k:('NaN library missing marker' if isinstance(v,float) and np.isnan(v) else v) for k,v in model.get_params().items()} save(output/'progress.json', dict(stage='FIT', job=job, rows=len(y), load_cpu_seconds=loaded_cpu)) before = time.process_time() model.fit(x, y, sample_weight=weights) fitted_cpu = time.process_time()-before if job['arm'] in HIST and model.n_iter_ != params['max_iter']: raise InputViolation('Unplanned early stop') learning_nodes = len(y); del x, y, weights score = None if job['kind']=='HPO': save(output/'progress.json', dict(stage='VALIDATION', job=job, fit_cpu_seconds=fitted_cpu)) score = validate(model, c, job, output, access) path = output/'model.joblib' joblib.dump(model, path, compress=3) if job['arm'] not in HIST: (output/'xgboost-config.json').write_text(model.get_booster().save_config()) result = dict(status='COMPLETE', job=job, config_index=config_index, params=params, validation_score=score, model_path=str(path), model_sha256=sha(path), load_cpu_seconds=loaded_cpu, fit_cpu_seconds=fitted_cpu, worker_cpu_seconds=time.process_time()-cpu, wall_seconds=time.monotonic()-start, learning_nodes=learning_nodes, learning_parts=len(rows), learning_components=len({r['component_id'] for r in rows}), access_sha256=sha(output/'access.json'), threadpools=pools, cpu_affinity=sorted(os.sched_getaffinity(0)), test_labels_read=False, new_decodes=0, retry=False, final_learning_only=job['kind']=='FINAL') save(output/'result.json', result); save(output/'progress.json', dict(stage='COMPLETE', job=job)) return 0 except Exception as exc: save(output/'access.json', access) save(output/'failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc(), global_stop=isinstance(exc, InputViolation), cpu_seconds=time.process_time()-cpu, wall_seconds=time.monotonic()-start, retry=False)) raise if __name__ == '__main__': parser = argparse.ArgumentParser(); parser.add_argument('root', type=Path); parser.add_argument('index', type=int) args = parser.parse_args(); raise SystemExit(run(args.root, args.index))