"""Shared-mode extrapolation with queue waiting charged separately from training.""" import math MARGIN = 1.25 SETUP = 60 CLEANUP = 120 USER_CAP = 180000 RULING = '1795022093.308479' def full_fold_bound(epoch_seconds, validation_seconds): if not math.isfinite(epoch_seconds) or epoch_seconds <= 0: raise ValueError('Positive measured epoch time required') if not math.isfinite(validation_seconds) or validation_seconds < 0: raise ValueError('Invalid validation time') return SETUP + MARGIN * (200 * epoch_seconds + 20 * validation_seconds) + CLEANUP def project(contract, charges, waits, histories, terminal): folds = set(range(5)) if set(charges) != folds or set(waits) != folds or set(histories) - folds or set(terminal) - folds: raise ValueError('Cost ledger fold identity') rows = {} for i, charged in charges.items(): waiting = waits[i] if not math.isfinite(charged) or not math.isfinite(waiting) or not 0 <= waiting <= charged: raise ValueError('Invalid queue and waiting charges') elapsed = charged - waiting job = contract['jobs'][i] history = histories.get(i, []) if i in terminal: training = elapsed elif history: epochs = sorted(r['epoch'] for r in history) finished = epochs[-1] if epochs != list(range(1, finished + 1)) or finished > 200: raise ValueError('Incomplete epoch history') observed = [r['train_epoch_seconds'] for r in history] diagnostics = [r.get('validation_seconds', 0) for r in history] if any(not math.isfinite(x) or x <= 0 for x in observed): raise ValueError('Invalid measured epoch') if any(not math.isfinite(x) or x < 0 for x in diagnostics): raise ValueError('Invalid validation time') epoch_time = max(job['shared_epoch_seconds_bound'], *observed) validation_time = max(job['validation_seconds_bound'], *diagnostics) training = elapsed + MARGIN * ((200-finished)*epoch_time + (20-finished//10)*validation_time) + CLEANUP else: training = max(elapsed, job['full_training_gpu_seconds_bound']) rows[str(i)] = dict(charged=charged, waiting_gpu_seconds=waiting, actual_training_with_setup_gpu_seconds=elapsed, projected_training_gpu_seconds=training, projected=waiting+training) actual = contract['preparation_charge_gpu_seconds'] + sum(charges.values()) projected = (contract['preparation_charge_gpu_seconds'] + contract['postfit_gpu_reserve'] + sum(row['projected'] for row in rows.values())) ok = projected <= contract['gpu_cap'] and actual + contract['postfit_gpu_reserve'] <= contract['gpu_cap'] ok &= all(charges[i] - waits[i] < contract['runaway_gpu_seconds'] for i in folds - set(terminal)) return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected, actual_training_and_preparation_gpu_seconds=actual, actual_waiting_gpu_seconds=sum(waits.values()), folds=rows, postfit_gpu_reserve=contract['postfit_gpu_reserve'], safety_factor=MARGIN)