63 lines
3.2 KiB
Python
63 lines
3.2 KiB
Python
"""Shared-mode extrapolation with queue waiting charged separately from training."""
|
|
|
|
import math
|
|
|
|
|
|
MARGIN = 1.25
|
|
SETUP = 60
|
|
CLEANUP = 120
|
|
USER_CAP = 180000
|
|
RULING = '1795022093.308479'
|
|
|
|
|
|
def full_fold_bound(epoch_seconds, validation_seconds):
|
|
if not math.isfinite(epoch_seconds) or epoch_seconds <= 0:
|
|
raise ValueError('Positive measured epoch time required')
|
|
if not math.isfinite(validation_seconds) or validation_seconds < 0:
|
|
raise ValueError('Invalid validation time')
|
|
return SETUP + MARGIN * (200 * epoch_seconds + 20 * validation_seconds) + CLEANUP
|
|
|
|
|
|
def project(contract, charges, waits, histories, terminal):
|
|
folds = set(range(5))
|
|
if set(charges) != folds or set(waits) != folds or set(histories) - folds or set(terminal) - folds:
|
|
raise ValueError('Cost ledger fold identity')
|
|
rows = {}
|
|
for i, charged in charges.items():
|
|
waiting = waits[i]
|
|
if not math.isfinite(charged) or not math.isfinite(waiting) or not 0 <= waiting <= charged:
|
|
raise ValueError('Invalid queue and waiting charges')
|
|
elapsed = charged - waiting
|
|
job = contract['jobs'][i]
|
|
history = histories.get(i, [])
|
|
if i in terminal:
|
|
training = elapsed
|
|
elif history:
|
|
epochs = sorted(r['epoch'] for r in history)
|
|
finished = epochs[-1]
|
|
if epochs != list(range(1, finished + 1)) or finished > 200:
|
|
raise ValueError('Incomplete epoch history')
|
|
observed = [r['train_epoch_seconds'] for r in history]
|
|
diagnostics = [r.get('validation_seconds', 0) for r in history]
|
|
if any(not math.isfinite(x) or x <= 0 for x in observed):
|
|
raise ValueError('Invalid measured epoch')
|
|
if any(not math.isfinite(x) or x < 0 for x in diagnostics):
|
|
raise ValueError('Invalid validation time')
|
|
epoch_time = max(job['shared_epoch_seconds_bound'], *observed)
|
|
validation_time = max(job['validation_seconds_bound'], *diagnostics)
|
|
training = elapsed + MARGIN * ((200-finished)*epoch_time + (20-finished//10)*validation_time) + CLEANUP
|
|
else:
|
|
training = max(elapsed, job['full_training_gpu_seconds_bound'])
|
|
rows[str(i)] = dict(charged=charged, waiting_gpu_seconds=waiting,
|
|
actual_training_with_setup_gpu_seconds=elapsed,
|
|
projected_training_gpu_seconds=training, projected=waiting+training)
|
|
|
|
actual = contract['preparation_charge_gpu_seconds'] + sum(charges.values())
|
|
projected = (contract['preparation_charge_gpu_seconds'] + contract['postfit_gpu_reserve']
|
|
+ sum(row['projected'] for row in rows.values()))
|
|
ok = projected <= contract['gpu_cap'] and actual + contract['postfit_gpu_reserve'] <= contract['gpu_cap']
|
|
ok &= all(charges[i] - waits[i] < contract['runaway_gpu_seconds'] for i in folds - set(terminal))
|
|
return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected,
|
|
actual_training_and_preparation_gpu_seconds=actual, actual_waiting_gpu_seconds=sum(waits.values()),
|
|
folds=rows, postfit_gpu_reserve=contract['postfit_gpu_reserve'], safety_factor=MARGIN)
|