Files
MeltMind/tools/phase3_item_b_shared_budget.py
T

63 lines
3.2 KiB
Python

"""Shared-mode extrapolation with queue waiting charged separately from training."""
import math
MARGIN = 1.25
SETUP = 60
CLEANUP = 120
USER_CAP = 180000
RULING = '1795022093.308479'
def full_fold_bound(epoch_seconds, validation_seconds):
if not math.isfinite(epoch_seconds) or epoch_seconds <= 0:
raise ValueError('Positive measured epoch time required')
if not math.isfinite(validation_seconds) or validation_seconds < 0:
raise ValueError('Invalid validation time')
return SETUP + MARGIN * (200 * epoch_seconds + 20 * validation_seconds) + CLEANUP
def project(contract, charges, waits, histories, terminal):
folds = set(range(5))
if set(charges) != folds or set(waits) != folds or set(histories) - folds or set(terminal) - folds:
raise ValueError('Cost ledger fold identity')
rows = {}
for i, charged in charges.items():
waiting = waits[i]
if not math.isfinite(charged) or not math.isfinite(waiting) or not 0 <= waiting <= charged:
raise ValueError('Invalid queue and waiting charges')
elapsed = charged - waiting
job = contract['jobs'][i]
history = histories.get(i, [])
if i in terminal:
training = elapsed
elif history:
epochs = sorted(r['epoch'] for r in history)
finished = epochs[-1]
if epochs != list(range(1, finished + 1)) or finished > 200:
raise ValueError('Incomplete epoch history')
observed = [r['train_epoch_seconds'] for r in history]
diagnostics = [r.get('validation_seconds', 0) for r in history]
if any(not math.isfinite(x) or x <= 0 for x in observed):
raise ValueError('Invalid measured epoch')
if any(not math.isfinite(x) or x < 0 for x in diagnostics):
raise ValueError('Invalid validation time')
epoch_time = max(job['shared_epoch_seconds_bound'], *observed)
validation_time = max(job['validation_seconds_bound'], *diagnostics)
training = elapsed + MARGIN * ((200-finished)*epoch_time + (20-finished//10)*validation_time) + CLEANUP
else:
training = max(elapsed, job['full_training_gpu_seconds_bound'])
rows[str(i)] = dict(charged=charged, waiting_gpu_seconds=waiting,
actual_training_with_setup_gpu_seconds=elapsed,
projected_training_gpu_seconds=training, projected=waiting+training)
actual = contract['preparation_charge_gpu_seconds'] + sum(charges.values())
projected = (contract['preparation_charge_gpu_seconds'] + contract['postfit_gpu_reserve']
+ sum(row['projected'] for row in rows.values()))
ok = projected <= contract['gpu_cap'] and actual + contract['postfit_gpu_reserve'] <= contract['gpu_cap']
ok &= all(charges[i] - waits[i] < contract['runaway_gpu_seconds'] for i in folds - set(terminal))
return dict(status='PASS' if ok else 'STOP_BUDGET', projected_gpu_seconds=projected,
actual_training_and_preparation_gpu_seconds=actual, actual_waiting_gpu_seconds=sum(waits.values()),
folds=rows, postfit_gpu_reserve=contract['postfit_gpu_reserve'], safety_factor=MARGIN)