92 lines
6.6 KiB
Python
92 lines
6.6 KiB
Python
"""CPU-only ledger/projection/observation tests for the new stopping rule."""
|
|
import copy
|
|
import inspect
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
import phase3_seed1_continuation_common as prior
|
|
import phase3_seed1_v2_common as new
|
|
import phase3_seed1_v2_observation as observation
|
|
from prepare_phase3_seed1_v2 import remaining_contract,OLD,PRIOR
|
|
|
|
class V2Tests(unittest.TestCase):
|
|
def setUp(self):
|
|
original=dict(jobs=[dict(id=i,gpu_seconds=22320,cpu_seconds=22320,validation_seconds_bound=12) for i in range(5)],phase3_prior_gpu_once=72421.31599938683,phase3_gpu_cap=612000,postfit_gpu_reserve=7200)
|
|
def stopped(a,b):return dict(status='STOPPED',never_attempted=[2,3,4],entries=[dict(id=0,gpu_seconds_charged=a),dict(id=1,gpu_seconds_charged=b)],training_gpu_seconds=a+b)
|
|
original_copy=copy.deepcopy(original)
|
|
self.c=remaining_contract(original,[(OLD,stopped(269.53611274203286,269.5359442579793)),(PRIOR,stopped(4317.711953648948,3625.5293972480576))])
|
|
self.assertEqual(original,original_copy)
|
|
def est(self,v,charged=100):return dict(projected_gpu_seconds=v,actual_charged_gpu_seconds=charged)
|
|
def test_both_old_attempts_once_and_initial_budget(self):
|
|
p=new.project_total(self.c,{},{})
|
|
self.assertEqual(p['status'],'PASS')
|
|
self.assertAlmostEqual(self.c['old_seed1_gpu_once'],8482.313407897018)
|
|
self.assertAlmostEqual(p['projected_gpu_seconds']+self.c['old_seed1_gpu_once'],167200)
|
|
self.assertAlmostEqual(p['projected_phase3_gpu_seconds'],72421.31599938683+167200)
|
|
self.assertAlmostEqual(p['pending_reserve_gpu_seconds'],self.c['train_gpu_cap']/5)
|
|
def test_one_measured_fold_sets_pending_median(self):
|
|
p=new.project_total(self.c,{0:self.est(24000)},{},{0:100})
|
|
self.assertEqual(p['projected_gpu_seconds'],5*24000+7200)
|
|
self.assertEqual(p['pending_reserve_gpu_seconds'],24000)
|
|
def test_even_median_and_two_started_count_once(self):
|
|
p=new.project_total(self.c,{0:self.est(20000),1:self.est(30000)},{},{0:1000,1:2000})
|
|
self.assertEqual(p['pending_reserve_gpu_seconds'],25000)
|
|
self.assertEqual(p['projected_gpu_seconds'],20000+30000+3*25000+7200)
|
|
def test_total_exceed_stops_even_below_runaway(self):
|
|
p=new.project_total(self.c,{0:self.est(31000)},{},{0:100})
|
|
self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION')
|
|
self.assertTrue(all(r['actual_within_runaway'] for r in p['rows']))
|
|
def test_projected_above40000_does_not_alone_stop(self):
|
|
p=new.project_total(self.c,{0:self.est(45000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:100})
|
|
self.assertEqual(p['status'],'PASS')
|
|
self.assertEqual(p['projected_gpu_seconds'],56200)
|
|
def test_actual_runaway_stops_within_total(self):
|
|
p=new.project_total(self.c,{0:self.est(10000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:40000.01})
|
|
self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION')
|
|
self.assertTrue(p['total_projection_within_cap']);self.assertFalse(p['rows'][0]['actual_within_runaway'])
|
|
def test_terminal_actual_replaces_prior_estimate(self):
|
|
p=new.project_total(self.c,{0:self.est(45000)},{0:dict(gpu_seconds_charged=20000)},{})
|
|
self.assertEqual(p['projected_gpu_seconds'],107200)
|
|
self.assertEqual(p['rows'][0]['basis'],'terminal_actual')
|
|
def test_active_unmeasured_not_free_and_actual_not_ignored(self):
|
|
p=new.project_total(self.c,{0:self.est(20000)},{},{0:100,1:100})
|
|
initial=self.c['train_gpu_cap']/5
|
|
self.assertEqual(p['rows'][1]['projected_gpu_seconds'],initial)
|
|
self.assertEqual(p['pending_reserve_gpu_seconds'],(20000+initial)/2)
|
|
q=new.project_total(self.c,{0:self.est(20000)},{},{0:35000})
|
|
self.assertEqual(q['rows'][0]['projected_gpu_seconds'],35000)
|
|
with self.assertRaises(ValueError):new.project_total(self.c,{0:self.est(float('nan'))},{})
|
|
with self.assertRaises(ValueError):new.project_total(self.c,{}, {},{5:100})
|
|
def test_projection_math_unchanged(self):
|
|
j=self.c['jobs'][0];obs=dict(epoch=34,charged=3620,max_epoch_seconds=134.2113543249434,max_validation_seconds=15)
|
|
self.assertEqual(new.project_cell(j,obs)['projected_gpu_seconds'],prior.project_cell(j,obs)['projected_gpu_seconds'])
|
|
def test_scientific_entry_and_guard_only_names_changed(self):
|
|
b=Path(__file__).resolve().parent
|
|
for suffix in ['worker','guard']:
|
|
s=(b/f'phase3_seed1_v2_{suffix}.py').read_text().replace('phase3_seed1_v2_common','phase3_seed1_continuation_common').replace('phase3_seed1_v2_worker.py','phase3_seed1_continuation_worker.py')
|
|
self.assertEqual(s,(b/f'phase3_seed1_continuation_{suffix}.py').read_text())
|
|
for name in ['fold_view','first_epoch_release','dispatchable','idle_eligible_gpus','assigned_job','available_gpus']:
|
|
self.assertEqual(inspect.getsource(getattr(new,name)),inspect.getsource(getattr(prior,name)))
|
|
for prefix in ['phase3_seed1','phase3_seed1_continuation','phase3_seed1_v2']:
|
|
for suffix in ['worker','guard','queue']:self.assertTrue(new.execution_command(['python',f'tools/{prefix}_{suffix}.py']))
|
|
def test_epoch_snapshot_marks_foreign_without_intervening(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root=Path(d);own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d]))
|
|
response=SimpleNamespace(stdout='GPU-A, 10, python, 800\nGPU-A, 20, pilot.py, 900\nGPU-B, 30, python, 1000\n')
|
|
with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=response) as run:
|
|
r=observation.epoch_host_observation(root,0,1,'GPU-A')
|
|
self.assertEqual(r['status'],'OBSERVED');self.assertEqual([p['pid'] for p in r['gpu_processes'] if p['foreign_on_assigned_gpu']],[20])
|
|
self.assertEqual(run.call_args.kwargs['timeout'],5)
|
|
self.assertTrue((root/'jobs/0/epoch-host-observations/001.json').exists())
|
|
def test_snapshot_error_retained_observation_only(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',side_effect=subprocess.TimeoutExpired('nvidia-smi',5)):
|
|
r=observation.epoch_host_observation(Path(d),1,2,'GPU-A')
|
|
self.assertEqual(r['status'],'OBSERVATION_INCOMPLETE');self.assertTrue(r['observation_only']);self.assertEqual(r['errors'][0]['stage'],'gpu_process_snapshot')
|
|
|
|
if __name__=='__main__':unittest.main()
|