63 lines
3.8 KiB
Python
63 lines
3.8 KiB
Python
"""Narrow CPU-only continuation delta checks; no datasets or CUDA execution."""
|
|
import ast
|
|
import copy
|
|
import inspect
|
|
import unittest
|
|
from pathlib import Path
|
|
from phase3_seed1_common import project_cell, project_total
|
|
|
|
TOOLS=Path(__file__).resolve().parent
|
|
|
|
class ContinuationTests(unittest.TestCase):
|
|
def test_worker_guard_queue_only_entry_names_change(self):
|
|
for suffix in ['worker','guard','queue']:
|
|
original=(TOOLS/f'phase3_seed1_{suffix}.py').read_text()
|
|
new=(TOOLS/f'phase3_seed1_continuation_{suffix}.py').read_text()
|
|
restored=new.replace('phase3_seed1_continuation_common','phase3_seed1_common').replace('tools/phase3_seed1_continuation_guard.py','tools/phase3_seed1_guard.py').replace('tools/phase3_seed1_continuation_worker.py','tools/phase3_seed1_worker.py')
|
|
self.assertEqual(original,restored)
|
|
|
|
def test_common_projection_idle_and_release_functions_unchanged(self):
|
|
import phase3_seed1_common as old
|
|
import phase3_seed1_continuation_common as new
|
|
for name in ['fold_view','project_cell','project_total','first_epoch_release','dispatchable','idle_eligible_gpus','assigned_job','available_gpus']:
|
|
self.assertEqual(inspect.getsource(getattr(old,name)),inspect.getsource(getattr(new,name)))
|
|
|
|
def test_duplicate_detection_includes_both_generations(self):
|
|
from phase3_seed1_continuation_common import execution_command
|
|
for prefix in ['phase3_seed1','phase3_seed1_continuation']:
|
|
for suffix in ['queue','guard','worker']:
|
|
self.assertTrue(execution_command(['python',f'tools/{prefix}_{suffix}.py']))
|
|
self.assertFalse(execution_command(['python','pilot.py']))
|
|
|
|
def budget_fixture(self):
|
|
from prepare_phase3_seed1_continuation import remaining_contract
|
|
old=dict(jobs=[dict(id=i,gpu_seconds=22320,cpu_seconds=22320,validation_seconds_bound=12) for i in range(5)],phase3_prior_gpu_once=72421.31599938683,phase3_gpu_cap=612000,postfit_gpu_reserve=7200)
|
|
terminal=dict(status='STOPPED',never_attempted=[2,3,4],entries=[dict(id=0,gpu_seconds_charged=269.53611274203286),dict(id=1,gpu_seconds_charged=269.5359442579793)],training_gpu_seconds=539.0720570000121)
|
|
original=copy.deepcopy(old);c=remaining_contract(old,terminal);self.assertEqual(old,original)
|
|
return c
|
|
|
|
def test_full_pending_caps_include_old_cost_once(self):
|
|
c=self.budget_fixture();p=project_total(c,{}, {})
|
|
self.assertEqual(p['status'],'PASS')
|
|
self.assertAlmostEqual(p['projected_gpu_seconds']+c['old_seed1_gpu_once'],167200,places=8)
|
|
self.assertAlmostEqual(p['projected_phase3_gpu_seconds'],72421.31599938683+167200,places=8)
|
|
self.assertEqual(sum(j['gpu_seconds'] for j in c['jobs']),c['train_gpu_cap'])
|
|
self.assertEqual(c['postfit_gpu_reserve'],7200)
|
|
|
|
def test_settled_new_cost_does_not_double_charge_old(self):
|
|
c=self.budget_fixture();terminal={i:dict(gpu_seconds_charged=10000+i) for i in range(5)}
|
|
p=project_total(c,{},terminal)
|
|
self.assertAlmostEqual(p['projected_gpu_seconds'],57210)
|
|
self.assertAlmostEqual(p['projected_phase3_gpu_seconds'],72421.31599938683+539.0720570000121+57210)
|
|
self.assertEqual(p['status'],'PASS')
|
|
|
|
def test_final_fold_or_total_overrun_stops(self):
|
|
c=self.budget_fixture();estimates={0:dict(projected_gpu_seconds=c['jobs'][0]['gpu_seconds']+.01)}
|
|
self.assertEqual(project_total(c,estimates,{})['status'],'STOP_BUDGET_PROJECTION')
|
|
observation=dict(epoch=2,charged=270,max_epoch_seconds=104.409218447,max_validation_seconds=0)
|
|
self.assertTrue(project_cell(c['jobs'][1],observation)['within_cap'])
|
|
observation['max_epoch_seconds']=140
|
|
self.assertFalse(project_cell(c['jobs'][1],observation)['within_cap'])
|
|
|
|
if __name__=='__main__':unittest.main()
|