Files
MeltMind/tools/test_phase3_seed1_v2.py
T

92 lines
6.6 KiB
Python

"""CPU-only ledger/projection/observation tests for the new stopping rule."""
import copy
import inspect
import json
from pathlib import Path
import subprocess
import tempfile
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import phase3_seed1_continuation_common as prior
import phase3_seed1_v2_common as new
import phase3_seed1_v2_observation as observation
from prepare_phase3_seed1_v2 import remaining_contract,OLD,PRIOR
class V2Tests(unittest.TestCase):
def setUp(self):
original=dict(jobs=[dict(id=i,gpu_seconds=22320,cpu_seconds=22320,validation_seconds_bound=12) for i in range(5)],phase3_prior_gpu_once=72421.31599938683,phase3_gpu_cap=612000,postfit_gpu_reserve=7200)
def stopped(a,b):return dict(status='STOPPED',never_attempted=[2,3,4],entries=[dict(id=0,gpu_seconds_charged=a),dict(id=1,gpu_seconds_charged=b)],training_gpu_seconds=a+b)
original_copy=copy.deepcopy(original)
self.c=remaining_contract(original,[(OLD,stopped(269.53611274203286,269.5359442579793)),(PRIOR,stopped(4317.711953648948,3625.5293972480576))])
self.assertEqual(original,original_copy)
def est(self,v,charged=100):return dict(projected_gpu_seconds=v,actual_charged_gpu_seconds=charged)
def test_both_old_attempts_once_and_initial_budget(self):
p=new.project_total(self.c,{},{})
self.assertEqual(p['status'],'PASS')
self.assertAlmostEqual(self.c['old_seed1_gpu_once'],8482.313407897018)
self.assertAlmostEqual(p['projected_gpu_seconds']+self.c['old_seed1_gpu_once'],167200)
self.assertAlmostEqual(p['projected_phase3_gpu_seconds'],72421.31599938683+167200)
self.assertAlmostEqual(p['pending_reserve_gpu_seconds'],self.c['train_gpu_cap']/5)
def test_one_measured_fold_sets_pending_median(self):
p=new.project_total(self.c,{0:self.est(24000)},{},{0:100})
self.assertEqual(p['projected_gpu_seconds'],5*24000+7200)
self.assertEqual(p['pending_reserve_gpu_seconds'],24000)
def test_even_median_and_two_started_count_once(self):
p=new.project_total(self.c,{0:self.est(20000),1:self.est(30000)},{},{0:1000,1:2000})
self.assertEqual(p['pending_reserve_gpu_seconds'],25000)
self.assertEqual(p['projected_gpu_seconds'],20000+30000+3*25000+7200)
def test_total_exceed_stops_even_below_runaway(self):
p=new.project_total(self.c,{0:self.est(31000)},{},{0:100})
self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION')
self.assertTrue(all(r['actual_within_runaway'] for r in p['rows']))
def test_projected_above40000_does_not_alone_stop(self):
p=new.project_total(self.c,{0:self.est(45000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:100})
self.assertEqual(p['status'],'PASS')
self.assertEqual(p['projected_gpu_seconds'],56200)
def test_actual_runaway_stops_within_total(self):
p=new.project_total(self.c,{0:self.est(10000)},{i:dict(gpu_seconds_charged=1000) for i in [1,2,3,4]},{0:40000.01})
self.assertEqual(p['status'],'STOP_BUDGET_PROJECTION')
self.assertTrue(p['total_projection_within_cap']);self.assertFalse(p['rows'][0]['actual_within_runaway'])
def test_terminal_actual_replaces_prior_estimate(self):
p=new.project_total(self.c,{0:self.est(45000)},{0:dict(gpu_seconds_charged=20000)},{})
self.assertEqual(p['projected_gpu_seconds'],107200)
self.assertEqual(p['rows'][0]['basis'],'terminal_actual')
def test_active_unmeasured_not_free_and_actual_not_ignored(self):
p=new.project_total(self.c,{0:self.est(20000)},{},{0:100,1:100})
initial=self.c['train_gpu_cap']/5
self.assertEqual(p['rows'][1]['projected_gpu_seconds'],initial)
self.assertEqual(p['pending_reserve_gpu_seconds'],(20000+initial)/2)
q=new.project_total(self.c,{0:self.est(20000)},{},{0:35000})
self.assertEqual(q['rows'][0]['projected_gpu_seconds'],35000)
with self.assertRaises(ValueError):new.project_total(self.c,{0:self.est(float('nan'))},{})
with self.assertRaises(ValueError):new.project_total(self.c,{}, {},{5:100})
def test_projection_math_unchanged(self):
j=self.c['jobs'][0];obs=dict(epoch=34,charged=3620,max_epoch_seconds=134.2113543249434,max_validation_seconds=15)
self.assertEqual(new.project_cell(j,obs)['projected_gpu_seconds'],prior.project_cell(j,obs)['projected_gpu_seconds'])
def test_scientific_entry_and_guard_only_names_changed(self):
b=Path(__file__).resolve().parent
for suffix in ['worker','guard']:
s=(b/f'phase3_seed1_v2_{suffix}.py').read_text().replace('phase3_seed1_v2_common','phase3_seed1_continuation_common').replace('phase3_seed1_v2_worker.py','phase3_seed1_continuation_worker.py')
self.assertEqual(s,(b/f'phase3_seed1_continuation_{suffix}.py').read_text())
for name in ['fold_view','first_epoch_release','dispatchable','idle_eligible_gpus','assigned_job','available_gpus']:
self.assertEqual(inspect.getsource(getattr(new,name)),inspect.getsource(getattr(prior,name)))
for prefix in ['phase3_seed1','phase3_seed1_continuation','phase3_seed1_v2']:
for suffix in ['worker','guard','queue']:self.assertTrue(new.execution_command(['python',f'tools/{prefix}_{suffix}.py']))
def test_epoch_snapshot_marks_foreign_without_intervening(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d]))
response=SimpleNamespace(stdout='GPU-A, 10, python, 800\nGPU-A, 20, pilot.py, 900\nGPU-B, 30, python, 1000\n')
with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=response) as run:
r=observation.epoch_host_observation(root,0,1,'GPU-A')
self.assertEqual(r['status'],'OBSERVED');self.assertEqual([p['pid'] for p in r['gpu_processes'] if p['foreign_on_assigned_gpu']],[20])
self.assertEqual(run.call_args.kwargs['timeout'],5)
self.assertTrue((root/'jobs/0/epoch-host-observations/001.json').exists())
def test_snapshot_error_retained_observation_only(self):
with tempfile.TemporaryDirectory() as d:
with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',side_effect=subprocess.TimeoutExpired('nvidia-smi',5)):
r=observation.epoch_host_observation(Path(d),1,2,'GPU-A')
self.assertEqual(r['status'],'OBSERVATION_INCOMPLETE');self.assertTrue(r['observation_only']);self.assertEqual(r['errors'][0]['stage'],'gpu_process_snapshot')
if __name__=='__main__':unittest.main()