From 652fd6825b6f20929d57cae471fee37887385242 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 18 Jun 2024 22:42:43 -0400 Subject: [PATCH 01/22] add ability to pass in multiple request rates --- setup.py | 1 + src/guidellm/executor/__init__.py | 4 ++-- src/guidellm/executor/executor.py | 2 +- src/guidellm/executor/profile_generator.py | 27 +++++++++++----------- src/guidellm/main.py | 6 +++-- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 9075d84..aba9e8d 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ def _setup_long_description() -> Tuple[str, str]: 'openai', 'requests', 'transformers', + 'click' ], extras_require={ 'dev': [ diff --git a/src/guidellm/executor/__init__.py b/src/guidellm/executor/__init__.py index eb0b292..8ad53a2 100644 --- a/src/guidellm/executor/__init__.py +++ b/src/guidellm/executor/__init__.py @@ -3,7 +3,7 @@ Profile, ProfileGenerationModes, ProfileGenerator, - SingleProfileGenerator, + MultiProfileGenerator, SweepProfileGenerator, ) @@ -12,6 +12,6 @@ "ProfileGenerationModes", "Profile", "ProfileGenerator", - "SingleProfileGenerator", + "MultiProfileGenerator", "SweepProfileGenerator", ] diff --git a/src/guidellm/executor/executor.py b/src/guidellm/executor/executor.py index b7b29d1..8d070e8 100644 --- a/src/guidellm/executor/executor.py +++ b/src/guidellm/executor/executor.py @@ -14,7 +14,7 @@ def __init__( self, request_generator: RequestGenerator, backend: Backend, - profile_mode: Union[str, ProfileGenerationModes] = "single", + profile_mode: Union[str, ProfileGenerationModes] = "multi", profile_args: Optional[Dict[str, Any]] = None, max_requests: Optional[int] = None, max_duration: Optional[float] = None, diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 0b9dd7b..b74cf25 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum -from typing import Optional, Union +from typing import List, Optional, Union import numpy @@ -12,13 +12,13 @@ "ProfileGenerationModes", "Profile", "ProfileGenerator", - "SingleProfileGenerator", + "MultiProfileGenerator", "SweepProfileGenerator", ] class ProfileGenerationModes(Enum): - SINGLE = "single" + MULTI = "multi" SWEEP = "sweep" @@ -61,25 +61,24 @@ def next_profile( pass -@ProfileGenerator.register_generator(ProfileGenerationModes.SINGLE) -class SingleProfileGenerator(ProfileGenerator): - def __init__(self, rate: float, rate_type: str, **kwargs): - super().__init__(ProfileGenerationModes.SINGLE) - self._rate = rate +@ProfileGenerator.register_generator(ProfileGenerationModes.MULTI) +class MultiProfileGenerator(ProfileGenerator): + def __init__(self, rate: List[float], rate_type: str, **kwargs): + super().__init__(ProfileGenerationModes.MULTI) + self._rates = rate + self._rate_index = 0 self._rate_type = rate_type self._generated = False def next_profile( self, current_report: TextGenerationBenchmarkReport ) -> Optional[Profile]: - if self._generated: + if self._rate_index >= len(self._rates): return None - self._generated = True - if self._rate_type == "constant": return Profile( - load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=self._rate + load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=self._rates[self._rate_index] ) if self._rate_type == "synchronous": @@ -89,7 +88,7 @@ def next_profile( if self._rate_type == "poisson": return Profile( - load_gen_mode=LoadGenerationModes.POISSON, load_gen_rate=self._rate + load_gen_mode=LoadGenerationModes.POISSON, load_gen_rate=self._rates[self._rate_index] ) raise ValueError(f"Invalid rate type: {self._rate_type}") @@ -151,4 +150,4 @@ def next_profile( load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=rate ) - return None + return None \ No newline at end of file diff --git a/src/guidellm/main.py b/src/guidellm/main.py index adf0aa9..5269cd1 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -46,8 +46,9 @@ @click.option( "--rate", type=float, - default="1.0", + default=[1.0], help="Rate to use for constant and poisson rate types", + multiple=True, ) @click.option( "--num-seconds", @@ -106,10 +107,11 @@ def main( raise ValueError(f"Unknown data type: {data_type}") # Create executor + profile_mode = rate_type if rate_type == "sweep" else "multi" executor = Executor( request_generator=request_generator, backend=backend, - profile_mode=rate_type, + profile_mode=profile_mode, profile_args={"rate_type": rate_type, "rate": rate}, max_requests=num_requests, max_duration=num_seconds, From bfcc329ff452f43927eca31c7b911ad9e1a45775 Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Wed, 26 Jun 2024 16:55:40 +0300 Subject: [PATCH 02/22] running tests command is added to the Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 53f3537..4f8e169 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ style: isort src tests flake8 src tests --max-line-length 88 -# test: -# pytest tests +test: + python -m pytest -s -vvv --cache-clear tests/ build: python setup.py sdist bdist_wheel From 44f8e414f409d7067d059671a8032b6a1aa2c384 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Mon, 1 Jul 2024 22:14:25 -0400 Subject: [PATCH 03/22] rename to fixed rate profile generator --- src/guidellm/executor/__init__.py | 4 ++-- src/guidellm/executor/executor.py | 2 +- src/guidellm/executor/profile_generator.py | 10 +++++----- src/guidellm/main.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/guidellm/executor/__init__.py b/src/guidellm/executor/__init__.py index 8ad53a2..67bfb50 100644 --- a/src/guidellm/executor/__init__.py +++ b/src/guidellm/executor/__init__.py @@ -3,7 +3,7 @@ Profile, ProfileGenerationModes, ProfileGenerator, - MultiProfileGenerator, + FixedRateProfileGenerator, SweepProfileGenerator, ) @@ -12,6 +12,6 @@ "ProfileGenerationModes", "Profile", "ProfileGenerator", - "MultiProfileGenerator", + "FixedRateProfileGenerator", "SweepProfileGenerator", ] diff --git a/src/guidellm/executor/executor.py b/src/guidellm/executor/executor.py index 8d070e8..48061f8 100644 --- a/src/guidellm/executor/executor.py +++ b/src/guidellm/executor/executor.py @@ -14,7 +14,7 @@ def __init__( self, request_generator: RequestGenerator, backend: Backend, - profile_mode: Union[str, ProfileGenerationModes] = "multi", + profile_mode: Union[str, ProfileGenerationModes] = "fixed_rate", profile_args: Optional[Dict[str, Any]] = None, max_requests: Optional[int] = None, max_duration: Optional[float] = None, diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index b74cf25..fc87433 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -12,13 +12,13 @@ "ProfileGenerationModes", "Profile", "ProfileGenerator", - "MultiProfileGenerator", + "FixedRateProfileGenerator", "SweepProfileGenerator", ] class ProfileGenerationModes(Enum): - MULTI = "multi" + FIXED = "fixed_rate" SWEEP = "sweep" @@ -61,10 +61,10 @@ def next_profile( pass -@ProfileGenerator.register_generator(ProfileGenerationModes.MULTI) -class MultiProfileGenerator(ProfileGenerator): +@ProfileGenerator.register_generator(ProfileGenerationModes.FIXED) +class FixedRateProfileGenerator(ProfileGenerator): def __init__(self, rate: List[float], rate_type: str, **kwargs): - super().__init__(ProfileGenerationModes.MULTI) + super().__init__(ProfileGenerationModes.FIXED) self._rates = rate self._rate_index = 0 self._rate_type = rate_type diff --git a/src/guidellm/main.py b/src/guidellm/main.py index 5269cd1..7f436d3 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -107,7 +107,7 @@ def main( raise ValueError(f"Unknown data type: {data_type}") # Create executor - profile_mode = rate_type if rate_type == "sweep" else "multi" + profile_mode = rate_type if rate_type == "sweep" else "fixed_rate" executor = Executor( request_generator=request_generator, backend=backend, From d8469cb3250ea232c18163c1a1f6c12d67f31166 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Mon, 1 Jul 2024 22:15:11 -0400 Subject: [PATCH 04/22] increment rate index --- src/guidellm/executor/profile_generator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index fc87433..6ee40a6 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -75,10 +75,13 @@ def next_profile( ) -> Optional[Profile]: if self._rate_index >= len(self._rates): return None + + current_rate = self._rates[self._rate_index] + self._rate_index += 1 if self._rate_type == "constant": return Profile( - load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=self._rates[self._rate_index] + load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=current_rate ) if self._rate_type == "synchronous": @@ -88,7 +91,7 @@ def next_profile( if self._rate_type == "poisson": return Profile( - load_gen_mode=LoadGenerationModes.POISSON, load_gen_rate=self._rates[self._rate_index] + load_gen_mode=LoadGenerationModes.POISSON, load_gen_rate=current_rate ) raise ValueError(f"Invalid rate type: {self._rate_type}") From eb03706bb48e1a815405a4979777d8d66b223a98 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Mon, 1 Jul 2024 22:23:39 -0400 Subject: [PATCH 05/22] combine constanct and poisson Profile creation --- src/guidellm/executor/profile_generator.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 6ee40a6..9b155c7 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -16,6 +16,10 @@ "SweepProfileGenerator", ] +RateTypeLoadGenModeMap = { + "constant": LoadGenerationModes.CONSTANT, + "poisson": LoadGenerationModes.POISSON, +} class ProfileGenerationModes(Enum): FIXED = "fixed_rate" @@ -79,19 +83,15 @@ def next_profile( current_rate = self._rates[self._rate_index] self._rate_index += 1 - if self._rate_type == "constant": - return Profile( - load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=current_rate - ) - if self._rate_type == "synchronous": return Profile( load_gen_mode=LoadGenerationModes.SYNCHRONOUS, load_gen_rate=None ) - - if self._rate_type == "poisson": + + if self._rate_type in {"constant", "poisson"}: + load_gen_mode = RateTypeLoadGenModeMap[self._rate_type] return Profile( - load_gen_mode=LoadGenerationModes.POISSON, load_gen_rate=current_rate + load_gen_mode=load_gen_mode, load_gen_rate=current_rate ) raise ValueError(f"Invalid rate type: {self._rate_type}") From 9a0c582ee98657673d7e5acfaa2496663bdd7ca8 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Mon, 1 Jul 2024 22:30:39 -0400 Subject: [PATCH 06/22] throw error if user passes in rate in synchronous mode --- src/guidellm/executor/profile_generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 9b155c7..44d30dd 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -69,6 +69,8 @@ def next_profile( class FixedRateProfileGenerator(ProfileGenerator): def __init__(self, rate: List[float], rate_type: str, **kwargs): super().__init__(ProfileGenerationModes.FIXED) + if rate_type == "synchronous" and rate.length > 0: + raise ValueError("custom rates are not supported in synchronous mode") self._rates = rate self._rate_index = 0 self._rate_type = rate_type From fb1ebcb55bb564c775cd8358d104779af02a1c43 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Mon, 1 Jul 2024 22:34:28 -0400 Subject: [PATCH 07/22] refactor to better handle creating profile generator --- src/guidellm/executor/executor.py | 5 ++++- src/guidellm/main.py | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/guidellm/executor/executor.py b/src/guidellm/executor/executor.py index 48061f8..95692b3 100644 --- a/src/guidellm/executor/executor.py +++ b/src/guidellm/executor/executor.py @@ -14,13 +14,16 @@ def __init__( self, request_generator: RequestGenerator, backend: Backend, - profile_mode: Union[str, ProfileGenerationModes] = "fixed_rate", + rate_type: str = "sweep", profile_args: Optional[Dict[str, Any]] = None, max_requests: Optional[int] = None, max_duration: Optional[float] = None, ): self.request_generator = request_generator self.backend = backend + profile_mode = "sweep" + if rate_type in {"synchronous", "constant", "poisson"}: + profile_mode = "fixed_rate" self.profile = ProfileGenerator.create_generator( profile_mode, **(profile_args or {}) ) diff --git a/src/guidellm/main.py b/src/guidellm/main.py index 7f436d3..cec3047 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -107,11 +107,10 @@ def main( raise ValueError(f"Unknown data type: {data_type}") # Create executor - profile_mode = rate_type if rate_type == "sweep" else "fixed_rate" executor = Executor( request_generator=request_generator, backend=backend, - profile_mode=profile_mode, + rate_type=rate_type, profile_args={"rate_type": rate_type, "rate": rate}, max_requests=num_requests, max_duration=num_seconds, From bf724226567244700fb12e84ae7f3df838951b63 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 2 Jul 2024 20:44:19 -0400 Subject: [PATCH 08/22] test base ProfileGenerator class --- src/guidellm/executor/profile_generator.py | 11 ++++---- tests/unit/executor/test_profile_generator.py | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 tests/unit/executor/test_profile_generator.py diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 44d30dd..0fb206b 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -47,9 +47,11 @@ def inner_wrapper(wrapped_class): def create_generator( mode: Union[str, ProfileGenerationModes], **kwargs ) -> "ProfileGenerator": - if isinstance(mode, str): - mode = ProfileGenerationModes(mode) - + mode_is_invalid = not isinstance(mode, str) or mode not in [m.value for m in ProfileGenerationModes] + if mode_is_invalid: + raise ValueError(f"Invalid profile generation mode: {mode}") + mode = ProfileGenerationModes(mode) + if mode not in ProfileGenerator._registry: raise ValueError(f"Invalid profile generation mode: {mode}") @@ -69,12 +71,11 @@ def next_profile( class FixedRateProfileGenerator(ProfileGenerator): def __init__(self, rate: List[float], rate_type: str, **kwargs): super().__init__(ProfileGenerationModes.FIXED) - if rate_type == "synchronous" and rate.length > 0: + if rate_type == "synchronous" and len(rate) > 0: raise ValueError("custom rates are not supported in synchronous mode") self._rates = rate self._rate_index = 0 self._rate_type = rate_type - self._generated = False def next_profile( self, current_report: TextGenerationBenchmarkReport diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py new file mode 100644 index 0000000..af69a3f --- /dev/null +++ b/tests/unit/executor/test_profile_generator.py @@ -0,0 +1,28 @@ +import pytest + +from guidellm.executor import (ProfileGenerator, FixedRateProfileGenerator, SweepProfileGenerator) + +def test_invalid_profile_generation_mode_error(): + rate = [1] + rate_type = "constant" + profile_mode = "burst" + with pytest.raises(ValueError, match=f"Invalid profile generation mode: {profile_mode}"): + ProfileGenerator.create_generator(profile_mode, **({ "rate": rate, "rate_type": rate_type})) + +def test_sweep_profile_generator_creation(): + profile = ProfileGenerator.create_generator("sweep", **({})) + assert isinstance(profile, SweepProfileGenerator) + assert profile._sync_run == False + assert profile._max_found == False + assert profile._pending_rates == None + assert profile._pending_rates == None + +def test_fixed_rate_profile_generator_creation(): + rate = [1] + rate_type = "constant" + profile = ProfileGenerator.create_generator("fixed_rate", **({ "rate": rate, "rate_type": rate_type})) + assert isinstance(profile, FixedRateProfileGenerator) + assert profile._rates == rate + assert profile._rate_type == rate_type + assert profile._rate_index == 0 + assert profile._rate_index == 0 \ No newline at end of file From 98f79c48e0a4016bc5b21afa08b3f0955600b035 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 9 Jul 2024 18:28:17 -0400 Subject: [PATCH 09/22] update fixed rate profile generator, add tests, set up initial executor test --- src/guidellm/executor/profile_generator.py | 21 +++++--- tests/unit/executor/test_executor.py | 17 ++++++ tests/unit/executor/test_profile_generator.py | 54 ++++++++++++++----- 3 files changed, 72 insertions(+), 20 deletions(-) create mode 100644 tests/unit/executor/test_executor.py diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 0fb206b..a9ef77c 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -69,30 +69,37 @@ def next_profile( @ProfileGenerator.register_generator(ProfileGenerationModes.FIXED) class FixedRateProfileGenerator(ProfileGenerator): - def __init__(self, rate: List[float], rate_type: str, **kwargs): + def __init__(self, rate_type: str, rate: Optional[List[float]] = None, **kwargs): super().__init__(ProfileGenerationModes.FIXED) - if rate_type == "synchronous" and len(rate) > 0: + if rate_type == "synchronous" and rate and len(rate) > 0: raise ValueError("custom rates are not supported in synchronous mode") self._rates = rate self._rate_index = 0 + self._generated = False self._rate_type = rate_type def next_profile( self, current_report: TextGenerationBenchmarkReport ) -> Optional[Profile]: - if self._rate_index >= len(self._rates): - return None + if self._rate_type == "synchronous": + if self._generated: + return None - current_rate = self._rates[self._rate_index] - self._rate_index += 1 + self._generated = True - if self._rate_type == "synchronous": return Profile( load_gen_mode=LoadGenerationModes.SYNCHRONOUS, load_gen_rate=None ) if self._rate_type in {"constant", "poisson"}: + if self._rate_index >= len(self._rates): + return None + + current_rate = self._rates[self._rate_index] + self._rate_index += 1 + load_gen_mode = RateTypeLoadGenModeMap[self._rate_type] + return Profile( load_gen_mode=load_gen_mode, load_gen_rate=current_rate ) diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py new file mode 100644 index 0000000..095f5ac --- /dev/null +++ b/tests/unit/executor/test_executor.py @@ -0,0 +1,17 @@ +from unittest.mock import MagicMock +from src.guidellm.backend.base import Backend +from src.guidellm.executor.executor import Executor +from src.guidellm.request.base import RequestGenerator + +def test_executor_creation(): + mock_request_generator = MagicMock(spec=RequestGenerator) + mock_backend = MagicMock(spec=Backend) + rate_type = "sweep" + profile_args = None + max_requests = None, + max_duration = None, + executor = Executor(mock_request_generator, mock_backend, rate_type, profile_args, max_requests, max_duration); + assert executor.request_generator == mock_request_generator + assert executor.backend == mock_backend + assert executor.max_requests == max_requests + assert executor.max_duration == max_duration diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index af69a3f..1c9f084 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -1,6 +1,8 @@ import pytest - +from unittest.mock import MagicMock from guidellm.executor import (ProfileGenerator, FixedRateProfileGenerator, SweepProfileGenerator) +from src.guidellm.core.result import TextGenerationBenchmarkReport +from src.guidellm.scheduler.load_generator import LoadGenerationModes def test_invalid_profile_generation_mode_error(): rate = [1] @@ -10,19 +12,45 @@ def test_invalid_profile_generation_mode_error(): ProfileGenerator.create_generator(profile_mode, **({ "rate": rate, "rate_type": rate_type})) def test_sweep_profile_generator_creation(): - profile = ProfileGenerator.create_generator("sweep", **({})) - assert isinstance(profile, SweepProfileGenerator) - assert profile._sync_run == False - assert profile._max_found == False - assert profile._pending_rates == None - assert profile._pending_rates == None + profile_generator = ProfileGenerator.create_generator("sweep", **({})) + assert isinstance(profile_generator, SweepProfileGenerator) + assert profile_generator._sync_run == False + assert profile_generator._max_found == False + assert profile_generator._pending_rates == None + assert profile_generator._pending_rates == None def test_fixed_rate_profile_generator_creation(): rate = [1] rate_type = "constant" - profile = ProfileGenerator.create_generator("fixed_rate", **({ "rate": rate, "rate_type": rate_type})) - assert isinstance(profile, FixedRateProfileGenerator) - assert profile._rates == rate - assert profile._rate_type == rate_type - assert profile._rate_index == 0 - assert profile._rate_index == 0 \ No newline at end of file + profile_generator = ProfileGenerator.create_generator("fixed_rate", **({ "rate": rate, "rate_type": rate_type})) + assert isinstance(profile_generator, FixedRateProfileGenerator) + assert profile_generator._rates == rate + assert profile_generator._rate_type == rate_type + assert profile_generator._rate_index == 0 + assert profile_generator._rate_index == 0 + +def test_synchronous_mode_rate_list_error(): + rate = [1] + rate_type = "synchronous" + with pytest.raises(ValueError, match="custom rates are not supported in synchronous mode"): + ProfileGenerator.create_generator("fixed_rate", **({ "rate": rate, "rate_type": rate_type})) + +def test_next_profile_with_multiple_rates(): + rates = [1, 2] + rate_type = "constant" + profile_generator = ProfileGenerator.create_generator("fixed_rate", **({ "rate": rates, "rate_type": rate_type})) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + for rate in rates: + current_profile = profile_generator.next_profile(mock_report) + assert current_profile.load_gen_rate == rate + assert current_profile.load_gen_mode.name == LoadGenerationModes.CONSTANT.name + assert profile_generator.next_profile(mock_report) == None + +def test_next_profile_with_sync_mode(): + rate_type = "synchronous" + profile_generator = ProfileGenerator.create_generator("fixed_rate", **({ "rate_type": rate_type})) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + current_profile = profile_generator.next_profile(mock_report) + assert current_profile.load_gen_rate == None + assert current_profile.load_gen_mode.name == LoadGenerationModes.SYNCHRONOUS.name + assert profile_generator.next_profile(mock_report) == None \ No newline at end of file From c0284a3fa2a39bbf0d6a6cf66a7f651738505b35 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Wed, 10 Jul 2024 17:02:41 -0400 Subject: [PATCH 10/22] test executor run method --- tests/unit/executor/test_executor.py | 55 +++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py index 095f5ac..42b7fa1 100644 --- a/tests/unit/executor/test_executor.py +++ b/tests/unit/executor/test_executor.py @@ -1,7 +1,10 @@ -from unittest.mock import MagicMock +import pytest +from unittest.mock import MagicMock, patch from src.guidellm.backend.base import Backend from src.guidellm.executor.executor import Executor +from src.guidellm.executor.profile_generator import Profile, ProfileGenerator from src.guidellm.request.base import RequestGenerator +from src.guidellm.scheduler.load_generator import LoadGenerationModes def test_executor_creation(): mock_request_generator = MagicMock(spec=RequestGenerator) @@ -15,3 +18,53 @@ def test_executor_creation(): assert executor.backend == mock_backend assert executor.max_requests == max_requests assert executor.max_duration == max_duration + + +@pytest.fixture +def mock_request_generator(): + return MagicMock(spec=RequestGenerator) + +@pytest.fixture +def mock_backend(): + return MagicMock(spec=Backend) + +@pytest.fixture +def mock_scheduler(): + with patch('src.guidellm.executor.executor.Scheduler') as MockScheduler: + yield MockScheduler + +def test_executor_run(mock_request_generator, mock_backend, mock_scheduler): + + mock_profile_generator = MagicMock(spec=ProfileGenerator) + profiles = [ + Profile(load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=1.0), + Profile(load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=2.0), + None + ] + mock_profile_generator.next_profile.side_effect = profiles + + with patch('src.guidellm.executor.executor.ProfileGenerator.create_generator', return_value=mock_profile_generator): + executor = Executor( + request_generator=mock_request_generator, + backend=mock_backend, + rate_type="constant", + profile_args={"rate_type": "constant", "rate": [1.0, 2.0]}, + max_requests=10, + max_duration=100 + ) + + mock_benchmark = MagicMock() + mock_scheduler.return_value.run.return_value = mock_benchmark + + report = executor.run() + + + assert mock_scheduler.call_count == 2 + assert len(report.benchmarks) == 2 + assert report.benchmarks[0] == mock_benchmark + assert report.benchmarks[1] == mock_benchmark + calls = mock_scheduler.call_args_list + assert calls[0][1]['load_gen_mode'] == LoadGenerationModes.CONSTANT + assert calls[0][1]['load_gen_rate'] == 1.0 + assert calls[1][1]['load_gen_mode'] == LoadGenerationModes.CONSTANT + assert calls[1][1]['load_gen_rate'] == 2.0 From d91551540e983b3818f722fb3f86be77d0b445e2 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Thu, 11 Jul 2024 17:40:14 -0400 Subject: [PATCH 11/22] test sweep profile generator --- tests/unit/executor/test_profile_generator.py | 91 +++++++++++++++++-- 1 file changed, 82 insertions(+), 9 deletions(-) diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index 1c9f084..8ffb6cc 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -1,7 +1,9 @@ +import numpy import pytest from unittest.mock import MagicMock from guidellm.executor import (ProfileGenerator, FixedRateProfileGenerator, SweepProfileGenerator) -from src.guidellm.core.result import TextGenerationBenchmarkReport +from src.guidellm.core.result import TextGenerationBenchmark, TextGenerationBenchmarkReport +from src.guidellm.executor import profile_generator from src.guidellm.scheduler.load_generator import LoadGenerationModes def test_invalid_profile_generation_mode_error(): @@ -11,13 +13,7 @@ def test_invalid_profile_generation_mode_error(): with pytest.raises(ValueError, match=f"Invalid profile generation mode: {profile_mode}"): ProfileGenerator.create_generator(profile_mode, **({ "rate": rate, "rate_type": rate_type})) -def test_sweep_profile_generator_creation(): - profile_generator = ProfileGenerator.create_generator("sweep", **({})) - assert isinstance(profile_generator, SweepProfileGenerator) - assert profile_generator._sync_run == False - assert profile_generator._max_found == False - assert profile_generator._pending_rates == None - assert profile_generator._pending_rates == None +# Fixed Rate Profile Generator def test_fixed_rate_profile_generator_creation(): rate = [1] @@ -53,4 +49,81 @@ def test_next_profile_with_sync_mode(): current_profile = profile_generator.next_profile(mock_report) assert current_profile.load_gen_rate == None assert current_profile.load_gen_mode.name == LoadGenerationModes.SYNCHRONOUS.name - assert profile_generator.next_profile(mock_report) == None \ No newline at end of file + assert profile_generator.next_profile(mock_report) == None + +# Sweep Profile Generator + +def test_sweep_profile_generator_creation(): + profile_generator = ProfileGenerator.create_generator("sweep", **({})) + assert isinstance(profile_generator, SweepProfileGenerator) + assert profile_generator._sync_run == False + assert profile_generator._max_found == False + assert profile_generator._pending_rates == None + assert profile_generator._pending_rates == None + +def test_first_profile_is_synchronous(): + profile_generator = ProfileGenerator.create_generator("sweep") + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + profile = profile_generator.next_profile(mock_report) + assert profile.load_gen_rate == None + assert profile.load_gen_mode.name == LoadGenerationModes.SYNCHRONOUS.name + +def test_rate_doubles(): + profile_generator = ProfileGenerator.create_generator("sweep") + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + mock_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_benchmark.overloaded = False + mock_benchmark.args_rate = 2.0 + mock_benchmark.request_rate = 2.0 + benchmarks = [ + mock_benchmark + ] + mock_report.benchmarks = benchmarks + profile = profile_generator.next_profile(mock_report) + + profile = profile_generator.next_profile(mock_report) + assert profile.load_gen_rate == 4.0 + +def test_max_found(): + profile_generator = ProfileGenerator.create_generator("sweep") + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + mock_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_benchmark.overloaded = False + mock_benchmark.args_rate = 2.0 + mock_benchmark.request_rate = 2.0 + mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_overloaded_benchmark.overloaded = True + mock_overloaded_benchmark.args_rate = 4.0 + mock_overloaded_benchmark.request_rate = 4.0 + benchmarks = [ + mock_benchmark, + mock_overloaded_benchmark + ] + mock_report.benchmarks = benchmarks + + profile_generator.next_profile(mock_report) + profile = profile_generator.next_profile(mock_report) + + # if benchmark wasn't overloaded, rate would have doubled to 8 + assert profile.load_gen_rate == 2.0 + +def test_pending_rates(): + profile_generator = ProfileGenerator.create_generator("sweep") + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + mock_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_benchmark.overloaded = False + mock_benchmark.args_rate = 2.0 + mock_benchmark.request_rate = 2.0 + mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_overloaded_benchmark.overloaded = True + mock_overloaded_benchmark.args_rate = 8.0 + mock_overloaded_benchmark.request_rate = 8.0 + benchmarks = [ + mock_benchmark, + mock_overloaded_benchmark + ] + mock_report.benchmarks = benchmarks + profile = profile_generator.next_profile(mock_report) + for expected_rate in numpy.linspace(2.0, 8.0, 10): + profile = profile_generator.next_profile(mock_report) + assert profile.load_gen_rate == expected_rate \ No newline at end of file From ffc0e7ba25d2ef0450c1f8aac9fbdd34811c8df8 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Mon, 15 Jul 2024 14:57:19 -0400 Subject: [PATCH 12/22] fix test indentation --- tests/unit/executor/test_executor.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py index 42b7fa1..b4a0ac5 100644 --- a/tests/unit/executor/test_executor.py +++ b/tests/unit/executor/test_executor.py @@ -7,17 +7,17 @@ from src.guidellm.scheduler.load_generator import LoadGenerationModes def test_executor_creation(): - mock_request_generator = MagicMock(spec=RequestGenerator) - mock_backend = MagicMock(spec=Backend) - rate_type = "sweep" - profile_args = None - max_requests = None, - max_duration = None, - executor = Executor(mock_request_generator, mock_backend, rate_type, profile_args, max_requests, max_duration); - assert executor.request_generator == mock_request_generator - assert executor.backend == mock_backend - assert executor.max_requests == max_requests - assert executor.max_duration == max_duration + mock_request_generator = MagicMock(spec=RequestGenerator) + mock_backend = MagicMock(spec=Backend) + rate_type = "sweep" + profile_args = None + max_requests = None, + max_duration = None, + executor = Executor(mock_request_generator, mock_backend, rate_type, profile_args, max_requests, max_duration); + assert executor.request_generator == mock_request_generator + assert executor.backend == mock_backend + assert executor.max_requests == max_requests + assert executor.max_duration == max_duration @pytest.fixture From b41b74ae77f6727f6843231d941cc6f0983a2a21 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Wed, 17 Jul 2024 09:52:56 -0400 Subject: [PATCH 13/22] fix some improper imports --- tests/unit/executor/test_executor.py | 14 +++++++------- tests/unit/executor/test_profile_generator.py | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py index b4a0ac5..80e2751 100644 --- a/tests/unit/executor/test_executor.py +++ b/tests/unit/executor/test_executor.py @@ -1,10 +1,10 @@ import pytest from unittest.mock import MagicMock, patch -from src.guidellm.backend.base import Backend -from src.guidellm.executor.executor import Executor -from src.guidellm.executor.profile_generator import Profile, ProfileGenerator -from src.guidellm.request.base import RequestGenerator -from src.guidellm.scheduler.load_generator import LoadGenerationModes +from guidellm.backend.base import Backend +from guidellm.executor.executor import Executor +from guidellm.executor.profile_generator import Profile, ProfileGenerator +from guidellm.request.base import RequestGenerator +from guidellm.scheduler.load_generator import LoadGenerationModes def test_executor_creation(): mock_request_generator = MagicMock(spec=RequestGenerator) @@ -30,7 +30,7 @@ def mock_backend(): @pytest.fixture def mock_scheduler(): - with patch('src.guidellm.executor.executor.Scheduler') as MockScheduler: + with patch('guidellm.executor.executor.Scheduler') as MockScheduler: yield MockScheduler def test_executor_run(mock_request_generator, mock_backend, mock_scheduler): @@ -43,7 +43,7 @@ def test_executor_run(mock_request_generator, mock_backend, mock_scheduler): ] mock_profile_generator.next_profile.side_effect = profiles - with patch('src.guidellm.executor.executor.ProfileGenerator.create_generator', return_value=mock_profile_generator): + with patch('guidellm.executor.executor.ProfileGenerator.create_generator', return_value=mock_profile_generator): executor = Executor( request_generator=mock_request_generator, backend=mock_backend, diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index 8ffb6cc..a65e995 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -2,9 +2,9 @@ import pytest from unittest.mock import MagicMock from guidellm.executor import (ProfileGenerator, FixedRateProfileGenerator, SweepProfileGenerator) -from src.guidellm.core.result import TextGenerationBenchmark, TextGenerationBenchmarkReport -from src.guidellm.executor import profile_generator -from src.guidellm.scheduler.load_generator import LoadGenerationModes +from guidellm.core.result import TextGenerationBenchmark, TextGenerationBenchmarkReport +from guidellm.executor import profile_generator +from guidellm.scheduler.load_generator import LoadGenerationModes def test_invalid_profile_generation_mode_error(): rate = [1] From 21774cd8160f4ef586e21564f7bf0ce318c5eba6 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 23 Jul 2024 16:03:35 -0400 Subject: [PATCH 14/22] wip --- pyproject.toml | 9 -- src/guidellm/executor/__init__.py | 2 + src/guidellm/executor/profile_generator.py | 22 +++-- src/guidellm/main.py | 8 +- src/guidellm/scheduler/load_generator.py | 2 +- tests/unit/executor/test_executor.py | 20 ++--- tests/unit/executor/test_profile_generator.py | 86 +++++++++---------- 7 files changed, 71 insertions(+), 78 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8746528..382f862 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,12 +99,3 @@ ignore_missing_imports=true line-length = 88 exclude = ["build", "dist", "env", ".venv"] lint.select = ["E", "F", "W"] - - -[tool.pytest.ini_options] -addopts = '-s -vvv --cache-clear --cov-report=term-missing --cov --cov-fail-under=75' -markers = [ - "smoke: quick tests to check basic functionality", - "sanity: detailed tests to ensure major functions work correctly", - "regression: tests to ensure that new changes do not break existing functionality" -] diff --git a/src/guidellm/executor/__init__.py b/src/guidellm/executor/__init__.py index f907217..83ecb31 100644 --- a/src/guidellm/executor/__init__.py +++ b/src/guidellm/executor/__init__.py @@ -1,5 +1,6 @@ from .executor import Executor from .profile_generator import ( + rate_type_to_load_gen_mode, Profile, ProfileGenerationMode, ProfileGenerator, @@ -8,6 +9,7 @@ ) __all__ = [ + "rate_type_to_load_gen_mode", "Executor", "ProfileGenerationMode", "Profile", diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 503b365..aae79d1 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -16,6 +16,12 @@ "SweepProfileGenerator", ] +rate_type_to_load_gen_mode = { + "synchronous": LoadGenerationMode.SYNCHRONOUS, + "constant": LoadGenerationMode.CONSTANT, + "poisson": LoadGenerationMode.POISSON +} + class ProfileGenerationMode(Enum): FIXED_RATE = "fixed_rate" SWEEP = "sweep" @@ -54,37 +60,37 @@ def next(self, current_report: TextGenerationBenchmarkReport) -> Optional[Profil pass -@ProfileGenerator.register_generator(ProfileGenerationMode.FIXED_RATE) +@ProfileGenerator.register(ProfileGenerationMode.FIXED_RATE) class FixedRateProfileGenerator(ProfileGenerator): - def __init__(self, rate_type: LoadGenerationMode, rates: Optional[List[float]] = None, **kwargs): + def __init__(self, load_gen_mode: Optional[LoadGenerationMode], rates: Optional[List[float]] = None, **kwargs): super().__init__(ProfileGenerationMode.FIXED_RATE) - if rate_type == "synchronous" and rates and len(rates) > 0: + if load_gen_mode == "synchronous" and rates and len(rates) > 0: raise ValueError("custom rates are not supported in synchronous mode") self._rates: Optional[List[float]] = rates - self._rate_type: LoadGenerationMode = rate_type + self._load_gen_mode: LoadGenerationMode = load_gen_mode self._generated: bool = False self._rate_index: int = 0 def next( self, current_report: TextGenerationBenchmarkReport ) -> Optional[Profile]: - if self._rate_type.name == LoadGenerationMode.SYNCHRONOUS.name: + if self._load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name: if self._generated: return None self._generated = True return Profile( load_gen_mode=LoadGenerationMode.SYNCHRONOUS, load_gen_rate=None ) - elif self._rate_type.name in {LoadGenerationMode.CONSTANT.name, LoadGenerationMode.POISSON.name}: + elif self._load_gen_mode.name in {LoadGenerationMode.CONSTANT.name, LoadGenerationMode.POISSON.name}: if self._rate_index >= len(self._rates): return None current_rate = self._rates[self._rate_index] self._rate_index += 1 return Profile( - load_gen_mode=self.rate_type, load_gen_rate=current_rate + load_gen_mode=self._load_gen_mode, load_gen_rate=current_rate ) - raise ValueError(f"Invalid rate type: {self._rate_type}") + raise ValueError(f"Invalid rate type: {self._load_gen_mode}") @ProfileGenerator.register(ProfileGenerationMode.SWEEP) diff --git a/src/guidellm/main.py b/src/guidellm/main.py index c556a6a..b9294ad 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -2,13 +2,14 @@ from guidellm.backend import Backend from guidellm.core import TextGenerationBenchmarkReport -from guidellm.executor import Executor +from guidellm.executor import Executor, rate_type_to_load_gen_mode from guidellm.request import ( EmulatedRequestGenerator, FileRequestGenerator, TransformersDatasetRequestGenerator, ) from guidellm.request.base import RequestGenerator +from guidellm.scheduler.load_generator import LoadGenerationMode @click.command() @@ -106,13 +107,14 @@ def main( ) else: raise ValueError(f"Unknown data type: {data_type}") - + + load_gen_mode = rate_type_to_load_gen_mode.get(rate_type, None) # Create executor executor = Executor( request_generator=request_generator, backend=backend, rate_type=rate_type, - profile_args={"rate_type": rate_type, "rate": rate}, + profile_args={"load_gen_mode": load_gen_mode, "rates": rate}, max_requests=num_requests, max_duration=num_seconds, ) diff --git a/src/guidellm/scheduler/load_generator.py b/src/guidellm/scheduler/load_generator.py index 3577951..9ea5198 100644 --- a/src/guidellm/scheduler/load_generator.py +++ b/src/guidellm/scheduler/load_generator.py @@ -16,7 +16,7 @@ class LoadGenerationMode(str, Enum): """ - SYNCHRONOUS = "sync" + SYNCHRONOUS = "synchronous" CONSTANT = "constant" POISSON = "poisson" diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py index 80e2751..1361444 100644 --- a/tests/unit/executor/test_executor.py +++ b/tests/unit/executor/test_executor.py @@ -1,10 +1,10 @@ import pytest from unittest.mock import MagicMock, patch from guidellm.backend.base import Backend -from guidellm.executor.executor import Executor -from guidellm.executor.profile_generator import Profile, ProfileGenerator +from guidellm.executor import Executor +from guidellm.executor import Profile, ProfileGenerator from guidellm.request.base import RequestGenerator -from guidellm.scheduler.load_generator import LoadGenerationModes +from guidellm.scheduler import LoadGenerationMode def test_executor_creation(): mock_request_generator = MagicMock(spec=RequestGenerator) @@ -13,7 +13,7 @@ def test_executor_creation(): profile_args = None max_requests = None, max_duration = None, - executor = Executor(mock_request_generator, mock_backend, rate_type, profile_args, max_requests, max_duration); + executor = Executor(mock_backend, mock_request_generator, rate_type, profile_args, max_requests, max_duration); assert executor.request_generator == mock_request_generator assert executor.backend == mock_backend assert executor.max_requests == max_requests @@ -37,13 +37,13 @@ def test_executor_run(mock_request_generator, mock_backend, mock_scheduler): mock_profile_generator = MagicMock(spec=ProfileGenerator) profiles = [ - Profile(load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=1.0), - Profile(load_gen_mode=LoadGenerationModes.CONSTANT, load_gen_rate=2.0), + Profile(load_gen_mode=LoadGenerationMode.CONSTANT, load_gen_rate=1.0), + Profile(load_gen_mode=LoadGenerationMode.CONSTANT, load_gen_rate=2.0), None ] - mock_profile_generator.next_profile.side_effect = profiles + mock_profile_generator.next.side_effect = profiles - with patch('guidellm.executor.executor.ProfileGenerator.create_generator', return_value=mock_profile_generator): + with patch('guidellm.executor.executor.ProfileGenerator.create', return_value=mock_profile_generator): executor = Executor( request_generator=mock_request_generator, backend=mock_backend, @@ -64,7 +64,7 @@ def test_executor_run(mock_request_generator, mock_backend, mock_scheduler): assert report.benchmarks[0] == mock_benchmark assert report.benchmarks[1] == mock_benchmark calls = mock_scheduler.call_args_list - assert calls[0][1]['load_gen_mode'] == LoadGenerationModes.CONSTANT + assert calls[0][1]['load_gen_mode'] == LoadGenerationMode.CONSTANT assert calls[0][1]['load_gen_rate'] == 1.0 - assert calls[1][1]['load_gen_mode'] == LoadGenerationModes.CONSTANT + assert calls[1][1]['load_gen_mode'] == LoadGenerationMode.CONSTANT assert calls[1][1]['load_gen_rate'] == 2.0 diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index a65e995..804cdc6 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -2,59 +2,51 @@ import pytest from unittest.mock import MagicMock from guidellm.executor import (ProfileGenerator, FixedRateProfileGenerator, SweepProfileGenerator) -from guidellm.core.result import TextGenerationBenchmark, TextGenerationBenchmarkReport -from guidellm.executor import profile_generator -from guidellm.scheduler.load_generator import LoadGenerationModes - -def test_invalid_profile_generation_mode_error(): - rate = [1] - rate_type = "constant" - profile_mode = "burst" - with pytest.raises(ValueError, match=f"Invalid profile generation mode: {profile_mode}"): - ProfileGenerator.create_generator(profile_mode, **({ "rate": rate, "rate_type": rate_type})) +from guidellm.core import TextGenerationBenchmark, TextGenerationBenchmarkReport +from guidellm.executor import profile_generator, ProfileGenerationMode +from guidellm.scheduler import LoadGenerationMode # Fixed Rate Profile Generator def test_fixed_rate_profile_generator_creation(): - rate = [1] - rate_type = "constant" - profile_generator = ProfileGenerator.create_generator("fixed_rate", **({ "rate": rate, "rate_type": rate_type})) + rates = [1] + load_gen_mode = LoadGenerationMode.CONSTANT + profile_generator = ProfileGenerator.create(ProfileGenerationMode.FIXED_RATE, **({ "rates": rates, "load_gen_mode": load_gen_mode})) assert isinstance(profile_generator, FixedRateProfileGenerator) - assert profile_generator._rates == rate - assert profile_generator._rate_type == rate_type - assert profile_generator._rate_index == 0 + assert profile_generator._rates == rates + assert profile_generator._load_gen_mode.name == load_gen_mode.name assert profile_generator._rate_index == 0 def test_synchronous_mode_rate_list_error(): - rate = [1] - rate_type = "synchronous" + rates = [1] + load_gen_mode = LoadGenerationMode.SYNCHRONOUS with pytest.raises(ValueError, match="custom rates are not supported in synchronous mode"): - ProfileGenerator.create_generator("fixed_rate", **({ "rate": rate, "rate_type": rate_type})) + ProfileGenerator.create(ProfileGenerationMode.FIXED_RATE, **({ "rates": rates, "load_gen_mode": load_gen_mode})) -def test_next_profile_with_multiple_rates(): +def test_next_with_multiple_rates(): rates = [1, 2] - rate_type = "constant" - profile_generator = ProfileGenerator.create_generator("fixed_rate", **({ "rate": rates, "rate_type": rate_type})) + load_gen_mode = LoadGenerationMode.CONSTANT + profile_generator = ProfileGenerator.create(ProfileGenerationMode.FIXED_RATE, **({ "rates": rates, "load_gen_mode": load_gen_mode})) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - for rate in rates: - current_profile = profile_generator.next_profile(mock_report) - assert current_profile.load_gen_rate == rate - assert current_profile.load_gen_mode.name == LoadGenerationModes.CONSTANT.name - assert profile_generator.next_profile(mock_report) == None + for rates in rates: + current_profile = profile_generator.next(mock_report) + assert current_profile.load_gen_rate == rates + assert current_profile.load_gen_mode.name == LoadGenerationMode.CONSTANT.name + assert profile_generator.next(mock_report) == None -def test_next_profile_with_sync_mode(): - rate_type = "synchronous" - profile_generator = ProfileGenerator.create_generator("fixed_rate", **({ "rate_type": rate_type})) +def test_next_with_sync_mode(): + load_gen_mode = LoadGenerationMode.SYNCHRONOUS + profile_generator = ProfileGenerator.create(ProfileGenerationMode.FIXED_RATE, **({ "load_gen_mode": load_gen_mode})) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - current_profile = profile_generator.next_profile(mock_report) + current_profile = profile_generator.next(mock_report) assert current_profile.load_gen_rate == None - assert current_profile.load_gen_mode.name == LoadGenerationModes.SYNCHRONOUS.name - assert profile_generator.next_profile(mock_report) == None + assert current_profile.load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name + assert profile_generator.next(mock_report) == None # Sweep Profile Generator def test_sweep_profile_generator_creation(): - profile_generator = ProfileGenerator.create_generator("sweep", **({})) + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP, **({})) assert isinstance(profile_generator, SweepProfileGenerator) assert profile_generator._sync_run == False assert profile_generator._max_found == False @@ -62,14 +54,14 @@ def test_sweep_profile_generator_creation(): assert profile_generator._pending_rates == None def test_first_profile_is_synchronous(): - profile_generator = ProfileGenerator.create_generator("sweep") + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - profile = profile_generator.next_profile(mock_report) + profile = profile_generator.next(mock_report) assert profile.load_gen_rate == None - assert profile.load_gen_mode.name == LoadGenerationModes.SYNCHRONOUS.name + assert profile.load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name def test_rate_doubles(): - profile_generator = ProfileGenerator.create_generator("sweep") + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) mock_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_benchmark.overloaded = False @@ -79,13 +71,13 @@ def test_rate_doubles(): mock_benchmark ] mock_report.benchmarks = benchmarks - profile = profile_generator.next_profile(mock_report) + profile = profile_generator.next(mock_report) - profile = profile_generator.next_profile(mock_report) + profile = profile_generator.next(mock_report) assert profile.load_gen_rate == 4.0 def test_max_found(): - profile_generator = ProfileGenerator.create_generator("sweep") + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) mock_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_benchmark.overloaded = False @@ -101,14 +93,14 @@ def test_max_found(): ] mock_report.benchmarks = benchmarks - profile_generator.next_profile(mock_report) - profile = profile_generator.next_profile(mock_report) + profile_generator.next(mock_report) + profile = profile_generator.next(mock_report) - # if benchmark wasn't overloaded, rate would have doubled to 8 + # if benchmark wasn't overloaded, rates would have doubled to 8 assert profile.load_gen_rate == 2.0 def test_pending_rates(): - profile_generator = ProfileGenerator.create_generator("sweep") + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) mock_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_benchmark.overloaded = False @@ -123,7 +115,7 @@ def test_pending_rates(): mock_overloaded_benchmark ] mock_report.benchmarks = benchmarks - profile = profile_generator.next_profile(mock_report) + profile = profile_generator.next(mock_report) for expected_rate in numpy.linspace(2.0, 8.0, 10): - profile = profile_generator.next_profile(mock_report) + profile = profile_generator.next(mock_report) assert profile.load_gen_rate == expected_rate \ No newline at end of file From 1d3a7cf7d9f61655c4a1efc76161b6ca7e2d307e Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 23 Jul 2024 16:09:08 -0400 Subject: [PATCH 15/22] wip --- .../executor/test_report_generation.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration/executor/test_report_generation.py b/tests/integration/executor/test_report_generation.py index 47d3130..fa68309 100644 --- a/tests/integration/executor/test_report_generation.py +++ b/tests/integration/executor/test_report_generation.py @@ -16,10 +16,10 @@ def test_executor_openai_single_report_generation_sync_mode( request_genrator = dummy.services.TestRequestGenerator( tokenizer="bert-base-uncased" ) - profile_generation_mode = ProfileGenerationMode.SINGLE + profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { "rate_type": LoadGenerationMode.SYNCHRONOUS, - "rate": 1.0, + "rate": [1.0], } executor = Executor( @@ -55,10 +55,10 @@ def test_executor_openai_single_report_generation_constant_mode_infinite( request_genrator = dummy.services.TestRequestGenerator( tokenizer="bert-base-uncased" ) - profile_generation_mode = ProfileGenerationMode.SINGLE + profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { "rate_type": LoadGenerationMode.CONSTANT, - "rate": 1.0, + "rate": [1.0], } executor = Executor( @@ -88,10 +88,10 @@ def test_executor_openai_single_report_generation_constant_mode_limited( request_genrator = dummy.services.TestRequestGenerator( tokenizer="bert-base-uncased" ) - profile_generation_mode = ProfileGenerationMode.SINGLE + profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { "rate_type": LoadGenerationMode.CONSTANT, - "rate": 1.0, + "rate": [1.0], } executor = Executor( @@ -124,10 +124,10 @@ def test_executor_openai_single_report_generation_constant_mode_failed( request_genrator = dummy.services.TestRequestGenerator( tokenizer="bert-base-uncased" ) - profile_generation_mode = ProfileGenerationMode.SINGLE + profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { "rate_type": LoadGenerationMode.CONSTANT, - "rate": 1.0, + "rate": [1.0], } executor = Executor( @@ -153,10 +153,10 @@ def test_executor_openai_single_report_generation_constant_mode_cancelled_report request_genrator = dummy.services.TestRequestGenerator( tokenizer="bert-base-uncased" ) - profile_generation_mode = ProfileGenerationMode.SINGLE + profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { "rate_type": LoadGenerationMode.CONSTANT, - "rate": 1.0, + "rate": [1.0], } executor = Executor( From 79c03cefefa9d6b266199162997fde502400a97d Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 23 Jul 2024 18:06:46 -0400 Subject: [PATCH 16/22] fix all broken tests --- src/guidellm/executor/__init__.py | 2 ++ src/guidellm/executor/executor.py | 9 +-------- src/guidellm/executor/profile_generator.py | 7 +++++++ src/guidellm/main.py | 6 ++++-- .../executor/test_report_generation.py | 19 +++++++++---------- tests/unit/executor/test_executor.py | 9 +++++---- tests/unit/executor/test_profile_generator.py | 10 +++++----- .../test_single_profile_generation_mode.py | 7 +++++-- 8 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/guidellm/executor/__init__.py b/src/guidellm/executor/__init__.py index 83ecb31..ee6d68b 100644 --- a/src/guidellm/executor/__init__.py +++ b/src/guidellm/executor/__init__.py @@ -1,6 +1,7 @@ from .executor import Executor from .profile_generator import ( rate_type_to_load_gen_mode, + rate_type_to_profile_mode, Profile, ProfileGenerationMode, ProfileGenerator, @@ -10,6 +11,7 @@ __all__ = [ "rate_type_to_load_gen_mode", + "rate_type_to_profile_mode", "Executor", "ProfileGenerationMode", "Profile", diff --git a/src/guidellm/executor/executor.py b/src/guidellm/executor/executor.py index c01993c..c24010d 100644 --- a/src/guidellm/executor/executor.py +++ b/src/guidellm/executor/executor.py @@ -20,20 +20,13 @@ def __init__( self, backend: Backend, request_generator: RequestGenerator, - rate_type: str = "sweep", + profile_mode: ProfileGenerationMode = ProfileGenerationMode.SWEEP, profile_args: Optional[Dict[str, Any]] = None, max_requests: Optional[int] = None, max_duration: Optional[float] = None, ): self.request_generator = request_generator self.backend = backend - profile_mode = "sweep" - if rate_type in {"synchronous", "constant", "poisson"}: - profile_mode = ProfileGenerationMode.FIXED_RATE - elif rate_type == "sweep": - profile_mode = ProfileGenerationMode.SWEEP - else: - raise ValueError("invalid rate type provided") self.profile_generator: ProfileGenerator = ProfileGenerator.create( profile_mode, **(profile_args or {}) ) diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index aae79d1..51dade0 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -22,10 +22,17 @@ "poisson": LoadGenerationMode.POISSON } + class ProfileGenerationMode(Enum): FIXED_RATE = "fixed_rate" SWEEP = "sweep" +rate_type_to_profile_mode = { + "synchronous": ProfileGenerationMode.FIXED_RATE, + "constant": ProfileGenerationMode.FIXED_RATE, + "poisson": ProfileGenerationMode.FIXED_RATE, + "sweep": ProfileGenerationMode.SWEEP, +} @dataclass class Profile: diff --git a/src/guidellm/main.py b/src/guidellm/main.py index b9294ad..1056343 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -2,7 +2,7 @@ from guidellm.backend import Backend from guidellm.core import TextGenerationBenchmarkReport -from guidellm.executor import Executor, rate_type_to_load_gen_mode +from guidellm.executor import Executor, rate_type_to_load_gen_mode, rate_type_to_profile_mode from guidellm.request import ( EmulatedRequestGenerator, FileRequestGenerator, @@ -108,12 +108,14 @@ def main( else: raise ValueError(f"Unknown data type: {data_type}") + profile_mode = rate_type_to_profile_mode(rate_type) load_gen_mode = rate_type_to_load_gen_mode.get(rate_type, None) + # Create executor executor = Executor( request_generator=request_generator, backend=backend, - rate_type=rate_type, + profile_mode=profile_mode, profile_args={"load_gen_mode": load_gen_mode, "rates": rate}, max_requests=num_requests, max_duration=num_seconds, diff --git a/tests/integration/executor/test_report_generation.py b/tests/integration/executor/test_report_generation.py index fa68309..b71410d 100644 --- a/tests/integration/executor/test_report_generation.py +++ b/tests/integration/executor/test_report_generation.py @@ -18,8 +18,7 @@ def test_executor_openai_single_report_generation_sync_mode( ) profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { - "rate_type": LoadGenerationMode.SYNCHRONOUS, - "rate": [1.0], + "load_gen_mode": LoadGenerationMode.SYNCHRONOUS, } executor = Executor( @@ -57,8 +56,8 @@ def test_executor_openai_single_report_generation_constant_mode_infinite( ) profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { - "rate_type": LoadGenerationMode.CONSTANT, - "rate": [1.0], + "load_gen_mode": LoadGenerationMode.CONSTANT, + "rates": [1.0], } executor = Executor( @@ -90,8 +89,8 @@ def test_executor_openai_single_report_generation_constant_mode_limited( ) profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { - "rate_type": LoadGenerationMode.CONSTANT, - "rate": [1.0], + "load_gen_mode": LoadGenerationMode.CONSTANT, + "rates": [1.0], } executor = Executor( @@ -126,8 +125,8 @@ def test_executor_openai_single_report_generation_constant_mode_failed( ) profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { - "rate_type": LoadGenerationMode.CONSTANT, - "rate": [1.0], + "load_gen_mode": LoadGenerationMode.CONSTANT, + "rates": [1.0], } executor = Executor( @@ -155,8 +154,8 @@ def test_executor_openai_single_report_generation_constant_mode_cancelled_report ) profile_generation_mode = ProfileGenerationMode.FIXED_RATE profile_generator_kwargs = { - "rate_type": LoadGenerationMode.CONSTANT, - "rate": [1.0], + "load_gen_mode": LoadGenerationMode.CONSTANT, + "rates": [1.0], } executor = Executor( diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py index 1361444..2e6e6f5 100644 --- a/tests/unit/executor/test_executor.py +++ b/tests/unit/executor/test_executor.py @@ -3,17 +3,18 @@ from guidellm.backend.base import Backend from guidellm.executor import Executor from guidellm.executor import Profile, ProfileGenerator +from guidellm.executor.profile_generator import ProfileGenerationMode from guidellm.request.base import RequestGenerator from guidellm.scheduler import LoadGenerationMode def test_executor_creation(): mock_request_generator = MagicMock(spec=RequestGenerator) mock_backend = MagicMock(spec=Backend) - rate_type = "sweep" + profile_mode = ProfileGenerationMode.SWEEP profile_args = None max_requests = None, max_duration = None, - executor = Executor(mock_backend, mock_request_generator, rate_type, profile_args, max_requests, max_duration); + executor = Executor(mock_backend, mock_request_generator, profile_mode, profile_args, max_requests, max_duration); assert executor.request_generator == mock_request_generator assert executor.backend == mock_backend assert executor.max_requests == max_requests @@ -47,8 +48,8 @@ def test_executor_run(mock_request_generator, mock_backend, mock_scheduler): executor = Executor( request_generator=mock_request_generator, backend=mock_backend, - rate_type="constant", - profile_args={"rate_type": "constant", "rate": [1.0, 2.0]}, + profile_mode=ProfileGenerationMode.FIXED_RATE, + profile_args={"load_gen_mode": LoadGenerationMode.CONSTANT, "rates": [1.0, 2.0]}, max_requests=10, max_duration=100 ) diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index 804cdc6..001a933 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -65,7 +65,7 @@ def test_rate_doubles(): mock_report = MagicMock(spec=TextGenerationBenchmarkReport) mock_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_benchmark.overloaded = False - mock_benchmark.args_rate = 2.0 + mock_benchmark.rate = 2.0 mock_benchmark.request_rate = 2.0 benchmarks = [ mock_benchmark @@ -81,11 +81,11 @@ def test_max_found(): mock_report = MagicMock(spec=TextGenerationBenchmarkReport) mock_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_benchmark.overloaded = False - mock_benchmark.args_rate = 2.0 + mock_benchmark.rate = 2.0 mock_benchmark.request_rate = 2.0 mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_overloaded_benchmark.overloaded = True - mock_overloaded_benchmark.args_rate = 4.0 + mock_overloaded_benchmark.rate = 4.0 mock_overloaded_benchmark.request_rate = 4.0 benchmarks = [ mock_benchmark, @@ -104,11 +104,11 @@ def test_pending_rates(): mock_report = MagicMock(spec=TextGenerationBenchmarkReport) mock_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_benchmark.overloaded = False - mock_benchmark.args_rate = 2.0 + mock_benchmark.rate = 2.0 mock_benchmark.request_rate = 2.0 mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_overloaded_benchmark.overloaded = True - mock_overloaded_benchmark.args_rate = 8.0 + mock_overloaded_benchmark.rate = 8.0 mock_overloaded_benchmark.request_rate = 8.0 benchmarks = [ mock_benchmark, diff --git a/tests/unit/executor/test_single_profile_generation_mode.py b/tests/unit/executor/test_single_profile_generation_mode.py index 380e3f4..e97bba6 100644 --- a/tests/unit/executor/test_single_profile_generation_mode.py +++ b/tests/unit/executor/test_single_profile_generation_mode.py @@ -24,12 +24,15 @@ def test_executor_single_profile_generator_benchmark_report( request_genrator = dummy.services.TestRequestGenerator( tokenizer="bert-base-uncased" ) - profile_generator_kwargs = {"rate_type": load_gen_mode, "rate": 1.0} + rates = [1.0] + if (load_gen_mode == LoadGenerationMode.SYNCHRONOUS): + rates = None + profile_generator_kwargs = {"load_gen_mode": load_gen_mode, "rates": rates} executor = Executor( backend=openai_backend_factory(), request_generator=request_genrator, - profile_mode=ProfileGenerationMode.SINGLE, + profile_mode=ProfileGenerationMode.FIXED_RATE, profile_args=profile_generator_kwargs, max_requests=1, max_duration=None, From 8833ae39f1f3efa8ea115a9314aea5898a3afb48 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 23 Jul 2024 18:11:30 -0400 Subject: [PATCH 17/22] run make style --- src/guidellm/executor/__init__.py | 6 +- src/guidellm/executor/profile_generator.py | 22 +- src/guidellm/main.py | 10 +- tests/unit/executor/test_executor.py | 52 +++-- tests/unit/executor/test_profile_generator.py | 210 ++++++++++-------- .../test_single_profile_generation_mode.py | 2 +- 6 files changed, 176 insertions(+), 126 deletions(-) diff --git a/src/guidellm/executor/__init__.py b/src/guidellm/executor/__init__.py index ee6d68b..0d17c2f 100644 --- a/src/guidellm/executor/__init__.py +++ b/src/guidellm/executor/__init__.py @@ -1,12 +1,12 @@ from .executor import Executor from .profile_generator import ( - rate_type_to_load_gen_mode, - rate_type_to_profile_mode, + FixedRateProfileGenerator, Profile, ProfileGenerationMode, ProfileGenerator, - FixedRateProfileGenerator, SweepProfileGenerator, + rate_type_to_load_gen_mode, + rate_type_to_profile_mode, ) __all__ = [ diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 51dade0..6b37aff 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -19,7 +19,7 @@ rate_type_to_load_gen_mode = { "synchronous": LoadGenerationMode.SYNCHRONOUS, "constant": LoadGenerationMode.CONSTANT, - "poisson": LoadGenerationMode.POISSON + "poisson": LoadGenerationMode.POISSON, } @@ -27,6 +27,7 @@ class ProfileGenerationMode(Enum): FIXED_RATE = "fixed_rate" SWEEP = "sweep" + rate_type_to_profile_mode = { "synchronous": ProfileGenerationMode.FIXED_RATE, "constant": ProfileGenerationMode.FIXED_RATE, @@ -34,6 +35,7 @@ class ProfileGenerationMode(Enum): "sweep": ProfileGenerationMode.SWEEP, } + @dataclass class Profile: load_gen_mode: LoadGenerationMode @@ -69,7 +71,12 @@ def next(self, current_report: TextGenerationBenchmarkReport) -> Optional[Profil @ProfileGenerator.register(ProfileGenerationMode.FIXED_RATE) class FixedRateProfileGenerator(ProfileGenerator): - def __init__(self, load_gen_mode: Optional[LoadGenerationMode], rates: Optional[List[float]] = None, **kwargs): + def __init__( + self, + load_gen_mode: Optional[LoadGenerationMode], + rates: Optional[List[float]] = None, + **kwargs, + ): super().__init__(ProfileGenerationMode.FIXED_RATE) if load_gen_mode == "synchronous" and rates and len(rates) > 0: raise ValueError("custom rates are not supported in synchronous mode") @@ -78,9 +85,7 @@ def __init__(self, load_gen_mode: Optional[LoadGenerationMode], rates: Optional[ self._generated: bool = False self._rate_index: int = 0 - def next( - self, current_report: TextGenerationBenchmarkReport - ) -> Optional[Profile]: + def next(self, current_report: TextGenerationBenchmarkReport) -> Optional[Profile]: if self._load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name: if self._generated: return None @@ -88,7 +93,10 @@ def next( return Profile( load_gen_mode=LoadGenerationMode.SYNCHRONOUS, load_gen_rate=None ) - elif self._load_gen_mode.name in {LoadGenerationMode.CONSTANT.name, LoadGenerationMode.POISSON.name}: + elif self._load_gen_mode.name in { + LoadGenerationMode.CONSTANT.name, + LoadGenerationMode.POISSON.name, + }: if self._rate_index >= len(self._rates): return None current_rate = self._rates[self._rate_index] @@ -154,4 +162,4 @@ def next(self, current_report: TextGenerationBenchmarkReport) -> Optional[Profil load_gen_mode=LoadGenerationMode.CONSTANT, load_gen_rate=rate ) - return None \ No newline at end of file + return None diff --git a/src/guidellm/main.py b/src/guidellm/main.py index 1056343..921e7db 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -2,7 +2,11 @@ from guidellm.backend import Backend from guidellm.core import TextGenerationBenchmarkReport -from guidellm.executor import Executor, rate_type_to_load_gen_mode, rate_type_to_profile_mode +from guidellm.executor import ( + Executor, + rate_type_to_load_gen_mode, + rate_type_to_profile_mode, +) from guidellm.request import ( EmulatedRequestGenerator, FileRequestGenerator, @@ -107,10 +111,10 @@ def main( ) else: raise ValueError(f"Unknown data type: {data_type}") - + profile_mode = rate_type_to_profile_mode(rate_type) load_gen_mode = rate_type_to_load_gen_mode.get(rate_type, None) - + # Create executor executor = Executor( request_generator=request_generator, diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py index 2e6e6f5..093c1e2 100644 --- a/tests/unit/executor/test_executor.py +++ b/tests/unit/executor/test_executor.py @@ -1,20 +1,29 @@ -import pytest from unittest.mock import MagicMock, patch + +import pytest + from guidellm.backend.base import Backend -from guidellm.executor import Executor -from guidellm.executor import Profile, ProfileGenerator +from guidellm.executor import Executor, Profile, ProfileGenerator from guidellm.executor.profile_generator import ProfileGenerationMode from guidellm.request.base import RequestGenerator from guidellm.scheduler import LoadGenerationMode + def test_executor_creation(): mock_request_generator = MagicMock(spec=RequestGenerator) mock_backend = MagicMock(spec=Backend) profile_mode = ProfileGenerationMode.SWEEP profile_args = None - max_requests = None, - max_duration = None, - executor = Executor(mock_backend, mock_request_generator, profile_mode, profile_args, max_requests, max_duration); + max_requests = (None,) + max_duration = (None,) + executor = Executor( + mock_backend, + mock_request_generator, + profile_mode, + profile_args, + max_requests, + max_duration, + ) assert executor.request_generator == mock_request_generator assert executor.backend == mock_backend assert executor.max_requests == max_requests @@ -25,33 +34,41 @@ def test_executor_creation(): def mock_request_generator(): return MagicMock(spec=RequestGenerator) + @pytest.fixture def mock_backend(): return MagicMock(spec=Backend) + @pytest.fixture def mock_scheduler(): - with patch('guidellm.executor.executor.Scheduler') as MockScheduler: + with patch("guidellm.executor.executor.Scheduler") as MockScheduler: yield MockScheduler -def test_executor_run(mock_request_generator, mock_backend, mock_scheduler): +def test_executor_run(mock_request_generator, mock_backend, mock_scheduler): mock_profile_generator = MagicMock(spec=ProfileGenerator) profiles = [ Profile(load_gen_mode=LoadGenerationMode.CONSTANT, load_gen_rate=1.0), Profile(load_gen_mode=LoadGenerationMode.CONSTANT, load_gen_rate=2.0), - None + None, ] mock_profile_generator.next.side_effect = profiles - - with patch('guidellm.executor.executor.ProfileGenerator.create', return_value=mock_profile_generator): + + with patch( + "guidellm.executor.executor.ProfileGenerator.create", + return_value=mock_profile_generator, + ): executor = Executor( request_generator=mock_request_generator, backend=mock_backend, profile_mode=ProfileGenerationMode.FIXED_RATE, - profile_args={"load_gen_mode": LoadGenerationMode.CONSTANT, "rates": [1.0, 2.0]}, + profile_args={ + "load_gen_mode": LoadGenerationMode.CONSTANT, + "rates": [1.0, 2.0], + }, max_requests=10, - max_duration=100 + max_duration=100, ) mock_benchmark = MagicMock() @@ -59,13 +76,12 @@ def test_executor_run(mock_request_generator, mock_backend, mock_scheduler): report = executor.run() - assert mock_scheduler.call_count == 2 assert len(report.benchmarks) == 2 assert report.benchmarks[0] == mock_benchmark assert report.benchmarks[1] == mock_benchmark calls = mock_scheduler.call_args_list - assert calls[0][1]['load_gen_mode'] == LoadGenerationMode.CONSTANT - assert calls[0][1]['load_gen_rate'] == 1.0 - assert calls[1][1]['load_gen_mode'] == LoadGenerationMode.CONSTANT - assert calls[1][1]['load_gen_rate'] == 2.0 + assert calls[0][1]["load_gen_mode"] == LoadGenerationMode.CONSTANT + assert calls[0][1]["load_gen_rate"] == 1.0 + assert calls[1][1]["load_gen_mode"] == LoadGenerationMode.CONSTANT + assert calls[1][1]["load_gen_rate"] == 2.0 diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index 001a933..233e826 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -1,121 +1,143 @@ +from unittest.mock import MagicMock + import numpy import pytest -from unittest.mock import MagicMock -from guidellm.executor import (ProfileGenerator, FixedRateProfileGenerator, SweepProfileGenerator) + from guidellm.core import TextGenerationBenchmark, TextGenerationBenchmarkReport -from guidellm.executor import profile_generator, ProfileGenerationMode +from guidellm.executor import ( + FixedRateProfileGenerator, + ProfileGenerationMode, + ProfileGenerator, + SweepProfileGenerator, + profile_generator, +) from guidellm.scheduler import LoadGenerationMode # Fixed Rate Profile Generator + def test_fixed_rate_profile_generator_creation(): - rates = [1] - load_gen_mode = LoadGenerationMode.CONSTANT - profile_generator = ProfileGenerator.create(ProfileGenerationMode.FIXED_RATE, **({ "rates": rates, "load_gen_mode": load_gen_mode})) - assert isinstance(profile_generator, FixedRateProfileGenerator) - assert profile_generator._rates == rates - assert profile_generator._load_gen_mode.name == load_gen_mode.name - assert profile_generator._rate_index == 0 + rates = [1] + load_gen_mode = LoadGenerationMode.CONSTANT + profile_generator = ProfileGenerator.create( + ProfileGenerationMode.FIXED_RATE, + **({"rates": rates, "load_gen_mode": load_gen_mode}), + ) + assert isinstance(profile_generator, FixedRateProfileGenerator) + assert profile_generator._rates == rates + assert profile_generator._load_gen_mode.name == load_gen_mode.name + assert profile_generator._rate_index == 0 + def test_synchronous_mode_rate_list_error(): - rates = [1] - load_gen_mode = LoadGenerationMode.SYNCHRONOUS - with pytest.raises(ValueError, match="custom rates are not supported in synchronous mode"): - ProfileGenerator.create(ProfileGenerationMode.FIXED_RATE, **({ "rates": rates, "load_gen_mode": load_gen_mode})) + rates = [1] + load_gen_mode = LoadGenerationMode.SYNCHRONOUS + with pytest.raises( + ValueError, match="custom rates are not supported in synchronous mode" + ): + ProfileGenerator.create( + ProfileGenerationMode.FIXED_RATE, + **({"rates": rates, "load_gen_mode": load_gen_mode}), + ) + def test_next_with_multiple_rates(): - rates = [1, 2] - load_gen_mode = LoadGenerationMode.CONSTANT - profile_generator = ProfileGenerator.create(ProfileGenerationMode.FIXED_RATE, **({ "rates": rates, "load_gen_mode": load_gen_mode})) - mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - for rates in rates: - current_profile = profile_generator.next(mock_report) - assert current_profile.load_gen_rate == rates - assert current_profile.load_gen_mode.name == LoadGenerationMode.CONSTANT.name - assert profile_generator.next(mock_report) == None + rates = [1, 2] + load_gen_mode = LoadGenerationMode.CONSTANT + profile_generator = ProfileGenerator.create( + ProfileGenerationMode.FIXED_RATE, + **({"rates": rates, "load_gen_mode": load_gen_mode}), + ) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + for rates in rates: + current_profile = profile_generator.next(mock_report) + assert current_profile.load_gen_rate == rates + assert current_profile.load_gen_mode.name == LoadGenerationMode.CONSTANT.name + assert profile_generator.next(mock_report) == None + def test_next_with_sync_mode(): - load_gen_mode = LoadGenerationMode.SYNCHRONOUS - profile_generator = ProfileGenerator.create(ProfileGenerationMode.FIXED_RATE, **({ "load_gen_mode": load_gen_mode})) - mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - current_profile = profile_generator.next(mock_report) - assert current_profile.load_gen_rate == None - assert current_profile.load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name - assert profile_generator.next(mock_report) == None + load_gen_mode = LoadGenerationMode.SYNCHRONOUS + profile_generator = ProfileGenerator.create( + ProfileGenerationMode.FIXED_RATE, **({"load_gen_mode": load_gen_mode}) + ) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + current_profile = profile_generator.next(mock_report) + assert current_profile.load_gen_rate == None + assert current_profile.load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name + assert profile_generator.next(mock_report) == None + # Sweep Profile Generator + def test_sweep_profile_generator_creation(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP, **({})) - assert isinstance(profile_generator, SweepProfileGenerator) - assert profile_generator._sync_run == False - assert profile_generator._max_found == False - assert profile_generator._pending_rates == None - assert profile_generator._pending_rates == None + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP, **({})) + assert isinstance(profile_generator, SweepProfileGenerator) + assert profile_generator._sync_run == False + assert profile_generator._max_found == False + assert profile_generator._pending_rates == None + assert profile_generator._pending_rates == None + def test_first_profile_is_synchronous(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) - mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - profile = profile_generator.next(mock_report) - assert profile.load_gen_rate == None - assert profile.load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + profile = profile_generator.next(mock_report) + assert profile.load_gen_rate == None + assert profile.load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name + def test_rate_doubles(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) - mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - mock_benchmark = MagicMock(spec=TextGenerationBenchmark) - mock_benchmark.overloaded = False - mock_benchmark.rate = 2.0 - mock_benchmark.request_rate = 2.0 - benchmarks = [ - mock_benchmark - ] - mock_report.benchmarks = benchmarks - profile = profile_generator.next(mock_report) - - profile = profile_generator.next(mock_report) - assert profile.load_gen_rate == 4.0 + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + mock_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_benchmark.overloaded = False + mock_benchmark.rate = 2.0 + mock_benchmark.request_rate = 2.0 + benchmarks = [mock_benchmark] + mock_report.benchmarks = benchmarks + profile = profile_generator.next(mock_report) + + profile = profile_generator.next(mock_report) + assert profile.load_gen_rate == 4.0 + def test_max_found(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) - mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - mock_benchmark = MagicMock(spec=TextGenerationBenchmark) - mock_benchmark.overloaded = False - mock_benchmark.rate = 2.0 - mock_benchmark.request_rate = 2.0 - mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark) - mock_overloaded_benchmark.overloaded = True - mock_overloaded_benchmark.rate = 4.0 - mock_overloaded_benchmark.request_rate = 4.0 - benchmarks = [ - mock_benchmark, - mock_overloaded_benchmark - ] - mock_report.benchmarks = benchmarks - - profile_generator.next(mock_report) - profile = profile_generator.next(mock_report) - - # if benchmark wasn't overloaded, rates would have doubled to 8 - assert profile.load_gen_rate == 2.0 + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + mock_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_benchmark.overloaded = False + mock_benchmark.rate = 2.0 + mock_benchmark.request_rate = 2.0 + mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_overloaded_benchmark.overloaded = True + mock_overloaded_benchmark.rate = 4.0 + mock_overloaded_benchmark.request_rate = 4.0 + benchmarks = [mock_benchmark, mock_overloaded_benchmark] + mock_report.benchmarks = benchmarks + + profile_generator.next(mock_report) + profile = profile_generator.next(mock_report) + + # if benchmark wasn't overloaded, rates would have doubled to 8 + assert profile.load_gen_rate == 2.0 + def test_pending_rates(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) - mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - mock_benchmark = MagicMock(spec=TextGenerationBenchmark) - mock_benchmark.overloaded = False - mock_benchmark.rate = 2.0 - mock_benchmark.request_rate = 2.0 - mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark) - mock_overloaded_benchmark.overloaded = True - mock_overloaded_benchmark.rate = 8.0 - mock_overloaded_benchmark.request_rate = 8.0 - benchmarks = [ - mock_benchmark, - mock_overloaded_benchmark - ] - mock_report.benchmarks = benchmarks - profile = profile_generator.next(mock_report) - for expected_rate in numpy.linspace(2.0, 8.0, 10): + profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) + mock_report = MagicMock(spec=TextGenerationBenchmarkReport) + mock_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_benchmark.overloaded = False + mock_benchmark.rate = 2.0 + mock_benchmark.request_rate = 2.0 + mock_overloaded_benchmark = MagicMock(spec=TextGenerationBenchmark) + mock_overloaded_benchmark.overloaded = True + mock_overloaded_benchmark.rate = 8.0 + mock_overloaded_benchmark.request_rate = 8.0 + benchmarks = [mock_benchmark, mock_overloaded_benchmark] + mock_report.benchmarks = benchmarks + profile = profile_generator.next(mock_report) + for expected_rate in numpy.linspace(2.0, 8.0, 10): profile = profile_generator.next(mock_report) - assert profile.load_gen_rate == expected_rate \ No newline at end of file + assert profile.load_gen_rate == expected_rate diff --git a/tests/unit/executor/test_single_profile_generation_mode.py b/tests/unit/executor/test_single_profile_generation_mode.py index e97bba6..54762b0 100644 --- a/tests/unit/executor/test_single_profile_generation_mode.py +++ b/tests/unit/executor/test_single_profile_generation_mode.py @@ -25,7 +25,7 @@ def test_executor_single_profile_generator_benchmark_report( tokenizer="bert-base-uncased" ) rates = [1.0] - if (load_gen_mode == LoadGenerationMode.SYNCHRONOUS): + if load_gen_mode == LoadGenerationMode.SYNCHRONOUS: rates = None profile_generator_kwargs = {"load_gen_mode": load_gen_mode, "rates": rates} From 2ec371baa3b8cece9b740a97781de80e001bb8ca Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 23 Jul 2024 18:51:49 -0400 Subject: [PATCH 18/22] fix linting issues --- src/guidellm/executor/profile_generator.py | 27 +++--- src/guidellm/main.py | 6 +- tests/unit/executor/test_executor.py | 4 +- ...est_fixed_rate_profile_generation_mode.py} | 4 +- tests/unit/executor/test_profile_generator.py | 86 ++++++++++--------- 5 files changed, 69 insertions(+), 58 deletions(-) rename tests/unit/executor/{test_single_profile_generation_mode.py => test_fixed_rate_profile_generation_mode.py} (94%) diff --git a/src/guidellm/executor/profile_generator.py b/src/guidellm/executor/profile_generator.py index 6b37aff..a6b3838 100644 --- a/src/guidellm/executor/profile_generator.py +++ b/src/guidellm/executor/profile_generator.py @@ -78,32 +78,33 @@ def __init__( **kwargs, ): super().__init__(ProfileGenerationMode.FIXED_RATE) - if load_gen_mode == "synchronous" and rates and len(rates) > 0: + if load_gen_mode == LoadGenerationMode.SYNCHRONOUS and rates and len(rates) > 0: raise ValueError("custom rates are not supported in synchronous mode") self._rates: Optional[List[float]] = rates - self._load_gen_mode: LoadGenerationMode = load_gen_mode + self._load_gen_mode = load_gen_mode self._generated: bool = False self._rate_index: int = 0 def next(self, current_report: TextGenerationBenchmarkReport) -> Optional[Profile]: - if self._load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name: + if self._load_gen_mode == LoadGenerationMode.SYNCHRONOUS: if self._generated: return None self._generated = True return Profile( load_gen_mode=LoadGenerationMode.SYNCHRONOUS, load_gen_rate=None ) - elif self._load_gen_mode.name in { - LoadGenerationMode.CONSTANT.name, - LoadGenerationMode.POISSON.name, + elif self._load_gen_mode in { + LoadGenerationMode.CONSTANT, + LoadGenerationMode.POISSON, }: - if self._rate_index >= len(self._rates): - return None - current_rate = self._rates[self._rate_index] - self._rate_index += 1 - return Profile( - load_gen_mode=self._load_gen_mode, load_gen_rate=current_rate - ) + if self._rates: + if self._rate_index >= len(self._rates): + return None + current_rate = self._rates[self._rate_index] + self._rate_index += 1 + return Profile( + load_gen_mode=self._load_gen_mode, load_gen_rate=current_rate + ) raise ValueError(f"Invalid rate type: {self._load_gen_mode}") diff --git a/src/guidellm/main.py b/src/guidellm/main.py index 921e7db..ad4a674 100644 --- a/src/guidellm/main.py +++ b/src/guidellm/main.py @@ -13,7 +13,6 @@ TransformersDatasetRequestGenerator, ) from guidellm.request.base import RequestGenerator -from guidellm.scheduler.load_generator import LoadGenerationMode @click.command() @@ -112,9 +111,10 @@ def main( else: raise ValueError(f"Unknown data type: {data_type}") - profile_mode = rate_type_to_profile_mode(rate_type) + profile_mode = rate_type_to_profile_mode.get(rate_type) load_gen_mode = rate_type_to_load_gen_mode.get(rate_type, None) - + if not profile_mode or not load_gen_mode: + raise ValueError("Invalid rate type") # Create executor executor = Executor( request_generator=request_generator, diff --git a/tests/unit/executor/test_executor.py b/tests/unit/executor/test_executor.py index 093c1e2..11f701e 100644 --- a/tests/unit/executor/test_executor.py +++ b/tests/unit/executor/test_executor.py @@ -14,8 +14,8 @@ def test_executor_creation(): mock_backend = MagicMock(spec=Backend) profile_mode = ProfileGenerationMode.SWEEP profile_args = None - max_requests = (None,) - max_duration = (None,) + max_requests = None + max_duration = None executor = Executor( mock_backend, mock_request_generator, diff --git a/tests/unit/executor/test_single_profile_generation_mode.py b/tests/unit/executor/test_fixed_rate_profile_generation_mode.py similarity index 94% rename from tests/unit/executor/test_single_profile_generation_mode.py rename to tests/unit/executor/test_fixed_rate_profile_generation_mode.py index 54762b0..07f5e45 100644 --- a/tests/unit/executor/test_single_profile_generation_mode.py +++ b/tests/unit/executor/test_fixed_rate_profile_generation_mode.py @@ -1,3 +1,5 @@ +from typing import List, Optional + import pytest from guidellm.core import TextGenerationBenchmark, TextGenerationBenchmarkReport @@ -24,7 +26,7 @@ def test_executor_single_profile_generator_benchmark_report( request_genrator = dummy.services.TestRequestGenerator( tokenizer="bert-base-uncased" ) - rates = [1.0] + rates: Optional[List[float]] = [1.0] if load_gen_mode == LoadGenerationMode.SYNCHRONOUS: rates = None profile_generator_kwargs = {"load_gen_mode": load_gen_mode, "rates": rates} diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index 233e826..a72a2b9 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -1,3 +1,4 @@ +from profile import Profile from unittest.mock import MagicMock import numpy @@ -9,7 +10,6 @@ ProfileGenerationMode, ProfileGenerator, SweepProfileGenerator, - profile_generator, ) from guidellm.scheduler import LoadGenerationMode @@ -17,20 +17,20 @@ def test_fixed_rate_profile_generator_creation(): - rates = [1] + rates = [1.0] load_gen_mode = LoadGenerationMode.CONSTANT - profile_generator = ProfileGenerator.create( + test_profile_generator = ProfileGenerator.create( ProfileGenerationMode.FIXED_RATE, **({"rates": rates, "load_gen_mode": load_gen_mode}), ) - assert isinstance(profile_generator, FixedRateProfileGenerator) - assert profile_generator._rates == rates - assert profile_generator._load_gen_mode.name == load_gen_mode.name - assert profile_generator._rate_index == 0 + assert isinstance(test_profile_generator, FixedRateProfileGenerator) + assert test_profile_generator._rates == rates + assert test_profile_generator._load_gen_mode == load_gen_mode + assert test_profile_generator._rate_index == 0 def test_synchronous_mode_rate_list_error(): - rates = [1] + rates = [1.0] load_gen_mode = LoadGenerationMode.SYNCHRONOUS with pytest.raises( ValueError, match="custom rates are not supported in synchronous mode" @@ -42,54 +42,59 @@ def test_synchronous_mode_rate_list_error(): def test_next_with_multiple_rates(): - rates = [1, 2] + rates = [1.0, 2.0] load_gen_mode = LoadGenerationMode.CONSTANT - profile_generator = ProfileGenerator.create( + test_profile_generator = ProfileGenerator.create( ProfileGenerationMode.FIXED_RATE, **({"rates": rates, "load_gen_mode": load_gen_mode}), ) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - for rates in rates: - current_profile = profile_generator.next(mock_report) - assert current_profile.load_gen_rate == rates - assert current_profile.load_gen_mode.name == LoadGenerationMode.CONSTANT.name - assert profile_generator.next(mock_report) == None + for rate in rates: + current_profile = test_profile_generator.next(mock_report) + assert current_profile is not None + assert current_profile.load_gen_rate == rate + assert current_profile.load_gen_mode == LoadGenerationMode.CONSTANT + assert test_profile_generator.next(mock_report) is None def test_next_with_sync_mode(): load_gen_mode = LoadGenerationMode.SYNCHRONOUS - profile_generator = ProfileGenerator.create( + test_profile_generator = ProfileGenerator.create( ProfileGenerationMode.FIXED_RATE, **({"load_gen_mode": load_gen_mode}) ) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - current_profile = profile_generator.next(mock_report) - assert current_profile.load_gen_rate == None - assert current_profile.load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name - assert profile_generator.next(mock_report) == None + current_profile = test_profile_generator.next(mock_report) + assert current_profile is not None + assert current_profile.load_gen_rate is None + assert current_profile.load_gen_mode == LoadGenerationMode.SYNCHRONOUS + assert test_profile_generator.next(mock_report) is None # Sweep Profile Generator def test_sweep_profile_generator_creation(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP, **({})) - assert isinstance(profile_generator, SweepProfileGenerator) - assert profile_generator._sync_run == False - assert profile_generator._max_found == False - assert profile_generator._pending_rates == None - assert profile_generator._pending_rates == None + test_profile_generator = ProfileGenerator.create( + ProfileGenerationMode.SWEEP, **({}) + ) + assert isinstance(test_profile_generator, SweepProfileGenerator) + assert not test_profile_generator._sync_run + assert not test_profile_generator._max_found + assert test_profile_generator._pending_rates is None + assert test_profile_generator._pending_rates is None def test_first_profile_is_synchronous(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) + test_profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) - profile = profile_generator.next(mock_report) - assert profile.load_gen_rate == None - assert profile.load_gen_mode.name == LoadGenerationMode.SYNCHRONOUS.name + profile = test_profile_generator.next(mock_report) + assert profile is not None + assert profile.load_gen_rate is None + assert profile.load_gen_mode == LoadGenerationMode.SYNCHRONOUS def test_rate_doubles(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) + test_profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) mock_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_benchmark.overloaded = False @@ -97,14 +102,15 @@ def test_rate_doubles(): mock_benchmark.request_rate = 2.0 benchmarks = [mock_benchmark] mock_report.benchmarks = benchmarks - profile = profile_generator.next(mock_report) + test_profile_generator.next(mock_report) - profile = profile_generator.next(mock_report) + profile = test_profile_generator.next(mock_report) + assert profile is not None assert profile.load_gen_rate == 4.0 def test_max_found(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) + test_profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) mock_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_benchmark.overloaded = False @@ -117,15 +123,16 @@ def test_max_found(): benchmarks = [mock_benchmark, mock_overloaded_benchmark] mock_report.benchmarks = benchmarks - profile_generator.next(mock_report) - profile = profile_generator.next(mock_report) + test_profile_generator.next(mock_report) + profile = test_profile_generator.next(mock_report) + assert profile is not None # if benchmark wasn't overloaded, rates would have doubled to 8 assert profile.load_gen_rate == 2.0 def test_pending_rates(): - profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) + test_profile_generator = ProfileGenerator.create(ProfileGenerationMode.SWEEP) mock_report = MagicMock(spec=TextGenerationBenchmarkReport) mock_benchmark = MagicMock(spec=TextGenerationBenchmark) mock_benchmark.overloaded = False @@ -137,7 +144,8 @@ def test_pending_rates(): mock_overloaded_benchmark.request_rate = 8.0 benchmarks = [mock_benchmark, mock_overloaded_benchmark] mock_report.benchmarks = benchmarks - profile = profile_generator.next(mock_report) + profile = test_profile_generator.next(mock_report) + assert profile is not None for expected_rate in numpy.linspace(2.0, 8.0, 10): - profile = profile_generator.next(mock_report) + profile = test_profile_generator.next(mock_report) assert profile.load_gen_rate == expected_rate From 2361916953532c97d640bebacb2fb2d9bfca2b5b Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 23 Jul 2024 19:10:40 -0400 Subject: [PATCH 19/22] remove unused import --- tests/unit/executor/test_profile_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index a72a2b9..2fdec79 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -1,4 +1,3 @@ -from profile import Profile from unittest.mock import MagicMock import numpy From ca8e5533e646b43c2f51069e649cd72b297351d3 Mon Sep 17 00:00:00 2001 From: dalthecow Date: Tue, 23 Jul 2024 19:14:07 -0400 Subject: [PATCH 20/22] fix type issue --- tests/unit/executor/test_profile_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/executor/test_profile_generator.py b/tests/unit/executor/test_profile_generator.py index 2fdec79..f0aecf1 100644 --- a/tests/unit/executor/test_profile_generator.py +++ b/tests/unit/executor/test_profile_generator.py @@ -144,7 +144,7 @@ def test_pending_rates(): benchmarks = [mock_benchmark, mock_overloaded_benchmark] mock_report.benchmarks = benchmarks profile = test_profile_generator.next(mock_report) - assert profile is not None for expected_rate in numpy.linspace(2.0, 8.0, 10): profile = test_profile_generator.next(mock_report) + assert profile is not None assert profile.load_gen_rate == expected_rate From ce460dec912cf60da81c1b27986c3feeb994524c Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Wed, 24 Jul 2024 16:17:36 +0300 Subject: [PATCH 21/22] pytest.init_options section is restored --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 382f862..26984d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,3 +99,12 @@ ignore_missing_imports=true line-length = 88 exclude = ["build", "dist", "env", ".venv"] lint.select = ["E", "F", "W"] + + +[tool.pytest.ini_options] + addopts = '-s -vvv --cache-clear --cov-report=term-missing --cov --cov-fail-under=75' + markers = [ + "smoke: quick tests to check basic functionality", + "sanity: detailed tests to ensure major functions work correctly", + "regression: tests to ensure that new changes do not break existing functionality" + ] From 3062fba2d10e5724de9df2f474a9618ed84b09d3 Mon Sep 17 00:00:00 2001 From: Dmytro Parfeniuk Date: Wed, 24 Jul 2024 16:19:42 +0300 Subject: [PATCH 22/22] format pyproject.toml file --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 26984d1..8746528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,9 +102,9 @@ lint.select = ["E", "F", "W"] [tool.pytest.ini_options] - addopts = '-s -vvv --cache-clear --cov-report=term-missing --cov --cov-fail-under=75' - markers = [ - "smoke: quick tests to check basic functionality", - "sanity: detailed tests to ensure major functions work correctly", - "regression: tests to ensure that new changes do not break existing functionality" - ] +addopts = '-s -vvv --cache-clear --cov-report=term-missing --cov --cov-fail-under=75' +markers = [ + "smoke: quick tests to check basic functionality", + "sanity: detailed tests to ensure major functions work correctly", + "regression: tests to ensure that new changes do not break existing functionality" +]