From fd122cb7594d69345e51dd9af7adf2fbdd33b344 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 2 Sep 2024 21:08:32 -0700 Subject: [PATCH 1/6] fix always read redis --- litellm/__init__.py | 3 +++ litellm/caching.py | 18 ++++++++++++++---- litellm/proxy/utils.py | 2 +- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index e220c89920..4ee604aede 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -146,6 +146,9 @@ enable_caching_on_provider_specific_optional_params: bool = ( caching: bool = ( False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 ) +always_read_redis: bool = ( + True # always use redis for rate limiting logic on litellm proxy +) caching_with_models: bool = ( False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 ) diff --git a/litellm/caching.py b/litellm/caching.py index d34686c2ad..8a25230582 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -1777,6 +1777,7 @@ class DualCache(BaseCache): redis_cache: Optional[RedisCache] = None, default_in_memory_ttl: Optional[float] = None, default_redis_ttl: Optional[float] = None, + always_read_redis: Optional[bool] = False, ) -> None: super().__init__() # If in_memory_cache is not provided, use the default InMemoryCache @@ -1788,6 +1789,7 @@ class DualCache(BaseCache): default_in_memory_ttl or litellm.default_in_memory_ttl ) self.default_redis_ttl = default_redis_ttl or litellm.default_redis_ttl + self.always_read_redis = always_read_redis def update_cache_ttl( self, default_in_memory_ttl: Optional[float], default_redis_ttl: Optional[float] @@ -1847,8 +1849,12 @@ class DualCache(BaseCache): if in_memory_result is not None: result = in_memory_result - if result is None and self.redis_cache is not None and local_only == False: - # If not found in in-memory cache, try fetching from Redis + if ( + (self.always_read_redis is True) + and self.redis_cache is not None + and local_only == False + ): + # If not found in in-memory cache or always_read_redis is True, try fetching from Redis redis_result = self.redis_cache.get_cache(key, **kwargs) if redis_result is not None: @@ -1911,8 +1917,12 @@ class DualCache(BaseCache): if in_memory_result is not None: result = in_memory_result - if result is None and self.redis_cache is not None and local_only == False: - # If not found in in-memory cache, try fetching from Redis + if ( + (self.always_read_redis is True) + and self.redis_cache is not None + and local_only == False + ): + # If not found in in-memory cache or always_read_redis is True, try fetching from Redis redis_result = await self.redis_cache.async_get_cache(key, **kwargs) if redis_result is not None: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ea830c1363..2423fb105a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -207,7 +207,7 @@ class ProxyLogging: self.call_details: dict = {} self.call_details["user_api_key_cache"] = user_api_key_cache self.internal_usage_cache = DualCache( - default_in_memory_ttl=1 + default_in_memory_ttl=1, always_read_redis=litellm.always_read_redis ) # ping redis cache every 1s self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler( self.internal_usage_cache From f5122e0325605e9dfb0bf2bc0178c74cf6376733 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 07:41:08 -0700 Subject: [PATCH 2/6] ci/cd run again --- litellm/tests/test_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 5fce984520..5e0f9d58cc 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -23,7 +23,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.prompt_templates.factory import anthropic_messages_pt -# litellm.num_retries = 3 +# litellm.num_retries=3 litellm.cache = None litellm.success_callback = [] user_message = "Write a short poem about the sky" From 657b7eef823f3c53f7b7695fac20aa028740620e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 07:57:23 -0700 Subject: [PATCH 3/6] add always read redis test --- litellm/tests/test_caching.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index 9b859b451a..db2e7e1c41 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -1958,3 +1958,30 @@ async def test_cache_default_off_acompletion(): ) print(f"Response4: {response4}") assert response3.id == response4.id + + +@pytest.mark.asyncio() +async def test_dual_cache_uses_redis(): + """ + + - Store diff values in redis and in memory cache + - call get cache + - Assert that value from redis is used + """ + litellm.set_verbose = True + from litellm.caching import DualCache, RedisCache + + current_usage = uuid.uuid4() + + _cache_obj = DualCache(redis_cache=RedisCache(), always_read_redis=True) + + # set cache + await _cache_obj.async_set_cache(key=f"current_usage: {current_usage}", value=10) + + # modify value of in memory cache + _cache_obj.in_memory_cache.cache_dict[f"current_usage: {current_usage}"] = 1 + + # get cache + value = await _cache_obj.async_get_cache(key=f"current_usage: {current_usage}") + print("value from dual cache", value) + assert value == 10 From db0ad40e98883f7a04b0449c9513403427ceda3e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 08:00:49 -0700 Subject: [PATCH 4/6] test for pl obj --- litellm/tests/test_caching.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index db2e7e1c41..660cf68ea9 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -1985,3 +1985,14 @@ async def test_dual_cache_uses_redis(): value = await _cache_obj.async_get_cache(key=f"current_usage: {current_usage}") print("value from dual cache", value) assert value == 10 + + +@pytest.mark.asyncio() +async def test_proxy_logging_setup(): + """ + Assert always_read_redis is True when used by internal usage cache + """ + from litellm.proxy.utils import ProxyLogging + + pl_obj = ProxyLogging() + assert pl_obj.internal_usage_cache.always_read_redis is True From ca5a117544e52e4ab29f9b3e47022e205d66036c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 08:01:55 -0700 Subject: [PATCH 5/6] dual cache use always read redis as True by default --- litellm/caching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/caching.py b/litellm/caching.py index 8a25230582..db2f93507d 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -1777,7 +1777,7 @@ class DualCache(BaseCache): redis_cache: Optional[RedisCache] = None, default_in_memory_ttl: Optional[float] = None, default_redis_ttl: Optional[float] = None, - always_read_redis: Optional[bool] = False, + always_read_redis: Optional[bool] = True, ) -> None: super().__init__() # If in_memory_cache is not provided, use the default InMemoryCache From 9c573cda0a80d3a25f6d5b2afa3732f977b6d757 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 10:59:15 -0700 Subject: [PATCH 6/6] fix test_proxy_logging_setup --- litellm/tests/test_caching.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index 660cf68ea9..a272d2dcf3 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -1992,7 +1992,8 @@ async def test_proxy_logging_setup(): """ Assert always_read_redis is True when used by internal usage cache """ + from litellm.caching import DualCache from litellm.proxy.utils import ProxyLogging - pl_obj = ProxyLogging() + pl_obj = ProxyLogging(user_api_key_cache=DualCache()) assert pl_obj.internal_usage_cache.always_read_redis is True