phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix always read redis

Browse source
This commit is contained in:
Ishaan Jaff 2024-09-02 21:08:32 -07:00
parent 0b51712aab
commit fd122cb759
3 changed files with 18 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

3
litellm/__init__.py
View file

@ -146,6 +146,9 @@ enable_caching_on_provider_specific_optional_params: bool = (
caching: bool = ( caching: bool = (
False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
) )
always_read_redis: bool = (
True # always use redis for rate limiting logic on litellm proxy
)
caching_with_models: bool = ( caching_with_models: bool = (
False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
) )

18
litellm/caching.py
View file

@ -1777,6 +1777,7 @@ class DualCache(BaseCache):
redis_cache: Optional[RedisCache] = None, redis_cache: Optional[RedisCache] = None,
default_in_memory_ttl: Optional[float] = None, default_in_memory_ttl: Optional[float] = None,
default_redis_ttl: Optional[float] = None, default_redis_ttl: Optional[float] = None,
always_read_redis: Optional[bool] = False,
) -> None: ) -> None:
super().__init__() super().__init__()
# If in_memory_cache is not provided, use the default InMemoryCache # If in_memory_cache is not provided, use the default InMemoryCache
@ -1788,6 +1789,7 @@ class DualCache(BaseCache):
default_in_memory_ttl or litellm.default_in_memory_ttl default_in_memory_ttl or litellm.default_in_memory_ttl
) )
self.default_redis_ttl = default_redis_ttl or litellm.default_redis_ttl self.default_redis_ttl = default_redis_ttl or litellm.default_redis_ttl
self.always_read_redis = always_read_redis
def update_cache_ttl( def update_cache_ttl(
self, default_in_memory_ttl: Optional[float], default_redis_ttl: Optional[float] self, default_in_memory_ttl: Optional[float], default_redis_ttl: Optional[float]
@ -1847,8 +1849,12 @@ class DualCache(BaseCache):
if in_memory_result is not None: if in_memory_result is not None:
result = in_memory_result result = in_memory_result
if result is None and self.redis_cache is not None and local_only == False: if (
# If not found in in-memory cache, try fetching from Redis (self.always_read_redis is True)
and self.redis_cache is not None
and local_only == False
):
# If not found in in-memory cache or always_read_redis is True, try fetching from Redis
redis_result = self.redis_cache.get_cache(key, **kwargs) redis_result = self.redis_cache.get_cache(key, **kwargs)
if redis_result is not None: if redis_result is not None:
@ -1911,8 +1917,12 @@ class DualCache(BaseCache):
if in_memory_result is not None: if in_memory_result is not None:
result = in_memory_result result = in_memory_result
if result is None and self.redis_cache is not None and local_only == False: if (
# If not found in in-memory cache, try fetching from Redis (self.always_read_redis is True)
and self.redis_cache is not None
and local_only == False
):
# If not found in in-memory cache or always_read_redis is True, try fetching from Redis
redis_result = await self.redis_cache.async_get_cache(key, **kwargs) redis_result = await self.redis_cache.async_get_cache(key, **kwargs)
if redis_result is not None: if redis_result is not None:

2
litellm/proxy/utils.py
View file

@ -207,7 +207,7 @@ class ProxyLogging:
self.call_details: dict = {} self.call_details: dict = {}
self.call_details["user_api_key_cache"] = user_api_key_cache self.call_details["user_api_key_cache"] = user_api_key_cache
self.internal_usage_cache = DualCache( self.internal_usage_cache = DualCache(
default_in_memory_ttl=1 default_in_memory_ttl=1, always_read_redis=litellm.always_read_redis
) # ping redis cache every 1s ) # ping redis cache every 1s
self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler( self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler(
self.internal_usage_cache self.internal_usage_cache