phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

perf: remove 'always_read_redis' - adding +830ms on each llm call (#6414)

Browse source 
* perf: remove 'always_read_redis' - adding +830ms on each llm call

* test: cleanup codestral tests - backend api unavailable
This commit is contained in:
Krish Dholakia 2024-10-24 17:48:36 -07:00 • committed by GitHub
parent 0f0470f574
commit d59f8f952d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 49 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

10
litellm/caching/dual_cache.py
View file

@ -32,7 +32,6 @@ class DualCache(BaseCache):
redis_cache: Optional[RedisCache] = None,
default_in_memory_ttl: Optional[float] = None,
default_redis_ttl: Optional[float] = None,
always_read_redis: Optional[bool] = True,
) -> None:
super().__init__()
# If in_memory_cache is not provided, use the default InMemoryCache
@ -44,7 +43,6 @@ class DualCache(BaseCache):
default_in_memory_ttl or litellm.default_in_memory_ttl
)
self.default_redis_ttl = default_redis_ttl or litellm.default_redis_ttl
self.always_read_redis = always_read_redis
def update_cache_ttl(
self, default_in_memory_ttl: Optional[float], default_redis_ttl: Optional[float]
@ -102,12 +100,8 @@ class DualCache(BaseCache):
if in_memory_result is not None:
result = in_memory_result
if (
(self.always_read_redis is True)
and self.redis_cache is not None
and local_only is False
):
# If not found in in-memory cache or always_read_redis is True, try fetching from Redis
if result is None and self.redis_cache is not None and local_only is False:
# If not found in in-memory cache, try fetching from Redis
redis_result = self.redis_cache.get_cache(key, **kwargs)
if redis_result is not None: