phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix in mem cache tests

Browse source
This commit is contained in:
Ishaan Jaff 2024-06-22 19:51:43 -07:00
parent fbef5013a1
commit 8a66e074ce
2 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/caching.py
View file

@ -64,7 +64,7 @@ class BaseCache:
class InMemoryCache(BaseCache): class InMemoryCache(BaseCache):
def __init__(self, default_ttl: Optional[float] = 60.0): def __init__(self, default_ttl: Optional[float] = 120.0):
""" """
default_ttl [float]: If default_ttl is 6 seconds, every 6 seconds the cache will be set to {} default_ttl [float]: If default_ttl is 6 seconds, every 6 seconds the cache will be set to {}
this is done to prevent overuse of System RAM this is done to prevent overuse of System RAM
@ -72,7 +72,7 @@ class InMemoryCache(BaseCache):
# if users don't provider one, use the default litellm cache # if users don't provider one, use the default litellm cache
self.cache_dict: dict = {} self.cache_dict: dict = {}
self.ttl_dict: dict = {} self.ttl_dict: dict = {}
self.default_ttl = default_ttl or 60.0 self.default_ttl = default_ttl or 120.0
self.last_cleaned = 0 # since this is in memory we need to periodically clean it up to not overuse the machines RAM self.last_cleaned = 0 # since this is in memory we need to periodically clean it up to not overuse the machines RAM
def set_cache(self, key, value, **kwargs): def set_cache(self, key, value, **kwargs):

2
litellm/router.py
View file

@ -282,7 +282,7 @@ class Router:
litellm.cache = litellm.Cache(type=cache_type, **cache_config) # type: ignore litellm.cache = litellm.Cache(type=cache_type, **cache_config) # type: ignore
self.cache_responses = cache_responses self.cache_responses = cache_responses
self.cache = DualCache( self.cache = DualCache(
redis_cache=redis_cache, in_memory_cache=InMemoryCache() redis_cache=redis_cache, in_memory_cache=InMemoryCache(default_ttl=86400)
) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc. ) # use a dual cache (Redis+In-Memory) for tracking cooldowns, usage, etc.
### SCHEDULER ### ### SCHEDULER ###