phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 19:24:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix InMemoryCache

Browse source
This commit is contained in:
Ishaan Jaff 2024-06-24 17:08:30 -07:00
parent 21fd91fe94
commit 81ef2c38dc
1 changed files with 0 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

15
litellm/caching.py
View file

@ -76,7 +76,6 @@ class InMemoryCache(BaseCache):
) )
self.ttl_dict: dict = {} self.ttl_dict: dict = {}
self.default_ttl = default_ttl or 120.0 self.default_ttl = default_ttl or 120.0
self.last_cleaned = 0 # since this is in memory we need to periodically clean it up to not overuse the machines RAM
def set_cache(self, key, value, **kwargs): def set_cache(self, key, value, **kwargs):
print_verbose("InMemoryCache: set_cache") print_verbose("InMemoryCache: set_cache")
@ -149,22 +148,8 @@ class InMemoryCache(BaseCache):
value = init_value + value value = init_value + value
await self.async_set_cache(key, value, **kwargs) await self.async_set_cache(key, value, **kwargs)
if time.time() - self.last_cleaned > self.default_ttl:
asyncio.create_task(self.clean_up_in_memory_cache())
return value return value
async def clean_up_in_memory_cache(self):
"""
Runs periodically to clean up the in-memory cache
- loop through all keys in cache, check if they are expired
- if yes, delete them
"""
self.cache_dict = {}
self.ttl_dict = {}
self.last_cleaned = time.time()
def flush_cache(self): def flush_cache(self):
self.cache_dict.clear() self.cache_dict.clear()
self.ttl_dict.clear() self.ttl_dict.clear()