phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) in mem redis reads

Browse source
This commit is contained in:
Ishaan Jaff 2024-03-26 09:10:49 -07:00
parent 9668824d77
commit 336fe2f876
1 changed files with 16 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

16
litellm/caching.py
View file

@ -119,6 +119,9 @@ class RedisCache(BaseCache):
# for high traffic, we store the redis results in memory and then batch write to redis # for high traffic, we store the redis results in memory and then batch write to redis
self.redis_batch_writing_buffer = [] self.redis_batch_writing_buffer = []
self.redis_batch_reading_buffer = []
self.redis_last_updated_read_buffer = None
self.redis_fetch_interval = 1 # fetch from redis every 1 second
self.redis_flush_size = redis_flush_size self.redis_flush_size = redis_flush_size
self.redis_version = "Unknown" self.redis_version = "Unknown"
try: try:
@ -253,11 +256,24 @@ class RedisCache(BaseCache):
traceback.print_exc() traceback.print_exc()
logging.debug("LiteLLM Caching: get() - Got exception from REDIS: ", e) logging.debug("LiteLLM Caching: get() - Got exception from REDIS: ", e)
def _should_fetch_from_redis(self):
if self.redis_last_updated_read_buffer is None:
return True
if (
time.time() - self.redis_last_updated_read_buffer
> self.redis_fetch_interval
):
return True
return False
async def async_get_cache(self, key, **kwargs): async def async_get_cache(self, key, **kwargs):
_redis_client = self.init_async_client() _redis_client = self.init_async_client()
async with _redis_client as redis_client: async with _redis_client as redis_client:
try: try:
print_verbose(f"Get Async Redis Cache: key: {key}") print_verbose(f"Get Async Redis Cache: key: {key}")
if self._should_fetch_from_redis():
self.redis_last_updated_read_buffer = time.time()
cached_response = await redis_client.get(key) cached_response = await redis_client.get(key)
print_verbose( print_verbose(
f"Got Async Redis Cache: key: {key}, cached_response {cached_response}" f"Got Async Redis Cache: key: {key}, cached_response {cached_response}"