phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 19:24:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(batch_redis_get.py): batch redis GET requests for a given key + call type

Browse source 
reduces number of redis requests. 85ms latency improvement over 3 minutes of load (19k requests).
This commit is contained in:
Krrish Dholakia 2024-03-15 14:54:16 -07:00
parent 8d1c60bfdc
commit 3680f16cd7
2 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

8
litellm/proxy/hooks/batch_redis_get.py
View file

@ -85,10 +85,10 @@ class _PROXY_BatchRedisRequests(CustomLogger):
)
## Add to cache
for key, value in key_value_dict.items():
_cache_key = f"{cache_key_name}:{key}"
cache.in_memory_cache.cache_dict[_cache_key] = value
if len(key_value_dict.items()) > 0:
await cache.in_memory_cache.async_set_cache_pipeline(
cache_list=list(key_value_dict.items()), ttl=60
)
## Set cache namespace if it's a miss
data["metadata"]["redis_namespace"] = cache_key_name
except HTTPException as e: