phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 03:34:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(batch_redis_get.py): batch redis GET requests for a given key + call type

Browse source 
reduces number of redis requests. 85ms latency improvement over 3 minutes of load (19k requests).
This commit is contained in:
Krrish Dholakia 2024-03-15 14:54:16 -07:00
parent 226953e1d8
commit 8a20ea795b
2 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/_new_secret_config.yaml
View file

@ -13,7 +13,7 @@ litellm_settings:
cache: true cache: true
cache_params: cache_params:
type: redis type: redis
# callbacks: ["batch_redis_requests"] callbacks: ["batch_redis_requests"]
general_settings: general_settings:
master_key: sk-1234 master_key: sk-1234

8
litellm/proxy/hooks/batch_redis_get.py
View file

@ -85,10 +85,10 @@ class _PROXY_BatchRedisRequests(CustomLogger):
) )
## Add to cache ## Add to cache
for key, value in key_value_dict.items(): if len(key_value_dict.items()) > 0:
_cache_key = f"{cache_key_name}:{key}" await cache.in_memory_cache.async_set_cache_pipeline(
cache.in_memory_cache.cache_dict[_cache_key] = value cache_list=list(key_value_dict.items()), ttl=60
)
## Set cache namespace if it's a miss ## Set cache namespace if it's a miss
data["metadata"]["redis_namespace"] = cache_key_name data["metadata"]["redis_namespace"] = cache_key_name
except HTTPException as e: except HTTPException as e: