phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

feat(batch_redis_get.py): batch redis GET requests for a given key + call type

Browse source 
reduces the number of GET requests we're making in high-throughput scenarios
This commit is contained in:
Krrish Dholakia 2024-03-15 14:40:11 -07:00
parent e033e84720
commit 226953e1d8
5 changed files with 189 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
litellm/proxy/proxy_server.py
View file

@ -1795,6 +1795,16 @@ class ProxyConfig:
_ENTERPRISE_PromptInjectionDetection()
)
imported_list.append(prompt_injection_detection_obj)
elif (
isinstance(callback, str)
and callback == "batch_redis_requests"
):
from litellm.proxy.hooks.batch_redis_get import (
_PROXY_BatchRedisRequests,
)
batch_redis_obj = _PROXY_BatchRedisRequests()
imported_list.append(batch_redis_obj)
else:
imported_list.append(
get_instance_fn(