phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Merge pull request #2542 from BerriAI/litellm_redis_perf_improvements

Browse source 
89% Caching improvement - Async Redis completion calls + batch redis GET requests for a given key + call type
This commit is contained in:
Krish Dholakia 2024-03-15 18:58:36 -07:00 • committed by GitHub
commit 4969ae0e9d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 287 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

10
litellm/proxy/proxy_server.py
View file

@ -1798,6 +1798,16 @@ class ProxyConfig:
_ENTERPRISE_PromptInjectionDetection()
)
imported_list.append(prompt_injection_detection_obj)
elif (
isinstance(callback, str)
and callback == "batch_redis_requests"
):
from litellm.proxy.hooks.batch_redis_get import (
_PROXY_BatchRedisRequests,
)
batch_redis_obj = _PROXY_BatchRedisRequests()
imported_list.append(batch_redis_obj)
else:
imported_list.append(
get_instance_fn(