phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(feat) redis-semantic cache on proxy

Browse source
This commit is contained in:
ishaan-jaff 2024-02-06 10:35:21 -08:00
parent bdc2091838
commit 2732c47b70
2 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
litellm/proxy/proxy_server.py
View file

@ -1135,7 +1135,7 @@ class ProxyConfig:
verbose_proxy_logger.debug(f"passed cache type={cache_type}") verbose_proxy_logger.debug(f"passed cache type={cache_type}")
if cache_type == "redis": if cache_type == "redis" or cache_type == "redis-semantic":
cache_host = litellm.get_secret("REDIS_HOST", None) cache_host = litellm.get_secret("REDIS_HOST", None)
cache_port = litellm.get_secret("REDIS_PORT", None) cache_port = litellm.get_secret("REDIS_PORT", None)
cache_password = litellm.get_secret("REDIS_PASSWORD", None) cache_password = litellm.get_secret("REDIS_PASSWORD", None)
@ -1162,6 +1162,9 @@ class ProxyConfig:
f"{blue_color_code}Cache Password:{reset_color_code} {cache_password}" f"{blue_color_code}Cache Password:{reset_color_code} {cache_password}"
) )
print() # noqa print() # noqa
if cache_type == "redis-semantic":
# by default this should always be async
cache_params.update({"redis_semantic_cache_use_async": True})
# users can pass os.environ/ variables on the proxy - we should read them from the env # users can pass os.environ/ variables on the proxy - we should read them from the env
for key, value in cache_params.items(): for key, value in cache_params.items():
@ -4067,7 +4070,6 @@ def _has_user_setup_sso():
async def shutdown_event(): async def shutdown_event():
global prisma_client, master_key, user_custom_auth, user_custom_key_generate global prisma_client, master_key, user_custom_auth, user_custom_key_generate
if prisma_client: if prisma_client:
verbose_proxy_logger.debug("Disconnecting from Prisma") verbose_proxy_logger.debug("Disconnecting from Prisma")
await prisma_client.disconnect() await prisma_client.disconnect()

1
requirements.txt
View file

@ -10,6 +10,7 @@ gunicorn==21.2.0 # server dep
boto3==1.28.58 # aws bedrock/sagemaker calls boto3==1.28.58 # aws bedrock/sagemaker calls
redis==4.6.0 # caching redis==4.6.0 # caching
redisvl==0.0.7 # semantic caching redisvl==0.0.7 # semantic caching
numpy==1.24.3 # semantic caching
prisma==0.11.0 # for db prisma==0.11.0 # for db
mangum==0.17.0 # for aws lambda functions mangum==0.17.0 # for aws lambda functions
google-generativeai==0.1.0 # for vertex ai calls google-generativeai==0.1.0 # for vertex ai calls