phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 03:34:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(feat) working semantic-cache on litellm proxy

Browse source
This commit is contained in:
ishaan-jaff 2024-02-06 08:52:57 -08:00
parent 33f5ab8ba5
commit 0ddcebbf52
1 changed files with 18 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

23
litellm/caching.py
View file

@ -266,21 +266,30 @@ class RedisSemanticCache(BaseCache):
if redis_url is None: if redis_url is None:
# if no url passed, check if host, port and password are passed, if not raise an Exception # if no url passed, check if host, port and password are passed, if not raise an Exception
if host is None or port is None or password is None: if host is None or port is None or password is None:
raise Exception(f"Redis host, port, and password must be provided") # try checking env for host, port and password
import os
host = os.getenv("REDIS_HOST")
port = os.getenv("REDIS_PORT")
password = os.getenv("REDIS_PASSWORD")
if host is None or port is None or password is None:
raise Exception("Redis host, port, and password must be provided")
redis_url = "redis://:" + password + "@" + host + ":" + port redis_url = "redis://:" + password + "@" + host + ":" + port
print_verbose(f"redis semantic-cache redis_url: {redis_url}") print_verbose(f"redis semantic-cache redis_url: {redis_url}")
if use_async == False: if use_async == False:
self.index = SearchIndex.from_dict(schema) self.index = SearchIndex.from_dict(schema)
self.index.connect(redis_url=redis_url) self.index.connect(redis_url=redis_url)
try:
self.index.create(overwrite=False) # don't overwrite existing index
except Exception as e:
print_verbose(f"Got exception creating semantic cache index: {str(e)}")
elif use_async == True: elif use_async == True:
schema["index"]["name"] = "litellm_semantic_cache_index_async" schema["index"]["name"] = "litellm_semantic_cache_index_async"
self.index = SearchIndex.from_dict(schema) self.index = SearchIndex.from_dict(schema)
self.index.connect(redis_url=redis_url, use_async=True) self.index.connect(redis_url=redis_url, use_async=True)
try:
self.index.create(overwrite=False) # don't overwrite existing index
except Exception as e:
print_verbose(f"Got exception creating semantic cache index: {str(e)}")
#
def _get_cache_logic(self, cached_response: Any): def _get_cache_logic(self, cached_response: Any):
""" """
Common 'get_cache_logic' across sync + async redis client implementations Common 'get_cache_logic' across sync + async redis client implementations
@ -397,6 +406,10 @@ class RedisSemanticCache(BaseCache):
async def async_set_cache(self, key, value, **kwargs): async def async_set_cache(self, key, value, **kwargs):
import numpy as np import numpy as np
try:
await self.index.acreate(overwrite=False) # don't overwrite existing index
except Exception as e:
print_verbose(f"Got exception creating semantic cache index: {str(e)}")
print_verbose(f"async redis semantic-cache set_cache, kwargs: {kwargs}") print_verbose(f"async redis semantic-cache set_cache, kwargs: {kwargs}")
# get the prompt # get the prompt