phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

with caching

Browse source
This commit is contained in:
ishaan-jaff 2023-08-10 15:14:58 -07:00
parent e9c2f3b55d
commit a1b64832a7
2 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

12
litellm/utils.py
View file

@ -724,29 +724,29 @@ def add_cache(messages, model_response):
global cache_collection global cache_collection
if cache_collection == None: if cache_collection == None:
make_collection() make_collection()
print("cache collection in add cache", cache_collection) # print("cache collection in add cache", cache_collection)
user_question = message_to_user_question(messages) user_question = message_to_user_question(messages)
cache_collection.add( cache_collection.add(
documents=[user_question], documents=[user_question],
metadatas=[{"model_response": str(model_response)}], metadatas=[{"model_response": str(model_response)}],
ids = [ str(uuid.uuid4())] ids = [ str(uuid.uuid4())]
) )
print("in add cache, peek()", cache_collection.peek()) # print("in add cache, peek()", cache_collection.peek())
return return
def get_cache(messages): def get_cache(messages):
print("in get cache") # print("in get cache")
try: try:
global cache_collection global cache_collection
if cache_collection == None: if cache_collection == None:
make_collection() make_collection()
print("cache collection", cache_collection) # print("cache collection", cache_collection)
user_question = message_to_user_question(messages) user_question = message_to_user_question(messages)
results = cache_collection.query( results = cache_collection.query(
query_texts=[user_question], query_texts=[user_question],
n_results=1 n_results=1
) )
print("query cache result", results) # print("query cache result", results)
distance = results['distances'][0][0] distance = results['distances'][0][0]
sim = (1 - distance) sim = (1 - distance)
if sim >= litellm.cache_similarity_threshold: if sim >= litellm.cache_similarity_threshold:
@ -757,5 +757,5 @@ def get_cache(messages):
# no hit # no hit
return None return None
except Exception as e: except Exception as e:
print("error in get cache", e) # print("error in get cache", e)
return None return None

2
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "litellm" name = "litellm"
version = "0.1.373" version = "0.1.374"
description = "Library to easily interface with LLM API providers" description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"] authors = ["BerriAI"]
license = "MIT License" license = "MIT License"