phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(feat )add semantic cache

Browse source
This commit is contained in:
ishaan-jaff 2024-02-05 12:28:21 -08:00
parent 646764f1d4
commit d4a799a3ca
2 changed files with 124 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

25
litellm/tests/test_caching.py
View file

@ -987,3 +987,28 @@ def test_cache_context_managers():
# test_cache_context_managers()
def test_redis_semantic_cache_completion():
litellm.set_verbose = False
random_number = random.randint(
1, 100000
) # add a random number to ensure it's always adding / reading from cache
messages = [
{"role": "user", "content": f"write a one sentence poem about: {random_number}"}
]
litellm.cache = Cache(
type="redis-semantic",
host=os.environ["REDIS_HOST"],
port=os.environ["REDIS_PORT"],
password=os.environ["REDIS_PASSWORD"],
)
print("test2 for Redis Caching - non streaming")
response1 = completion(model="gpt-3.5-turbo", messages=messages, max_tokens=20)
# response2 = completion(
# model="gpt-3.5-turbo", messages=messages,max_tokens=20
# )
# test_redis_cache_completion()