phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 10:14:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(test) caching

Browse source
This commit is contained in:
ishaan-jaff 2023-11-23 18:17:11 -08:00
parent 3660fb1f7f
commit b6f8dbbc0e
1 changed files with 24 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

34
litellm/tests/test_caching.py
View file

@ -155,28 +155,42 @@ def test_redis_cache_completion():
litellm.set_verbose = False
random_number = random.randint(1, 100000) # add a random number to ensure it's always adding / reading from cache
messages = [{"role": "user", "content": f"who is ishaan CTO of litellm, call all llm apis{random_number}"}]
messages = [{"role": "user", "content": f"write a one sentence {random_number}"}]
litellm.cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD'])
print("test2 for caching")
response1 = completion(model="gpt-3.5-turbo", messages=messages, caching=True)
response2 = completion(model="gpt-3.5-turbo", messages=messages, caching=True)
response3 = completion(model="command-nightly", messages=messages, caching=True)
response1 = completion(model="gpt-3.5-turbo", messages=messages, caching=True, max_tokens=10, seed=1222)
response2 = completion(model="gpt-3.5-turbo", messages=messages, caching=True, max_tokens=10, seed=1222)
response3 = completion(model="gpt-3.5-turbo", messages=messages, caching=True, temperature=0.1)
response4 = completion(model="command-nightly", messages=messages, caching=True)
print("\nresponse 1", response1)
print("\nresponse 2", response2)
print("\nresponse 3", response3)
print("\nresponse 4", response4)
litellm.cache = None
if response3['choices'][0]['message']['content'] == response2['choices'][0]['message']['content']:
# if models are different, it should not return cached response
print(f"response2: {response2}")
print(f"response3: {response3}")
pytest.fail(f"Error occurred:")
"""
1 & 2 should be exactly the same
1 & 3 should be different, since input params are diff
1 & 4 should be diff, since models are diff
"""
if response1['choices'][0]['message']['content'] != response2['choices'][0]['message']['content']: # 1 and 2 should be the same
# 1&2 have the exact same input params. This MUST Be a CACHE HIT
print(f"response1: {response1}")
print(f"response2: {response2}")
pytest.fail(f"Error occurred:")
if response1['choices'][0]['message']['content'] == response3['choices'][0]['message']['content']:
# if input params like seed, max_tokens are diff it should NOT be a cache hit
print(f"response1: {response1}")
print(f"response3: {response3}")
pytest.fail(f"Response 1 == response 3. Same model, diff params shoudl not cache Error occurred:")
if response1['choices'][0]['message']['content'] == response4['choices'][0]['message']['content']:
# if models are different, it should not return cached response
print(f"response1: {response1}")
print(f"response4: {response4}")
pytest.fail(f"Error occurred:")
# test_redis_cache_completion()
test_redis_cache_completion()
# redis cache with custom keys
def custom_get_cache_key(*args, **kwargs):