phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix caching

Browse source
This commit is contained in:
ishaan-jaff 2023-08-26 21:05:34 -07:00
parent 28b32dbe8b
commit d4fd0b2010
2 changed files with 16 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

8
litellm/tests/test_caching.py
View file

@ -14,7 +14,6 @@ from litellm import embedding, completion
messages = [{"role": "user", "content": "who is ishaan Github? "}] messages = [{"role": "user", "content": "who is ishaan Github? "}]
# test if response cached # test if response cached
def test_caching(): def test_caching():
try: try:
@ -36,6 +35,7 @@ def test_caching():
def test_caching_with_models(): def test_caching_with_models():
litellm.caching_with_models = True litellm.caching_with_models = True
response1 = completion(model="gpt-3.5-turbo", messages=messages)
response2 = completion(model="gpt-3.5-turbo", messages=messages) response2 = completion(model="gpt-3.5-turbo", messages=messages)
response3 = completion(model="command-nightly", messages=messages) response3 = completion(model="command-nightly", messages=messages)
print(f"response2: {response2}") print(f"response2: {response2}")
@ -46,6 +46,12 @@ def test_caching_with_models():
print(f"response2: {response2}") print(f"response2: {response2}")
print(f"response3: {response3}") print(f"response3: {response3}")
pytest.fail(f"Error occurred:") pytest.fail(f"Error occurred:")
if response1 != response2:
print(f"response1: {response1}")
print(f"response2: {response2}")
pytest.fail(f"Error occurred:")
# test_caching_with_models()
def test_gpt_cache(): def test_gpt_cache():

12
litellm/utils.py
View file

@ -410,19 +410,20 @@ def client(original_function):
def check_cache(*args, **kwargs): def check_cache(*args, **kwargs):
try: # never block execution try: # never block execution
prompt = get_prompt(*args, **kwargs) prompt = get_prompt(*args, **kwargs)
if (prompt != None and prompt if (prompt != None): # check if messages / prompt exists
in local_cache): # check if messages / prompt exists
if litellm.caching_with_models: if litellm.caching_with_models:
# if caching with model names is enabled, key is prompt + model name # if caching with model names is enabled, key is prompt + model name
if ("model" in kwargs and kwargs["model"] if ("model" in kwargs):
in local_cache[prompt]["models"]):
cache_key = prompt + kwargs["model"] cache_key = prompt + kwargs["model"]
if cache_key in local_cache:
return local_cache[cache_key] return local_cache[cache_key]
else: # caching only with prompts else: # caching only with prompts
if prompt in local_cache:
result = local_cache[prompt] result = local_cache[prompt]
return result return result
else: else:
return None return None
return None # default to return None
except: except:
return None return None
@ -430,8 +431,7 @@ def client(original_function):
try: # never block execution try: # never block execution
prompt = get_prompt(*args, **kwargs) prompt = get_prompt(*args, **kwargs)
if litellm.caching_with_models: # caching with model + prompt if litellm.caching_with_models: # caching with model + prompt
if ("model" in kwargs if ("model" in kwargs):
and kwargs["model"] in local_cache[prompt]["models"]):
cache_key = prompt + kwargs["model"] cache_key = prompt + kwargs["model"]
local_cache[cache_key] = result local_cache[cache_key] = result
else: # caching based only on prompts else: # caching based only on prompts