phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

test(test_streaming.py): set cache to none

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-10 15:04:01 -08:00
parent abee5a0e05
commit 40edb546dc
4 changed files with 240 additions and 238 deletions
Download patch file Download diff file Expand all files Collapse all files

186
litellm/tests/test_caching.py
View file

@ -352,98 +352,98 @@ def test_hosted_cache():
# test_hosted_cache() # test_hosted_cache()
def test_redis_cache_with_ttl(): # def test_redis_cache_with_ttl():
cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD']) # cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD'])
sample_model_response_object_str = """{ # sample_model_response_object_str = """{
"choices": [ # "choices": [
{ # {
"finish_reason": "stop", # "finish_reason": "stop",
"index": 0, # "index": 0,
"message": { # "message": {
"role": "assistant", # "role": "assistant",
"content": "I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic." # "content": "I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic."
} # }
} # }
], # ],
"created": 1691429984.3852863, # "created": 1691429984.3852863,
"model": "claude-instant-1", # "model": "claude-instant-1",
"usage": { # "usage": {
"prompt_tokens": 18, # "prompt_tokens": 18,
"completion_tokens": 23, # "completion_tokens": 23,
"total_tokens": 41 # "total_tokens": 41
} # }
}""" # }"""
sample_model_response_object = { # sample_model_response_object = {
"choices": [ # "choices": [
{ # {
"finish_reason": "stop", # "finish_reason": "stop",
"index": 0, # "index": 0,
"message": { # "message": {
"role": "assistant", # "role": "assistant",
"content": "I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic." # "content": "I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic."
} # }
} # }
], # ],
"created": 1691429984.3852863, # "created": 1691429984.3852863,
"model": "claude-instant-1", # "model": "claude-instant-1",
"usage": { # "usage": {
"prompt_tokens": 18, # "prompt_tokens": 18,
"completion_tokens": 23, # "completion_tokens": 23,
"total_tokens": 41 # "total_tokens": 41
} # }
} # }
cache.add_cache(cache_key="test_key", result=sample_model_response_object_str, ttl=1) # cache.add_cache(cache_key="test_key", result=sample_model_response_object_str, ttl=1)
cached_value = cache.get_cache(cache_key="test_key") # cached_value = cache.get_cache(cache_key="test_key")
print(f"cached-value: {cached_value}") # print(f"cached-value: {cached_value}")
assert cached_value['choices'][0]['message']['content'] == sample_model_response_object['choices'][0]['message']['content'] # assert cached_value['choices'][0]['message']['content'] == sample_model_response_object['choices'][0]['message']['content']
time.sleep(2) # time.sleep(2)
assert cache.get_cache(cache_key="test_key") is None # assert cache.get_cache(cache_key="test_key") is None
# test_redis_cache_with_ttl() # # test_redis_cache_with_ttl()
def test_in_memory_cache_with_ttl(): # def test_in_memory_cache_with_ttl():
cache = Cache(type="local") # cache = Cache(type="local")
sample_model_response_object_str = """{ # sample_model_response_object_str = """{
"choices": [ # "choices": [
{ # {
"finish_reason": "stop", # "finish_reason": "stop",
"index": 0, # "index": 0,
"message": { # "message": {
"role": "assistant", # "role": "assistant",
"content": "I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic." # "content": "I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic."
} # }
} # }
], # ],
"created": 1691429984.3852863, # "created": 1691429984.3852863,
"model": "claude-instant-1", # "model": "claude-instant-1",
"usage": { # "usage": {
"prompt_tokens": 18, # "prompt_tokens": 18,
"completion_tokens": 23, # "completion_tokens": 23,
"total_tokens": 41 # "total_tokens": 41
} # }
}""" # }"""
sample_model_response_object = { # sample_model_response_object = {
"choices": [ # "choices": [
{ # {
"finish_reason": "stop", # "finish_reason": "stop",
"index": 0, # "index": 0,
"message": { # "message": {
"role": "assistant", # "role": "assistant",
"content": "I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic." # "content": "I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic."
} # }
} # }
], # ],
"created": 1691429984.3852863, # "created": 1691429984.3852863,
"model": "claude-instant-1", # "model": "claude-instant-1",
"usage": { # "usage": {
"prompt_tokens": 18, # "prompt_tokens": 18,
"completion_tokens": 23, # "completion_tokens": 23,
"total_tokens": 41 # "total_tokens": 41
} # }
} # }
cache.add_cache(cache_key="test_key", result=sample_model_response_object_str, ttl=1) # cache.add_cache(cache_key="test_key", result=sample_model_response_object_str, ttl=1)
cached_value = cache.get_cache(cache_key="test_key") # cached_value = cache.get_cache(cache_key="test_key")
assert cached_value['choices'][0]['message']['content'] == sample_model_response_object['choices'][0]['message']['content'] # assert cached_value['choices'][0]['message']['content'] == sample_model_response_object['choices'][0]['message']['content']
time.sleep(2) # time.sleep(2)
assert cache.get_cache(cache_key="test_key") is None # assert cache.get_cache(cache_key="test_key") is None
# test_in_memory_cache_with_ttl() # # test_in_memory_cache_with_ttl()

152
litellm/tests/test_client.py
View file

@ -1,90 +1,90 @@
#### What this tests #### # #### What this tests ####
# This tests error logging (with custom user functions) for the `completion` + `embedding` endpoints w/ callbacks # # This tests error logging (with custom user functions) for the `completion` + `embedding` endpoints w/ callbacks
# This only tests posthog, sentry, and helicone # # This only tests posthog, sentry, and helicone
import sys, os # import sys, os
import traceback # import traceback
import pytest # import pytest
sys.path.insert( # sys.path.insert(
0, os.path.abspath("../..") # 0, os.path.abspath("../..")
) # Adds the parent directory to the system path # ) # Adds the parent directory to the system path
import litellm # import litellm
from litellm import embedding, completion # from litellm import embedding, completion
litellm.success_callback = ["posthog", "helicone"] # litellm.success_callback = ["posthog", "helicone"]
litellm.failure_callback = ["sentry", "posthog"] # litellm.failure_callback = ["sentry", "posthog"]
litellm.set_verbose = True # litellm.set_verbose = True
def logger_fn(model_call_object: dict): # def logger_fn(model_call_object: dict):
# print(f"model call details: {model_call_object}") # # print(f"model call details: {model_call_object}")
pass # pass
user_message = "Hello, how are you?" # user_message = "Hello, how are you?"
messages = [{"content": user_message, "role": "user"}] # messages = [{"content": user_message, "role": "user"}]
def test_completion_openai(): # def test_completion_openai():
try:
print("running query")
response = completion(
model="gpt-3.5-turbo", messages=messages, logger_fn=logger_fn
)
print(f"response: {response}")
# Add any assertions here to check the response
except Exception as e:
traceback.print_exc()
pytest.fail(f"Error occurred: {e}")
def test_completion_claude():
try:
response = completion(
model="claude-instant-1", messages=messages, logger_fn=logger_fn
)
# Add any assertions here to check the response
except Exception as e:
pytest.fail(f"Error occurred: {e}")
def test_completion_non_openai():
try:
response = completion(
model="claude-instant-1", messages=messages, logger_fn=logger_fn
)
# Add any assertions here to check the response
except Exception as e:
pytest.fail(f"Error occurred: {e}")
def test_embedding_openai():
try:
response = embedding(
model="text-embedding-ada-002", input=[user_message], logger_fn=logger_fn
)
# Add any assertions here to check the response
print(f"response: {str(response)[:50]}")
except Exception as e:
pytest.fail(f"Error occurred: {e}")
def test_bad_azure_embedding():
try:
response = embedding(
model="chatgpt-test", input=[user_message], logger_fn=logger_fn
)
# Add any assertions here to check the response
print(f"response: {str(response)[:50]}")
except Exception as e:
pass
# def test_good_azure_embedding():
# try: # try:
# response = embedding(model='azure/azure-embedding-model', input=[user_message], logger_fn=logger_fn) # print("running query")
# response = completion(
# model="gpt-3.5-turbo", messages=messages, logger_fn=logger_fn
# )
# print(f"response: {response}")
# # Add any assertions here to check the response
# except Exception as e:
# traceback.print_exc()
# pytest.fail(f"Error occurred: {e}")
# def test_completion_claude():
# try:
# response = completion(
# model="claude-instant-1", messages=messages, logger_fn=logger_fn
# )
# # Add any assertions here to check the response
# except Exception as e:
# pytest.fail(f"Error occurred: {e}")
# def test_completion_non_openai():
# try:
# response = completion(
# model="claude-instant-1", messages=messages, logger_fn=logger_fn
# )
# # Add any assertions here to check the response
# except Exception as e:
# pytest.fail(f"Error occurred: {e}")
# def test_embedding_openai():
# try:
# response = embedding(
# model="text-embedding-ada-002", input=[user_message], logger_fn=logger_fn
# )
# # Add any assertions here to check the response # # Add any assertions here to check the response
# print(f"response: {str(response)[:50]}") # print(f"response: {str(response)[:50]}")
# except Exception as e: # except Exception as e:
# pytest.fail(f"Error occurred: {e}") # pytest.fail(f"Error occurred: {e}")
# def test_bad_azure_embedding():
# try:
# response = embedding(
# model="chatgpt-test", input=[user_message], logger_fn=logger_fn
# )
# # Add any assertions here to check the response
# print(f"response: {str(response)[:50]}")
# except Exception as e:
# pass
# # def test_good_azure_embedding():
# # try:
# # response = embedding(model='azure/azure-embedding-model', input=[user_message], logger_fn=logger_fn)
# # # Add any assertions here to check the response
# # print(f"response: {str(response)[:50]}")
# # except Exception as e:
# # pytest.fail(f"Error occurred: {e}")

130
litellm/tests/test_router.py
View file

@ -56,82 +56,82 @@ load_dotenv()
# test_openai_only() # test_openai_only()
def test_multiple_deployments(): # def test_multiple_deployments():
import concurrent # import concurrent
# litellm.set_verbose=True # # litellm.set_verbose=True
futures = {} # futures = {}
model_list = [{ # list of model deployments # model_list = [{ # list of model deployments
"model_name": "gpt-3.5-turbo", # openai model name # "model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call # "litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-2", # "model": "azure/chatgpt-v-2",
"api_key": os.getenv("AZURE_API_KEY"), # "api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"), # "api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE") # "api_base": os.getenv("AZURE_API_BASE")
}, # },
"tpm": 240000, # "tpm": 240000,
"rpm": 1800 # "rpm": 1800
}, { # }, {
"model_name": "gpt-3.5-turbo", # openai model name # "model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call # "litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-functioncalling", # "model": "azure/chatgpt-functioncalling",
"api_key": os.getenv("AZURE_API_KEY"), # "api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"), # "api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE") # "api_base": os.getenv("AZURE_API_BASE")
}, # },
"tpm": 240000, # "tpm": 240000,
"rpm": 1800 # "rpm": 1800
}, { # }, {
"model_name": "gpt-3.5-turbo", # openai model name # "model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call # "litellm_params": { # params for litellm completion/embedding call
"model": "gpt-3.5-turbo", # "model": "gpt-3.5-turbo",
"api_key": os.getenv("OPENAI_API_KEY"), # "api_key": os.getenv("OPENAI_API_KEY"),
}, # },
"tpm": 1000000, # "tpm": 1000000,
"rpm": 9000 # "rpm": 9000
}] # }]
router = Router(model_list=model_list, redis_host=os.getenv("REDIS_HOST"), redis_password=os.getenv("REDIS_PASSWORD"), redis_port=int(os.getenv("REDIS_PORT"))) # type: ignore # router = Router(model_list=model_list, redis_host=os.getenv("REDIS_HOST"), redis_password=os.getenv("REDIS_PASSWORD"), redis_port=int(os.getenv("REDIS_PORT"))) # type: ignore
results = [] # results = []
with ThreadPoolExecutor(max_workers=10) as executor: # with ThreadPoolExecutor(max_workers=10) as executor:
kwargs = { # kwargs = {
"model": "gpt-3.5-turbo", # "model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": """Context: # "messages": [{"role": "user", "content": """Context:
In the historical era of Ancient Greece, a multitude of significant individuals lived, contributing immensely to various disciplines like science, politics, philosophy, and literature. For instance, Socrates, a renowned philosopher, primarily focused on ethics. His notable method, the Socratic Method, involved acknowledging one's own ignorance to stimulate critical thinking and illuminate ideas. His student, Plato, another prominent figure, founded the Academy in Athens. He proposed theories on justice, beauty, and equality, and also introduced the theory of forms, which is pivotal to understanding his philosophical insights. Another student of Socrates, Xenophon, distinguished himself more in the domain of history and military affairs. # In the historical era of Ancient Greece, a multitude of significant individuals lived, contributing immensely to various disciplines like science, politics, philosophy, and literature. For instance, Socrates, a renowned philosopher, primarily focused on ethics. His notable method, the Socratic Method, involved acknowledging one's own ignorance to stimulate critical thinking and illuminate ideas. His student, Plato, another prominent figure, founded the Academy in Athens. He proposed theories on justice, beauty, and equality, and also introduced the theory of forms, which is pivotal to understanding his philosophical insights. Another student of Socrates, Xenophon, distinguished himself more in the domain of history and military affairs.
Aristotle, who studied under Plato, led an equally remarkable life. His extensive works have been influential across various domains, including science, logic, metaphysics, ethics, and politics. Perhaps most notably, a substantial portion of the Western intellectual tradition traces back to his writings. He later tutored Alexander the Great who went on to create one of the most vast empires in the world. # Aristotle, who studied under Plato, led an equally remarkable life. His extensive works have been influential across various domains, including science, logic, metaphysics, ethics, and politics. Perhaps most notably, a substantial portion of the Western intellectual tradition traces back to his writings. He later tutored Alexander the Great who went on to create one of the most vast empires in the world.
In the domain of mathematics, Pythagoras and Euclid made significant contributions. Pythagoras is best known for the Pythagorean theorem, a fundamental principle in geometry, while Euclid, often regarded as the father of geometry, wrote "The Elements", a collection of definitions, axioms, theorems, and proofs. # In the domain of mathematics, Pythagoras and Euclid made significant contributions. Pythagoras is best known for the Pythagorean theorem, a fundamental principle in geometry, while Euclid, often regarded as the father of geometry, wrote "The Elements", a collection of definitions, axioms, theorems, and proofs.
Apart from these luminaries, the period also saw a number of influential political figures. Pericles, a prominent and influential Greek statesman, orator, and general of Athens during the Golden Age, specifically between the Persian and Peloponnesian wars, played a significant role in developing the Athenian democracy. # Apart from these luminaries, the period also saw a number of influential political figures. Pericles, a prominent and influential Greek statesman, orator, and general of Athens during the Golden Age, specifically between the Persian and Peloponnesian wars, played a significant role in developing the Athenian democracy.
The Ancient Greek era also witnessed extraordinary advancements in arts and literature. Homer, credited with the creation of the epic poems 'The Iliad' and 'The Odyssey,' is considered one of the greatest poets in history. The tragedies of Sophocles, Aeschylus, and Euripides left an indelible mark on the field of drama, and the comedies of Aristophanes remain influential even today. # The Ancient Greek era also witnessed extraordinary advancements in arts and literature. Homer, credited with the creation of the epic poems 'The Iliad' and 'The Odyssey,' is considered one of the greatest poets in history. The tragedies of Sophocles, Aeschylus, and Euripides left an indelible mark on the field of drama, and the comedies of Aristophanes remain influential even today.
--- # ---
Question: # Question:
Who among the mentioned figures from Ancient Greece contributed to the domain of mathematics and what are their significant contributions?"""}], # Who among the mentioned figures from Ancient Greece contributed to the domain of mathematics and what are their significant contributions?"""}],
} # }
for _ in range(10): # for _ in range(10):
future = executor.submit(router.completion, **kwargs) # future = executor.submit(router.completion, **kwargs)
futures[future] = future # futures[future] = future
# Retrieve the results from the futures # # Retrieve the results from the futures
while futures: # while futures:
done, not_done = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED) # done, not_done = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
for future in done: # for future in done:
try: # try:
result = future.result() # result = future.result()
print(f"result: {result}") # print(f"result: {result}")
results.append(result) # results.append(result)
del futures[future] # del futures[future]
except Exception as e: # except Exception as e:
print(f"Exception: {e}; traceback: {traceback.format_exc()}") # print(f"Exception: {e}; traceback: {traceback.format_exc()}")
del futures[future] # remove the done future # del futures[future] # remove the done future
# Check results # # Check results
print(results) # print(results)
# test_multiple_deployments() # test_multiple_deployments()

10
litellm/tests/test_streaming.py
View file

@ -518,6 +518,7 @@ def test_completion_palm_stream():
def test_completion_claude_stream_bad_key(): def test_completion_claude_stream_bad_key():
try: try:
litellm.cache = None
api_key = "bad-key" api_key = "bad-key"
messages = [ messages = [
{"role": "system", "content": "You are a helpful assistant."}, {"role": "system", "content": "You are a helpful assistant."},
@ -538,14 +539,15 @@ def test_completion_claude_stream_bad_key():
complete_response += chunk complete_response += chunk
if complete_response.strip() == "": if complete_response.strip() == "":
raise Exception("Empty response received") raise Exception("Empty response received")
print(f"completion_response: {complete_response}") print(f"1234completion_response: {complete_response}")
raise Exception("Auth error not raised")
except AuthenticationError as e: except AuthenticationError as e:
pass print("Auth Error raised")
except Exception as e: except Exception as e:
pytest.fail(f"Error occurred: {e}") pytest.fail(f"Error occurred: {e}")
# test_completion_claude_stream_bad_key() test_completion_claude_stream_bad_key()
# test_completion_replicate_stream() # test_completion_replicate_stream()
# def test_completion_vertexai_stream(): # def test_completion_vertexai_stream():
@ -912,7 +914,7 @@ def test_openai_chat_completion_complete_response_call():
print(f"error occurred: {traceback.format_exc()}") print(f"error occurred: {traceback.format_exc()}")
pass pass
test_openai_chat_completion_complete_response_call() # test_openai_chat_completion_complete_response_call()
def test_openai_text_completion_call(): def test_openai_text_completion_call():
try: try: