phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixes to batch testing

Browse source
This commit is contained in:
Krrish Dholakia 2023-08-16 11:46:53 -07:00
parent fd6367ca25
commit c58c52db04
3 changed files with 15 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

7
litellm/testing.py
View file

@ -27,16 +27,17 @@ def testing_batch_completion(*args, **kwargs):
else: else:
kwargs_modified["messages"] = message_list kwargs_modified["messages"] = message_list
future = executor.submit(litellm.completion, *args_modified, **kwargs_modified) future = executor.submit(litellm.completion, *args_modified, **kwargs_modified)
completions.append(future) completions.append((future, message_list))
# Retrieve the results and calculate elapsed time for each completion call # Retrieve the results and calculate elapsed time for each completion call
for future in completions: for completion in completions:
future, message_list = completion
start_time = time.time() start_time = time.time()
try: try:
result = future.result() result = future.result()
end_time = time.time() end_time = time.time()
elapsed_time = end_time - start_time elapsed_time = end_time - start_time
result_dict = {"status": "succeeded", "response": future.result(), "response_time": elapsed_time} result_dict = {"status": "succeeded", "response": future.result(), "prompt": message_list, "response_time": elapsed_time}
results.append(result_dict) results.append(result_dict)
except Exception as e: except Exception as e:
end_time = time.time() end_time = time.time()

20
litellm/tests/test_load_test_model.py
View file

@ -4,19 +4,19 @@ sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the
import litellm import litellm
from litellm import load_test_model, testing_batch_completion from litellm import load_test_model, testing_batch_completion
## Load Test Model # ## Load Test Model
model="gpt-3.5-turbo" # model="gpt-3.5-turbo"
result = load_test_model(model=model, num_calls=5) # result = load_test_model(model=model, num_calls=5)
print(result) # print(result)
print(len(result["results"])) # print(len(result["results"]))
## Duration Test Model # ## Duration Test Model
model="gpt-3.5-turbo" # model="gpt-3.5-turbo"
result = load_test_model(model=model, num_calls=5, duration=15, interval=15) # duration test the model for 2 minutes, sending 5 calls every 15s # result = load_test_model(model=model, num_calls=5, duration=15, interval=15) # duration test the model for 2 minutes, sending 5 calls every 15s
print(result) # print(result)
## Quality Test across Model ## Quality Test across Model
models = ["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-4", "claude-instant-1", {"model": "replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781", "custom_llm_provider": "replicate"}] models = ["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-4", "claude-instant-1", {"model": "replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781", "custom_llm_provider": "replicate"}]
messages = [[{"role": "user", "content": "What is your name?"}]] messages = [[{"role": "user", "content": "What is your name?"}], [{"role": "user", "content": "Hey, how's it going?"}]]
result = testing_batch_completion(models=models, messages=messages) result = testing_batch_completion(models=models, messages=messages)
print(result) print(result)

2
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "litellm" name = "litellm"
version = "0.1.402" version = "0.1.403"
description = "Library to easily interface with LLM API providers" description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"] authors = ["BerriAI"]
license = "MIT License" license = "MIT License"