phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 19:54:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

show litedebugger for streaming objects

Browse source
This commit is contained in:
Krrish Dholakia 2023-08-26 10:28:27 -07:00
parent a883dab374
commit 47917edef4
6 changed files with 84 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
litellm/__pycache__/__init__.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/__pycache__/main.cpython-311.pyc
View file

Binary file not shown.

BIN
litellm/__pycache__/utils.cpython-311.pyc
View file

Binary file not shown.

36
litellm/integrations/litedebugger.py
View file

@ -1,6 +1,5 @@
import requests, traceback, json, os
class LiteDebugger:
user_email = None
dashboard_url = None
@ -35,6 +34,13 @@ class LiteDebugger:
print_verbose(
f"LiteLLMDebugger: Logging - Enters input logging function for model {model}"
)
def remove_key_value(dictionary, key):
new_dict = dictionary.copy() # Create a copy of the original dictionary
new_dict.pop(key) # Remove the specified key-value pair from the copy
return new_dict
updated_litellm_params = remove_key_value(litellm_params, "logger_fn")
litellm_data_obj = {
"model": model,
"messages": messages,
@ -42,9 +48,12 @@ class LiteDebugger:
"status": "initiated",
"litellm_call_id": litellm_call_id,
"user_email": self.user_email,
"litellm_params": litellm_params,
"litellm_params": updated_litellm_params,
"optional_params": optional_params
}
print_verbose(
f"LiteLLMDebugger: Logging - logged data obj {litellm_data_obj}"
)
response = requests.post(
url=self.api_url,
headers={"content-type": "application/json"},
@ -91,7 +100,7 @@ class LiteDebugger:
):
try:
print_verbose(
f"LiteLLMDebugger: Logging - Enters handler logging function for model {model}"
f"LiteLLMDebugger: Logging - Enters handler logging function for model {model} with response object {response_obj}"
)
total_cost = 0 # [TODO] implement cost tracking
response_time = (end_time - start_time).total_seconds()
@ -101,7 +110,7 @@ class LiteDebugger:
"model": response_obj["model"],
"total_cost": total_cost,
"messages": messages,
"response": response_obj["choices"][0]["message"]["content"],
"response": response['choices'][0]['message']['content'],
"end_user": end_user,
"litellm_call_id": litellm_call_id,
"status": "success",
@ -136,6 +145,25 @@ class LiteDebugger:
headers={"content-type": "application/json"},
data=json.dumps(litellm_data_obj),
)
elif isinstance(response_obj, object) and response_obj.__class__.__name__ == "CustomStreamWrapper":
litellm_data_obj = {
"response_time": response_time,
"total_cost": total_cost,
"messages": messages,
"response": "Streamed response",
"end_user": end_user,
"litellm_call_id": litellm_call_id,
"status": "success",
"user_email": self.user_email,
}
print_verbose(
f"LiteDebugger: Logging - final data object: {litellm_data_obj}"
)
response = requests.post(
url=self.api_url,
headers={"content-type": "application/json"},
data=json.dumps(litellm_data_obj),
)
elif "error" in response_obj:
if "Unable to map your input to a model." in response_obj["error"]:
total_cost = 0

87
litellm/tests/test_streaming.py
View file

@ -3,14 +3,14 @@
import sys, os
import traceback
import time
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import litellm
from litellm import completion
litellm.set_verbose = False
litellm.set_verbose = True
score = 0
@ -22,54 +22,25 @@ def logger_fn(model_call_object: dict):
user_message = "Hello, how are you?"
messages = [{"content": user_message, "role": "user"}]
# test on openai completion call
try:
response = completion(
model="gpt-3.5-turbo", messages=messages, stream=True, logger_fn=logger_fn
)
for chunk in response:
print(chunk["choices"][0]["delta"])
score += 1
except:
print(f"error occurred: {traceback.format_exc()}")
pass
# test on azure completion call
try:
response = completion(
model="azure/chatgpt-test", messages=messages, stream=True, logger_fn=logger_fn
)
for chunk in response:
print(chunk["choices"][0]["delta"])
score += 1
except:
print(f"error occurred: {traceback.format_exc()}")
pass
# test on anthropic completion call
try:
start_time = time.time()
response = completion(
model="claude-instant-1", messages=messages, stream=True, logger_fn=logger_fn
)
for chunk in response:
chunk_time = time.time()
print(f"time since initial request: {chunk_time - start_time:.2f}")
print(chunk["choices"][0]["delta"])
score += 1
except:
print(f"error occurred: {traceback.format_exc()}")
pass
# # test on huggingface completion call
# # test on openai completion call
# try:
# response = completion(
# model="meta-llama/Llama-2-7b-chat-hf",
# messages=messages,
# custom_llm_provider="huggingface",
# custom_api_base="https://s7c7gytn18vnu4tw.us-east-1.aws.endpoints.huggingface.cloud",
# stream=True,
# logger_fn=logger_fn,
# model="gpt-3.5-turbo", messages=messages, stream=True, logger_fn=logger_fn
# )
# for chunk in response:
# print(chunk["choices"][0]["delta"])
@ -77,3 +48,47 @@ except:
# except:
# print(f"error occurred: {traceback.format_exc()}")
# pass
# # test on azure completion call
# try:
# response = completion(
# model="azure/chatgpt-test", messages=messages, stream=True, logger_fn=logger_fn
# )
# for chunk in response:
# print(chunk["choices"][0]["delta"])
# score += 1
# except:
# print(f"error occurred: {traceback.format_exc()}")
# pass
# # test on anthropic completion call
# try:
# response = completion(
# model="claude-instant-1", messages=messages, stream=True, logger_fn=logger_fn
# )
# for chunk in response:
# print(chunk["choices"][0]["delta"])
# score += 1
# except:
# print(f"error occurred: {traceback.format_exc()}")
# pass
# # # test on huggingface completion call
# # try:
# # response = completion(
# # model="meta-llama/Llama-2-7b-chat-hf",
# # messages=messages,
# # custom_llm_provider="huggingface",
# # custom_api_base="https://s7c7gytn18vnu4tw.us-east-1.aws.endpoints.huggingface.cloud",
# # stream=True,
# # logger_fn=logger_fn,
# # )
# # for chunk in response:
# # print(chunk["choices"][0]["delta"])
# # score += 1
# # except:
# # print(f"error occurred: {traceback.format_exc()}")
# # pass

2
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "0.1.484"
version = "0.1.485"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT License"