show litedebugger for streaming objects

litellm/integrations/litedebugger.py

@@ -1,6 +1,5 @@
 import requests, traceback, json, os
 
-
 class LiteDebugger:
     user_email = None
     dashboard_url = None
@@ -35,6 +34,13 @@ class LiteDebugger:
             print_verbose(
                 f"LiteLLMDebugger: Logging - Enters input logging function for model {model}"
             )
+            def remove_key_value(dictionary, key):
+                new_dict = dictionary.copy()  # Create a copy of the original dictionary
+                new_dict.pop(key)  # Remove the specified key-value pair from the copy
+                return new_dict
+            
+            updated_litellm_params = remove_key_value(litellm_params, "logger_fn")
+
             litellm_data_obj = {
                 "model": model,
                 "messages": messages,
@@ -42,9 +48,12 @@ class LiteDebugger:
                 "status": "initiated",
                 "litellm_call_id": litellm_call_id,
                 "user_email": self.user_email,
-                "litellm_params": litellm_params,
+                "litellm_params": updated_litellm_params,
                 "optional_params": optional_params
             }
+            print_verbose(
+                f"LiteLLMDebugger: Logging - logged data obj {litellm_data_obj}"
+            )
             response = requests.post(
                 url=self.api_url,
                 headers={"content-type": "application/json"},
@@ -91,7 +100,7 @@ class LiteDebugger:
     ):
         try:
             print_verbose(
-                f"LiteLLMDebugger: Logging - Enters handler logging function for model {model}"
+                f"LiteLLMDebugger: Logging - Enters handler logging function for model {model} with response object {response_obj}"
             )
             total_cost = 0  # [TODO] implement cost tracking
             response_time = (end_time - start_time).total_seconds()
@@ -101,7 +110,7 @@ class LiteDebugger:
                     "model": response_obj["model"],
                     "total_cost": total_cost,
                     "messages": messages,
-                    "response": response_obj["choices"][0]["message"]["content"],
+                    "response": response['choices'][0]['message']['content'],
                     "end_user": end_user,
                     "litellm_call_id": litellm_call_id,
                     "status": "success",
@@ -136,6 +145,25 @@ class LiteDebugger:
                     headers={"content-type": "application/json"},
                     data=json.dumps(litellm_data_obj),
                 )
+            elif isinstance(response_obj, object) and response_obj.__class__.__name__ == "CustomStreamWrapper":
+                litellm_data_obj = {
+                    "response_time": response_time,
+                    "total_cost": total_cost,
+                    "messages": messages,
+                    "response": "Streamed response",
+                    "end_user": end_user,
+                    "litellm_call_id": litellm_call_id,
+                    "status": "success",
+                    "user_email": self.user_email,
+                }
+                print_verbose(
+                    f"LiteDebugger: Logging - final data object: {litellm_data_obj}"
+                )
+                response = requests.post(
+                    url=self.api_url,
+                    headers={"content-type": "application/json"},
+                    data=json.dumps(litellm_data_obj),
+                )
             elif "error" in response_obj:
                 if "Unable to map your input to a model." in response_obj["error"]:
                     total_cost = 0

litellm/tests/test_streaming.py

@@ -3,14 +3,14 @@
 
 import sys, os
 import traceback
-
+import time 
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
 import litellm
 from litellm import completion
 
-litellm.set_verbose = False
+litellm.set_verbose = True
 
 score = 0
 
@@ -22,54 +22,25 @@ def logger_fn(model_call_object: dict):
 user_message = "Hello, how are you?"
 messages = [{"content": user_message, "role": "user"}]
 
-# test on openai completion call
-try:
-    response = completion(
-        model="gpt-3.5-turbo", messages=messages, stream=True, logger_fn=logger_fn
-    )
-    for chunk in response:
-        print(chunk["choices"][0]["delta"])
-    score += 1
-except:
-    print(f"error occurred: {traceback.format_exc()}")
-    pass
-
-
-# test on azure completion call
-try:
-    response = completion(
-        model="azure/chatgpt-test", messages=messages, stream=True, logger_fn=logger_fn
-    )
-    for chunk in response:
-        print(chunk["choices"][0]["delta"])
-    score += 1
-except:
-    print(f"error occurred: {traceback.format_exc()}")
-    pass
-
-
 # test on anthropic completion call
 try:
+    start_time = time.time()
     response = completion(
         model="claude-instant-1", messages=messages, stream=True, logger_fn=logger_fn
     )
     for chunk in response:
+        chunk_time = time.time() 
+        print(f"time since initial request: {chunk_time - start_time:.2f}")
         print(chunk["choices"][0]["delta"])
     score += 1
 except:
     print(f"error occurred: {traceback.format_exc()}")
     pass
 
-
+# # test on openai completion call
-# # test on huggingface completion call
 # try:
 #     response = completion(
-#         model="meta-llama/Llama-2-7b-chat-hf",
+#         model="gpt-3.5-turbo", messages=messages, stream=True, logger_fn=logger_fn
-#         messages=messages,
-#         custom_llm_provider="huggingface",
-#         custom_api_base="https://s7c7gytn18vnu4tw.us-east-1.aws.endpoints.huggingface.cloud",
-#         stream=True,
-#         logger_fn=logger_fn,
 #     )
 #     for chunk in response:
 #         print(chunk["choices"][0]["delta"])
@@ -77,3 +48,47 @@ except:
 # except:
 #     print(f"error occurred: {traceback.format_exc()}")
 #     pass
+
+
+# # test on azure completion call
+# try:
+#     response = completion(
+#         model="azure/chatgpt-test", messages=messages, stream=True, logger_fn=logger_fn
+#     )
+#     for chunk in response:
+#         print(chunk["choices"][0]["delta"])
+#     score += 1
+# except:
+#     print(f"error occurred: {traceback.format_exc()}")
+#     pass
+
+
+# # test on anthropic completion call
+# try:
+#     response = completion(
+#         model="claude-instant-1", messages=messages, stream=True, logger_fn=logger_fn
+#     )
+#     for chunk in response:
+#         print(chunk["choices"][0]["delta"])
+#     score += 1
+# except:
+#     print(f"error occurred: {traceback.format_exc()}")
+#     pass
+
+
+# # # test on huggingface completion call
+# # try:
+# #     response = completion(
+# #         model="meta-llama/Llama-2-7b-chat-hf",
+# #         messages=messages,
+# #         custom_llm_provider="huggingface",
+# #         custom_api_base="https://s7c7gytn18vnu4tw.us-east-1.aws.endpoints.huggingface.cloud",
+# #         stream=True,
+# #         logger_fn=logger_fn,
+# #     )
+# #     for chunk in response:
+# #         print(chunk["choices"][0]["delta"])
+# #     score += 1
+# # except:
+# #     print(f"error occurred: {traceback.format_exc()}")
+# #     pass

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "0.1.484"
+version = "0.1.485"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"