add metadata logging to promptlayer

docs/my-website/docs/observability/promptlayer_integration.md

@@ -25,7 +25,7 @@ Complete code
 from litellm import completion
 
 ## set env variables
-os.environ["PROMPTLAYER_API_KEY"] = "your"
+os.environ["PROMPTLAYER_API_KEY"] = "your-promptlayer-key"
 
 os.environ["OPENAI_API_KEY"], os.environ["COHERE_API_KEY"] = "", ""
 
@@ -38,3 +38,33 @@ response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content
 #cohere call
 response = completion(model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}])
 ```
+
+### Logging Metadata 
+
+You can also log completion call metadata to Promptlayer. 
+
+You can add metadata to a completion call through the metadata param: 
+```python 
+completion(model,messages, metadata={"model": "ai21"})
+```
+
+**Complete Code**
+```python
+from litellm import completion
+
+## set env variables
+os.environ["PROMPTLAYER_API_KEY"] = "your-promptlayer-key"
+
+os.environ["OPENAI_API_KEY"], os.environ["COHERE_API_KEY"] = "", ""
+
+# set callbacks
+litellm.success_callback = ["promptlayer"]
+
+#openai call - log llm provider is openai
+response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}], metadata={"provider": "openai"})
+
+#cohere call - log llm provider is cohere
+response = completion(model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}], metadata={"provider": "cohere"})
+```
+
+Credits to [Nick Bradford](https://github.com/nsbradford), from [Vim-GPT](https://github.com/nsbradford/VimGPT), for the suggestion. 

litellm/integrations/prompt_layer.py

@@ -41,8 +41,25 @@ class PromptLayerLogger:
                 },
             )
             print_verbose(
-                f"Prompt Layer Logging: success - final response object: {request_response}"
+                f"Prompt Layer Logging: success - final response object: {request_response.text}"
             )
+            response_json = request_response.json()
+            if "success" not in request_response.json(): 
+                raise Exception("Promptlayer did not successfully log the response!") 
+
+            if "request_id" in response_json:
+                print(kwargs["litellm_params"]["metadata"])
+                if kwargs["litellm_params"]["metadata"] is not None:
+                    response = requests.post(
+                        "https://api.promptlayer.com/rest/track-metadata",
+                        json={
+                            "request_id": response_json["request_id"],
+                            "api_key": self.key,
+                            "metadata": kwargs["litellm_params"]["metadata"]
+                        },
+                    )
+                    print_verbose(f"Prompt Layer Logging: success - metadata post response object: {response.text}")
+
         except:
             print_verbose(f"error: Prompt Layer Error - {traceback.format_exc()}")
             pass

litellm/main.py

@@ -171,6 +171,7 @@ def completion(
     litellm_logging_obj=None,
     use_client=False,
     id=None, # this is an optional param to tag individual completion calls 
+    metadata: Optional[dict]=None,
     # model specific optional params
     top_k=40,# used by text-bison only
     task: Optional[str]="text-generation-inference", # used by huggingface inference endpoints
@@ -201,6 +202,7 @@ def completion(
         frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far.
         logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion.
         user (str, optional):  A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse.
+        metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc. 
 
         LITELLM Specific Params
         mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None).
@@ -276,7 +278,8 @@ def completion(
             api_base=api_base,
             litellm_call_id=litellm_call_id,
             model_alias_map=litellm.model_alias_map,
-            completion_call_id=id
+            completion_call_id=id,
+            metadata=metadata
         )
         logging.update_environment_variables(model=model, user=user, optional_params=optional_params, litellm_params=litellm_params)
         if custom_llm_provider == "azure":

litellm/tests/test_promptlayer_integration.py

@@ -13,18 +13,46 @@ import time
 
 
 
-def test_promptlayer_logging():
+# def test_promptlayer_logging():
+#     try:
+#         # Redirect stdout
+#         old_stdout = sys.stdout
+#         sys.stdout = new_stdout = io.StringIO()
+
+
+#         response = completion(model="claude-instant-1.2",
+#                               messages=[{
+#                                   "role": "user",
+#                                   "content": "Hi 👋 - i'm claude"
+#                               }])
+
+#         # Restore stdout
+#         time.sleep(1)
+#         sys.stdout = old_stdout
+#         output = new_stdout.getvalue().strip()
+#         print(output)
+#         if "LiteLLM: Prompt Layer Logging: success" not in output:
+#             raise Exception("Required log message not found!")
+
+#     except Exception as e:
+#         print(e)
+
+# test_promptlayer_logging()
+
+
+def test_promptlayer_logging_with_metadata():
     try:
         # Redirect stdout
         old_stdout = sys.stdout
         sys.stdout = new_stdout = io.StringIO()
 
 
-        response = completion(model="claude-instant-1.2",
+        response = completion(model="j2-light",
                               messages=[{
                                   "role": "user",
-                                  "content": "Hi 👋 - i'm openai"
+                                  "content": "Hi 👋 - i'm ai21"
-                              }])
+                              }], 
+                              metadata={"model": "ai21"})
 
         # Restore stdout
         time.sleep(1)
@@ -37,7 +65,7 @@ def test_promptlayer_logging():
     except Exception as e:
         print(e)
 
-test_promptlayer_logging()
+# test_promptlayer_logging_with_metadata()
 
 
 

litellm/utils.py

@@ -177,7 +177,7 @@ class CallTypes(Enum):
 
 # Logging function -> log the exact model details + what's being sent | Non-Blocking
 class Logging:
-    global supabaseClient, liteDebuggerClient
+    global supabaseClient, liteDebuggerClient, promptLayerLogger
 
     def __init__(self, model, messages, stream, call_type, start_time, litellm_call_id, function_id):
         if call_type not in [item.value for item in CallTypes]:
@@ -395,6 +395,15 @@ class Logging:
                                 self.litellm_params["stream_response"][litellm_call_id] = new_model_response
                             #print("adding to cache for", litellm_call_id)                              
                             litellm.cache.add_cache(self.litellm_params["stream_response"][litellm_call_id], **self.model_call_details)
+                    if callback == "promptlayer":
+                        print_verbose("reaches promptlayer for logging!")
+                        promptLayerLogger.log_event(
+                            kwargs=self.model_call_details,
+                            response_obj=result,
+                            start_time=start_time,
+                            end_time=end_time,
+                            print_verbose=print_verbose,
+                        )
 
                 except Exception as e:
                     print_verbose(
@@ -827,7 +836,8 @@ def get_litellm_params(
     api_base=None,
     litellm_call_id=None,
     model_alias_map=None,
-    completion_call_id=None
+    completion_call_id=None,
+    metadata=None
 ):
     litellm_params = {
         "return_async": return_async,
@@ -840,6 +850,7 @@ def get_litellm_params(
         "litellm_call_id": litellm_call_id,
         "model_alias_map": model_alias_map,
         "completion_call_id": completion_call_id,
+        "metadata": metadata,
         "stream_response": {} # litellm_call_id: ModelResponse Dict
     }
 
@@ -1630,15 +1641,6 @@ def handle_success(args, kwargs, result, start_time, end_time):
                         run_id=kwargs["litellm_call_id"],
                         print_verbose=print_verbose,
                     )
-                elif callback == "promptlayer":
-                    print_verbose("reaches promptlayer for logging!")
-                    promptLayerLogger.log_event(
-                        kwargs=kwargs,
-                        response_obj=result,
-                        start_time=start_time,
-                        end_time=end_time,
-                        print_verbose=print_verbose,
-                    )
                 elif callback == "langfuse":
                     print_verbose("reaches langfuse for logging!")
                     langFuseLogger.log_event(

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "0.1.721"
+version = "0.1.722"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"