diff --git a/docs/my-website/docs/observability/promptlayer_integration.md b/docs/my-website/docs/observability/promptlayer_integration.md index 55a8bbb7b5..676c0107fc 100644 --- a/docs/my-website/docs/observability/promptlayer_integration.md +++ b/docs/my-website/docs/observability/promptlayer_integration.md @@ -25,7 +25,7 @@ Complete code from litellm import completion ## set env variables -os.environ["PROMPTLAYER_API_KEY"] = "your" +os.environ["PROMPTLAYER_API_KEY"] = "your-promptlayer-key" os.environ["OPENAI_API_KEY"], os.environ["COHERE_API_KEY"] = "", "" @@ -38,3 +38,33 @@ response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content #cohere call response = completion(model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}]) ``` + +### Logging Metadata + +You can also log completion call metadata to Promptlayer. + +You can add metadata to a completion call through the metadata param: +```python +completion(model,messages, metadata={"model": "ai21"}) +``` + +**Complete Code** +```python +from litellm import completion + +## set env variables +os.environ["PROMPTLAYER_API_KEY"] = "your-promptlayer-key" + +os.environ["OPENAI_API_KEY"], os.environ["COHERE_API_KEY"] = "", "" + +# set callbacks +litellm.success_callback = ["promptlayer"] + +#openai call - log llm provider is openai +response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}], metadata={"provider": "openai"}) + +#cohere call - log llm provider is cohere +response = completion(model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}], metadata={"provider": "cohere"}) +``` + +Credits to [Nick Bradford](https://github.com/nsbradford), from [Vim-GPT](https://github.com/nsbradford/VimGPT), for the suggestion. \ No newline at end of file diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index 24ef31d5f0..9a91ccf3bc 100644 Binary files a/litellm/__pycache__/main.cpython-311.pyc and b/litellm/__pycache__/main.cpython-311.pyc differ diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index dbb232368b..de1a3929e5 100644 Binary files a/litellm/__pycache__/utils.cpython-311.pyc and b/litellm/__pycache__/utils.cpython-311.pyc differ diff --git a/litellm/integrations/prompt_layer.py b/litellm/integrations/prompt_layer.py index c70fcb3334..cf0b7fd57c 100644 --- a/litellm/integrations/prompt_layer.py +++ b/litellm/integrations/prompt_layer.py @@ -41,8 +41,25 @@ class PromptLayerLogger: }, ) print_verbose( - f"Prompt Layer Logging: success - final response object: {request_response}" + f"Prompt Layer Logging: success - final response object: {request_response.text}" ) + response_json = request_response.json() + if "success" not in request_response.json(): + raise Exception("Promptlayer did not successfully log the response!") + + if "request_id" in response_json: + print(kwargs["litellm_params"]["metadata"]) + if kwargs["litellm_params"]["metadata"] is not None: + response = requests.post( + "https://api.promptlayer.com/rest/track-metadata", + json={ + "request_id": response_json["request_id"], + "api_key": self.key, + "metadata": kwargs["litellm_params"]["metadata"] + }, + ) + print_verbose(f"Prompt Layer Logging: success - metadata post response object: {response.text}") + except: print_verbose(f"error: Prompt Layer Error - {traceback.format_exc()}") pass diff --git a/litellm/main.py b/litellm/main.py index 546d8a7345..538c60b8db 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -171,6 +171,7 @@ def completion( litellm_logging_obj=None, use_client=False, id=None, # this is an optional param to tag individual completion calls + metadata: Optional[dict]=None, # model specific optional params top_k=40,# used by text-bison only task: Optional[str]="text-generation-inference", # used by huggingface inference endpoints @@ -201,6 +202,7 @@ def completion( frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far. logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion. user (str, optional): A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse. + metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc. LITELLM Specific Params mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None). @@ -276,7 +278,8 @@ def completion( api_base=api_base, litellm_call_id=litellm_call_id, model_alias_map=litellm.model_alias_map, - completion_call_id=id + completion_call_id=id, + metadata=metadata ) logging.update_environment_variables(model=model, user=user, optional_params=optional_params, litellm_params=litellm_params) if custom_llm_provider == "azure": diff --git a/litellm/tests/test_promptlayer_integration.py b/litellm/tests/test_promptlayer_integration.py index 975434764d..6d3bbe105c 100644 --- a/litellm/tests/test_promptlayer_integration.py +++ b/litellm/tests/test_promptlayer_integration.py @@ -13,18 +13,46 @@ import time -def test_promptlayer_logging(): +# def test_promptlayer_logging(): +# try: +# # Redirect stdout +# old_stdout = sys.stdout +# sys.stdout = new_stdout = io.StringIO() + + +# response = completion(model="claude-instant-1.2", +# messages=[{ +# "role": "user", +# "content": "Hi 👋 - i'm claude" +# }]) + +# # Restore stdout +# time.sleep(1) +# sys.stdout = old_stdout +# output = new_stdout.getvalue().strip() +# print(output) +# if "LiteLLM: Prompt Layer Logging: success" not in output: +# raise Exception("Required log message not found!") + +# except Exception as e: +# print(e) + +# test_promptlayer_logging() + + +def test_promptlayer_logging_with_metadata(): try: # Redirect stdout old_stdout = sys.stdout sys.stdout = new_stdout = io.StringIO() - response = completion(model="claude-instant-1.2", + response = completion(model="j2-light", messages=[{ "role": "user", - "content": "Hi 👋 - i'm openai" - }]) + "content": "Hi 👋 - i'm ai21" + }], + metadata={"model": "ai21"}) # Restore stdout time.sleep(1) @@ -37,7 +65,7 @@ def test_promptlayer_logging(): except Exception as e: print(e) -test_promptlayer_logging() +# test_promptlayer_logging_with_metadata() diff --git a/litellm/utils.py b/litellm/utils.py index 137e4f9742..ead50964fe 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -177,7 +177,7 @@ class CallTypes(Enum): # Logging function -> log the exact model details + what's being sent | Non-Blocking class Logging: - global supabaseClient, liteDebuggerClient + global supabaseClient, liteDebuggerClient, promptLayerLogger def __init__(self, model, messages, stream, call_type, start_time, litellm_call_id, function_id): if call_type not in [item.value for item in CallTypes]: @@ -395,6 +395,15 @@ class Logging: self.litellm_params["stream_response"][litellm_call_id] = new_model_response #print("adding to cache for", litellm_call_id) litellm.cache.add_cache(self.litellm_params["stream_response"][litellm_call_id], **self.model_call_details) + if callback == "promptlayer": + print_verbose("reaches promptlayer for logging!") + promptLayerLogger.log_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + ) except Exception as e: print_verbose( @@ -827,7 +836,8 @@ def get_litellm_params( api_base=None, litellm_call_id=None, model_alias_map=None, - completion_call_id=None + completion_call_id=None, + metadata=None ): litellm_params = { "return_async": return_async, @@ -840,6 +850,7 @@ def get_litellm_params( "litellm_call_id": litellm_call_id, "model_alias_map": model_alias_map, "completion_call_id": completion_call_id, + "metadata": metadata, "stream_response": {} # litellm_call_id: ModelResponse Dict } @@ -1630,15 +1641,6 @@ def handle_success(args, kwargs, result, start_time, end_time): run_id=kwargs["litellm_call_id"], print_verbose=print_verbose, ) - elif callback == "promptlayer": - print_verbose("reaches promptlayer for logging!") - promptLayerLogger.log_event( - kwargs=kwargs, - response_obj=result, - start_time=start_time, - end_time=end_time, - print_verbose=print_verbose, - ) elif callback == "langfuse": print_verbose("reaches langfuse for logging!") langFuseLogger.log_event( diff --git a/pyproject.toml b/pyproject.toml index 8cb6f889eb..73b6beea35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.721" +version = "0.1.722" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License"