From 8b6d686e5247fb20d9bd8295c494260bb2ce766e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Apr 2024 10:01:09 -0700 Subject: [PATCH 1/7] feat - turn_off_message_logging --- litellm/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index b9d9891ca..9a8b73447 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -45,6 +45,7 @@ _async_failure_callback: List[Callable] = ( ) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] +turn_off_message_logging: Optional[bool] = False ## end of callbacks ############# email: Optional[str] = ( From 10da35675f14aecdd1dee8d0d2ad772ba90322db Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Apr 2024 10:03:07 -0700 Subject: [PATCH 2/7] feat- turn off message logging --- litellm/utils.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index 8c3863344..4fe232666 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1527,6 +1527,32 @@ class Logging: else: callbacks = litellm.success_callback + # check if user opted out of logging message/response to callbacks + if litellm.turn_off_message_logging == True: + # remove messages, prompts, input, response from logging + self.model_call_details["messages"] = "redacted-by-litellm" + self.model_call_details["prompt"] = "" + self.model_call_details["input"] = "" + + # response cleaning + # ChatCompletion Responses + if ( + self.stream + and "complete_streaming_response" in self.model_call_details + ): + _streaming_response = self.model_call_details[ + "complete_streaming_response" + ] + for choice in _streaming_response.choices: + choice.message.content = "redacted-by-litellm" + else: + for choice in result.choices: + choice.message.content = "redacted-by-litellm" + + # Embedding Responses + + # Text Completion Responses + for callback in callbacks: try: litellm_params = self.model_call_details.get("litellm_params", {}) From 743dfdb950ea9e5b6842419ca4d9ef814d9ec048 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Apr 2024 10:03:34 -0700 Subject: [PATCH 3/7] test - redacting messages from langfuse --- litellm/tests/test_alangfuse.py | 72 ++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/litellm/tests/test_alangfuse.py b/litellm/tests/test_alangfuse.py index c1d8123c7..12e5f9587 100644 --- a/litellm/tests/test_alangfuse.py +++ b/litellm/tests/test_alangfuse.py @@ -161,40 +161,54 @@ async def make_async_calls(): return total_time -# def test_langfuse_logging_async_text_completion(): -# try: -# pre_langfuse_setup() -# litellm.set_verbose = False -# litellm.success_callback = ["langfuse"] +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_langfuse_logging_without_request_response(stream): + try: + import uuid -# async def _test_langfuse(): -# response = await litellm.atext_completion( -# model="gpt-3.5-turbo-instruct", -# prompt="this is a test", -# max_tokens=5, -# temperature=0.7, -# timeout=5, -# user="test_user", -# stream=True -# ) -# async for chunk in response: -# print() -# print(chunk) -# await asyncio.sleep(1) -# return response + _unique_trace_name = f"litellm-test-{str(uuid.uuid4())}" + litellm.set_verbose = True + litellm.turn_off_message_logging = True + litellm.success_callback = ["langfuse"] + response = await litellm.acompletion( + model="gpt-3.5-turbo", + mock_response="It's simple to use and easy to get started", + messages=[{"role": "user", "content": "Hi 👋 - i'm claude"}], + max_tokens=10, + temperature=0.2, + stream=stream, + metadata={"trace_id": _unique_trace_name}, + ) + print(response) + if stream: + async for chunk in response: + print(chunk) -# response = asyncio.run(_test_langfuse()) -# print(f"response: {response}") + await asyncio.sleep(3) -# # # check langfuse.log to see if there was a failed response -# search_logs("langfuse.log") -# except litellm.Timeout as e: -# pass -# except Exception as e: -# pytest.fail(f"An exception occurred - {e}") + import langfuse + langfuse_client = langfuse.Langfuse( + public_key=os.environ["LANGFUSE_PUBLIC_KEY"], + secret_key=os.environ["LANGFUSE_SECRET_KEY"], + ) -# test_langfuse_logging_async_text_completion() + # get trace with _unique_trace_name + trace = langfuse_client.get_generations(trace_id=_unique_trace_name) + + print("trace_from_langfuse", trace) + + _trace_data = trace.data + + assert _trace_data[0].input == {"messages": "redacted-by-litellm"} + assert _trace_data[0].output == { + "role": "assistant", + "content": "redacted-by-litellm", + } + + except Exception as e: + pytest.fail(f"An exception occurred - {e}") @pytest.mark.skip(reason="beta test - checking langfuse output") From b2111a97e2ca727a5e11943ae7391387ae0e615d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Apr 2024 10:51:17 -0700 Subject: [PATCH 4/7] fix use redact_message_input_output_from_logging --- litellm/utils.py | 90 +++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 4fe232666..8c064bcca 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1212,7 +1212,6 @@ class Logging: print_verbose( f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {traceback.format_exc()}" ) - # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made callbacks = litellm.input_callback + self.dynamic_input_callbacks for callback in callbacks: @@ -1229,29 +1228,17 @@ class Logging: litellm_call_id=self.litellm_params["litellm_call_id"], print_verbose=print_verbose, ) - - elif callback == "lite_debugger": - print_verbose( - f"reaches litedebugger for logging! - model_call_details {self.model_call_details}" - ) - model = self.model_call_details["model"] - messages = self.model_call_details["input"] - print_verbose(f"liteDebuggerClient: {liteDebuggerClient}") - liteDebuggerClient.input_log_event( - model=model, - messages=messages, - end_user=self.model_call_details.get("user", "default"), - litellm_call_id=self.litellm_params["litellm_call_id"], - litellm_params=self.model_call_details["litellm_params"], - optional_params=self.model_call_details["optional_params"], - print_verbose=print_verbose, - call_type=self.call_type, - ) elif callback == "sentry" and add_breadcrumb: - print_verbose("reaches sentry breadcrumbing") + details_to_log = copy.deepcopy(self.model_call_details) + if litellm.turn_off_message_logging: + # make a copy of the _model_Call_details and log it + details_to_log.pop("messages", None) + details_to_log.pop("input", None) + details_to_log.pop("prompt", None) + add_breadcrumb( category="litellm.llm_call", - message=f"Model Call Details pre-call: {self.model_call_details}", + message=f"Model Call Details pre-call: {details_to_log}", level="info", ) elif isinstance(callback, CustomLogger): # custom logger class @@ -1315,7 +1302,7 @@ class Logging: print_verbose( f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {traceback.format_exc()}" ) - + self.redact_message_input_output_from_logging(result=original_response) # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made callbacks = litellm.input_callback + self.dynamic_input_callbacks @@ -1527,31 +1514,7 @@ class Logging: else: callbacks = litellm.success_callback - # check if user opted out of logging message/response to callbacks - if litellm.turn_off_message_logging == True: - # remove messages, prompts, input, response from logging - self.model_call_details["messages"] = "redacted-by-litellm" - self.model_call_details["prompt"] = "" - self.model_call_details["input"] = "" - - # response cleaning - # ChatCompletion Responses - if ( - self.stream - and "complete_streaming_response" in self.model_call_details - ): - _streaming_response = self.model_call_details[ - "complete_streaming_response" - ] - for choice in _streaming_response.choices: - choice.message.content = "redacted-by-litellm" - else: - for choice in result.choices: - choice.message.content = "redacted-by-litellm" - - # Embedding Responses - - # Text Completion Responses + self.redact_message_input_output_from_logging(result=result) for callback in callbacks: try: @@ -2097,6 +2060,9 @@ class Logging: callbacks.append(callback) else: callbacks = litellm._async_success_callback + + self.redact_message_input_output_from_logging(result=result) + print_verbose(f"Async success callbacks: {callbacks}") for callback in callbacks: # check if callback can run for this request @@ -2258,7 +2224,10 @@ class Logging: start_time=start_time, end_time=end_time, ) + result = None # result sent to all loggers, init this to None incase it's not created + + self.redact_message_input_output_from_logging(result=result) for callback in litellm.failure_callback: try: if callback == "lite_debugger": @@ -2443,6 +2412,33 @@ class Logging: f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {traceback.format_exc()}" ) + def redact_message_input_output_from_logging(self, result): + """ + Removes messages, prompts, input, response from logging. This modifies the data in-place + only redacts when litellm.turn_off_message_logging == True + """ + # check if user opted out of logging message/response to callbacks + if litellm.turn_off_message_logging == True: + # remove messages, prompts, input, response from logging + self.model_call_details["messages"] = "redacted-by-litellm" + self.model_call_details["prompt"] = "" + self.model_call_details["input"] = "" + + # response cleaning + # ChatCompletion Responses + if self.stream and "complete_streaming_response" in self.model_call_details: + _streaming_response = self.model_call_details[ + "complete_streaming_response" + ] + for choice in _streaming_response.choices: + choice.message.content = "redacted-by-litellm" + else: + if result is not None: + if isinstance(result, litellm.ModelResponse): + if hasattr(result, "choices"): + for choice in result.choices: + choice.message.content = "redacted-by-litellm" + def exception_logging( additional_args={}, From 4ce27e121934ad0b062dbe20576a8d7181baacd3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Apr 2024 11:23:08 -0700 Subject: [PATCH 5/7] fix - sentry data redaction --- litellm/utils.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 8c064bcca..984212b43 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1320,9 +1320,17 @@ class Logging: ) elif callback == "sentry" and add_breadcrumb: print_verbose("reaches sentry breadcrumbing") + + details_to_log = copy.deepcopy(self.model_call_details) + if litellm.turn_off_message_logging: + # make a copy of the _model_Call_details and log it + details_to_log.pop("messages", None) + details_to_log.pop("input", None) + details_to_log.pop("prompt", None) + add_breadcrumb( category="litellm.llm_call", - message=f"Model Call Details post-call: {self.model_call_details}", + message=f"Model Call Details post-call: {details_to_log}", level="info", ) elif isinstance(callback, CustomLogger): # custom logger class @@ -2620,9 +2628,15 @@ def function_setup( dynamic_success_callbacks = kwargs.pop("success_callback") if add_breadcrumb: + details_to_log = copy.deepcopy(kwargs) + if litellm.turn_off_message_logging: + # make a copy of the _model_Call_details and log it + details_to_log.pop("messages", None) + details_to_log.pop("input", None) + details_to_log.pop("prompt", None) add_breadcrumb( category="litellm.llm_call", - message=f"Positional Args: {args}, Keyword Args: {kwargs}", + message=f"Positional Args: {args}, Keyword Args: {details_to_log}", level="info", ) if "logger_fn" in kwargs: From 01478c9148bfa02b6c520825478b7f18dd68d1c6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Apr 2024 11:23:35 -0700 Subject: [PATCH 6/7] docs - langfuse redact messages --- docs/my-website/docs/observability/langfuse_integration.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index 50b016d09..fe210e6b7 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -167,6 +167,9 @@ messages = [ chat(messages) ``` +## Redacting Messages, Response Content from Langfuse Logging + +Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to langfuse, but request metadata will still be logged. ## Troubleshooting & Errors ### Data not getting logged to Langfuse ? From f55838d1853d0773c7acfbdaf173f3de1922955e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 27 Apr 2024 11:23:46 -0700 Subject: [PATCH 7/7] sentry redact messages --- docs/my-website/docs/observability/sentry.md | 4 ++++ docs/my-website/docs/proxy/logging.md | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/docs/my-website/docs/observability/sentry.md b/docs/my-website/docs/observability/sentry.md index 255dd55cf..5877db661 100644 --- a/docs/my-website/docs/observability/sentry.md +++ b/docs/my-website/docs/observability/sentry.md @@ -40,5 +40,9 @@ response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content print(response) ``` +## Redacting Messages, Response Content from Sentry Logging + +Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to sentry, but request metadata will still be logged. + [Let us know](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+) if you need any additional options from Sentry. diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 48a5955b1..1a5a7f0f0 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -569,6 +569,22 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ All requests made with these keys will log data to their team-specific logging. +### Redacting Messages, Response Content from Langfuse Logging + +Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to langfuse, but request metadata will still be logged. + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + success_callback: ["langfuse"] + turn_off_message_logging: True +``` + + + ## Logging Proxy Input/Output - DataDog We will use the `--config` to set `litellm.success_callback = ["datadog"]` this will log all successfull LLM calls to DataDog