From 2d10f48c4328a1bf77d3c74b65af5ae779b538c2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 2 Dec 2024 23:01:42 -0800 Subject: [PATCH] (fixes) datadog logging - handle 1MB max log size on DD (#6996) * fix dd truncate_standard_logging_payload_content * dd truncate_standard_logging_payload_content * fix test_datadog_payload_content_truncation * add clear msg on _truncate_text * test_truncate_standard_logging_payload * fix linting error * fix linting errors --- litellm/integrations/datadog/datadog.py | 5 + litellm/litellm_core_utils/litellm_logging.py | 54 ++++++++ litellm/proxy/proxy_config.yaml | 13 +- .../create_mock_standard_logging_payload.py | 131 ++++++++++++++++++ tests/logging_callback_tests/test_datadog.py | 54 ++++++++ .../test_standard_logging_payload.py | 50 ++++++- 6 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 tests/logging_callback_tests/create_mock_standard_logging_payload.py diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index ffbc46720c..dcc1fd6b76 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -262,6 +262,10 @@ class DataDogLogger(CustomBatchLogger): """ import json + from litellm.litellm_core_utils.litellm_logging import ( + truncate_standard_logging_payload_content, + ) + standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get( "standard_logging_object", None ) @@ -273,6 +277,7 @@ class DataDogLogger(CustomBatchLogger): status = DataDogStatus.ERROR # Build the initial payload + truncate_standard_logging_payload_content(standard_logging_object) make_json_serializable(standard_logging_object) json_payload = json.dumps(standard_logging_object) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7af10ef73e..d49a848845 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2842,6 +2842,60 @@ def get_standard_logging_object_payload( return None +def truncate_standard_logging_payload_content( + standard_logging_object: StandardLoggingPayload, +): + """ + Truncate error strings and message content in logging payload + + Some loggers like DataDog have a limit on the size of the payload. (1MB) + + This function truncates the error string and the message content if they exceed a certain length. + """ + MAX_STR_LENGTH = 10_000 + + # Truncate fields that might exceed max length + fields_to_truncate = ["error_str", "messages", "response"] + for field in fields_to_truncate: + _truncate_field( + standard_logging_object=standard_logging_object, + field_name=field, + max_length=MAX_STR_LENGTH, + ) + + +def _truncate_text(text: str, max_length: int) -> str: + """Truncate text if it exceeds max_length""" + return ( + text[:max_length] + + "...truncated by litellm, this logger does not support large content" + if len(text) > max_length + else text + ) + + +def _truncate_field( + standard_logging_object: StandardLoggingPayload, field_name: str, max_length: int +) -> None: + """ + Helper function to truncate a field in the logging payload + + This converts the field to a string and then truncates it if it exceeds the max length. + + Why convert to string ? + 1. User was sending a poorly formatted list for `messages` field, we could not predict where they would send content + - Converting to string and then truncating the logged content catches this + 2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user + """ + field_value = standard_logging_object.get(field_name) # type: ignore + if field_value: + str_value = str(field_value) + if len(str_value) > max_length: + standard_logging_object[field_name] = _truncate_text( # type: ignore + text=str_value, max_length=max_length + ) + + def get_standard_logging_metadata( metadata: Optional[Dict[str, Any]] ) -> StandardLoggingMetadata: diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 968cb8b392..1ed363b8d7 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,5 +1,14 @@ -include: - - model_config.yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + - model_name: anthropic/* + litellm_params: + model: anthropic/fake + api_base: https://exampleanthropicendpoint-production.up.railway.app/ + + litellm_settings: callbacks: ["datadog"] diff --git a/tests/logging_callback_tests/create_mock_standard_logging_payload.py b/tests/logging_callback_tests/create_mock_standard_logging_payload.py new file mode 100644 index 0000000000..2fd6a4ffa8 --- /dev/null +++ b/tests/logging_callback_tests/create_mock_standard_logging_payload.py @@ -0,0 +1,131 @@ +import io +import os +import sys + + +sys.path.insert(0, os.path.abspath("../..")) + +import asyncio +import gzip +import json +import logging +import time +from unittest.mock import AsyncMock, patch + +import pytest + +import litellm +from litellm import completion +from litellm._logging import verbose_logger +from litellm.integrations.datadog.datadog import * +from datetime import datetime, timedelta +from litellm.types.utils import ( + StandardLoggingPayload, + StandardLoggingModelInformation, + StandardLoggingMetadata, + StandardLoggingHiddenParams, +) + +verbose_logger.setLevel(logging.DEBUG) + + +def create_standard_logging_payload() -> StandardLoggingPayload: + return StandardLoggingPayload( + id="test_id", + call_type="completion", + response_cost=0.1, + response_cost_failure_debug_info=None, + status="success", + total_tokens=30, + prompt_tokens=20, + completion_tokens=10, + startTime=1234567890.0, + endTime=1234567891.0, + completionStartTime=1234567890.5, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-3.5-turbo", model_map_value=None + ), + model="gpt-3.5-turbo", + model_id="model-123", + model_group="openai-gpt", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_org_id=None, + user_api_key_alias="test_alias", + user_api_key_team_id="test_team", + user_api_key_user_id="test_user", + user_api_key_team_alias="test_team_alias", + spend_logs_metadata=None, + requester_ip_address="127.0.0.1", + requester_metadata=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address="127.0.0.1", + messages=[{"role": "user", "content": "Hello, world!"}], + response={"choices": [{"message": {"content": "Hi there!"}}]}, + error_str=None, + model_parameters={"stream": True}, + hidden_params=StandardLoggingHiddenParams( + model_id="model-123", + cache_key=None, + api_base="https://api.openai.com", + response_cost="0.1", + additional_headers=None, + ), + ) + + +def create_standard_logging_payload_with_long_content() -> StandardLoggingPayload: + return StandardLoggingPayload( + id="test_id", + call_type="completion", + response_cost=0.1, + response_cost_failure_debug_info=None, + status="success", + total_tokens=30, + prompt_tokens=20, + completion_tokens=10, + startTime=1234567890.0, + endTime=1234567891.0, + completionStartTime=1234567890.5, + model_map_information=StandardLoggingModelInformation( + model_map_key="gpt-3.5-turbo", model_map_value=None + ), + model="gpt-3.5-turbo", + model_id="model-123", + model_group="openai-gpt", + api_base="https://api.openai.com", + metadata=StandardLoggingMetadata( + user_api_key_hash="test_hash", + user_api_key_org_id=None, + user_api_key_alias="test_alias", + user_api_key_team_id="test_team", + user_api_key_user_id="test_user", + user_api_key_team_alias="test_team_alias", + spend_logs_metadata=None, + requester_ip_address="127.0.0.1", + requester_metadata=None, + ), + cache_hit=False, + cache_key=None, + saved_cache_cost=0.0, + request_tags=[], + end_user=None, + requester_ip_address="127.0.0.1", + messages=[{"role": "user", "content": "Hello, world!" * 80000}], + response={"choices": [{"message": {"content": "Hi there!" * 80000}}]}, + error_str="error_str" * 80000, + model_parameters={"stream": True}, + hidden_params=StandardLoggingHiddenParams( + model_id="model-123", + cache_key=None, + api_base="https://api.openai.com", + response_cost="0.1", + additional_headers=None, + ), + ) diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index bc98065d72..ab53e99069 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -388,3 +388,57 @@ async def test_datadog_payload_environment_variables(): except Exception as e: pytest.fail(f"Test failed with exception: {str(e)}") + + +@pytest.mark.asyncio +async def test_datadog_payload_content_truncation(): + """ + Test that DataDog payload correctly truncates long content + + DataDog has a limit of 1MB for the logged payload size. + """ + dd_logger = DataDogLogger() + + # Create a standard payload with very long content + standard_payload = create_standard_logging_payload() + long_content = "x" * 80_000 # Create string longer than MAX_STR_LENGTH (10_000) + + # Modify payload with long content + standard_payload["error_str"] = long_content + standard_payload["messages"] = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": long_content, + "detail": "low", + }, + } + ], + } + ] + standard_payload["response"] = {"choices": [{"message": {"content": long_content}}]} + + # Create the payload + dd_payload = dd_logger.create_datadog_logging_payload( + kwargs={"standard_logging_object": standard_payload}, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + print("dd_payload", json.dumps(dd_payload, indent=2)) + + # Parse the message back to dict to verify truncation + message_dict = json.loads(dd_payload["message"]) + + # Verify truncation of fields + assert len(message_dict["error_str"]) < 10_100, "error_str not truncated correctly" + assert ( + len(str(message_dict["messages"])) < 10_100 + ), "messages not truncated correctly" + assert ( + len(str(message_dict["response"])) < 10_100 + ), "response not truncated correctly" diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index fca22d079a..533c3a70b1 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -18,12 +18,20 @@ import time import pytest import litellm from litellm.types.utils import ( + StandardLoggingPayload, Usage, StandardLoggingMetadata, StandardLoggingModelInformation, StandardLoggingHiddenParams, ) -from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from create_mock_standard_logging_payload import ( + create_standard_logging_payload, + create_standard_logging_payload_with_long_content, +) +from litellm.litellm_core_utils.litellm_logging import ( + StandardLoggingPayloadSetup, + truncate_standard_logging_payload_content, +) @pytest.mark.parametrize( @@ -321,6 +329,45 @@ def test_get_final_response_obj(): litellm.turn_off_message_logging = False + +def test_truncate_standard_logging_payload(): + """ + 1. original messages, response, and error_str should NOT BE MODIFIED, since these are from kwargs + 2. the `messages`, `response`, and `error_str` in new standard_logging_payload should be truncated + """ + standard_logging_payload: StandardLoggingPayload = ( + create_standard_logging_payload_with_long_content() + ) + original_messages = standard_logging_payload["messages"] + len_original_messages = len(str(original_messages)) + original_response = standard_logging_payload["response"] + len_original_response = len(str(original_response)) + original_error_str = standard_logging_payload["error_str"] + len_original_error_str = len(str(original_error_str)) + + truncate_standard_logging_payload_content(standard_logging_payload) + + # Original messages, response, and error_str should NOT BE MODIFIED + assert standard_logging_payload["messages"] != original_messages + assert standard_logging_payload["response"] != original_response + assert standard_logging_payload["error_str"] != original_error_str + assert len_original_messages == len(str(original_messages)) + assert len_original_response == len(str(original_response)) + assert len_original_error_str == len(str(original_error_str)) + + print( + "logged standard_logging_payload", + json.dumps(standard_logging_payload, indent=2), + ) + + # Logged messages, response, and error_str should be truncated + # assert len of messages is less than 10_500 + assert len(str(standard_logging_payload["messages"])) < 10_500 + # assert len of response is less than 10_500 + assert len(str(standard_logging_payload["response"])) < 10_500 + # assert len of error_str is less than 10_500 + assert len(str(standard_logging_payload["error_str"])) < 10_500 + def test_strip_trailing_slash(): common_api_base = "https://api.test.com" assert ( @@ -331,3 +378,4 @@ def test_strip_trailing_slash(): StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base) == common_api_base ) +