(fixes) datadog logging - handle 1MB max log size on DD (#6996)

* fix dd truncate_standard_logging_payload_content

* dd truncate_standard_logging_payload_content

* fix test_datadog_payload_content_truncation

* add clear msg on _truncate_text

* test_truncate_standard_logging_payload

* fix linting error

* fix linting errors

litellm/integrations/datadog/datadog.py

@@ -262,6 +262,10 @@ class DataDogLogger(CustomBatchLogger):
         """
         import json
 
+        from litellm.litellm_core_utils.litellm_logging import (
+            truncate_standard_logging_payload_content,
+        )
+
         standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get(
             "standard_logging_object", None
         )
@@ -273,6 +277,7 @@ class DataDogLogger(CustomBatchLogger):
             status = DataDogStatus.ERROR
 
         # Build the initial payload
+        truncate_standard_logging_payload_content(standard_logging_object)
         make_json_serializable(standard_logging_object)
         json_payload = json.dumps(standard_logging_object)
 

litellm/litellm_core_utils/litellm_logging.py

@@ -2842,6 +2842,60 @@ def get_standard_logging_object_payload(
         return None
 
 
+def truncate_standard_logging_payload_content(
+    standard_logging_object: StandardLoggingPayload,
+):
+    """
+    Truncate error strings and message content in logging payload
+
+    Some loggers like DataDog have a limit on the size of the payload. (1MB)
+
+    This function truncates the error string and the message content if they exceed a certain length.
+    """
+    MAX_STR_LENGTH = 10_000
+
+    # Truncate fields that might exceed max length
+    fields_to_truncate = ["error_str", "messages", "response"]
+    for field in fields_to_truncate:
+        _truncate_field(
+            standard_logging_object=standard_logging_object,
+            field_name=field,
+            max_length=MAX_STR_LENGTH,
+        )
+
+
+def _truncate_text(text: str, max_length: int) -> str:
+    """Truncate text if it exceeds max_length"""
+    return (
+        text[:max_length]
+        + "...truncated by litellm, this logger does not support large content"
+        if len(text) > max_length
+        else text
+    )
+
+
+def _truncate_field(
+    standard_logging_object: StandardLoggingPayload, field_name: str, max_length: int
+) -> None:
+    """
+    Helper function to truncate a field in the logging payload
+
+    This converts the field to a string and then truncates it if it exceeds the max length.
+
+    Why convert to string ?
+    1. User was sending a poorly formatted list for `messages` field, we could not predict where they would send content
+        - Converting to string and then truncating the logged content catches this
+    2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
+    """
+    field_value = standard_logging_object.get(field_name)  # type: ignore
+    if field_value:
+        str_value = str(field_value)
+        if len(str_value) > max_length:
+            standard_logging_object[field_name] = _truncate_text(  # type: ignore
+                text=str_value, max_length=max_length
+            )
+
+
 def get_standard_logging_metadata(
     metadata: Optional[Dict[str, Any]]
 ) -> StandardLoggingMetadata:

litellm/proxy/proxy_config.yaml

@@ -1,5 +1,14 @@
-include:
+model_list:
-  - model_config.yaml
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+  - model_name: anthropic/*
+    litellm_params:
+      model: anthropic/fake
+      api_base: https://exampleanthropicendpoint-production.up.railway.app/
+
+
 
 litellm_settings:
   callbacks: ["datadog"] 

tests/logging_callback_tests/create_mock_standard_logging_payload.py

@@ -0,0 +1,131 @@
+import io
+import os
+import sys
+
+
+sys.path.insert(0, os.path.abspath("../.."))
+
+import asyncio
+import gzip
+import json
+import logging
+import time
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+import litellm
+from litellm import completion
+from litellm._logging import verbose_logger
+from litellm.integrations.datadog.datadog import *
+from datetime import datetime, timedelta
+from litellm.types.utils import (
+    StandardLoggingPayload,
+    StandardLoggingModelInformation,
+    StandardLoggingMetadata,
+    StandardLoggingHiddenParams,
+)
+
+verbose_logger.setLevel(logging.DEBUG)
+
+
+def create_standard_logging_payload() -> StandardLoggingPayload:
+    return StandardLoggingPayload(
+        id="test_id",
+        call_type="completion",
+        response_cost=0.1,
+        response_cost_failure_debug_info=None,
+        status="success",
+        total_tokens=30,
+        prompt_tokens=20,
+        completion_tokens=10,
+        startTime=1234567890.0,
+        endTime=1234567891.0,
+        completionStartTime=1234567890.5,
+        model_map_information=StandardLoggingModelInformation(
+            model_map_key="gpt-3.5-turbo", model_map_value=None
+        ),
+        model="gpt-3.5-turbo",
+        model_id="model-123",
+        model_group="openai-gpt",
+        api_base="https://api.openai.com",
+        metadata=StandardLoggingMetadata(
+            user_api_key_hash="test_hash",
+            user_api_key_org_id=None,
+            user_api_key_alias="test_alias",
+            user_api_key_team_id="test_team",
+            user_api_key_user_id="test_user",
+            user_api_key_team_alias="test_team_alias",
+            spend_logs_metadata=None,
+            requester_ip_address="127.0.0.1",
+            requester_metadata=None,
+        ),
+        cache_hit=False,
+        cache_key=None,
+        saved_cache_cost=0.0,
+        request_tags=[],
+        end_user=None,
+        requester_ip_address="127.0.0.1",
+        messages=[{"role": "user", "content": "Hello, world!"}],
+        response={"choices": [{"message": {"content": "Hi there!"}}]},
+        error_str=None,
+        model_parameters={"stream": True},
+        hidden_params=StandardLoggingHiddenParams(
+            model_id="model-123",
+            cache_key=None,
+            api_base="https://api.openai.com",
+            response_cost="0.1",
+            additional_headers=None,
+        ),
+    )
+
+
+def create_standard_logging_payload_with_long_content() -> StandardLoggingPayload:
+    return StandardLoggingPayload(
+        id="test_id",
+        call_type="completion",
+        response_cost=0.1,
+        response_cost_failure_debug_info=None,
+        status="success",
+        total_tokens=30,
+        prompt_tokens=20,
+        completion_tokens=10,
+        startTime=1234567890.0,
+        endTime=1234567891.0,
+        completionStartTime=1234567890.5,
+        model_map_information=StandardLoggingModelInformation(
+            model_map_key="gpt-3.5-turbo", model_map_value=None
+        ),
+        model="gpt-3.5-turbo",
+        model_id="model-123",
+        model_group="openai-gpt",
+        api_base="https://api.openai.com",
+        metadata=StandardLoggingMetadata(
+            user_api_key_hash="test_hash",
+            user_api_key_org_id=None,
+            user_api_key_alias="test_alias",
+            user_api_key_team_id="test_team",
+            user_api_key_user_id="test_user",
+            user_api_key_team_alias="test_team_alias",
+            spend_logs_metadata=None,
+            requester_ip_address="127.0.0.1",
+            requester_metadata=None,
+        ),
+        cache_hit=False,
+        cache_key=None,
+        saved_cache_cost=0.0,
+        request_tags=[],
+        end_user=None,
+        requester_ip_address="127.0.0.1",
+        messages=[{"role": "user", "content": "Hello, world!" * 80000}],
+        response={"choices": [{"message": {"content": "Hi there!" * 80000}}]},
+        error_str="error_str" * 80000,
+        model_parameters={"stream": True},
+        hidden_params=StandardLoggingHiddenParams(
+            model_id="model-123",
+            cache_key=None,
+            api_base="https://api.openai.com",
+            response_cost="0.1",
+            additional_headers=None,
+        ),
+    )

tests/logging_callback_tests/test_datadog.py

@@ -388,3 +388,57 @@ async def test_datadog_payload_environment_variables():
 
     except Exception as e:
         pytest.fail(f"Test failed with exception: {str(e)}")
+
+
+@pytest.mark.asyncio
+async def test_datadog_payload_content_truncation():
+    """
+    Test that DataDog payload correctly truncates long content
+
+    DataDog has a limit of 1MB for the logged payload size.
+    """
+    dd_logger = DataDogLogger()
+
+    # Create a standard payload with very long content
+    standard_payload = create_standard_logging_payload()
+    long_content = "x" * 80_000  # Create string longer than MAX_STR_LENGTH (10_000)
+
+    # Modify payload with long content
+    standard_payload["error_str"] = long_content
+    standard_payload["messages"] = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": long_content,
+                        "detail": "low",
+                    },
+                }
+            ],
+        }
+    ]
+    standard_payload["response"] = {"choices": [{"message": {"content": long_content}}]}
+
+    # Create the payload
+    dd_payload = dd_logger.create_datadog_logging_payload(
+        kwargs={"standard_logging_object": standard_payload},
+        response_obj=None,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+    )
+
+    print("dd_payload", json.dumps(dd_payload, indent=2))
+
+    # Parse the message back to dict to verify truncation
+    message_dict = json.loads(dd_payload["message"])
+
+    # Verify truncation of fields
+    assert len(message_dict["error_str"]) < 10_100, "error_str not truncated correctly"
+    assert (
+        len(str(message_dict["messages"])) < 10_100
+    ), "messages not truncated correctly"
+    assert (
+        len(str(message_dict["response"])) < 10_100
+    ), "response not truncated correctly"

tests/logging_callback_tests/test_standard_logging_payload.py

@@ -18,12 +18,20 @@ import time
 import pytest
 import litellm
 from litellm.types.utils import (
+    StandardLoggingPayload,
     Usage,
     StandardLoggingMetadata,
     StandardLoggingModelInformation,
     StandardLoggingHiddenParams,
 )
-from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+from create_mock_standard_logging_payload import (
+    create_standard_logging_payload,
+    create_standard_logging_payload_with_long_content,
+)
+from litellm.litellm_core_utils.litellm_logging import (
+    StandardLoggingPayloadSetup,
+    truncate_standard_logging_payload_content,
+)
 
 
 @pytest.mark.parametrize(
@@ -321,6 +329,45 @@ def test_get_final_response_obj():
         litellm.turn_off_message_logging = False
 
 
+
+def test_truncate_standard_logging_payload():
+    """
+    1. original messages, response, and error_str should NOT BE MODIFIED, since these are from kwargs
+    2. the `messages`, `response`, and `error_str` in new standard_logging_payload should be truncated
+    """
+    standard_logging_payload: StandardLoggingPayload = (
+        create_standard_logging_payload_with_long_content()
+    )
+    original_messages = standard_logging_payload["messages"]
+    len_original_messages = len(str(original_messages))
+    original_response = standard_logging_payload["response"]
+    len_original_response = len(str(original_response))
+    original_error_str = standard_logging_payload["error_str"]
+    len_original_error_str = len(str(original_error_str))
+
+    truncate_standard_logging_payload_content(standard_logging_payload)
+
+    # Original messages, response, and error_str should NOT BE MODIFIED
+    assert standard_logging_payload["messages"] != original_messages
+    assert standard_logging_payload["response"] != original_response
+    assert standard_logging_payload["error_str"] != original_error_str
+    assert len_original_messages == len(str(original_messages))
+    assert len_original_response == len(str(original_response))
+    assert len_original_error_str == len(str(original_error_str))
+
+    print(
+        "logged standard_logging_payload",
+        json.dumps(standard_logging_payload, indent=2),
+    )
+
+    # Logged messages, response, and error_str should be truncated
+    # assert len of messages is less than 10_500
+    assert len(str(standard_logging_payload["messages"])) < 10_500
+    # assert len of response is less than 10_500
+    assert len(str(standard_logging_payload["response"])) < 10_500
+    # assert len of error_str is less than 10_500
+    assert len(str(standard_logging_payload["error_str"])) < 10_500
+
 def test_strip_trailing_slash():
     common_api_base = "https://api.test.com"
     assert (
@@ -331,3 +378,4 @@ def test_strip_trailing_slash():
         StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base)
         == common_api_base
     )
+