phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fixes) datadog logging - handle 1MB max log size on DD (#6996)

Browse source 
* fix dd truncate_standard_logging_payload_content

* dd truncate_standard_logging_payload_content

* fix test_datadog_payload_content_truncation

* add clear msg on _truncate_text

* test_truncate_standard_logging_payload

* fix linting error

* fix linting errors
This commit is contained in:
Ishaan Jaff 2024-12-02 23:01:42 -08:00 • committed by GitHub
parent 9617e7433d
commit 2d10f48c43
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 304 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/integrations/datadog/datadog.py
View file

@ -262,6 +262,10 @@ class DataDogLogger(CustomBatchLogger):
"""
import json
from litellm.litellm_core_utils.litellm_logging import (
truncate_standard_logging_payload_content,
)
standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object", None
)
@ -273,6 +277,7 @@ class DataDogLogger(CustomBatchLogger):
status = DataDogStatus.ERROR
# Build the initial payload
truncate_standard_logging_payload_content(standard_logging_object)
make_json_serializable(standard_logging_object)
json_payload = json.dumps(standard_logging_object)

54
litellm/litellm_core_utils/litellm_logging.py
View file

@ -2842,6 +2842,60 @@ def get_standard_logging_object_payload(
return None
def truncate_standard_logging_payload_content(
standard_logging_object: StandardLoggingPayload,
):
"""
Truncate error strings and message content in logging payload
Some loggers like DataDog have a limit on the size of the payload. (1MB)
This function truncates the error string and the message content if they exceed a certain length.
"""
MAX_STR_LENGTH = 10_000
# Truncate fields that might exceed max length
fields_to_truncate = ["error_str", "messages", "response"]
for field in fields_to_truncate:
_truncate_field(
standard_logging_object=standard_logging_object,
field_name=field,
max_length=MAX_STR_LENGTH,
)
def _truncate_text(text: str, max_length: int) -> str:
"""Truncate text if it exceeds max_length"""
return (
text[:max_length]
+ "...truncated by litellm, this logger does not support large content"
if len(text) > max_length
else text
)
def _truncate_field(
standard_logging_object: StandardLoggingPayload, field_name: str, max_length: int
) -> None:
"""
Helper function to truncate a field in the logging payload
This converts the field to a string and then truncates it if it exceeds the max length.
Why convert to string ?
1. User was sending a poorly formatted list for `messages` field, we could not predict where they would send content
- Converting to string and then truncating the logged content catches this
2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
"""
field_value = standard_logging_object.get(field_name) # type: ignore
if field_value:
str_value = str(field_value)
if len(str_value) > max_length:
standard_logging_object[field_name] = _truncate_text( # type: ignore
text=str_value, max_length=max_length
)
def get_standard_logging_metadata(
metadata: Optional[Dict[str, Any]]
) -> StandardLoggingMetadata:

13
litellm/proxy/proxy_config.yaml
View file

@ -1,5 +1,14 @@
include:
- model_config.yaml
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_base: https://exampleopenaiendpoint-production.up.railway.app/
- model_name: anthropic/*
litellm_params:
model: anthropic/fake
api_base: https://exampleanthropicendpoint-production.up.railway.app/
litellm_settings:
callbacks: ["datadog"]

131
tests/logging_callback_tests/create_mock_standard_logging_payload.py Normal file
View file

@ -0,0 +1,131 @@
import io
import os
import sys
sys.path.insert(0, os.path.abspath("../.."))
import asyncio
import gzip
import json
import logging
import time
from unittest.mock import AsyncMock, patch
import pytest
import litellm
from litellm import completion
from litellm._logging import verbose_logger
from litellm.integrations.datadog.datadog import *
from datetime import datetime, timedelta
from litellm.types.utils import (
StandardLoggingPayload,
StandardLoggingModelInformation,
StandardLoggingMetadata,
StandardLoggingHiddenParams,
)
verbose_logger.setLevel(logging.DEBUG)
def create_standard_logging_payload() -> StandardLoggingPayload:
return StandardLoggingPayload(
id="test_id",
call_type="completion",
response_cost=0.1,
response_cost_failure_debug_info=None,
status="success",
total_tokens=30,
prompt_tokens=20,
completion_tokens=10,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=1234567890.5,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-3.5-turbo", model_map_value=None
),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai-gpt",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_org_id=None,
user_api_key_alias="test_alias",
user_api_key_team_id="test_team",
user_api_key_user_id="test_user",
user_api_key_team_alias="test_team_alias",
spend_logs_metadata=None,
requester_ip_address="127.0.0.1",
requester_metadata=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address="127.0.0.1",
messages=[{"role": "user", "content": "Hello, world!"}],
response={"choices": [{"message": {"content": "Hi there!"}}]},
error_str=None,
model_parameters={"stream": True},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.1",
additional_headers=None,
),
)
def create_standard_logging_payload_with_long_content() -> StandardLoggingPayload:
return StandardLoggingPayload(
id="test_id",
call_type="completion",
response_cost=0.1,
response_cost_failure_debug_info=None,
status="success",
total_tokens=30,
prompt_tokens=20,
completion_tokens=10,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=1234567890.5,
model_map_information=StandardLoggingModelInformation(
model_map_key="gpt-3.5-turbo", model_map_value=None
),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai-gpt",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_org_id=None,
user_api_key_alias="test_alias",
user_api_key_team_id="test_team",
user_api_key_user_id="test_user",
user_api_key_team_alias="test_team_alias",
spend_logs_metadata=None,
requester_ip_address="127.0.0.1",
requester_metadata=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address="127.0.0.1",
messages=[{"role": "user", "content": "Hello, world!" * 80000}],
response={"choices": [{"message": {"content": "Hi there!" * 80000}}]},
error_str="error_str" * 80000,
model_parameters={"stream": True},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.1",
additional_headers=None,
),
)

54
tests/logging_callback_tests/test_datadog.py
View file

@ -388,3 +388,57 @@ async def test_datadog_payload_environment_variables():
except Exception as e:
pytest.fail(f"Test failed with exception: {str(e)}")
@pytest.mark.asyncio
async def test_datadog_payload_content_truncation():
"""
Test that DataDog payload correctly truncates long content
DataDog has a limit of 1MB for the logged payload size.
"""
dd_logger = DataDogLogger()
# Create a standard payload with very long content
standard_payload = create_standard_logging_payload()
long_content = "x" * 80_000 # Create string longer than MAX_STR_LENGTH (10_000)
# Modify payload with long content
standard_payload["error_str"] = long_content
standard_payload["messages"] = [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": long_content,
"detail": "low",
},
}
],
}
]
standard_payload["response"] = {"choices": [{"message": {"content": long_content}}]}
# Create the payload
dd_payload = dd_logger.create_datadog_logging_payload(
kwargs={"standard_logging_object": standard_payload},
response_obj=None,
start_time=datetime.now(),
end_time=datetime.now(),
)
print("dd_payload", json.dumps(dd_payload, indent=2))
# Parse the message back to dict to verify truncation
message_dict = json.loads(dd_payload["message"])
# Verify truncation of fields
assert len(message_dict["error_str"]) < 10_100, "error_str not truncated correctly"
assert (
len(str(message_dict["messages"])) < 10_100
), "messages not truncated correctly"
assert (
len(str(message_dict["response"])) < 10_100
), "response not truncated correctly"

50
tests/logging_callback_tests/test_standard_logging_payload.py
View file

@ -18,12 +18,20 @@ import time
import pytest
import litellm
from litellm.types.utils import (
StandardLoggingPayload,
Usage,
StandardLoggingMetadata,
StandardLoggingModelInformation,
StandardLoggingHiddenParams,
)
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from create_mock_standard_logging_payload import (
create_standard_logging_payload,
create_standard_logging_payload_with_long_content,
)
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
truncate_standard_logging_payload_content,
)
@pytest.mark.parametrize(
@ -321,6 +329,45 @@ def test_get_final_response_obj():
litellm.turn_off_message_logging = False
def test_truncate_standard_logging_payload():
"""
1. original messages, response, and error_str should NOT BE MODIFIED, since these are from kwargs
2. the `messages`, `response`, and `error_str` in new standard_logging_payload should be truncated
"""
standard_logging_payload: StandardLoggingPayload = (
create_standard_logging_payload_with_long_content()
)
original_messages = standard_logging_payload["messages"]
len_original_messages = len(str(original_messages))
original_response = standard_logging_payload["response"]
len_original_response = len(str(original_response))
original_error_str = standard_logging_payload["error_str"]
len_original_error_str = len(str(original_error_str))
truncate_standard_logging_payload_content(standard_logging_payload)
# Original messages, response, and error_str should NOT BE MODIFIED
assert standard_logging_payload["messages"] != original_messages
assert standard_logging_payload["response"] != original_response
assert standard_logging_payload["error_str"] != original_error_str
assert len_original_messages == len(str(original_messages))
assert len_original_response == len(str(original_response))
assert len_original_error_str == len(str(original_error_str))
print(
"logged standard_logging_payload",
json.dumps(standard_logging_payload, indent=2),
)
# Logged messages, response, and error_str should be truncated
# assert len of messages is less than 10_500
assert len(str(standard_logging_payload["messages"])) < 10_500
# assert len of response is less than 10_500
assert len(str(standard_logging_payload["response"])) < 10_500
# assert len of error_str is less than 10_500
assert len(str(standard_logging_payload["error_str"])) < 10_500
def test_strip_trailing_slash():
common_api_base = "https://api.test.com"
assert (
@ -331,3 +378,4 @@ def test_strip_trailing_slash():
StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base)
== common_api_base
)