From 217681eb5e5ad7d1a25ad0c7cd4f9c149a5ccc39 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Tue, 22 Apr 2025 23:58:43 -0700 Subject: [PATCH] Litellm dev 04 22 2025 p1 (#10206) * fix(openai.py): initial commit adding generic event type for openai responses api streaming Ensures handling for undocumented event types - e.g. "response.reasoning_summary_part.added" * fix(transformation.py): handle unknown openai response type * fix(datadog_llm_observability.py): handle dict[str, any] -> dict[str, str] conversion Fixes https://github.com/BerriAI/litellm/issues/9494 * test: add more unit testing * test: add unit test * fix(common_utils.py): fix message with content list * test: update testing --- .../integrations/datadog/datadog_llm_obs.py | 16 ++++- .../prompt_templates/common_utils.py | 31 +++++++++- .../llms/openai/responses/transformation.py | 2 +- litellm/proxy/_new_secret_config.yaml | 2 +- litellm/proxy/proxy_server.py | 2 +- litellm/responses/streaming_iterator.py | 4 +- litellm/types/integrations/datadog_llm_obs.py | 4 +- litellm/types/llms/openai.py | 12 +++- ...ore_utils_prompt_templates_common_utils.py | 61 +++++++++++++++++++ .../test_openai_responses_transformation.py | 19 ++++++ .../types/llms/test_types_llms_openai.py | 21 +++++++ tests/llm_translation/test_openai.py | 1 + 12 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 tests/litellm/types/llms/test_types_llms_openai.py diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index e4e074bab7..bbb042c57b 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -13,10 +13,15 @@ import uuid from datetime import datetime from typing import Any, Dict, List, Optional, Union +import httpx + import litellm from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.integrations.datadog.datadog import DataDogLogger +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + handle_any_messages_to_chat_completion_str_messages_conversion, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -106,7 +111,6 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): }, ) - response.raise_for_status() if response.status_code != 202: raise Exception( f"DataDogLLMObs: Unexpected response - status_code: {response.status_code}, text: {response.text}" @@ -116,6 +120,10 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): f"DataDogLLMObs: Successfully sent batch - status_code: {response.status_code}" ) self.log_queue.clear() + except httpx.HTTPStatusError as e: + verbose_logger.exception( + f"DataDogLLMObs: Error sending batch - {e.response.text}" + ) except Exception as e: verbose_logger.exception(f"DataDogLLMObs: Error sending batch - {str(e)}") @@ -133,7 +141,11 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): metadata = kwargs.get("litellm_params", {}).get("metadata", {}) - input_meta = InputMeta(messages=messages) # type: ignore + input_meta = InputMeta( + messages=handle_any_messages_to_chat_completion_str_messages_conversion( + messages + ) + ) output_meta = OutputMeta(messages=self._get_response_messages(response_obj)) meta = Meta( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 40cd4e286b..963ab33f52 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,7 +6,7 @@ import io import mimetypes import re from os import PathLike -from typing import Dict, List, Literal, Mapping, Optional, Union, cast +from typing import Any, Dict, List, Literal, Mapping, Optional, Union, cast from litellm.types.llms.openai import ( AllMessageValues, @@ -32,6 +32,35 @@ DEFAULT_ASSISTANT_CONTINUE_MESSAGE = ChatCompletionAssistantMessage( ) +def handle_any_messages_to_chat_completion_str_messages_conversion( + messages: Any, +) -> List[Dict[str, str]]: + """ + Handles any messages to chat completion str messages conversion + + Relevant Issue: https://github.com/BerriAI/litellm/issues/9494 + """ + import json + + if isinstance(messages, list): + try: + return cast( + List[Dict[str, str]], + handle_messages_with_content_list_to_str_conversion(messages), + ) + except Exception: + return [{"input": json.dumps(message, default=str)} for message in messages] + elif isinstance(messages, dict): + try: + return [{"input": json.dumps(messages, default=str)}] + except Exception: + return [{"input": str(messages)}] + elif isinstance(messages, str): + return [{"input": messages}] + else: + return [{"input": str(messages)}] + + def handle_messages_with_content_list_to_str_conversion( messages: List[AllMessageValues], ) -> List[AllMessageValues]: diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index d4a443aedb..ab16a5647d 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -187,7 +187,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): model_class = event_models.get(cast(ResponsesAPIStreamEvents, event_type)) if not model_class: - raise ValueError(f"Unknown event type: {event_type}") + return GenericEvent return model_class diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 8516e9c25b..ad9ae99f7a 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -33,7 +33,7 @@ model_list: litellm_settings: num_retries: 0 - callbacks: ["prometheus"] + callbacks: ["datadog_llm_observability"] check_provider_endpoint: true files_settings: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4ca6b35db4..fd32a62ee4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1296,7 +1296,7 @@ class ProxyConfig: config=config, base_dir=os.path.dirname(os.path.abspath(file_path or "")) ) - verbose_proxy_logger.debug(f"loaded config={json.dumps(config, indent=4)}") + # verbose_proxy_logger.debug(f"loaded config={json.dumps(config, indent=4)}") return config def _process_includes(self, config: dict, base_dir: str) -> dict: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 3e12761ba0..a111fbec09 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -44,12 +44,12 @@ class BaseResponsesAPIStreamingIterator: self.responses_api_provider_config = responses_api_provider_config self.completed_response: Optional[ResponsesAPIStreamingResponse] = None self.start_time = datetime.now() - + # set request kwargs self.litellm_metadata = litellm_metadata self.custom_llm_provider = custom_llm_provider - def _process_chunk(self, chunk): + def _process_chunk(self, chunk) -> Optional[ResponsesAPIStreamingResponse]: """Process a single chunk of data from the stream""" if not chunk: return None diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index 9298b157d2..1a0f7df501 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -8,7 +8,9 @@ from typing import Any, Dict, List, Literal, Optional, TypedDict class InputMeta(TypedDict): - messages: List[Any] + messages: List[ + Dict[str, str] + ] # Relevant Issue: https://github.com/BerriAI/litellm/issues/9494 class OutputMeta(TypedDict): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 24aebf12af..dc45ebe5cc 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -50,7 +50,7 @@ from openai.types.responses.response_create_params import ( ToolParam, ) from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall -from pydantic import BaseModel, Discriminator, Field, PrivateAttr +from pydantic import BaseModel, ConfigDict, Discriminator, Field, PrivateAttr from typing_extensions import Annotated, Dict, Required, TypedDict, override from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject @@ -1013,6 +1013,9 @@ class ResponsesAPIStreamEvents(str, Enum): RESPONSE_FAILED = "response.failed" RESPONSE_INCOMPLETE = "response.incomplete" + # Part added + RESPONSE_PART_ADDED = "response.reasoning_summary_part.added" + # Output item events OUTPUT_ITEM_ADDED = "response.output_item.added" OUTPUT_ITEM_DONE = "response.output_item.done" @@ -1200,6 +1203,12 @@ class ErrorEvent(BaseLiteLLMOpenAIResponseObject): param: Optional[str] +class GenericEvent(BaseLiteLLMOpenAIResponseObject): + type: str + + model_config = ConfigDict(extra="allow", protected_namespaces=()) + + # Union type for all possible streaming responses ResponsesAPIStreamingResponse = Annotated[ Union[ @@ -1226,6 +1235,7 @@ ResponsesAPIStreamingResponse = Annotated[ WebSearchCallSearchingEvent, WebSearchCallCompletedEvent, ErrorEvent, + GenericEvent, ], Discriminator("type"), ] diff --git a/tests/litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index aca0e02e17..1d349a44e5 100644 --- a/tests/litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -11,6 +11,7 @@ sys.path.insert( from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_format_from_file_id, + handle_any_messages_to_chat_completion_str_messages_conversion, update_messages_with_model_file_ids, ) @@ -64,3 +65,63 @@ def test_update_messages_with_model_file_ids(): ], } ] + + +def test_handle_any_messages_to_chat_completion_str_messages_conversion_list(): + # Test with list of messages + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + ] + result = handle_any_messages_to_chat_completion_str_messages_conversion(messages) + assert len(result) == 2 + assert result[0] == messages[0] + assert result[1] == messages[1] + + +def test_handle_any_messages_to_chat_completion_str_messages_conversion_list_infinite_loop(): + # Test that list handling doesn't cause infinite recursion + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + ] + # This should complete without stack overflow + result = handle_any_messages_to_chat_completion_str_messages_conversion(messages) + assert len(result) == 2 + assert result[0] == messages[0] + assert result[1] == messages[1] + + +def test_handle_any_messages_to_chat_completion_str_messages_conversion_dict(): + # Test with single dictionary message + message = {"role": "user", "content": "Hello"} + result = handle_any_messages_to_chat_completion_str_messages_conversion(message) + assert len(result) == 1 + assert result[0]["input"] == json.dumps(message) + + +def test_handle_any_messages_to_chat_completion_str_messages_conversion_str(): + # Test with string message + message = "Hello" + result = handle_any_messages_to_chat_completion_str_messages_conversion(message) + assert len(result) == 1 + assert result[0]["input"] == message + + +def test_handle_any_messages_to_chat_completion_str_messages_conversion_other(): + # Test with non-string/dict/list type + message = 123 + result = handle_any_messages_to_chat_completion_str_messages_conversion(message) + assert len(result) == 1 + assert result[0]["input"] == "123" + + +def test_handle_any_messages_to_chat_completion_str_messages_conversion_complex(): + # Test with complex nested structure + message = { + "role": "user", + "content": {"text": "Hello", "metadata": {"timestamp": "2024-01-01"}}, + } + result = handle_any_messages_to_chat_completion_str_messages_conversion(message) + assert len(result) == 1 + assert result[0]["input"] == json.dumps(message) diff --git a/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py index 3b9ae72da7..04b9de5616 100644 --- a/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -252,3 +252,22 @@ class TestOpenAIResponsesAPIConfig: ) assert result == "https://custom-openai.example.com/v1/responses" + + def test_get_event_model_class_generic_event(self): + """Test that get_event_model_class returns the correct event model class""" + from litellm.types.llms.openai import GenericEvent + + event_type = "test" + result = self.config.get_event_model_class(event_type) + assert result == GenericEvent + + def test_transform_streaming_response_generic_event(self): + """Test that transform_streaming_response returns the correct event model class""" + from litellm.types.llms.openai import GenericEvent + + chunk = {"type": "test", "test": "test"} + result = self.config.transform_streaming_response( + model=self.model, parsed_chunk=chunk, logging_obj=self.logging_obj + ) + assert isinstance(result, GenericEvent) + assert result.type == "test" diff --git a/tests/litellm/types/llms/test_types_llms_openai.py b/tests/litellm/types/llms/test_types_llms_openai.py new file mode 100644 index 0000000000..86c2cb3f1a --- /dev/null +++ b/tests/litellm/types/llms/test_types_llms_openai.py @@ -0,0 +1,21 @@ +import asyncio +import os +import sys +from typing import Optional +from unittest.mock import AsyncMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../..")) +import json + +import litellm + + +def test_generic_event(): + from litellm.types.llms.openai import GenericEvent + + event = {"type": "test", "test": "test"} + event = GenericEvent(**event) + assert event.type == "test" + assert event.test == "test" diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 295bdb46f1..a470b53589 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -470,3 +470,4 @@ class TestOpenAIGPT4OAudioTranscription(BaseLLMAudioTranscriptionTest): def get_custom_llm_provider(self) -> litellm.LlmProviders: return litellm.LlmProviders.OPENAI +