diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index a37d816770..ab335ca7c1 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -1,6 +1,6 @@ """ - call /messages on Anthropic API -- Make streaming + non-streaming request - just pass it through direct to Anthropic. No need to do anything special here +- Make streaming + non-streaming request - just pass it through direct to Anthropic. No need to do anything special here - Ensure requests are logged in the DB - stream + non-stream """ @@ -43,7 +43,9 @@ class AnthropicMessagesHandler: from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) - from litellm.proxy.pass_through_endpoints.types import EndpointType + from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + EndpointType, + ) # Create success handler object passthrough_success_handler_obj = PassThroughEndpointLogging() @@ -98,11 +100,11 @@ async def anthropic_messages( api_base=optional_params.api_base, api_key=optional_params.api_key, ) - anthropic_messages_provider_config: Optional[ - BaseAnthropicMessagesConfig - ] = ProviderConfigManager.get_provider_anthropic_messages_config( - model=model, - provider=litellm.LlmProviders(_custom_llm_provider), + anthropic_messages_provider_config: Optional[BaseAnthropicMessagesConfig] = ( + ProviderConfigManager.get_provider_anthropic_messages_config( + model=model, + provider=litellm.LlmProviders(_custom_llm_provider), + ) ) if anthropic_messages_provider_config is None: raise ValueError( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index d6f2a01712..0edccdf89b 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -13,7 +13,9 @@ from litellm.llms.anthropic.chat.handler import ( from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body -from litellm.proxy.pass_through_endpoints.types import PassthroughStandardLoggingPayload +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, +) from litellm.types.utils import ModelResponse, TextCompletionResponse if TYPE_CHECKING: @@ -122,9 +124,9 @@ class AnthropicPassthroughLoggingHandler: litellm_model_response.id = logging_obj.litellm_call_id litellm_model_response.model = model logging_obj.model_call_details["model"] = model - logging_obj.model_call_details[ - "custom_llm_provider" - ] = litellm.LlmProviders.ANTHROPIC.value + logging_obj.model_call_details["custom_llm_provider"] = ( + litellm.LlmProviders.ANTHROPIC.value + ) return kwargs except Exception as e: verbose_proxy_logger.exception( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py index cba558248d..1acaf5f361 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py @@ -14,11 +14,13 @@ from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, ) from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.proxy.pass_through_endpoints.types import PassthroughStandardLoggingPayload from litellm.types.passthrough_endpoints.assembly_ai import ( ASSEMBLY_AI_MAX_POLLING_ATTEMPTS, ASSEMBLY_AI_POLLING_INTERVAL, ) +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, +) class AssemblyAITranscriptResponse(TypedDict, total=False): diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py index e73b652d83..b9df8ecede 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/base_passthrough_logging_handler.py @@ -13,7 +13,9 @@ from litellm.litellm_core_utils.litellm_logging import ( from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body -from litellm.proxy.pass_through_endpoints.types import PassthroughStandardLoggingPayload +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, +) from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse if TYPE_CHECKING: diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 2c11e4a2dd..d4260a0300 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -8,6 +8,7 @@ import httpx from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import PassThroughEndpointLoggingResultValues +from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.types.utils import StandardPassThroughResponseObject from .llm_provider_handlers.anthropic_passthrough_logging_handler import ( @@ -17,7 +18,6 @@ from .llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) from .success_handler import PassThroughEndpointLogging -from .types import EndpointType class PassThroughStreamingHandler: diff --git a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py index 90fcc273e1..fabd798791 100644 --- a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py +++ b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py @@ -14,19 +14,31 @@ sys.path.insert( import litellm from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.passthrough_endpoints.pass_through_endpoints import PassthroughStandardLoggingPayload from litellm.integrations.custom_logger import CustomLogger from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request class TestCustomLogger(CustomLogger): + def __init__(self): + self.logged_kwargs: Optional[dict] = None + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): print("in async log success event kwargs", json.dumps(kwargs, indent=4, default=str)) - pass + self.logged_kwargs = kwargs @pytest.mark.asyncio async def test_assistants_passthrough_logging(): test_custom_logger = TestCustomLogger() litellm._async_success_callback = [test_custom_logger] + TARGET_URL = "https://api.openai.com/v1/assistants" + REQUEST_BODY = { + "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.", + "name": "Math Tutor", + "tools": [{"type": "code_interpreter"}], + "model": "gpt-4o" + } + result = await pass_through_request( request=Request( scope={ @@ -41,7 +53,7 @@ async def test_assistants_passthrough_logging(): ] }, ), - target="https://api.openai.com/v1/assistants", + target=TARGET_URL, custom_headers={ "Content-Type": "application/json", "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}", @@ -53,12 +65,7 @@ async def test_assistants_passthrough_logging(): team_id="test", end_user_id="test", ), - custom_body={ - "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.", - "name": "Math Tutor", - "tools": [{"type": "code_interpreter"}], - "model": "gpt-4o" - }, + custom_body=REQUEST_BODY, forward_headers=False, merge_query_params=False, ) @@ -67,7 +74,17 @@ async def test_assistants_passthrough_logging(): print("result status code", result.status_code) print("result content", result.body) + await asyncio.sleep(1) + + assert test_custom_logger.logged_kwargs is not None + passthrough_logging_payload: Optional[PassthroughStandardLoggingPayload] = test_custom_logger.logged_kwargs["passthrough_logging_payload"] + assert passthrough_logging_payload is not None + assert passthrough_logging_payload["url"] == TARGET_URL + assert passthrough_logging_payload["request_body"] == REQUEST_BODY + + # assert that the response body content matches the response body content + client_facing_response_body = json.loads(result.body) + assert passthrough_logging_payload["response_body"] == client_facing_response_body - await asyncio.sleep(5) diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index cb9db00324..c79bf9482e 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -17,7 +17,7 @@ import pytest import litellm from typing import AsyncGenerator from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy.pass_through_endpoints.types import EndpointType +from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) @@ -34,7 +34,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( _init_kwargs_for_pass_through_endpoint, _update_metadata_with_tags_in_header, ) -from litellm.proxy.pass_through_endpoints.types import PassthroughStandardLoggingPayload +from litellm.types.passthrough_endpoints.pass_through_endpoints import PassthroughStandardLoggingPayload @pytest.fixture diff --git a/tests/pass_through_unit_tests/test_unit_test_streaming.py b/tests/pass_through_unit_tests/test_unit_test_streaming.py index 710687acc8..d3e0b6b0b0 100644 --- a/tests/pass_through_unit_tests/test_unit_test_streaming.py +++ b/tests/pass_through_unit_tests/test_unit_test_streaming.py @@ -13,7 +13,8 @@ import pytest import litellm from typing import AsyncGenerator from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy.pass_through_endpoints.types import EndpointType +from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType +from litellm.types.passthrough_endpoints.pass_through_endpoints import PassthroughStandardLoggingPayload from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, )