diff --git a/.circleci/config.yml b/.circleci/config.yml index b0a369a35..db7c4ef5b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -771,6 +771,7 @@ jobs: - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py - run: python ./tests/documentation_tests/test_env_keys.py - run: python ./tests/documentation_tests/test_api_docs.py + - run: python ./tests/code_coverage_tests/ensure_async_clients_test.py - run: helm lint ./deploy/charts/litellm-helm db_migration_disable_update_check: diff --git a/docs/my-website/docs/completion/json_mode.md b/docs/my-website/docs/completion/json_mode.md index 51f76b7a6..379775bf2 100644 --- a/docs/my-website/docs/completion/json_mode.md +++ b/docs/my-website/docs/completion/json_mode.md @@ -76,6 +76,8 @@ Works for: - Vertex AI models (Gemini + Anthropic) - Bedrock Models - Anthropic API Models +- Groq Models +- Ollama Models diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md index 5250ea403..603e04dd9 100644 --- a/docs/my-website/docs/embedding/supported_embedding.md +++ b/docs/my-website/docs/embedding/supported_embedding.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Embedding Models +# Embeddings ## Quick Start ```python diff --git a/docs/my-website/docs/image_generation.md b/docs/my-website/docs/image_generation.md index 5a7ef6f4f..958ff4c02 100644 --- a/docs/my-website/docs/image_generation.md +++ b/docs/my-website/docs/image_generation.md @@ -1,4 +1,4 @@ -# Image Generation +# Images ## Quick Start diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index d4660b807..b3bfe333c 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -10,6 +10,35 @@ LiteLLM supports all anthropic models. - `claude-2.1` - `claude-instant-1.2` + +| Property | Details | +|-------|-------| +| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. | +| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`) | +| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview) | +| API Endpoint for Provider | https://api.anthropic.com | +| Supported Endpoints | `/chat/completions` | + + +## Supported OpenAI Parameters + +Check this in code, [here](../completion/input.md#translated-openai-params) + +``` +"stream", +"stop", +"temperature", +"top_p", +"max_tokens", +"max_completion_tokens", +"tools", +"tool_choice", +"extra_headers", +"parallel_tool_calls", +"response_format", +"user" +``` + :::info Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed. @@ -1006,20 +1035,3 @@ curl http://0.0.0.0:4000/v1/chat/completions \ - -## All Supported OpenAI Params - -``` -"stream", -"stop", -"temperature", -"top_p", -"max_tokens", -"max_completion_tokens", -"tools", -"tool_choice", -"extra_headers", -"parallel_tool_calls", -"response_format", -"user" -``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 50cc83c08..f01402299 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -199,46 +199,52 @@ const sidebars = { ], }, - { - type: "category", - label: "Guides", - link: { - type: "generated-index", - title: "Chat Completions", - description: "Details on the completion() function", - slug: "/completion", - }, - items: [ - "completion/input", - "completion/provider_specific_params", - "completion/json_mode", - "completion/prompt_caching", - "completion/audio", - "completion/vision", - "completion/predict_outputs", - "completion/prefix", - "completion/drop_params", - "completion/prompt_formatting", - "completion/output", - "completion/usage", - "exception_mapping", - "completion/stream", - "completion/message_trimming", - "completion/function_call", - "completion/model_alias", - "completion/batching", - "completion/mock_requests", - "completion/reliable_completions", - ], - }, { type: "category", label: "Supported Endpoints", items: [ + { + type: "category", + label: "Chat", + link: { + type: "generated-index", + title: "Chat Completions", + description: "Details on the completion() function", + slug: "/completion", + }, + items: [ + "completion/input", + "completion/provider_specific_params", + "completion/json_mode", + "completion/prompt_caching", + "completion/audio", + "completion/vision", + "completion/predict_outputs", + "completion/prefix", + "completion/drop_params", + "completion/prompt_formatting", + "completion/output", + "completion/usage", + "exception_mapping", + "completion/stream", + "completion/message_trimming", + "completion/function_call", + "completion/model_alias", + "completion/batching", + "completion/mock_requests", + "completion/reliable_completions", + ], + }, "embedding/supported_embedding", "image_generation", - "audio_transcription", - "text_to_speech", + { + type: "category", + label: "Audio", + "items": [ + "audio_transcription", + "text_to_speech", + ] + }, "rerank", "assistants", "batches", diff --git a/litellm/__init__.py b/litellm/__init__.py index 9a8c56a56..c978b24ee 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -133,7 +133,7 @@ use_client: bool = False ssl_verify: Union[str, bool] = True ssl_certificate: Optional[str] = None disable_streaming_logging: bool = False -in_memory_llm_clients_cache: dict = {} +in_memory_llm_clients_cache: InMemoryCache = InMemoryCache() safe_memory_mode: bool = False enable_azure_ad_token_refresh: Optional[bool] = False ### DEFAULT AZURE API VERSION ### diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 5c18ff512..483121c38 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1793,7 +1793,7 @@ class CustomStreamWrapper: or self.custom_llm_provider == "bedrock" or self.custom_llm_provider == "triton" or self.custom_llm_provider == "watsonx" - or self.custom_llm_provider in litellm.openai_compatible_endpoints + or self.custom_llm_provider in litellm.openai_compatible_providers or self.custom_llm_provider in litellm._custom_providers ): async for chunk in self.completion_stream: diff --git a/litellm/llms/AzureOpenAI/azure.py b/litellm/llms/AzureOpenAI/azure.py index 39dea14e2..f6a1790b6 100644 --- a/litellm/llms/AzureOpenAI/azure.py +++ b/litellm/llms/AzureOpenAI/azure.py @@ -12,7 +12,11 @@ from typing_extensions import overload import litellm from litellm.caching.caching import DualCache from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) from litellm.types.utils import EmbeddingResponse from litellm.utils import ( CustomStreamWrapper, @@ -977,7 +981,10 @@ class AzureChatCompletion(BaseLLM): else: _params["timeout"] = httpx.Timeout(timeout=600.0, connect=5.0) - async_handler = AsyncHTTPHandler(**_params) # type: ignore + async_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.AZURE, + params=_params, + ) else: async_handler = client # type: ignore diff --git a/litellm/llms/OpenAI/chat/o1_handler.py b/litellm/llms/OpenAI/chat/o1_handler.py index 55dfe3715..5ff53a896 100644 --- a/litellm/llms/OpenAI/chat/o1_handler.py +++ b/litellm/llms/OpenAI/chat/o1_handler.py @@ -17,22 +17,6 @@ from litellm.utils import CustomStreamWrapper class OpenAIO1ChatCompletion(OpenAIChatCompletion): - async def mock_async_streaming( - self, - response: Any, - model: Optional[str], - logging_obj: Any, - ): - model_response = await response - completion_stream = MockResponseIterator(model_response=model_response) - streaming_response = CustomStreamWrapper( - completion_stream=completion_stream, - model=model, - custom_llm_provider="openai", - logging_obj=logging_obj, - ) - return streaming_response - def completion( self, model_response: ModelResponse, @@ -54,7 +38,7 @@ class OpenAIO1ChatCompletion(OpenAIChatCompletion): custom_llm_provider: Optional[str] = None, drop_params: Optional[bool] = None, ): - stream: Optional[bool] = optional_params.pop("stream", False) + # stream: Optional[bool] = optional_params.pop("stream", False) response = super().completion( model_response, timeout, @@ -76,20 +60,4 @@ class OpenAIO1ChatCompletion(OpenAIChatCompletion): drop_params, ) - if stream is True: - if asyncio.iscoroutine(response): - return self.mock_async_streaming( - response=response, model=model, logging_obj=logging_obj # type: ignore - ) - - completion_stream = MockResponseIterator(model_response=response) - streaming_response = CustomStreamWrapper( - completion_stream=completion_stream, - model=model, - custom_llm_provider="openai", - logging_obj=logging_obj, - ) - - return streaming_response - else: - return response + return response diff --git a/litellm/llms/OpenAI/openai.py b/litellm/llms/OpenAI/openai.py index 7d701d26c..057340b51 100644 --- a/litellm/llms/OpenAI/openai.py +++ b/litellm/llms/OpenAI/openai.py @@ -18,6 +18,7 @@ import litellm from litellm import LlmProviders from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.custom_httpx.http_handler import _DEFAULT_TTL_FOR_HTTPX_CLIENTS from litellm.secret_managers.main import get_secret_str from litellm.types.utils import ProviderField from litellm.utils import ( @@ -562,8 +563,9 @@ class OpenAIChatCompletion(BaseLLM): _cache_key = f"hashed_api_key={hashed_api_key},api_base={api_base},timeout={timeout},max_retries={max_retries},organization={organization},is_async={is_async}" - if _cache_key in litellm.in_memory_llm_clients_cache: - return litellm.in_memory_llm_clients_cache[_cache_key] + _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key) + if _cached_client: + return _cached_client if is_async: _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI( api_key=api_key, @@ -584,7 +586,11 @@ class OpenAIChatCompletion(BaseLLM): ) ## SAVE CACHE KEY - litellm.in_memory_llm_clients_cache[_cache_key] = _new_client + litellm.in_memory_llm_clients_cache.set_cache( + key=_cache_key, + value=_new_client, + ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + ) return _new_client else: diff --git a/litellm/llms/anthropic/completion.py b/litellm/llms/anthropic/completion.py index 89a50db6a..dc06401d6 100644 --- a/litellm/llms/anthropic/completion.py +++ b/litellm/llms/anthropic/completion.py @@ -13,7 +13,11 @@ import httpx import requests import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) from litellm.utils import CustomStreamWrapper, ModelResponse, Usage from ..base import BaseLLM @@ -162,7 +166,10 @@ class AnthropicTextCompletion(BaseLLM): client=None, ): if client is None: - client = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) + client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.ANTHROPIC, + params={"timeout": httpx.Timeout(timeout=600.0, connect=5.0)}, + ) response = await client.post(api_base, headers=headers, data=json.dumps(data)) @@ -198,7 +205,10 @@ class AnthropicTextCompletion(BaseLLM): client=None, ): if client is None: - client = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) + client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.ANTHROPIC, + params={"timeout": httpx.Timeout(timeout=600.0, connect=5.0)}, + ) response = await client.post(api_base, headers=headers, data=json.dumps(data)) diff --git a/litellm/llms/azure_ai/embed/handler.py b/litellm/llms/azure_ai/embed/handler.py index 638a77479..2946a84dd 100644 --- a/litellm/llms/azure_ai/embed/handler.py +++ b/litellm/llms/azure_ai/embed/handler.py @@ -74,7 +74,10 @@ class AzureAIEmbedding(OpenAIChatCompletion): client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, ) -> EmbeddingResponse: if client is None or not isinstance(client, AsyncHTTPHandler): - client = AsyncHTTPHandler(timeout=timeout, concurrent_limit=1) + client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.AZURE_AI, + params={"timeout": timeout}, + ) url = "{}/images/embeddings".format(api_base) diff --git a/litellm/llms/clarifai.py b/litellm/llms/clarifai.py index 2011c0bee..61d445423 100644 --- a/litellm/llms/clarifai.py +++ b/litellm/llms/clarifai.py @@ -9,7 +9,10 @@ import httpx import requests import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, +) from litellm.utils import Choices, CustomStreamWrapper, Message, ModelResponse, Usage from .prompt_templates.factory import custom_prompt, prompt_factory @@ -185,7 +188,10 @@ async def async_completion( headers={}, ): - async_handler = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) + async_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.CLARIFAI, + params={"timeout": 600.0}, + ) response = await async_handler.post( url=model, headers=headers, data=json.dumps(data) ) diff --git a/litellm/llms/cohere/embed/handler.py b/litellm/llms/cohere/embed/handler.py index 95cbec225..5b224c375 100644 --- a/litellm/llms/cohere/embed/handler.py +++ b/litellm/llms/cohere/embed/handler.py @@ -11,7 +11,11 @@ import requests # type: ignore import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) from litellm.types.llms.bedrock import CohereEmbeddingRequest from litellm.utils import Choices, Message, ModelResponse, Usage @@ -71,7 +75,10 @@ async def async_embedding( ) ## COMPLETION CALL if client is None: - client = AsyncHTTPHandler(concurrent_limit=1, timeout=timeout) + client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.COHERE, + params={"timeout": timeout}, + ) try: response = await client.post(api_base, headers=headers, data=json.dumps(data)) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 020af7e90..f1b78ea63 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -7,6 +7,7 @@ import httpx from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport import litellm +from litellm.caching import InMemoryCache from .types import httpxSpecialProvider @@ -26,6 +27,7 @@ headers = { # https://www.python-httpx.org/advanced/timeouts _DEFAULT_TIMEOUT = httpx.Timeout(timeout=5.0, connect=5.0) +_DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour class AsyncHTTPHandler: @@ -476,8 +478,9 @@ def get_async_httpx_client( pass _cache_key_name = "async_httpx_client" + _params_key_name + llm_provider - if _cache_key_name in litellm.in_memory_llm_clients_cache: - return litellm.in_memory_llm_clients_cache[_cache_key_name] + _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key_name) + if _cached_client: + return _cached_client if params is not None: _new_client = AsyncHTTPHandler(**params) @@ -485,7 +488,11 @@ def get_async_httpx_client( _new_client = AsyncHTTPHandler( timeout=httpx.Timeout(timeout=600.0, connect=5.0) ) - litellm.in_memory_llm_clients_cache[_cache_key_name] = _new_client + litellm.in_memory_llm_clients_cache.set_cache( + key=_cache_key_name, + value=_new_client, + ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + ) return _new_client @@ -505,13 +512,18 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler: pass _cache_key_name = "httpx_client" + _params_key_name - if _cache_key_name in litellm.in_memory_llm_clients_cache: - return litellm.in_memory_llm_clients_cache[_cache_key_name] + _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key_name) + if _cached_client: + return _cached_client if params is not None: _new_client = HTTPHandler(**params) else: _new_client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) - litellm.in_memory_llm_clients_cache[_cache_key_name] = _new_client + litellm.in_memory_llm_clients_cache.set_cache( + key=_cache_key_name, + value=_new_client, + ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + ) return _new_client diff --git a/litellm/llms/databricks/chat.py b/litellm/llms/databricks/chat.py index 79e885646..e752f4d98 100644 --- a/litellm/llms/databricks/chat.py +++ b/litellm/llms/databricks/chat.py @@ -393,7 +393,10 @@ class DatabricksChatCompletion(BaseLLM): if timeout is None: timeout = httpx.Timeout(timeout=600.0, connect=5.0) - self.async_handler = AsyncHTTPHandler(timeout=timeout) + self.async_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.DATABRICKS, + params={"timeout": timeout}, + ) try: response = await self.async_handler.post( @@ -610,7 +613,10 @@ class DatabricksChatCompletion(BaseLLM): response = None try: if client is None or isinstance(client, AsyncHTTPHandler): - self.async_client = AsyncHTTPHandler(timeout=timeout) # type: ignore + self.async_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.DATABRICKS, + params={"timeout": timeout}, + ) else: self.async_client = client diff --git a/litellm/llms/fine_tuning_apis/vertex_ai.py b/litellm/llms/fine_tuning_apis/vertex_ai.py index 11d052191..fd418103e 100644 --- a/litellm/llms/fine_tuning_apis/vertex_ai.py +++ b/litellm/llms/fine_tuning_apis/vertex_ai.py @@ -5,9 +5,14 @@ from typing import Any, Coroutine, Literal, Optional, Union import httpx from openai.types.fine_tuning.fine_tuning_job import FineTuningJob, Hyperparameters +import litellm from litellm._logging import verbose_logger from litellm.llms.base import BaseLLM -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) from litellm.llms.vertex_ai_and_google_ai_studio.gemini.vertex_and_google_ai_studio_gemini import ( VertexLLM, ) @@ -26,8 +31,9 @@ class VertexFineTuningAPI(VertexLLM): def __init__(self) -> None: super().__init__() - self.async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=600.0, connect=5.0) + self.async_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.VERTEX_AI, + params={"timeout": 600.0}, ) def convert_response_created_at(self, response: ResponseTuningJob): diff --git a/litellm/llms/groq/chat/handler.py b/litellm/llms/groq/chat/handler.py index f4a16abc8..1fe87844c 100644 --- a/litellm/llms/groq/chat/handler.py +++ b/litellm/llms/groq/chat/handler.py @@ -6,55 +6,68 @@ from typing import Any, Callable, Optional, Union from httpx._config import Timeout +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.utils import CustomStreamingDecoder from litellm.utils import ModelResponse from ...groq.chat.transformation import GroqChatConfig -from ...OpenAI.openai import OpenAIChatCompletion +from ...openai_like.chat.handler import OpenAILikeChatHandler -class GroqChatCompletion(OpenAIChatCompletion): +class GroqChatCompletion(OpenAILikeChatHandler): def __init__(self, **kwargs): super().__init__(**kwargs) def completion( self, + *, + model: str, + messages: list, + api_base: str, + custom_llm_provider: str, + custom_prompt_dict: dict, model_response: ModelResponse, - timeout: Union[float, Timeout], + print_verbose: Callable, + encoding, + api_key: Optional[str], + logging_obj, optional_params: dict, - logging_obj: Any, - model: Optional[str] = None, - messages: Optional[list] = None, - print_verbose: Optional[Callable[..., Any]] = None, - api_key: Optional[str] = None, - api_base: Optional[str] = None, - acompletion: bool = False, + acompletion=None, litellm_params=None, logger_fn=None, headers: Optional[dict] = None, - custom_prompt_dict: dict = {}, - client=None, - organization: Optional[str] = None, - custom_llm_provider: Optional[str] = None, - drop_params: Optional[bool] = None, + timeout: Optional[Union[float, Timeout]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + custom_endpoint: Optional[bool] = None, + streaming_decoder: Optional[CustomStreamingDecoder] = None, + fake_stream: bool = False ): messages = GroqChatConfig()._transform_messages(messages) # type: ignore + + if optional_params.get("stream") is True: + fake_stream = GroqChatConfig()._should_fake_stream(optional_params) + else: + fake_stream = False + return super().completion( - model_response, - timeout, - optional_params, - logging_obj, - model, - messages, - print_verbose, - api_key, - api_base, - acompletion, - litellm_params, - logger_fn, - headers, - custom_prompt_dict, - client, - organization, - custom_llm_provider, - drop_params, + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + acompletion=acompletion, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + timeout=timeout, + client=client, + custom_endpoint=custom_endpoint, + streaming_decoder=streaming_decoder, + fake_stream=fake_stream, ) diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 4baba7657..dddc56a2c 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -2,6 +2,7 @@ Translate from OpenAI's `/v1/chat/completions` to Groq's `/v1/chat/completions` """ +import json import types from typing import List, Optional, Tuple, Union @@ -9,7 +10,12 @@ from pydantic import BaseModel import litellm from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantMessage, + ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, +) from ...OpenAI.chat.gpt_transformation import OpenAIGPTConfig @@ -99,3 +105,69 @@ class GroqChatConfig(OpenAIGPTConfig): ) # type: ignore dynamic_api_key = api_key or get_secret_str("GROQ_API_KEY") return api_base, dynamic_api_key + + def _should_fake_stream(self, optional_params: dict) -> bool: + """ + Groq doesn't support 'response_format' while streaming + """ + if optional_params.get("response_format") is not None: + return True + + return False + + def _create_json_tool_call_for_response_format( + self, + json_schema: dict, + ): + """ + Handles creating a tool call for getting responses in JSON format. + + Args: + json_schema (Optional[dict]): The JSON schema the response should be in + + Returns: + AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format + """ + return ChatCompletionToolParam( + type="function", + function=ChatCompletionToolParamFunctionChunk( + name="json_tool_call", + parameters=json_schema, + ), + ) + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool = False, + ) -> dict: + _response_format = non_default_params.get("response_format") + if _response_format is not None and isinstance(_response_format, dict): + json_schema: Optional[dict] = None + if "response_schema" in _response_format: + json_schema = _response_format["response_schema"] + elif "json_schema" in _response_format: + json_schema = _response_format["json_schema"]["schema"] + """ + When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + - You usually want to provide a single tool + - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool + - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective. + """ + if json_schema is not None: + _tool_choice = { + "type": "function", + "function": {"name": "json_tool_call"}, + } + _tool = self._create_json_tool_call_for_response_format( + json_schema=json_schema, + ) + optional_params["tools"] = [_tool] + optional_params["tool_choice"] = _tool_choice + optional_params["json_mode"] = True + non_default_params.pop("response_format", None) + return super().map_openai_params( + non_default_params, optional_params, model, drop_params + ) diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index 907d72a60..8b45f1ae7 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -263,7 +263,11 @@ def get_hf_task_for_model(model: str) -> Tuple[hf_tasks, str]: return "text-generation-inference", model # default to tgi -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) def get_hf_task_embedding_for_model( @@ -301,7 +305,9 @@ async def async_get_hf_task_embedding_for_model( task_type, hf_tasks_embeddings ) ) - http_client = AsyncHTTPHandler(concurrent_limit=1) + http_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.HUGGINGFACE, + ) model_info = await http_client.get(url=api_base) @@ -1067,7 +1073,9 @@ class Huggingface(BaseLLM): ) ## COMPLETION CALL if client is None: - client = AsyncHTTPHandler(concurrent_limit=1) + client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.HUGGINGFACE, + ) response = await client.post(api_base, headers=headers, data=json.dumps(data)) diff --git a/litellm/llms/ollama.py b/litellm/llms/ollama.py index 842d946c6..896b93be5 100644 --- a/litellm/llms/ollama.py +++ b/litellm/llms/ollama.py @@ -164,6 +164,30 @@ class OllamaConfig: "response_format", ] + def map_openai_params( + self, optional_params: dict, non_default_params: dict + ) -> dict: + for param, value in non_default_params.items(): + if param == "max_tokens": + optional_params["num_predict"] = value + if param == "stream": + optional_params["stream"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "seed": + optional_params["seed"] = value + if param == "top_p": + optional_params["top_p"] = value + if param == "frequency_penalty": + optional_params["repeat_penalty"] = value + if param == "stop": + optional_params["stop"] = value + if param == "response_format" and isinstance(value, dict): + if value["type"] == "json_object": + optional_params["format"] = "json" + + return optional_params + def _supports_function_calling(self, ollama_model_info: dict) -> bool: """ Check if the 'template' field in the ollama_model_info contains a 'tools' or 'function' key. diff --git a/litellm/llms/openai_like/chat/handler.py b/litellm/llms/openai_like/chat/handler.py index 0dbc3a978..baa970304 100644 --- a/litellm/llms/openai_like/chat/handler.py +++ b/litellm/llms/openai_like/chat/handler.py @@ -17,7 +17,9 @@ import httpx # type: ignore import requests # type: ignore import litellm +from litellm import LlmProviders from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -25,9 +27,19 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.llms.databricks.streaming_utils import ModelResponseIterator from litellm.types.utils import CustomStreamingDecoder, ModelResponse -from litellm.utils import CustomStreamWrapper, EmbeddingResponse +from litellm.utils import ( + Choices, + CustomStreamWrapper, + EmbeddingResponse, + Message, + ProviderConfigManager, + TextCompletionResponse, + Usage, + convert_to_model_response_object, +) from ..common_utils import OpenAILikeBase, OpenAILikeError +from .transformation import OpenAILikeChatConfig async def make_call( @@ -39,16 +51,22 @@ async def make_call( messages: list, logging_obj, streaming_decoder: Optional[CustomStreamingDecoder] = None, + fake_stream: bool = False, ): if client is None: client = litellm.module_level_aclient - response = await client.post(api_base, headers=headers, data=data, stream=True) + response = await client.post( + api_base, headers=headers, data=data, stream=not fake_stream + ) if streaming_decoder is not None: completion_stream: Any = streaming_decoder.aiter_bytes( response.aiter_bytes(chunk_size=1024) ) + elif fake_stream: + model_response = ModelResponse(**response.json()) + completion_stream = MockResponseIterator(model_response=model_response) else: completion_stream = ModelResponseIterator( streaming_response=response.aiter_lines(), sync_stream=False @@ -73,11 +91,12 @@ def make_sync_call( messages: list, logging_obj, streaming_decoder: Optional[CustomStreamingDecoder] = None, + fake_stream: bool = False, ): if client is None: client = litellm.module_level_client # Create a new client if none provided - response = client.post(api_base, headers=headers, data=data, stream=True) + response = client.post(api_base, headers=headers, data=data, stream=not fake_stream) if response.status_code != 200: raise OpenAILikeError(status_code=response.status_code, message=response.read()) @@ -86,6 +105,9 @@ def make_sync_call( completion_stream = streaming_decoder.iter_bytes( response.iter_bytes(chunk_size=1024) ) + elif fake_stream: + model_response = ModelResponse(**response.json()) + completion_stream = MockResponseIterator(model_response=model_response) else: completion_stream = ModelResponseIterator( streaming_response=response.iter_lines(), sync_stream=True @@ -126,8 +148,8 @@ class OpenAILikeChatHandler(OpenAILikeBase): headers={}, client: Optional[AsyncHTTPHandler] = None, streaming_decoder: Optional[CustomStreamingDecoder] = None, + fake_stream: bool = False, ) -> CustomStreamWrapper: - data["stream"] = True completion_stream = await make_call( client=client, @@ -169,6 +191,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): logger_fn=None, headers={}, timeout: Optional[Union[float, httpx.Timeout]] = None, + json_mode: bool = False, ) -> ModelResponse: if timeout is None: timeout = httpx.Timeout(timeout=600.0, connect=5.0) @@ -181,8 +204,6 @@ class OpenAILikeChatHandler(OpenAILikeBase): api_base, headers=headers, data=json.dumps(data), timeout=timeout ) response.raise_for_status() - - response_json = response.json() except httpx.HTTPStatusError as e: raise OpenAILikeError( status_code=e.response.status_code, @@ -193,22 +214,26 @@ class OpenAILikeChatHandler(OpenAILikeBase): except Exception as e: raise OpenAILikeError(status_code=500, message=str(e)) - logging_obj.post_call( - input=messages, - api_key="", - original_response=response_json, - additional_args={"complete_input_dict": data}, + return OpenAILikeChatConfig._transform_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + optional_params=optional_params, + api_key=api_key, + data=data, + messages=messages, + print_verbose=print_verbose, + encoding=encoding, + json_mode=json_mode, + custom_llm_provider=custom_llm_provider, + base_model=base_model, ) - response = ModelResponse(**response_json) - - response.model = custom_llm_provider + "/" + (response.model or "") - - if base_model is not None: - response._hidden_params["model"] = base_model - return response def completion( self, + *, model: str, messages: list, api_base: str, @@ -230,6 +255,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): streaming_decoder: Optional[ CustomStreamingDecoder ] = None, # if openai-compatible api needs custom stream decoder - e.g. sagemaker + fake_stream: bool = False, ): custom_endpoint = custom_endpoint or optional_params.pop( "custom_endpoint", None @@ -243,13 +269,24 @@ class OpenAILikeChatHandler(OpenAILikeBase): headers=headers, ) - stream: bool = optional_params.get("stream", None) or False - optional_params["stream"] = stream + stream: bool = optional_params.pop("stream", None) or False + extra_body = optional_params.pop("extra_body", {}) + json_mode = optional_params.pop("json_mode", None) + optional_params.pop("max_retries", None) + if not fake_stream: + optional_params["stream"] = stream + + if messages is not None and custom_llm_provider is not None: + provider_config = ProviderConfigManager.get_provider_config( + model=model, provider=LlmProviders(custom_llm_provider) + ) + messages = provider_config._transform_messages(messages) data = { "model": model, "messages": messages, **optional_params, + **extra_body, } ## LOGGING @@ -288,6 +325,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): client=client, custom_llm_provider=custom_llm_provider, streaming_decoder=streaming_decoder, + fake_stream=fake_stream, ) else: return self.acompletion_function( @@ -327,6 +365,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): messages=messages, logging_obj=logging_obj, streaming_decoder=streaming_decoder, + fake_stream=fake_stream, ) # completion_stream.__iter__() return CustomStreamWrapper( @@ -344,7 +383,6 @@ class OpenAILikeChatHandler(OpenAILikeBase): ) response.raise_for_status() - response_json = response.json() except httpx.HTTPStatusError as e: raise OpenAILikeError( status_code=e.response.status_code, @@ -356,17 +394,19 @@ class OpenAILikeChatHandler(OpenAILikeBase): ) except Exception as e: raise OpenAILikeError(status_code=500, message=str(e)) - logging_obj.post_call( - input=messages, - api_key="", - original_response=response_json, - additional_args={"complete_input_dict": data}, + return OpenAILikeChatConfig._transform_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + optional_params=optional_params, + api_key=api_key, + data=data, + messages=messages, + print_verbose=print_verbose, + encoding=encoding, + json_mode=json_mode, + custom_llm_provider=custom_llm_provider, + base_model=base_model, ) - response = ModelResponse(**response_json) - - response.model = custom_llm_provider + "/" + (response.model or "") - - if base_model is not None: - response._hidden_params["model"] = base_model - - return response diff --git a/litellm/llms/openai_like/chat/transformation.py b/litellm/llms/openai_like/chat/transformation.py new file mode 100644 index 000000000..c355cf330 --- /dev/null +++ b/litellm/llms/openai_like/chat/transformation.py @@ -0,0 +1,98 @@ +""" +OpenAI-like chat completion transformation +""" + +import types +from typing import List, Optional, Tuple, Union + +import httpx +from pydantic import BaseModel + +import litellm +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage +from litellm.types.utils import ModelResponse + +from ....utils import _remove_additional_properties, _remove_strict_from_schema +from ...OpenAI.chat.gpt_transformation import OpenAIGPTConfig + + +class OpenAILikeChatConfig(OpenAIGPTConfig): + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = api_base or get_secret_str("OPENAI_LIKE_API_BASE") # type: ignore + dynamic_api_key = ( + api_key or get_secret_str("OPENAI_LIKE_API_KEY") or "" + ) # vllm does not require an api key + return api_base, dynamic_api_key + + @staticmethod + def _convert_tool_response_to_message( + message: ChatCompletionAssistantMessage, json_mode: bool + ) -> ChatCompletionAssistantMessage: + """ + if json_mode is true, convert the returned tool call response to a content with json str + + e.g. input: + + {"role": "assistant", "tool_calls": [{"id": "call_5ms4", "type": "function", "function": {"name": "json_tool_call", "arguments": "{\"key\": \"question\", \"value\": \"What is the capital of France?\"}"}}]} + + output: + + {"role": "assistant", "content": "{\"key\": \"question\", \"value\": \"What is the capital of France?\"}"} + """ + if not json_mode: + return message + + _tool_calls = message.get("tool_calls") + + if _tool_calls is None or len(_tool_calls) != 1: + return message + + message["content"] = _tool_calls[0]["function"].get("arguments") or "" + message["tool_calls"] = None + + return message + + @staticmethod + def _transform_response( + model: str, + response: httpx.Response, + model_response: ModelResponse, + stream: bool, + logging_obj: litellm.litellm_core_utils.litellm_logging.Logging, # type: ignore + optional_params: dict, + api_key: Optional[str], + data: Union[dict, str], + messages: List, + print_verbose, + encoding, + json_mode: bool, + custom_llm_provider: str, + base_model: Optional[str], + ) -> ModelResponse: + response_json = response.json() + logging_obj.post_call( + input=messages, + api_key="", + original_response=response_json, + additional_args={"complete_input_dict": data}, + ) + + if json_mode: + for choice in response_json["choices"]: + message = OpenAILikeChatConfig._convert_tool_response_to_message( + choice.get("message"), json_mode + ) + choice["message"] = message + + returned_response = ModelResponse(**response_json) + + returned_response.model = ( + custom_llm_provider + "/" + (returned_response.model or "") + ) + + if base_model is not None: + returned_response._hidden_params["model"] = base_model + return returned_response diff --git a/litellm/llms/openai_like/embedding/handler.py b/litellm/llms/openai_like/embedding/handler.py index ce0860724..e786b5db8 100644 --- a/litellm/llms/openai_like/embedding/handler.py +++ b/litellm/llms/openai_like/embedding/handler.py @@ -45,7 +45,10 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): response = None try: if client is None or isinstance(client, AsyncHTTPHandler): - self.async_client = AsyncHTTPHandler(timeout=timeout) # type: ignore + self.async_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + params={"timeout": timeout}, + ) else: self.async_client = client @@ -62,7 +65,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase): except httpx.HTTPStatusError as e: raise OpenAILikeError( status_code=e.response.status_code, - message=response.text if response else str(e), + message=e.response.text if e.response else str(e), ) except httpx.TimeoutException: raise OpenAILikeError( diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index 96796f9dc..e80964551 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -19,7 +19,10 @@ import litellm.litellm_core_utils import litellm.litellm_core_utils.litellm_logging from litellm import verbose_logger from litellm.litellm_core_utils.core_helpers import map_finish_reason -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, +) from litellm.utils import Choices, CustomStreamWrapper, Message, ModelResponse, Usage from .base import BaseLLM @@ -549,7 +552,10 @@ class PredibaseChatCompletion(BaseLLM): headers={}, ) -> ModelResponse: - async_handler = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=timeout)) + async_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.PREDIBASE, + params={"timeout": timeout}, + ) try: response = await async_handler.post( api_base, headers=headers, data=json.dumps(data) diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 29028e053..45b7a6c5b 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -943,17 +943,10 @@ def _gemini_tool_call_invoke_helper( name = function_call_params.get("name", "") or "" arguments = function_call_params.get("arguments", "") arguments_dict = json.loads(arguments) - function_call: Optional[litellm.types.llms.vertex_ai.FunctionCall] = None - for k, v in arguments_dict.items(): - inferred_protocol_value = infer_protocol_value(value=v) - _field = litellm.types.llms.vertex_ai.Field( - key=k, value={inferred_protocol_value: v} - ) - _fields = litellm.types.llms.vertex_ai.FunctionCallArgs(fields=_field) - function_call = litellm.types.llms.vertex_ai.FunctionCall( - name=name, - args=_fields, - ) + function_call = litellm.types.llms.vertex_ai.FunctionCall( + name=name, + args=arguments_dict, + ) return function_call @@ -978,54 +971,26 @@ def convert_to_gemini_tool_call_invoke( }, """ """ - Gemini tool call invokes: - https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/function-calling#submit-api-output - content { - role: "model" - parts [ + Gemini tool call invokes: + { + "role": "model", + "parts": [ { - function_call { - name: "get_current_weather" - args { - fields { - key: "unit" - value { - string_value: "fahrenheit" - } - } - fields { - key: "predicted_temperature" - value { - number_value: 45 - } - } - fields { - key: "location" - value { - string_value: "Boston, MA" - } - } - } - }, - { - function_call { - name: "get_current_weather" - args { - fields { - key: "location" - value { - string_value: "San Francisco" - } - } - } + "functionCall": { + "name": "get_current_weather", + "args": { + "unit": "fahrenheit", + "predicted_temperature": 45, + "location": "Boston, MA", } + } } - ] + ] } """ """ - - json.load the arguments - - iterate through arguments -> create a FunctionCallArgs for each field + - json.load the arguments """ try: _parts_list: List[litellm.types.llms.vertex_ai.PartType] = [] @@ -1128,16 +1093,8 @@ def convert_to_gemini_tool_call_result( # We can't determine from openai message format whether it's a successful or # error call result so default to the successful result template - inferred_content_value = infer_protocol_value(value=content_str) - - _field = litellm.types.llms.vertex_ai.Field( - key="content", value={inferred_content_value: content_str} - ) - - _function_call_args = litellm.types.llms.vertex_ai.FunctionCallArgs(fields=_field) - _function_response = litellm.types.llms.vertex_ai.FunctionResponse( - name=name, response=_function_call_args # type: ignore + name=name, response={"content": content_str} # type: ignore ) _part = litellm.types.llms.vertex_ai.PartType(function_response=_function_response) diff --git a/litellm/llms/replicate.py b/litellm/llms/replicate.py index 094110234..2e9bbb333 100644 --- a/litellm/llms/replicate.py +++ b/litellm/llms/replicate.py @@ -9,7 +9,10 @@ import httpx # type: ignore import requests # type: ignore import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, +) from litellm.utils import CustomStreamWrapper, ModelResponse, Usage from .prompt_templates.factory import custom_prompt, prompt_factory @@ -325,7 +328,7 @@ def handle_prediction_response_streaming(prediction_url, api_token, print_verbos async def async_handle_prediction_response_streaming( prediction_url, api_token, print_verbose ): - http_handler = AsyncHTTPHandler(concurrent_limit=1) + http_handler = get_async_httpx_client(llm_provider=litellm.LlmProviders.REPLICATE) previous_output = "" output_string = "" @@ -560,7 +563,9 @@ async def async_completion( logging_obj, print_verbose, ) -> Union[ModelResponse, CustomStreamWrapper]: - http_handler = AsyncHTTPHandler(concurrent_limit=1) + http_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.REPLICATE, + ) prediction_url = await async_start_prediction( version_id, input_data, diff --git a/litellm/llms/text_completion_codestral.py b/litellm/llms/text_completion_codestral.py index 21582d26c..d3c1ae3cb 100644 --- a/litellm/llms/text_completion_codestral.py +++ b/litellm/llms/text_completion_codestral.py @@ -18,7 +18,10 @@ import litellm from litellm import verbose_logger from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, +) from litellm.types.llms.databricks import GenericStreamingChunk from litellm.utils import ( Choices, @@ -479,8 +482,9 @@ class CodestralTextCompletion(BaseLLM): headers={}, ) -> TextCompletionResponse: - async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=timeout), concurrent_limit=1 + async_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.TEXT_COMPLETION_CODESTRAL, + params={"timeout": timeout}, ) try: diff --git a/litellm/llms/triton.py b/litellm/llms/triton.py index be4179ccc..efd0d0a2d 100644 --- a/litellm/llms/triton.py +++ b/litellm/llms/triton.py @@ -8,7 +8,11 @@ import httpx # type: ignore import requests # type: ignore import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) from litellm.utils import ( Choices, CustomStreamWrapper, @@ -50,8 +54,8 @@ class TritonChatCompletion(BaseLLM): logging_obj: Any, api_key: Optional[str] = None, ) -> EmbeddingResponse: - async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=600.0, connect=5.0) + async_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.TRITON, params={"timeout": 600.0} ) response = await async_handler.post(url=api_base, data=json.dumps(data)) @@ -261,7 +265,9 @@ class TritonChatCompletion(BaseLLM): model_response, type_of_model, ) -> ModelResponse: - handler = AsyncHTTPHandler() + handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.TRITON, params={"timeout": 600.0} + ) if stream: return self._ahandle_stream( # type: ignore handler, api_base, data_for_triton, model, logging_obj diff --git a/litellm/llms/vertex_ai_and_google_ai_studio/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai_and_google_ai_studio/context_caching/vertex_ai_context_caching.py index e60a17052..b9be8a3bd 100644 --- a/litellm/llms/vertex_ai_and_google_ai_studio/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai_and_google_ai_studio/context_caching/vertex_ai_context_caching.py @@ -6,7 +6,11 @@ import httpx import litellm from litellm.caching.caching import Cache, LiteLLMCacheType from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) from litellm.llms.OpenAI.openai import AllMessageValues from litellm.types.llms.vertex_ai import ( CachedContentListAllResponseBody, @@ -331,6 +335,13 @@ class ContextCachingEndpoints(VertexBase): if cached_content is not None: return messages, cached_content + cached_messages, non_cached_messages = separate_cached_messages( + messages=messages + ) + + if len(cached_messages) == 0: + return messages, None + ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( gemini_api_key=api_key, @@ -347,22 +358,12 @@ class ContextCachingEndpoints(VertexBase): headers.update(extra_headers) if client is None or not isinstance(client, AsyncHTTPHandler): - _params = {} - if timeout is not None: - if isinstance(timeout, float) or isinstance(timeout, int): - timeout = httpx.Timeout(timeout) - _params["timeout"] = timeout - client = AsyncHTTPHandler(**_params) # type: ignore + client = get_async_httpx_client( + params={"timeout": timeout}, llm_provider=litellm.LlmProviders.VERTEX_AI + ) else: client = client - cached_messages, non_cached_messages = separate_cached_messages( - messages=messages - ) - - if len(cached_messages) == 0: - return messages, None - ## CHECK IF CACHED ALREADY generated_cache_key = local_cache_obj.get_cache_key(messages=cached_messages) google_cache_name = await self.async_check_cache( diff --git a/litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py index 39c63dbb3..f2fc599ed 100644 --- a/litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py @@ -1026,7 +1026,9 @@ async def make_call( logging_obj, ): if client is None: - client = AsyncHTTPHandler() # Create a new client if none provided + client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.VERTEX_AI, + ) try: response = await client.post(api_base, headers=headers, data=data, stream=True) diff --git a/litellm/llms/vertex_ai_and_google_ai_studio/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai_and_google_ai_studio/gemini_embeddings/batch_embed_content_handler.py index 314e129c2..8e2d1f39a 100644 --- a/litellm/llms/vertex_ai_and_google_ai_studio/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai_and_google_ai_studio/gemini_embeddings/batch_embed_content_handler.py @@ -7,8 +7,13 @@ from typing import Any, List, Literal, Optional, Union import httpx +import litellm from litellm import EmbeddingResponse -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) from litellm.types.llms.openai import EmbeddingInput from litellm.types.llms.vertex_ai import ( VertexAIBatchEmbeddingsRequestBody, @@ -150,7 +155,10 @@ class GoogleBatchEmbeddings(VertexLLM): else: _params["timeout"] = httpx.Timeout(timeout=600.0, connect=5.0) - async_handler: AsyncHTTPHandler = AsyncHTTPHandler(**_params) # type: ignore + async_handler: AsyncHTTPHandler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.VERTEX_AI, + params={"timeout": timeout}, + ) else: async_handler = client # type: ignore diff --git a/litellm/llms/vertex_ai_and_google_ai_studio/image_generation/image_generation_handler.py b/litellm/llms/vertex_ai_and_google_ai_studio/image_generation/image_generation_handler.py index 1531464c8..6cb5771e6 100644 --- a/litellm/llms/vertex_ai_and_google_ai_studio/image_generation/image_generation_handler.py +++ b/litellm/llms/vertex_ai_and_google_ai_studio/image_generation/image_generation_handler.py @@ -5,7 +5,11 @@ import httpx from openai.types.image import Image import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) from litellm.llms.vertex_ai_and_google_ai_studio.gemini.vertex_and_google_ai_studio_gemini import ( VertexLLM, ) @@ -156,7 +160,10 @@ class VertexImageGeneration(VertexLLM): else: _params["timeout"] = httpx.Timeout(timeout=600.0, connect=5.0) - self.async_handler = AsyncHTTPHandler(**_params) # type: ignore + self.async_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.VERTEX_AI, + params={"timeout": timeout}, + ) else: self.async_handler = client # type: ignore diff --git a/litellm/llms/vertex_ai_and_google_ai_studio/multimodal_embeddings/embedding_handler.py b/litellm/llms/vertex_ai_and_google_ai_studio/multimodal_embeddings/embedding_handler.py index d8af891b0..27b77fdd9 100644 --- a/litellm/llms/vertex_ai_and_google_ai_studio/multimodal_embeddings/embedding_handler.py +++ b/litellm/llms/vertex_ai_and_google_ai_studio/multimodal_embeddings/embedding_handler.py @@ -5,7 +5,11 @@ import httpx import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, +) from litellm.llms.vertex_ai_and_google_ai_studio.gemini.vertex_and_google_ai_studio_gemini import ( VertexAIError, VertexLLM, @@ -172,7 +176,10 @@ class VertexMultimodalEmbedding(VertexLLM): if isinstance(timeout, float) or isinstance(timeout, int): timeout = httpx.Timeout(timeout) _params["timeout"] = timeout - client = AsyncHTTPHandler(**_params) # type: ignore + client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.VERTEX_AI, + params={"timeout": timeout}, + ) else: client = client # type: ignore diff --git a/litellm/llms/vertex_ai_and_google_ai_studio/vertex_ai_non_gemini.py b/litellm/llms/vertex_ai_and_google_ai_studio/vertex_ai_non_gemini.py index 80295ec40..829bf6528 100644 --- a/litellm/llms/vertex_ai_and_google_ai_studio/vertex_ai_non_gemini.py +++ b/litellm/llms/vertex_ai_and_google_ai_studio/vertex_ai_non_gemini.py @@ -14,6 +14,7 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.llms.custom_httpx.http_handler import _DEFAULT_TTL_FOR_HTTPX_CLIENTS from litellm.llms.prompt_templates.factory import ( convert_to_anthropic_image_obj, convert_to_gemini_tool_call_invoke, @@ -93,11 +94,15 @@ def _get_client_cache_key( def _get_client_from_cache(client_cache_key: str): - return litellm.in_memory_llm_clients_cache.get(client_cache_key, None) + return litellm.in_memory_llm_clients_cache.get_cache(client_cache_key) def _set_client_in_cache(client_cache_key: str, vertex_llm_model: Any): - litellm.in_memory_llm_clients_cache[client_cache_key] = vertex_llm_model + litellm.in_memory_llm_clients_cache.set_cache( + key=client_cache_key, + value=vertex_llm_model, + ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS, + ) def completion( # noqa: PLR0915 diff --git a/litellm/llms/watsonx/chat/handler.py b/litellm/llms/watsonx/chat/handler.py index b016bb0a7..932946d3c 100644 --- a/litellm/llms/watsonx/chat/handler.py +++ b/litellm/llms/watsonx/chat/handler.py @@ -57,6 +57,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler): def completion( self, + *, model: str, messages: list, api_base: str, @@ -75,9 +76,8 @@ class WatsonXChatHandler(OpenAILikeChatHandler): timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, custom_endpoint: Optional[bool] = None, - streaming_decoder: Optional[ - CustomStreamingDecoder - ] = None, # if openai-compatible api needs custom stream decoder - e.g. sagemaker + streaming_decoder: Optional[CustomStreamingDecoder] = None, + fake_stream: bool = False, ): api_params = _get_api_params(optional_params, print_verbose=print_verbose) diff --git a/litellm/llms/watsonx/completion/handler.py b/litellm/llms/watsonx/completion/handler.py index fda25ba0f..9618f6342 100644 --- a/litellm/llms/watsonx/completion/handler.py +++ b/litellm/llms/watsonx/completion/handler.py @@ -24,7 +24,10 @@ import httpx # type: ignore import requests # type: ignore import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.watsonx import WatsonXAIEndpoint from litellm.utils import EmbeddingResponse, ModelResponse, Usage, map_finish_reason @@ -710,10 +713,13 @@ class RequestManager: if stream: request_params["stream"] = stream try: - self.async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout( - timeout=request_params.pop("timeout", 600.0), connect=5.0 - ), + self.async_handler = get_async_httpx_client( + llm_provider=litellm.LlmProviders.WATSONX, + params={ + "timeout": httpx.Timeout( + timeout=request_params.pop("timeout", 600.0), connect=5.0 + ), + }, ) if "json" in request_params: request_params["data"] = json.dumps(request_params.pop("json", {})) diff --git a/litellm/main.py b/litellm/main.py index 32055eb9d..5d433eb36 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1495,8 +1495,8 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, # type: ignore custom_prompt_dict=custom_prompt_dict, client=client, # pass AsyncOpenAI, OpenAI client - organization=organization, custom_llm_provider=custom_llm_provider, + encoding=encoding, ) elif ( model in litellm.open_ai_chat_completion_models @@ -3182,6 +3182,7 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: or custom_llm_provider == "azure_ai" or custom_llm_provider == "together_ai" or custom_llm_provider == "openai_like" + or custom_llm_provider == "jina_ai" ): # currently implemented aiohttp calls for just azure and openai, soon all. # Await normally init_response = await loop.run_in_executor(None, func_with_context) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 606a2756b..a56472f7f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1745,7 +1745,8 @@ "output_cost_per_token": 0.00000080, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama3-8b-8192": { "max_tokens": 8192, @@ -1755,7 +1756,74 @@ "output_cost_per_token": 0.00000008, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-1b-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000004, + "output_cost_per_token": 0.00000004, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-3b-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000006, + "output_cost_per_token": 0.00000006, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-11b-text-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000018, + "output_cost_per_token": 0.00000018, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-11b-vision-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000018, + "output_cost_per_token": 0.00000018, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-90b-text-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000009, + "output_cost_per_token": 0.0000009, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-90b-vision-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000009, + "output_cost_per_token": 0.0000009, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama3-70b-8192": { "max_tokens": 8192, @@ -1765,7 +1833,8 @@ "output_cost_per_token": 0.00000079, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama-3.1-8b-instant": { "max_tokens": 8192, @@ -1775,7 +1844,8 @@ "output_cost_per_token": 0.00000008, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama-3.1-70b-versatile": { "max_tokens": 8192, @@ -1785,7 +1855,8 @@ "output_cost_per_token": 0.00000079, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama-3.1-405b-reasoning": { "max_tokens": 8192, @@ -1795,7 +1866,8 @@ "output_cost_per_token": 0.00000079, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/mixtral-8x7b-32768": { "max_tokens": 32768, @@ -1805,7 +1877,8 @@ "output_cost_per_token": 0.00000024, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/gemma-7b-it": { "max_tokens": 8192, @@ -1815,7 +1888,8 @@ "output_cost_per_token": 0.00000007, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/gemma2-9b-it": { "max_tokens": 8192, @@ -1825,7 +1899,8 @@ "output_cost_per_token": 0.00000020, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama3-groq-70b-8192-tool-use-preview": { "max_tokens": 8192, @@ -1835,7 +1910,8 @@ "output_cost_per_token": 0.00000089, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama3-groq-8b-8192-tool-use-preview": { "max_tokens": 8192, @@ -1845,7 +1921,8 @@ "output_cost_per_token": 0.00000019, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "cerebras/llama3.1-8b": { "max_tokens": 128000, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 1155e0466..974b091cf 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -12,7 +12,6 @@ model_list: vertex_ai_project: "adroit-crow-413218" vertex_ai_location: "us-east5" - router_settings: model_group_alias: "gpt-4-turbo": # Aliased model name diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index c75c1e66c..9496776a8 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -192,6 +192,10 @@ class RouteChecks: return True if "/langfuse/" in route: return True + if "/anthropic/" in route: + return True + if "/azure/" in route: + return True return False @staticmethod diff --git a/litellm/proxy/vertex_ai_endpoints/google_ai_studio_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py similarity index 98% rename from litellm/proxy/vertex_ai_endpoints/google_ai_studio_endpoints.py rename to litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 29594a858..3f4643afc 100644 --- a/litellm/proxy/vertex_ai_endpoints/google_ai_studio_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -2,10 +2,8 @@ What is this? Provider-specific Pass-Through Endpoints -""" -""" -1. Create pass-through endpoints for any LITELLM_BASE_URL/gemini/ map to https://generativelanguage.googleapis.com/ +Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. """ import ast diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1551330d1..9d7c120a7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -203,6 +203,9 @@ from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config +from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + router as llm_passthrough_router, +) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( initialize_pass_through_endpoints, ) @@ -233,9 +236,6 @@ from litellm.proxy.utils import ( reset_budget, update_spend, ) -from litellm.proxy.vertex_ai_endpoints.google_ai_studio_endpoints import ( - router as gemini_router, -) from litellm.proxy.vertex_ai_endpoints.langfuse_endpoints import ( router as langfuse_router, ) @@ -9128,7 +9128,7 @@ app.include_router(router) app.include_router(rerank_router) app.include_router(fine_tuning_router) app.include_router(vertex_router) -app.include_router(gemini_router) +app.include_router(llm_passthrough_router) app.include_router(langfuse_router) app.include_router(pass_through_router) app.include_router(health_router) diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index d55cf3ec6..54d4c1af2 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -13,23 +13,14 @@ from typing_extensions import ( ) -class Field(TypedDict): - key: str - value: Dict[str, Any] - - -class FunctionCallArgs(TypedDict): - fields: Field - - class FunctionResponse(TypedDict): name: str - response: FunctionCallArgs + response: Optional[dict] class FunctionCall(TypedDict): name: str - args: FunctionCallArgs + args: Optional[dict] class FileDataType(TypedDict): diff --git a/litellm/utils.py b/litellm/utils.py index 2dce9db89..003971142 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1739,15 +1739,15 @@ def supports_response_schema(model: str, custom_llm_provider: Optional[str]) -> Does not raise error. Defaults to 'False'. Outputs logging.error. """ + ## GET LLM PROVIDER ## + model, custom_llm_provider, _, _ = get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider + ) + + if custom_llm_provider == "predibase": # predibase supports this globally + return True + try: - ## GET LLM PROVIDER ## - model, custom_llm_provider, _, _ = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider - ) - - if custom_llm_provider == "predibase": # predibase supports this globally - return True - ## GET MODEL INFO model_info = litellm.get_model_info( model=model, custom_llm_provider=custom_llm_provider @@ -1755,12 +1755,17 @@ def supports_response_schema(model: str, custom_llm_provider: Optional[str]) -> if model_info.get("supports_response_schema", False) is True: return True - return False except Exception: - verbose_logger.error( - f"Model not supports response_schema. You passed model={model}, custom_llm_provider={custom_llm_provider}." + ## check if provider supports response schema globally + supported_params = get_supported_openai_params( + model=model, + custom_llm_provider=custom_llm_provider, + request_type="chat_completion", ) - return False + if supported_params is not None and "response_schema" in supported_params: + return True + + return False def supports_function_calling( @@ -2710,6 +2715,7 @@ def get_optional_params( # noqa: PLR0915 non_default_params["response_format"] = type_to_response_format_param( response_format=non_default_params["response_format"] ) + if "tools" in non_default_params and isinstance( non_default_params, list ): # fixes https://github.com/BerriAI/litellm/issues/4933 @@ -3259,24 +3265,14 @@ def get_optional_params( # noqa: PLR0915 ) _check_valid_arg(supported_params=supported_params) - if max_tokens is not None: - optional_params["num_predict"] = max_tokens - if stream: - optional_params["stream"] = stream - if temperature is not None: - optional_params["temperature"] = temperature - if seed is not None: - optional_params["seed"] = seed - if top_p is not None: - optional_params["top_p"] = top_p - if frequency_penalty is not None: - optional_params["repeat_penalty"] = frequency_penalty - if stop is not None: - optional_params["stop"] = stop - if response_format is not None and response_format["type"] == "json_object": - optional_params["format"] = "json" + optional_params = litellm.OllamaConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + ) elif custom_llm_provider == "ollama_chat": - supported_params = litellm.OllamaChatConfig().get_supported_openai_params() + supported_params = get_supported_openai_params( + model=model, custom_llm_provider=custom_llm_provider + ) _check_valid_arg(supported_params=supported_params) @@ -3494,24 +3490,16 @@ def get_optional_params( # noqa: PLR0915 ) _check_valid_arg(supported_params=supported_params) - if temperature is not None: - optional_params["temperature"] = temperature - if max_tokens is not None: - optional_params["max_tokens"] = max_tokens - if top_p is not None: - optional_params["top_p"] = top_p - if stream is not None: - optional_params["stream"] = stream - if stop is not None: - optional_params["stop"] = stop - if tools is not None: - optional_params["tools"] = tools - if tool_choice is not None: - optional_params["tool_choice"] = tool_choice - if response_format is not None: - optional_params["response_format"] = response_format - if seed is not None: - optional_params["seed"] = seed + optional_params = litellm.GroqChatConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) elif custom_llm_provider == "deepseek": supported_params = get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider @@ -6178,5 +6166,7 @@ class ProviderConfigManager: return litellm.OpenAIO1Config() elif litellm.LlmProviders.DEEPSEEK == provider: return litellm.DeepSeekChatConfig() + elif litellm.LlmProviders.GROQ == provider: + return litellm.GroqChatConfig() return OpenAIGPTConfig() diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 606a2756b..a56472f7f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1745,7 +1745,8 @@ "output_cost_per_token": 0.00000080, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama3-8b-8192": { "max_tokens": 8192, @@ -1755,7 +1756,74 @@ "output_cost_per_token": 0.00000008, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-1b-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000004, + "output_cost_per_token": 0.00000004, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-3b-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000006, + "output_cost_per_token": 0.00000006, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-11b-text-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000018, + "output_cost_per_token": 0.00000018, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-11b-vision-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000018, + "output_cost_per_token": 0.00000018, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-90b-text-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000009, + "output_cost_per_token": 0.0000009, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true + }, + "groq/llama-3.2-90b-vision-preview": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000009, + "output_cost_per_token": 0.0000009, + "litellm_provider": "groq", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama3-70b-8192": { "max_tokens": 8192, @@ -1765,7 +1833,8 @@ "output_cost_per_token": 0.00000079, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama-3.1-8b-instant": { "max_tokens": 8192, @@ -1775,7 +1844,8 @@ "output_cost_per_token": 0.00000008, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama-3.1-70b-versatile": { "max_tokens": 8192, @@ -1785,7 +1855,8 @@ "output_cost_per_token": 0.00000079, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama-3.1-405b-reasoning": { "max_tokens": 8192, @@ -1795,7 +1866,8 @@ "output_cost_per_token": 0.00000079, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/mixtral-8x7b-32768": { "max_tokens": 32768, @@ -1805,7 +1877,8 @@ "output_cost_per_token": 0.00000024, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/gemma-7b-it": { "max_tokens": 8192, @@ -1815,7 +1888,8 @@ "output_cost_per_token": 0.00000007, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/gemma2-9b-it": { "max_tokens": 8192, @@ -1825,7 +1899,8 @@ "output_cost_per_token": 0.00000020, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama3-groq-70b-8192-tool-use-preview": { "max_tokens": 8192, @@ -1835,7 +1910,8 @@ "output_cost_per_token": 0.00000089, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "groq/llama3-groq-8b-8192-tool-use-preview": { "max_tokens": 8192, @@ -1845,7 +1921,8 @@ "output_cost_per_token": 0.00000019, "litellm_provider": "groq", "mode": "chat", - "supports_function_calling": true + "supports_function_calling": true, + "supports_response_schema": true }, "cerebras/llama3.1-8b": { "max_tokens": 128000, diff --git a/pyproject.toml b/pyproject.toml index 3e69461ae..d5cf3fb92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.52.12" +version = "1.52.13" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.52.12" +version = "1.52.13" version_files = [ "pyproject.toml:^version" ] diff --git a/tests/code_coverage_tests/ensure_async_clients_test.py b/tests/code_coverage_tests/ensure_async_clients_test.py new file mode 100644 index 000000000..a509e5509 --- /dev/null +++ b/tests/code_coverage_tests/ensure_async_clients_test.py @@ -0,0 +1,88 @@ +import ast +import os + +ALLOWED_FILES = [ + # local files + "../../litellm/__init__.py", + "../../litellm/llms/custom_httpx/http_handler.py", + # when running on ci/cd + "./litellm/__init__.py", + "./litellm/llms/custom_httpx/http_handler.py", +] + +warning_msg = "this is a serious violation that can impact latency. Creating Async clients per request can add +500ms per request" + + +def check_for_async_http_handler(file_path): + """ + Checks if AsyncHttpHandler is instantiated in the given file. + Returns a list of line numbers where AsyncHttpHandler is used. + """ + print("..checking file=", file_path) + if file_path in ALLOWED_FILES: + return [] + with open(file_path, "r") as file: + try: + tree = ast.parse(file.read()) + except SyntaxError: + print(f"Warning: Syntax error in file {file_path}") + return [] + + violations = [] + target_names = [ + "AsyncHttpHandler", + "AsyncHTTPHandler", + "AsyncClient", + "httpx.AsyncClient", + ] # Add variations here + for node in ast.walk(tree): + if isinstance(node, ast.Call): + if isinstance(node.func, ast.Name) and node.func.id.lower() in [ + name.lower() for name in target_names + ]: + raise ValueError( + f"found violation in file {file_path} line: {node.lineno}. Please use `get_async_httpx_client` instead. {warning_msg}" + ) + return violations + + +def scan_directory_for_async_handler(base_dir): + """ + Scans all Python files in the directory tree for AsyncHttpHandler usage. + Returns a dict of files and line numbers where violations were found. + """ + violations = {} + + for root, _, files in os.walk(base_dir): + for file in files: + if file.endswith(".py"): + file_path = os.path.join(root, file) + file_violations = check_for_async_http_handler(file_path) + if file_violations: + violations[file_path] = file_violations + + return violations + + +def test_no_async_http_handler_usage(): + """ + Test to ensure AsyncHttpHandler is not used anywhere in the codebase. + """ + base_dir = "./litellm" # Adjust this path as needed + + # base_dir = "../../litellm" # LOCAL TESTING + violations = scan_directory_for_async_handler(base_dir) + + if violations: + violation_messages = [] + for file_path, line_numbers in violations.items(): + violation_messages.append( + f"Found AsyncHttpHandler in {file_path} at lines: {line_numbers}" + ) + raise AssertionError( + "AsyncHttpHandler usage detected:\n" + "\n".join(violation_messages) + ) + + +if __name__ == "__main__": + test_no_async_http_handler_usage() diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index 692a0e4e9..6605b3e3d 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -8,6 +8,7 @@ import traceback from dotenv import load_dotenv from openai.types.image import Image +from litellm.caching import InMemoryCache logging.basicConfig(level=logging.DEBUG) load_dotenv() @@ -107,7 +108,7 @@ class TestVertexImageGeneration(BaseImageGenTest): # comment this when running locally load_vertex_ai_credentials() - litellm.in_memory_llm_clients_cache = {} + litellm.in_memory_llm_clients_cache = InMemoryCache() return { "model": "vertex_ai/imagegeneration@006", "vertex_ai_project": "adroit-crow-413218", @@ -118,13 +119,13 @@ class TestVertexImageGeneration(BaseImageGenTest): class TestBedrockSd3(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: - litellm.in_memory_llm_clients_cache = {} + litellm.in_memory_llm_clients_cache = InMemoryCache() return {"model": "bedrock/stability.sd3-large-v1:0"} class TestBedrockSd1(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: - litellm.in_memory_llm_clients_cache = {} + litellm.in_memory_llm_clients_cache = InMemoryCache() return {"model": "bedrock/stability.sd3-large-v1:0"} @@ -181,7 +182,7 @@ def test_image_generation_azure_dall_e_3(): @pytest.mark.asyncio async def test_aimage_generation_bedrock_with_optional_params(): try: - litellm.in_memory_llm_clients_cache = {} + litellm.in_memory_llm_clients_cache = InMemoryCache() response = await litellm.aimage_generation( prompt="A cute baby sea otter", model="bedrock/stability.stable-diffusion-xl-v1", diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 74fff60a4..88fce6dac 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -49,7 +49,7 @@ class BaseLLMChatTest(ABC): ) assert response is not None except litellm.InternalServerError: - pass + pytest.skip("Model is overloaded") # for OpenAI the content contains the JSON schema, so we need to assert that the content is not None assert response.choices[0].message.content is not None @@ -92,7 +92,9 @@ class BaseLLMChatTest(ABC): # relevant issue: https://github.com/BerriAI/litellm/issues/6741 assert response.choices[0].message.content is not None + @pytest.mark.flaky(retries=6, delay=1) def test_json_response_pydantic_obj(self): + litellm.set_verbose = True from pydantic import BaseModel from litellm.utils import supports_response_schema @@ -119,6 +121,11 @@ class BaseLLMChatTest(ABC): response_format=TestModel, ) assert res is not None + + print(res.choices[0].message) + + assert res.choices[0].message.content is not None + assert res.choices[0].message.tool_calls is None except litellm.InternalServerError: pytest.skip("Model is overloaded") @@ -140,12 +147,15 @@ class BaseLLMChatTest(ABC): }, ] - response = litellm.completion( - **base_completion_call_args, - messages=messages, - response_format={"type": "json_object"}, - stream=True, - ) + try: + response = litellm.completion( + **base_completion_call_args, + messages=messages, + response_format={"type": "json_object"}, + stream=True, + ) + except litellm.InternalServerError: + pytest.skip("Model is overloaded") print(response) @@ -161,6 +171,25 @@ class BaseLLMChatTest(ABC): assert content is not None assert len(content) > 0 + @pytest.fixture + def tool_call_no_arguments(self): + return { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_2c384bc6-de46-4f29-8adc-60dd5805d305", + "function": {"name": "Get-FAQ", "arguments": "{}"}, + "type": "function", + } + ], + } + + @abstractmethod + def test_tool_call_no_arguments(self, tool_call_no_arguments): + """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" + pass + @pytest.fixture def pdf_messages(self): import base64 diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index d6ee074b1..812291767 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -697,6 +697,15 @@ class TestAnthropicCompletion(BaseLLMChatTest): assert _document_validation["source"]["media_type"] == "application/pdf" assert _document_validation["source"]["type"] == "base64" + def test_tool_call_no_arguments(self, tool_call_no_arguments): + """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" + from litellm.llms.prompt_templates.factory import ( + convert_to_anthropic_tool_invoke, + ) + + result = convert_to_anthropic_tool_invoke([tool_call_no_arguments]) + print(result) + def test_convert_tool_response_to_message_with_values(): """Test converting a tool response with 'values' key to a message""" diff --git a/tests/llm_translation/test_deepseek_completion.py b/tests/llm_translation/test_deepseek_completion.py index b0f7ee663..17b0a340b 100644 --- a/tests/llm_translation/test_deepseek_completion.py +++ b/tests/llm_translation/test_deepseek_completion.py @@ -7,3 +7,7 @@ class TestDeepSeekChatCompletion(BaseLLMChatTest): return { "model": "deepseek/deepseek-chat", } + + def test_tool_call_no_arguments(self, tool_call_no_arguments): + """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" + pass diff --git a/tests/llm_translation/test_groq.py b/tests/llm_translation/test_groq.py new file mode 100644 index 000000000..359787b2d --- /dev/null +++ b/tests/llm_translation/test_groq.py @@ -0,0 +1,12 @@ +from base_llm_unit_tests import BaseLLMChatTest + + +class TestGroq(BaseLLMChatTest): + def get_base_completion_call_args(self) -> dict: + return { + "model": "groq/llama-3.1-70b-versatile", + } + + def test_tool_call_no_arguments(self, tool_call_no_arguments): + """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" + pass diff --git a/tests/llm_translation/test_mistral_api.py b/tests/llm_translation/test_mistral_api.py index b2cb36541..bb8cb3c60 100644 --- a/tests/llm_translation/test_mistral_api.py +++ b/tests/llm_translation/test_mistral_api.py @@ -32,3 +32,7 @@ class TestMistralCompletion(BaseLLMChatTest): def get_base_completion_call_args(self) -> dict: litellm.set_verbose = True return {"model": "mistral/mistral-small-latest"} + + def test_tool_call_no_arguments(self, tool_call_no_arguments): + """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" + pass diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 7fe8baeb5..34ecdfaca 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -952,3 +952,17 @@ def test_lm_studio_embedding_params(): drop_params=True, ) assert len(optional_params) == 0 + + +def test_ollama_pydantic_obj(): + from pydantic import BaseModel + + class ResponseFormat(BaseModel): + x: str + y: str + + get_optional_params( + model="qwen2:0.5b", + custom_llm_provider="ollama", + response_format=ResponseFormat, + ) diff --git a/tests/llm_translation/test_vertex.py b/tests/llm_translation/test_vertex.py index 73960020d..3e1087536 100644 --- a/tests/llm_translation/test_vertex.py +++ b/tests/llm_translation/test_vertex.py @@ -306,6 +306,8 @@ def test_multiple_function_call(): ) assert len(r.choices) > 0 + print(mock_post.call_args.kwargs["json"]) + assert mock_post.call_args.kwargs["json"] == { "contents": [ {"role": "user", "parts": [{"text": "do test"}]}, @@ -313,28 +315,8 @@ def test_multiple_function_call(): "role": "model", "parts": [ {"text": "test"}, - { - "function_call": { - "name": "test", - "args": { - "fields": { - "key": "arg", - "value": {"string_value": "test"}, - } - }, - } - }, - { - "function_call": { - "name": "test2", - "args": { - "fields": { - "key": "arg", - "value": {"string_value": "test2"}, - } - }, - } - }, + {"function_call": {"name": "test", "args": {"arg": "test"}}}, + {"function_call": {"name": "test2", "args": {"arg": "test2"}}}, ], }, { @@ -342,23 +324,13 @@ def test_multiple_function_call(): { "function_response": { "name": "test", - "response": { - "fields": { - "key": "content", - "value": {"string_value": "42"}, - } - }, + "response": {"content": "42"}, } }, { "function_response": { "name": "test2", - "response": { - "fields": { - "key": "content", - "value": {"string_value": "15"}, - } - }, + "response": {"content": "15"}, } }, ] @@ -441,34 +413,16 @@ def test_multiple_function_call_changed_text_pos(): assert len(resp.choices) > 0 mock_post.assert_called_once() + print(mock_post.call_args.kwargs["json"]["contents"]) + assert mock_post.call_args.kwargs["json"]["contents"] == [ {"role": "user", "parts": [{"text": "do test"}]}, { "role": "model", "parts": [ {"text": "test"}, - { - "function_call": { - "name": "test", - "args": { - "fields": { - "key": "arg", - "value": {"string_value": "test"}, - } - }, - } - }, - { - "function_call": { - "name": "test2", - "args": { - "fields": { - "key": "arg", - "value": {"string_value": "test2"}, - } - }, - } - }, + {"function_call": {"name": "test", "args": {"arg": "test"}}}, + {"function_call": {"name": "test2", "args": {"arg": "test2"}}}, ], }, { @@ -476,23 +430,13 @@ def test_multiple_function_call_changed_text_pos(): { "function_response": { "name": "test2", - "response": { - "fields": { - "key": "content", - "value": {"string_value": "15"}, - } - }, + "response": {"content": "15"}, } }, { "function_response": { "name": "test", - "response": { - "fields": { - "key": "content", - "value": {"string_value": "42"}, - } - }, + "response": {"content": "42"}, } }, ] @@ -1354,3 +1298,20 @@ def test_vertex_embedding_url(model, expected_url): assert url == expected_url assert endpoint == "predict" + + +from base_llm_unit_tests import BaseLLMChatTest + + +class TestVertexGemini(BaseLLMChatTest): + def get_base_completion_call_args(self) -> dict: + return {"model": "gemini/gemini-1.5-flash"} + + def test_tool_call_no_arguments(self, tool_call_no_arguments): + """Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833""" + from litellm.llms.prompt_templates.factory import ( + convert_to_gemini_tool_call_invoke, + ) + + result = convert_to_gemini_tool_call_invoke(tool_call_no_arguments) + print(result) diff --git a/tests/local_testing/test_alangfuse.py b/tests/local_testing/test_alangfuse.py index 8c69f567b..ec0cb335e 100644 --- a/tests/local_testing/test_alangfuse.py +++ b/tests/local_testing/test_alangfuse.py @@ -12,6 +12,7 @@ sys.path.insert(0, os.path.abspath("../..")) import litellm from litellm import completion +from litellm.caching import InMemoryCache litellm.num_retries = 3 litellm.success_callback = ["langfuse"] @@ -29,15 +30,20 @@ def langfuse_client(): f"{os.environ['LANGFUSE_PUBLIC_KEY']}-{os.environ['LANGFUSE_SECRET_KEY']}" ) # use a in memory langfuse client for testing, RAM util on ci/cd gets too high when we init many langfuse clients - if _langfuse_cache_key in litellm.in_memory_llm_clients_cache: - langfuse_client = litellm.in_memory_llm_clients_cache[_langfuse_cache_key] + + _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_langfuse_cache_key) + if _cached_client: + langfuse_client = _cached_client else: langfuse_client = langfuse.Langfuse( public_key=os.environ["LANGFUSE_PUBLIC_KEY"], secret_key=os.environ["LANGFUSE_SECRET_KEY"], host=None, ) - litellm.in_memory_llm_clients_cache[_langfuse_cache_key] = langfuse_client + litellm.in_memory_llm_clients_cache.set_cache( + key=_langfuse_cache_key, + value=langfuse_client, + ) print("NEW LANGFUSE CLIENT") diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index f801a53ce..50a39b242 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -2867,6 +2867,7 @@ def test_gemini_function_call_parameter_in_messages(): print(e) # mock_client.assert_any_call() + assert { "contents": [ { @@ -2879,12 +2880,7 @@ def test_gemini_function_call_parameter_in_messages(): { "function_call": { "name": "search", - "args": { - "fields": { - "key": "queries", - "value": {"list_value": ["weather in boston"]}, - } - }, + "args": {"queries": ["weather in boston"]}, } } ], @@ -2895,12 +2891,7 @@ def test_gemini_function_call_parameter_in_messages(): "function_response": { "name": "search", "response": { - "fields": { - "key": "content", - "value": { - "string_value": "The current weather in Boston is 22°F." - }, - } + "content": "The current weather in Boston is 22°F." }, } } @@ -2935,6 +2926,7 @@ def test_gemini_function_call_parameter_in_messages(): def test_gemini_function_call_parameter_in_messages_2(): + litellm.set_verbose = True from litellm.llms.vertex_ai_and_google_ai_studio.gemini.transformation import ( _gemini_convert_messages_with_history, ) @@ -2958,6 +2950,7 @@ def test_gemini_function_call_parameter_in_messages_2(): returned_contents = _gemini_convert_messages_with_history(messages=messages) + print(f"returned_contents: {returned_contents}") assert returned_contents == [ { "role": "user", @@ -2970,12 +2963,7 @@ def test_gemini_function_call_parameter_in_messages_2(): { "function_call": { "name": "search", - "args": { - "fields": { - "key": "queries", - "value": {"list_value": ["weather in boston"]}, - } - }, + "args": {"queries": ["weather in boston"]}, } }, ], @@ -2986,12 +2974,7 @@ def test_gemini_function_call_parameter_in_messages_2(): "function_response": { "name": "search", "response": { - "fields": { - "key": "content", - "value": { - "string_value": "The weather in Boston is 100 degrees." - }, - } + "content": "The weather in Boston is 100 degrees." }, } } diff --git a/tests/local_testing/test_ollama.py b/tests/local_testing/test_ollama.py index de41e24b8..34c0791c3 100644 --- a/tests/local_testing/test_ollama.py +++ b/tests/local_testing/test_ollama.py @@ -67,7 +67,8 @@ def test_ollama_json_mode(): assert converted_params == { "temperature": 0.5, "format": "json", - }, f"{converted_params} != {'temperature': 0.5, 'format': 'json'}" + "stream": False, + }, f"{converted_params} != {'temperature': 0.5, 'format': 'json', 'stream': False}" except Exception as e: pytest.fail(f"Error occurred: {e}") diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index cd5e8f6b2..20867e766 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -1450,7 +1450,7 @@ async def test_mistral_on_router(): { "model_name": "gpt-3.5-turbo", "litellm_params": { - "model": "mistral/mistral-medium", + "model": "mistral/mistral-small-latest", }, }, ] diff --git a/tests/local_testing/test_router_batch_completion.py b/tests/local_testing/test_router_batch_completion.py index 3de61c0a6..065730d48 100644 --- a/tests/local_testing/test_router_batch_completion.py +++ b/tests/local_testing/test_router_batch_completion.py @@ -64,6 +64,7 @@ async def test_batch_completion_multiple_models(mode): models_in_responses = [] print(f"response: {response}") for individual_response in response: + print(f"individual_response: {individual_response}") _model = individual_response["model"] models_in_responses.append(_model) diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index 0bc6953f9..757ff4d61 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -683,7 +683,7 @@ def test_completion_ollama_hosted_stream(): [ # "claude-3-5-haiku-20241022", # "claude-2", - # "mistral/mistral-medium", + # "mistral/mistral-small-latest", "openrouter/openai/gpt-4o-mini", ], ) diff --git a/tests/local_testing/test_utils.py b/tests/local_testing/test_utils.py index 6e7b0ff05..52946ca30 100644 --- a/tests/local_testing/test_utils.py +++ b/tests/local_testing/test_utils.py @@ -749,6 +749,7 @@ def test_convert_model_response_object(): ("gemini/gemini-1.5-pro", True), ("predibase/llama3-8b-instruct", True), ("gpt-3.5-turbo", False), + ("groq/llama3-70b-8192", True), ], ) def test_supports_response_schema(model, expected_bool): diff --git a/tests/proxy_admin_ui_tests/test_route_check_unit_tests.py b/tests/proxy_admin_ui_tests/test_route_check_unit_tests.py index 001cc0640..a8bba211f 100644 --- a/tests/proxy_admin_ui_tests/test_route_check_unit_tests.py +++ b/tests/proxy_admin_ui_tests/test_route_check_unit_tests.py @@ -27,6 +27,9 @@ from fastapi import HTTPException, Request import pytest from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + router as llm_passthrough_router, +) # Replace the actual hash_token function with our mock import litellm.proxy.auth.route_checks @@ -56,12 +59,21 @@ def test_is_llm_api_route(): assert RouteChecks.is_llm_api_route("/vertex-ai/text") is True assert RouteChecks.is_llm_api_route("/gemini/generate") is True assert RouteChecks.is_llm_api_route("/cohere/generate") is True + assert RouteChecks.is_llm_api_route("/anthropic/messages") is True + assert RouteChecks.is_llm_api_route("/anthropic/v1/messages") is True + assert RouteChecks.is_llm_api_route("/azure/endpoint") is True # check non-matching routes assert RouteChecks.is_llm_api_route("/some/random/route") is False assert RouteChecks.is_llm_api_route("/key/regenerate/82akk800000000jjsk") is False assert RouteChecks.is_llm_api_route("/key/82akk800000000jjsk/delete") is False + # check all routes in llm_passthrough_router, ensure they are considered llm api routes + for route in llm_passthrough_router.routes: + route_path = str(route.path) + print("route_path", route_path) + assert RouteChecks.is_llm_api_route(route_path) is True + # Test _route_matches_pattern def test_route_matches_pattern(): diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index b1c00ce75..d70962858 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -1794,7 +1794,7 @@ async def test_add_callback_via_key_litellm_pre_call_utils_langsmith( async def test_gemini_pass_through_endpoint(): from starlette.datastructures import URL - from litellm.proxy.vertex_ai_endpoints.google_ai_studio_endpoints import ( + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( Request, Response, gemini_proxy_route,