phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 18:24:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Allow passing thinking param to litellm proxy via client sdk + Code QA Refactor on get_optional_params (get correct values) (#9386)

Browse source 
* fix(litellm_proxy/chat/transformation.py): support 'thinking' param

Fixes https://github.com/BerriAI/litellm/issues/9380

* feat(azure/gpt_transformation.py): add azure audio model support

Closes https://github.com/BerriAI/litellm/issues/6305

* fix(utils.py): use provider_config in common functions

* fix(utils.py): add missing provider configs to get_chat_provider_config

* test: fix test

* fix: fix path

* feat(utils.py): make bedrock invoke nova config baseconfig compatible

* fix: fix linting errors

* fix(azure_ai/transformation.py): remove buggy optional param filtering for azure ai

Removes incorrect check for support tool choice when calling azure ai - prevented calling models with response_format unless on litell model cost map

* fix(amazon_cohere_transformation.py): fix bedrock invoke cohere transformation to inherit from coherechatconfig

* test: fix azure ai tool choice mapping

* fix: fix model cost map to add 'supports_tool_choice' to cohere models

* fix(get_supported_openai_params.py): check if custom llm provider in llm providers

* fix(get_supported_openai_params.py): fix llm provider in list check

* fix: fix ruff check errors

* fix: support defs when calling bedrock nova

* fix(factory.py): fix test
This commit is contained in:
Krish Dholakia 2025-04-07 21:04:11 -07:00 • committed by GitHub
parent fcf17d114f
commit ac9f03beae
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 278 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

23
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -1,8 +1,9 @@
from typing import Literal, Optional
import litellm
from litellm import LlmProviders
from litellm._logging import verbose_logger
from litellm.exceptions import BadRequestError
from litellm.types.utils import LlmProviders, LlmProvidersSet
def get_supported_openai_params( # noqa: PLR0915
@ -30,6 +31,23 @@ def get_supported_openai_params( # noqa: PLR0915
except BadRequestError:
return None
if custom_llm_provider in LlmProvidersSet:
provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
model=model, provider=LlmProviders(custom_llm_provider)
)
elif custom_llm_provider.split("/")[0] in LlmProvidersSet:
provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
model=model, provider=LlmProviders(custom_llm_provider.split("/")[0])
)
else:
provider_config = None
if provider_config and request_type == "chat_completion":
verbose_logger.info(
f"using provider_config: {provider_config} for checking supported params"
)
return provider_config.get_supported_openai_params(model=model)
if custom_llm_provider == "bedrock":
return litellm.AmazonConverseConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "ollama":
@ -226,7 +244,8 @@ def get_supported_openai_params( # noqa: PLR0915
provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
model=model, provider=LlmProviders.CUSTOM
)
return provider_config.get_supported_openai_params(model=model)
if provider_config:
return provider_config.get_supported_openai_params(model=model)
elif request_type == "embeddings":
return None
elif request_type == "transcription":

33
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -348,3 +348,36 @@ def update_messages_with_model_file_ids(
)
file_object_file_field["file_id"] = provider_file_id
return messages
def unpack_defs(schema, defs):
properties = schema.get("properties", None)
if properties is None:
return
for name, value in properties.items():
ref_key = value.get("$ref", None)
if ref_key is not None:
ref = defs[ref_key.split("defs/")[-1]]
unpack_defs(ref, defs)
properties[name] = ref
continue
anyof = value.get("anyOf", None)
if anyof is not None:
for i, atype in enumerate(anyof):
ref_key = atype.get("$ref", None)
if ref_key is not None:
ref = defs[ref_key.split("defs/")[-1]]
unpack_defs(ref, defs)
anyof[i] = ref
continue
items = value.get("items", None)
if items is not None:
ref_key = items.get("$ref", None)
if ref_key is not None:
ref = defs[ref_key.split("defs/")[-1]]
unpack_defs(ref, defs)
value["items"] = ref
continue

9
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -3442,6 +3442,8 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
}
]
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs
tool_block_list: List[BedrockToolBlock] = []
for tool in tools:
parameters = tool.get("function", {}).get(
@ -3455,6 +3457,13 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]:
description = tool.get("function", {}).get(
"description", name
) # converse api requires a description
defs = parameters.pop("$defs", {})
defs_copy = copy.deepcopy(defs)
# flatten the defs
for _, value in defs_copy.items():
unpack_defs(value, defs_copy)
unpack_defs(parameters, defs_copy)
tool_input_schema = BedrockToolInputSchemaBlock(json=parameters)
tool_spec = BedrockToolSpecBlock(
inputSchema=tool_input_schema, name=name, description=description

4
litellm/llms/anthropic/chat/handler.py
View file

@ -307,6 +307,10 @@ class AnthropicChatCompletion(BaseLLM):
model=model,
provider=LlmProviders(custom_llm_provider),
)
if config is None:
raise ValueError(
f"Provider config not found for model: {model} and provider: {custom_llm_provider}"
)
data = config.transform_request(
model=model,

1
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -235,7 +235,6 @@ class AmazonConverseConfig(BaseConfig):
optional_params: dict,
model: str,
drop_params: bool,
messages: Optional[List[AllMessageValues]] = None,
) -> dict:
is_thinking_enabled = self.is_thinking_enabled(non_default_params)

29
litellm/llms/bedrock/chat/invoke_transformations/amazon_cohere_transformation.py
View file

@ -1,13 +1,13 @@
import types
from typing import List, Optional
from litellm.llms.base_llm.chat.transformation import BaseConfig
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
from litellm.llms.cohere.chat.transformation import CohereChatConfig
class AmazonCohereConfig(AmazonInvokeConfig, BaseConfig):
class AmazonCohereConfig(AmazonInvokeConfig, CohereChatConfig):
"""
Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=command
@ -19,7 +19,6 @@ class AmazonCohereConfig(AmazonInvokeConfig, BaseConfig):
"""
max_tokens: Optional[int] = None
temperature: Optional[float] = None
return_likelihood: Optional[str] = None
def __init__(
@ -55,11 +54,10 @@ class AmazonCohereConfig(AmazonInvokeConfig, BaseConfig):
}
def get_supported_openai_params(self, model: str) -> List[str]:
return [
"max_tokens",
"temperature",
"stream",
]
supported_params = CohereChatConfig.get_supported_openai_params(
self, model=model
)
return supported_params
def map_openai_params(
self,
@ -68,11 +66,10 @@ class AmazonCohereConfig(AmazonInvokeConfig, BaseConfig):
model: str,
drop_params: bool,
) -> dict:
for k, v in non_default_params.items():
if k == "stream":
optional_params["stream"] = v
if k == "temperature":
optional_params["temperature"] = v
if k == "max_tokens":
optional_params["max_tokens"] = v
return optional_params
return CohereChatConfig.map_openai_params(
self,
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
drop_params=drop_params,
)

57
litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py
View file

@ -6,14 +6,21 @@ Inherits from `AmazonConverseConfig`
Nova + Invoke API Tutorial: https://docs.aws.amazon.com/nova/latest/userguide/using-invoke-api.html
"""
from typing import List
from typing import Any, List, Optional
import httpx
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.llms.bedrock import BedrockInvokeNovaRequest
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
from ..converse_transformation import AmazonConverseConfig
from .base_invoke_transformation import AmazonInvokeConfig
class AmazonInvokeNovaConfig(litellm.AmazonConverseConfig):
class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
"""
Config for sending `nova` requests to `/bedrock/invoke/`
"""
@ -21,6 +28,20 @@ class AmazonInvokeNovaConfig(litellm.AmazonConverseConfig):
def __init__(self, **kwargs):
super().__init__(**kwargs)
def get_supported_openai_params(self, model: str) -> list:
return AmazonConverseConfig.get_supported_openai_params(self, model)
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
return AmazonConverseConfig.map_openai_params(
self, non_default_params, optional_params, model, drop_params
)
def transform_request(
self,
model: str,
@ -29,7 +50,8 @@ class AmazonInvokeNovaConfig(litellm.AmazonConverseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
_transformed_nova_request = super().transform_request(
_transformed_nova_request = AmazonConverseConfig.transform_request(
self,
model=model,
messages=messages,
optional_params=optional_params,
@ -45,6 +67,35 @@ class AmazonInvokeNovaConfig(litellm.AmazonConverseConfig):
)
return bedrock_invoke_nova_request
def transform_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ModelResponse,
logging_obj: Logging,
request_data: dict,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: Any,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> litellm.ModelResponse:
return AmazonConverseConfig.transform_response(
self,
model,
raw_response,
model_response,
logging_obj,
request_data,
messages,
optional_params,
litellm_params,
encoding,
api_key,
json_mode,
)
def _filter_allowed_fields(
self, bedrock_invoke_nova_request: BedrockInvokeNovaRequest
) -> dict:

4
litellm/llms/custom_httpx/aiohttp_handler.py
View file

@ -218,6 +218,10 @@ class BaseLLMAIOHTTPHandler:
provider_config = ProviderConfigManager.get_provider_chat_config(
model=model, provider=litellm.LlmProviders(custom_llm_provider)
)
if provider_config is None:
raise ValueError(
f"Provider config not found for model: {model} and provider: {custom_llm_provider}"
)
# get config from model, custom llm provider
headers = provider_config.validate_environment(
api_key=api_key,

4
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -234,6 +234,10 @@ class BaseLLMHTTPHandler:
provider_config = ProviderConfigManager.get_provider_chat_config(
model=model, provider=litellm.LlmProviders(custom_llm_provider)
)
if provider_config is None:
raise ValueError(
f"Provider config not found for model: {model} and provider: {custom_llm_provider}"
)
# get config from model, custom llm provider
headers = provider_config.validate_environment(

5
litellm/llms/litellm_proxy/chat/transformation.py
View file

@ -10,6 +10,11 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class LiteLLMProxyChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> List:
list = super().get_supported_openai_params(model)
list.append("thinking")
return list
def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:

34
litellm/llms/vertex_ai/common_utils.py
View file

@ -6,6 +6,7 @@ import httpx
import litellm
from litellm import supports_response_schema, supports_system_messages, verbose_logger
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.vertex_ai import PartType, Schema
@ -229,39 +230,6 @@ def filter_schema_fields(
return result
def unpack_defs(schema, defs):
properties = schema.get("properties", None)
if properties is None:
return
for name, value in properties.items():
ref_key = value.get("$ref", None)
if ref_key is not None:
ref = defs[ref_key.split("defs/")[-1]]
unpack_defs(ref, defs)
properties[name] = ref
continue
anyof = value.get("anyOf", None)
if anyof is not None:
for i, atype in enumerate(anyof):
ref_key = atype.get("$ref", None)
if ref_key is not None:
ref = defs[ref_key.split("defs/")[-1]]
unpack_defs(ref, defs)
anyof[i] = ref
continue
items = value.get("items", None)
if items is not None:
ref_key = items.get("$ref", None)
if ref_key is not None:
ref = defs[ref_key.split("defs/")[-1]]
unpack_defs(ref, defs)
value["items"] = ref
continue
def convert_anyof_null_to_nullable(schema, depth=0):
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError(

6
litellm/llms/vertex_ai/vertex_ai_partner_models/ai21/transformation.py
View file

@ -2,9 +2,10 @@ import types
from typing import Optional
import litellm
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
class VertexAIAi21Config:
class VertexAIAi21Config(OpenAIGPTConfig):
"""
Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21
@ -40,9 +41,6 @@ class VertexAIAi21Config:
and v is not None
}
def get_supported_openai_params(self):
return litellm.OpenAIConfig().get_supported_openai_params(model="gpt-3.5-turbo")
def map_openai_params(
self,
non_default_params: dict,

24
litellm/model_prices_and_context_window_backup.json
View file

@ -2300,6 +2300,18 @@
"source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview",
"supports_tool_choice": true
},
"azure_ai/mistral-large-latest": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 0.000002,
"output_cost_per_token": 0.000006,
"litellm_provider": "azure_ai",
"supports_function_calling": true,
"mode": "chat",
"source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview",
"supports_tool_choice": true
},
"azure_ai/ministral-3b": {
"max_tokens": 4096,
"max_input_tokens": 128000,
@ -8444,7 +8456,8 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.0000020,
"litellm_provider": "bedrock",
"mode": "chat"
"mode": "chat",
"supports_tool_choice": true
},
"bedrock/*/1-month-commitment/cohere.command-text-v14": {
"max_tokens": 4096,
@ -8471,7 +8484,8 @@
"input_cost_per_token": 0.0000003,
"output_cost_per_token": 0.0000006,
"litellm_provider": "bedrock",
"mode": "chat"
"mode": "chat",
"supports_tool_choice": true
},
"bedrock/*/1-month-commitment/cohere.command-light-text-v14": {
"max_tokens": 4096,
@ -8498,7 +8512,8 @@
"input_cost_per_token": 0.0000030,
"output_cost_per_token": 0.000015,
"litellm_provider": "bedrock",
"mode": "chat"
"mode": "chat",
"supports_tool_choice": true
},
"cohere.command-r-v1:0": {
"max_tokens": 4096,
@ -8507,7 +8522,8 @@
"input_cost_per_token": 0.0000005,
"output_cost_per_token": 0.0000015,
"litellm_provider": "bedrock",
"mode": "chat"
"mode": "chat",
"supports_tool_choice": true
},
"cohere.embed-english-v3": {
"max_tokens": 512,

38
litellm/utils.py
View file

@ -3376,7 +3376,6 @@ def get_optional_params( # noqa: PLR0915
if drop_params is not None and isinstance(drop_params, bool)
else False
),
messages=messages,
)
elif "anthropic" in bedrock_base_model and bedrock_route == "invoke":
@ -3719,6 +3718,17 @@ def get_optional_params( # noqa: PLR0915
else False
),
)
elif provider_config is not None:
optional_params = provider_config.map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
drop_params=(
drop_params
if drop_params is not None and isinstance(drop_params, bool)
else False
),
)
else: # assume passing in params for openai-like api
optional_params = litellm.OpenAILikeChatConfig().map_openai_params(
non_default_params=non_default_params,
@ -6202,7 +6212,7 @@ class ProviderConfigManager:
@staticmethod
def get_provider_chat_config( # noqa: PLR0915
model: str, provider: LlmProviders
) -> BaseConfig:
) -> Optional[BaseConfig]:
"""
Returns the provider config for a given provider.
"""
@ -6233,9 +6243,22 @@ class ProviderConfigManager:
return litellm.AnthropicConfig()
elif litellm.LlmProviders.ANTHROPIC_TEXT == provider:
return litellm.AnthropicTextConfig()
elif litellm.LlmProviders.VERTEX_AI_BETA == provider:
return litellm.VertexGeminiConfig()
elif litellm.LlmProviders.VERTEX_AI == provider:
if "claude" in model:
if "gemini" in model:
return litellm.VertexGeminiConfig()
elif "claude" in model:
return litellm.VertexAIAnthropicConfig()
elif model in litellm.vertex_mistral_models:
if "codestral" in model:
return litellm.CodestralTextCompletionConfig()
else:
return litellm.MistralConfig()
elif model in litellm.vertex_ai_ai21_models:
return litellm.VertexAIAi21Config()
else: # use generic openai-like param mapping
return litellm.VertexAILlama3Config()
elif litellm.LlmProviders.CLOUDFLARE == provider:
return litellm.CloudflareChatConfig()
elif litellm.LlmProviders.SAGEMAKER_CHAT == provider:
@ -6258,7 +6281,6 @@ class ProviderConfigManager:
litellm.LlmProviders.CUSTOM == provider
or litellm.LlmProviders.CUSTOM_OPENAI == provider
or litellm.LlmProviders.OPENAI_LIKE == provider
or litellm.LlmProviders.LITELLM_PROXY == provider
):
return litellm.OpenAILikeChatConfig()
elif litellm.LlmProviders.AIOHTTP_OPENAI == provider:
@ -6363,9 +6385,15 @@ class ProviderConfigManager:
return litellm.AmazonMistralConfig()
elif bedrock_invoke_provider == "deepseek_r1": # deepseek models on bedrock
return litellm.AmazonDeepSeekR1Config()
elif bedrock_invoke_provider == "nova":
return litellm.AmazonInvokeNovaConfig()
else:
return litellm.AmazonInvokeConfig()
return litellm.OpenAIGPTConfig()
elif litellm.LlmProviders.LITELLM_PROXY == provider:
return litellm.LiteLLMProxyChatConfig()
elif litellm.LlmProviders.OPENAI == provider:
return litellm.OpenAIGPTConfig()
return None
@staticmethod
def get_provider_embedding_config(

24
model_prices_and_context_window.json
View file

@ -2300,6 +2300,18 @@
"source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview",
"supports_tool_choice": true
},
"azure_ai/mistral-large-latest": {
"max_tokens": 4096,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 0.000002,
"output_cost_per_token": 0.000006,
"litellm_provider": "azure_ai",
"supports_function_calling": true,
"mode": "chat",
"source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview",
"supports_tool_choice": true
},
"azure_ai/ministral-3b": {
"max_tokens": 4096,
"max_input_tokens": 128000,
@ -8444,7 +8456,8 @@
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.0000020,
"litellm_provider": "bedrock",
"mode": "chat"
"mode": "chat",
"supports_tool_choice": true
},
"bedrock/*/1-month-commitment/cohere.command-text-v14": {
"max_tokens": 4096,
@ -8471,7 +8484,8 @@
"input_cost_per_token": 0.0000003,
"output_cost_per_token": 0.0000006,
"litellm_provider": "bedrock",
"mode": "chat"
"mode": "chat",
"supports_tool_choice": true
},
"bedrock/*/1-month-commitment/cohere.command-light-text-v14": {
"max_tokens": 4096,
@ -8498,7 +8512,8 @@
"input_cost_per_token": 0.0000030,
"output_cost_per_token": 0.000015,
"litellm_provider": "bedrock",
"mode": "chat"
"mode": "chat",
"supports_tool_choice": true
},
"cohere.command-r-v1:0": {
"max_tokens": 4096,
@ -8507,7 +8522,8 @@
"input_cost_per_token": 0.0000005,
"output_cost_per_token": 0.0000015,
"litellm_provider": "bedrock",
"mode": "chat"
"mode": "chat",
"supports_tool_choice": true
},
"cohere.embed-english-v3": {
"max_tokens": 512,

24
tests/litellm_utils_tests/log.txt Normal file
View file

@ -0,0 +1,24 @@
============================= test session starts ==============================
platform darwin -- Python 3.13.1, pytest-8.3.5, pluggy-1.5.0 -- /Users/krrishdholakia/Documents/litellm/myenv/bin/python3.13
cachedir: .pytest_cache
rootdir: /Users/krrishdholakia/Documents/litellm
configfile: pyproject.toml
plugins: respx-0.22.0, postgresql-7.0.1, anyio-4.4.0, asyncio-0.26.0, mock-3.14.0, ddtrace-2.19.0rc1
asyncio: mode=Mode.STRICT, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function
collecting ... collected 3 items
test_supports_tool_choice.py::test_check_provider_match PASSED [ 33%]
test_supports_tool_choice.py::test_supports_tool_choice PASSED [ 66%]
test_supports_tool_choice.py::test_supports_tool_choice_simple_tests PASSED [100%]
=============================== warnings summary ===============================
../../myenv/lib/python3.13/site-packages/pydantic/_internal/_config.py:295
/Users/krrishdholakia/Documents/litellm/myenv/lib/python3.13/site-packages/pydantic/_internal/_config.py:295: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/
warnings.warn(DEPRECATION_MESSAGE, DeprecationWarning)
../../litellm/caching/llm_caching_handler.py:17
/Users/krrishdholakia/Documents/litellm/litellm/caching/llm_caching_handler.py:17: DeprecationWarning: There is no current event loop
event_loop = asyncio.get_event_loop()
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
======================== 3 passed, 2 warnings in 0.92s =========================

19
tests/litellm_utils_tests/test_supports_tool_choice.py
View file

@ -117,9 +117,9 @@ async def test_supports_tool_choice():
"""
# Load model prices
litellm._turn_on_debug()
local_path = "../../model_prices_and_context_window.json"
prod_path = "./model_prices_and_context_window.json"
with open(prod_path, "r") as f:
# path = "../../model_prices_and_context_window.json"
path = "./model_prices_and_context_window.json"
with open(path, "r") as f:
model_prices = json.load(f)
litellm.model_cost = model_prices
config_manager = ProviderConfigManager()
@ -137,8 +137,10 @@ async def test_supports_tool_choice():
or model_name in block_list
or "azure/eu" in model_name
or "azure/us" in model_name
or "codestral" in model_name
or "o1" in model_name
or "o3" in model_name
or "mistral" in model_name
):
continue
@ -152,8 +154,11 @@ async def test_supports_tool_choice():
print("LLM provider", provider)
provider_enum = LlmProviders(provider)
config = config_manager.get_provider_chat_config(model, provider_enum)
supported_params = config.get_supported_openai_params(model)
print("supported_params", supported_params)
if config:
supported_params = config.get_supported_openai_params(model)
print("supported_params", supported_params)
else:
raise Exception(f"No config found for {model_name}, provider: {provider}")
# Check tool_choice support
supports_tool_choice_result = litellm.utils.supports_tool_choice(
@ -162,7 +167,5 @@ async def test_supports_tool_choice():
tool_choice_in_params = "tool_choice" in supported_params
assert supports_tool_choice_result == tool_choice_in_params, (
f"Tool choice support mismatch for {model_name}:\n"
f"supports_tool_choice() returned: {supports_tool_choice_result}\n"
f"tool_choice in supported params: {tool_choice_in_params}"
f"Tool choice support mismatch for {model_name}. supports_tool_choice() returned: {supports_tool_choice_result}, tool_choice in supported params: {tool_choice_in_params}"
)

1
tests/llm_translation/base_llm_unit_tests.py
View file

@ -474,6 +474,7 @@ class BaseLLMChatTest(ABC):
"""
PROD Test: ensure nested json schema sent to proxy works as expected.
"""
litellm._turn_on_debug()
from pydantic import BaseModel
from litellm.utils import supports_response_schema
from litellm.llms.base_llm.base_utils import type_to_response_format_param

6
tests/llm_translation/test_max_completion_tokens.py
View file

@ -150,9 +150,11 @@ def test_all_model_configs():
{"max_completion_tokens": 10}, {}, "llama3", drop_params=False
) == {"max_tokens": 10}
assert "max_completion_tokens" in VertexAIAi21Config().get_supported_openai_params()
assert "max_completion_tokens" in VertexAIAi21Config().get_supported_openai_params(
model="jamba-1.5-mini@001"
)
assert VertexAIAi21Config().map_openai_params(
{"max_completion_tokens": 10}, {}, "llama3", drop_params=False
{"max_completion_tokens": 10}, {}, "jamba-1.5-mini@001", drop_params=False
) == {"max_tokens": 10}
from litellm.llms.fireworks_ai.chat.transformation import (

8
tests/llm_translation/test_optional_params.py
View file

@ -1404,6 +1404,13 @@ def test_azure_modalities_param():
assert optional_params["modalities"] == ["text", "audio"]
assert optional_params["audio"] == {"type": "audio_input", "input": "test.wav"}
def test_litellm_proxy_thinking_param():
optional_params = get_optional_params(
model="gpt-4o",
custom_llm_provider="litellm_proxy",
thinking={"type": "enabled", "budget_tokens": 1024},
)
assert optional_params["thinking"] == {"type": "enabled", "budget_tokens": 1024}
def test_gemini_modalities_param():
optional_params = get_optional_params(
@ -1441,3 +1448,4 @@ def test_anthropic_unified_reasoning_content(model, provider):
reasoning_effort="high",
)
assert optional_params["thinking"] == {"type": "enabled", "budget_tokens": 4096}

11
tests/local_testing/test_completion.py
View file

@ -880,6 +880,9 @@ def test_completion_azure_mistral_large_function_calling(provider):
This primarily tests if the 'Function()' pydantic object correctly handles argument param passed in as a dict vs. string
"""
litellm.set_verbose = True
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
model_cost = litellm.get_model_cost_map(url="")
tools = [
{
"type": "function",
@ -1903,16 +1906,16 @@ def test_completion_openai():
@pytest.mark.parametrize(
"model, api_version",
[
("gpt-4o-2024-08-06", None),
("azure/chatgpt-v-2", None),
# ("gpt-4o-2024-08-06", None),
# ("azure/chatgpt-v-2", None),
("bedrock/anthropic.claude-3-sonnet-20240229-v1:0", None),
("azure/gpt-4o", "2024-08-01-preview"),
# ("azure/gpt-4o", "2024-08-01-preview"),
],
)
@pytest.mark.flaky(retries=3, delay=1)
def test_completion_openai_pydantic(model, api_version):
try:
litellm.set_verbose = True
litellm._turn_on_debug()
from pydantic import BaseModel
messages = [