diff --git a/docs/my-website/docs/pass_through/anthropic_completion.md b/docs/my-website/docs/pass_through/anthropic_completion.md index b64cd1ece..0c6a5f1b6 100644 --- a/docs/my-website/docs/pass_through/anthropic_completion.md +++ b/docs/my-website/docs/pass_through/anthropic_completion.md @@ -1,4 +1,4 @@ -# Anthropic SDK +# Anthropic `/v1/messages` Pass-through endpoints for Anthropic - call provider-specific endpoint, in native format (no translation). diff --git a/litellm/__init__.py b/litellm/__init__.py index 04b594ca1..9a8c56a56 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1132,6 +1132,7 @@ from .llms.AzureOpenAI.chat.gpt_transformation import AzureOpenAIConfig from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig from .llms.deepseek.chat.transformation import DeepSeekChatConfig from .llms.lm_studio.chat.transformation import LMStudioChatConfig +from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig from .llms.perplexity.chat.transformation import PerplexityChatConfig from .llms.AzureOpenAI.chat.o1_transformation import AzureOpenAIO1Config from .llms.watsonx.completion.handler import IBMWatsonXAIConfig diff --git a/litellm/llms/lm_studio/embed/transformation.py b/litellm/llms/lm_studio/embed/transformation.py new file mode 100644 index 000000000..17b2173a7 --- /dev/null +++ b/litellm/llms/lm_studio/embed/transformation.py @@ -0,0 +1,54 @@ +""" +Transformation logic from OpenAI /v1/embeddings format to LM Studio's `/v1/embeddings` format. + +Why separate file? Make it easy to see how transformation works + +Docs - https://lmstudio.ai/docs/basics/server +""" + +import types +from typing import List, Optional, Tuple + +from litellm import LlmProviders +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import Embedding, EmbeddingResponse, Usage + + +class LmStudioEmbeddingConfig: + """ + Reference: https://lmstudio.ai/docs/basics/server + """ + + def __init__( + self, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self) -> List[str]: + return [] + + def map_openai_params( + self, non_default_params: dict, optional_params: dict + ) -> dict: + return optional_params diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d9efa6f9a..f5851ded9 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1131,7 +1131,6 @@ class KeyManagementSettings(LiteLLMBase): """ If True, virtual keys created by litellm will be stored in the secret manager """ - prefix_for_stored_virtual_keys: str = "litellm/" """ If set, this prefix will be used for stored virtual keys in the secret manager diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8d3afa33f..7d29032c6 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -280,6 +280,22 @@ def allowed_routes_check( return False +def allowed_route_check_inside_route( + user_api_key_dict: UserAPIKeyAuth, + requested_user_id: Optional[str], +) -> bool: + ret_val = True + if ( + user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN + and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ): + ret_val = False + if requested_user_id is not None and user_api_key_dict.user_id is not None: + if user_api_key_dict.user_id == requested_user_id: + ret_val = True + return ret_val + + def get_actual_routes(allowed_routes: list) -> list: actual_routes: list = [] for route_name in allowed_routes: diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index bdecc77b0..7becd3260 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -26,7 +26,6 @@ from litellm.proxy._types import ( # NOTE: This is the prefix for all virtual keys stored in AWS Secrets Manager LITELLM_PREFIX_STORED_VIRTUAL_KEYS = "litellm/" - class KeyManagementEventHooks: @staticmethod diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index ec6949936..251fa648e 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -39,7 +39,10 @@ from litellm.proxy._types import ( UpdateTeamRequest, UserAPIKeyAuth, ) -from litellm.proxy.auth.auth_checks import get_team_object +from litellm.proxy.auth.auth_checks import ( + allowed_route_check_inside_route, + get_team_object, +) from litellm.proxy.auth.user_api_key_auth import _is_user_proxy_admin, user_api_key_auth from litellm.proxy.management_helpers.utils import ( add_new_member, @@ -1280,10 +1283,8 @@ async def list_team( prisma_client, ) - if ( - user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN - and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY - and user_api_key_dict.user_id != user_id + if not allowed_route_check_inside_route( + user_api_key_dict=user_api_key_dict, requested_user_id=user_id ): raise HTTPException( status_code=401, diff --git a/litellm/utils.py b/litellm/utils.py index f4f31e6cf..cb8a53354 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2385,6 +2385,16 @@ def get_optional_params_embeddings( # noqa: PLR0915 ) final_params = {**optional_params, **kwargs} return final_params + elif custom_llm_provider == "lm_studio": + supported_params = ( + litellm.LmStudioEmbeddingConfig().get_supported_openai_params() + ) + _check_valid_arg(supported_params=supported_params) + optional_params = litellm.LmStudioEmbeddingConfig().map_openai_params( + non_default_params=non_default_params, optional_params={} + ) + final_params = {**optional_params, **kwargs} + return final_params elif custom_llm_provider == "bedrock": # if dimensions is in non_default_params -> pass it for model=bedrock/amazon.titan-embed-text-v2 if "amazon.titan-embed-text-v1" in model: diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index c9527c830..029e91513 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -942,3 +942,12 @@ def test_forward_user_param(): ) assert optional_params["metadata"]["user_id"] == "test_user" + +def test_lm_studio_embedding_params(): + optional_params = get_optional_params_embeddings( + model="lm_studio/gemma2-9b-it", + custom_llm_provider="lm_studio", + dimensions=1024, + drop_params=True, + ) + assert len(optional_params) == 0 diff --git a/tests/local_testing/test_user_api_key_auth.py b/tests/local_testing/test_user_api_key_auth.py index 31daa358a..1a129489c 100644 --- a/tests/local_testing/test_user_api_key_auth.py +++ b/tests/local_testing/test_user_api_key_auth.py @@ -387,3 +387,31 @@ def test_is_api_route_allowed(route, user_role, expected_result): pass else: raise e + + +from litellm.proxy._types import LitellmUserRoles + + +@pytest.mark.parametrize( + "user_role, auth_user_id, requested_user_id, expected_result", + [ + (LitellmUserRoles.PROXY_ADMIN, "1234", None, True), + (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, None, "1234", True), + (LitellmUserRoles.TEAM, "1234", None, False), + (LitellmUserRoles.TEAM, None, None, False), + (LitellmUserRoles.TEAM, "1234", "1234", True), + ], +) +def test_allowed_route_inside_route( + user_role, auth_user_id, requested_user_id, expected_result +): + from litellm.proxy.auth.auth_checks import allowed_route_check_inside_route + from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles + + assert ( + allowed_route_check_inside_route( + user_api_key_dict=UserAPIKeyAuth(user_role=user_role, user_id=auth_user_id), + requested_user_id=requested_user_id, + ) + == expected_result + ) diff --git a/tests/proxy_unit_tests/test_key_generate_prisma.py b/tests/proxy_unit_tests/test_key_generate_prisma.py index 4de451642..8ad773d63 100644 --- a/tests/proxy_unit_tests/test_key_generate_prisma.py +++ b/tests/proxy_unit_tests/test_key_generate_prisma.py @@ -3469,6 +3469,7 @@ async def test_key_generate_with_secret_manager_call(prisma_client): """ from litellm.secret_managers.aws_secret_manager_v2 import AWSSecretsManagerV2 from litellm.proxy._types import KeyManagementSystem, KeyManagementSettings + from litellm.proxy.hooks.key_management_event_hooks import ( LITELLM_PREFIX_STORED_VIRTUAL_KEYS, ) @@ -3517,6 +3518,7 @@ async def test_key_generate_with_secret_manager_call(prisma_client): await asyncio.sleep(2) # read from the secret manager + result = await aws_secret_manager_client.async_read_secret( secret_name=f"{litellm._key_management_settings.prefix_for_stored_virtual_keys}/{key_alias}" ) @@ -3537,6 +3539,7 @@ async def test_key_generate_with_secret_manager_call(prisma_client): await asyncio.sleep(2) # Assert the key is deleted from the secret manager + result = await aws_secret_manager_client.async_read_secret( secret_name=f"{litellm._key_management_settings.prefix_for_stored_virtual_keys}/{key_alias}" )