diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 69815102f6..a4a8618082 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -29,4 +29,4 @@ litellm_settings: failure_callback: ["langfuse"] langfuse_public_key: os.environ/LANGFUSE_PROJECT3_PUBLIC langfuse_secret: os.environ/LANGFUSE_PROJECT3_SECRET - langfuse_host: os.environ/LANGFUSE_HOST + langfuse_host: os.environ/LANGFUSE_HOST \ No newline at end of file diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index a226462ff7..a9a9db294c 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2237,3 +2237,6 @@ class ProxyStateVariables(TypedDict): """ spend_logs_row_count: int + + +UI_TEAM_ID = "litellm-dashboard" diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 1c6031da6d..5ca4a6251f 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -370,7 +370,7 @@ async def user_info( ## REMOVE HASHED TOKEN INFO before returning ## returned_keys = _process_keys_for_user_info(keys=keys, all_teams=teams_1) - team_list.sort(key=lambda x: (getattr(x, "team_alias", ""))) + team_list.sort(key=lambda x: (getattr(x, "team_alias", "") or "")) _user_info = ( user_info.model_dump() if isinstance(user_info, BaseModel) else user_info ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 580f23687a..6a94d5bd4f 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -85,6 +85,11 @@ def _is_allowed_to_create_key( ) if team_id is not None: + if ( + user_api_key_dict.team_id is not None + and user_api_key_dict.team_id == UI_TEAM_ID + ): + return True # handle https://github.com/BerriAI/litellm/issues/7482 assert ( user_api_key_dict.team_id == team_id ), "User can only create keys for their own team. Got={}, Your Team ID={}".format( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 397d6938fd..ad8408001b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -295,6 +295,7 @@ from fastapi import ( Header, HTTPException, Path, + Query, Request, Response, UploadFile, @@ -6622,6 +6623,20 @@ async def model_info_v1( # noqa: PLR0915 return {"data": all_models} +def _get_model_group_info( + llm_router: Router, all_models_str: List[str], model_group: Optional[str] +) -> List[ModelGroupInfo]: + model_groups: List[ModelGroupInfo] = [] + for model in all_models_str: + if model_group is not None and model_group != model: + continue + + _model_group_info = llm_router.get_model_group_info(model_group=model) + if _model_group_info is not None: + model_groups.append(_model_group_info) + return model_groups + + @router.get( "/model_group/info", tags=["model management"], @@ -6629,14 +6644,17 @@ async def model_info_v1( # noqa: PLR0915 ) async def model_group_info( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + model_group: Optional[str] = None, ): """ Get information about all the deployments on litellm proxy, including config.yaml descriptions (except api key and api base) - - /models returns all deployments. Proxy Admins can use this to list all deployments setup on the proxy - /model_group/info returns all model groups. End users of proxy should use /model_group/info since those models will be used for /chat/completions, /embeddings, etc. + - /model_group/info?model_group=rerank-english-v3.0 returns all model groups for a specific model group (`model_name` in config.yaml) + + Example Request (All Models): ```shell curl -X 'GET' \ 'http://localhost:4000/model_group/info' \ @@ -6644,6 +6662,24 @@ async def model_group_info( -H 'x-api-key: sk-1234' ``` + Example Request (Specific Model Group): + ```shell + curl -X 'GET' \ + 'http://localhost:4000/model_group/info?model_group=rerank-english-v3.0' \ + -H 'accept: application/json' \ + -H 'Authorization: Bearer sk-1234' + ``` + + Example Request (Specific Wildcard Model Group): (e.g. `model_name: openai/*` on config.yaml) + ```shell + curl -X 'GET' \ + 'http://localhost:4000/model_group/info?model_group=openai/tts-1' + -H 'accept: application/json' \ + -H 'Authorization: Bearersk-1234' + ``` + + Learn how to use and set wildcard models [here](https://docs.litellm.ai/docs/wildcard_routing) + Example Response: ```json { @@ -6796,13 +6832,9 @@ async def model_group_info( infer_model_from_keys=general_settings.get("infer_model_from_keys", False), ) - model_groups: List[ModelGroupInfo] = [] - - for model in all_models_str: - - _model_group_info = llm_router.get_model_group_info(model_group=model) - if _model_group_info is not None: - model_groups.append(_model_group_info) + model_groups: List[ModelGroupInfo] = _get_model_group_info( + llm_router=llm_router, all_models_str=all_models_str, model_group=model_group + ) return {"data": model_groups} diff --git a/litellm/types/router.py b/litellm/types/router.py index f1a1f44480..8c671fe52c 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -509,13 +509,17 @@ class ModelGroupInfo(BaseModel): input_cost_per_token: Optional[float] = None output_cost_per_token: Optional[float] = None mode: Optional[ - Literal[ - "chat", - "embedding", - "completion", - "image_generation", - "audio_transcription", - "rerank", + Union[ + str, + Literal[ + "chat", + "embedding", + "completion", + "image_generation", + "audio_transcription", + "rerank", + "moderations", + ], ] ] = Field(default="chat") tpm: Optional[int] = None diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 9f3533ab6c..29c02c7cc5 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -209,7 +209,6 @@ def test_model_info_bedrock_converse(monkeypatch): """ monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") litellm.model_cost = litellm.get_model_cost_map(url="") - try: # Load whitelist models from file with open("whitelisted_bedrock_models.txt", "r") as file: diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py index 71579dd15b..7027330db6 100644 --- a/tests/proxy_unit_tests/test_proxy_server.py +++ b/tests/proxy_unit_tests/test_proxy_server.py @@ -1914,8 +1914,10 @@ async def test_proxy_model_group_alias_checks(prisma_client, hidden): resp = await model_group_info( user_api_key_dict=UserAPIKeyAuth(models=[]), ) + print(f"resp: {resp}") models = resp["data"] is_model_alias_in_list = False + print(f"model_alias: {model_alias}, models: {models}") for item in models: if model_alias == item.model_group: is_model_alias_in_list = True diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 3b3da7b982..d1b797b34a 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -1191,3 +1191,64 @@ def test_litellm_verification_token_view_response_with_budget_table( getattr(resp, expected_user_api_key_auth_key) == expected_user_api_key_auth_value ) + + +def test_is_allowed_to_create_key(): + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _is_allowed_to_create_key, + ) + + assert ( + _is_allowed_to_create_key( + user_api_key_dict=UserAPIKeyAuth( + user_id="test_user_id", user_role=LitellmUserRoles.PROXY_ADMIN + ), + user_id="test_user_id", + team_id="test_team_id", + ) + is True + ) + + assert ( + _is_allowed_to_create_key( + user_api_key_dict=UserAPIKeyAuth( + user_id="test_user_id", + user_role=LitellmUserRoles.INTERNAL_USER, + team_id="litellm-dashboard", + ), + user_id="test_user_id", + team_id="test_team_id", + ) + is True + ) + + +def test_get_model_group_info(): + from litellm.proxy.proxy_server import _get_model_group_info + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "openai/tts-1", + "litellm_params": { + "model": "openai/tts-1", + "api_key": "sk-1234", + }, + }, + { + "model_name": "openai/gpt-3.5-turbo", + "litellm_params": { + "model": "openai/gpt-3.5-turbo", + "api_key": "sk-1234", + }, + }, + ] + ) + model_list = _get_model_group_info( + llm_router=router, + all_models_str=["openai/tts-1", "openai/gpt-3.5-turbo"], + model_group="openai/tts-1", + ) + assert len(model_list) == 1