From 0a2a51a5a5ad15c26422f70e18be250a94ae4a63 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Fri, 28 Feb 2025 23:23:03 -0800 Subject: [PATCH] UI - Allow admin to control default model access for internal users (#8912) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(create_user_button.tsx): allow admin to set models user has access to, on invite Enables controlling model access on invite * feat(auth_checks.py): enforce 'no-model-access' special model name on backend prevent user from calling models if default key has no model access * fix(chat_ui.tsx): allow user to input custom model * fix(chat_ui.tsx): pull available models based on models key has access to * style(create_user_button.tsx): move default model inside 'personal key creation' accordion * fix(chat_ui.tsx): fix linting error * test(test_auth_checks.py): add unit-test for special model name * docs(internal_user_endpoints.py): update docstring * fix test_moderations_bad_model * Litellm dev 02 27 2025 p6 (#8891) * fix(http_parsing_utils.py): orjson can throw errors on some emoji's in text, default to json.loads * fix(sagemaker/handler.py): support passing model id on async streaming * fix(litellm_pre_call_utils.py): Fixes https://github.com/BerriAI/litellm/issues/7237 * Fix calling claude via invoke route + response_format support for claude on invoke route (#8908) * fix(anthropic_claude3_transformation.py): fix amazon anthropic claude 3 tool calling transformation on invoke route move to using anthropic config as base * fix(utils.py): expose anthropic config via providerconfigmanager * fix(llm_http_handler.py): support json mode on async completion calls * fix(invoke_handler/make_call): support json mode for anthropic called via bedrock invoke * fix(anthropic/): handle 'response_format: {"type": "text"}` + migrate amazon claude 3 invoke config to inherit from anthropic config Prevents error when passing in 'response_format: {"type": "text"} * test: fix test * fix(utils.py): fix base invoke provider check * fix(anthropic_claude3_transformation.py): don't pass 'stream' param * fix: fix linting errors * fix(converse_transformation.py): handle response_format type=text for converse * converse_transformation: pass 'description' if set in response_format (#8907) * test(test_bedrock_completion.py): e2e test ensuring tool description is passed in * fix(converse_transformation.py): pass description, if set * fix(transformation.py): Fixes https://github.com/BerriAI/litellm/issues/8767#issuecomment-2689887663 * Fix bedrock passing `response_format: {"type": "text"}` (#8900) * fix(converse_transformation.py): ignore type: text, value in response_format no-op for bedrock * fix(converse_transformation.py): handle adding response format value to tools * fix(base_invoke_transformation.py): fix 'get_bedrock_invoke_provider' to handle cross-region-inferencing models * test(test_bedrock_completion.py): add unit testing for bedrock invoke provider logic * test: update test * fix(exception_mapping_utils.py): add context window exceeded error handling for databricks provider route * fix(fireworks_ai/): support passing tools + response_format together * fix: cleanup * fix(base_invoke_transformation.py): fix imports * (Feat) - Show Error Logs on LiteLLM UI (#8904) * fix test_moderations_bad_model * use async_post_call_failure_hook * basic logging errors in DB * show status on ui * show status on ui * ui show request / response side by side * stash fixes * working, track raw request * track error info in metadata * fix showing error / request / response logs * show traceback on error viewer * ui with traceback of error * fix async_post_call_failure_hook * fix(http_parsing_utils.py): orjson can throw errors on some emoji's in text, default to json.loads * test_get_error_information * fix code quality * rename proxy track cost callback test * _should_store_errors_in_spend_logs * feature flag error logs * Revert "_should_store_errors_in_spend_logs" This reverts commit 7f345df47762ff3be04e6fde2f13e70019ede4ee. * Revert "feature flag error logs" This reverts commit 0e90c022bbea3550f169118d81e60d711a4024fe. * test_spend_logs_payload * fix OTEL log_db_metrics * fix import json * fix ui linting error * test_async_post_call_failure_hook * test_chat_completion_bad_model_with_spend_logs --------- Co-authored-by: Krrish Dholakia * ui new build * test_chat_completion_bad_model_with_spend_logs * docs(release_cycle.md): document release cycle * bump: version 1.62.0 → 1.62.1 --------- Co-authored-by: Ishaan Jaff --- litellm/proxy/_types.py | 1 + litellm/proxy/auth/auth_checks.py | 9 ++++ .../internal_user_endpoints.py | 2 +- tests/proxy_unit_tests/test_auth_checks.py | 24 ++++++++++ .../src/components/chat_ui.tsx | 35 ++++++++++++-- .../src/components/create_user_button.tsx | 46 +++++++++++++++++-- 6 files changed, 109 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 1b5faf3f9c..68663ec01c 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1714,6 +1714,7 @@ class WebhookEvent(CallInfo): class SpecialModelNames(enum.Enum): all_team_models = "all-team-models" all_proxy_models = "all-proxy-models" + no_default_models = "no-default-models" class InvitationNew(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 0590bcb50a..183b5609d0 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -38,6 +38,7 @@ from litellm.proxy._types import ( ProxyErrorTypes, ProxyException, RoleBasedPermissions, + SpecialModelNames, UserAPIKeyAuth, ) from litellm.proxy.auth.route_checks import RouteChecks @@ -1083,6 +1084,14 @@ async def can_user_call_model( if user_object is None: return True + if SpecialModelNames.no_default_models.value in user_object.models: + raise ProxyException( + message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}", + type=ProxyErrorTypes.key_model_access_denied, + param="model", + code=status.HTTP_401_UNAUTHORIZED, + ) + return await _can_object_call_model( model=model, llm_router=llm_router, diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index a414f48847..36c12ada3f 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -127,7 +127,7 @@ async def new_user( - user_role: Optional[str] - Specify a user role - "proxy_admin", "proxy_admin_viewer", "internal_user", "internal_user_viewer", "team", "customer". Info about each role here: `https://github.com/BerriAI/litellm/litellm/proxy/_types.py#L20` - max_budget: Optional[float] - Specify max budget for a given user. - budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). - - models: Optional[list] - Model_name's a user is allowed to call. (if empty, key is allowed to call all models) + - models: Optional[list] - Model_name's a user is allowed to call. (if empty, key is allowed to call all models). Set to ['no-default-models'] to block all model access. Restricting user to only team-based model access. - tpm_limit: Optional[int] - Specify tpm limit for a given user (Tokens per minute) - rpm_limit: Optional[int] - Specify rpm limit for a given user (Requests per minute) - auto_create_key: bool - Default=True. Flag used for returning a key as part of the /user/new response diff --git a/tests/proxy_unit_tests/test_auth_checks.py b/tests/proxy_unit_tests/test_auth_checks.py index 0a8ebbe018..8e9618297e 100644 --- a/tests/proxy_unit_tests/test_auth_checks.py +++ b/tests/proxy_unit_tests/test_auth_checks.py @@ -550,6 +550,30 @@ async def test_can_user_call_model(): await can_user_call_model(**args) +@pytest.mark.asyncio +async def test_can_user_call_model_with_no_default_models(): + from litellm.proxy.auth.auth_checks import can_user_call_model + from litellm.proxy._types import ProxyException, SpecialModelNames + from unittest.mock import MagicMock + + args = { + "model": "anthropic-claude", + "llm_router": MagicMock(), + "user_object": LiteLLM_UserTable( + user_id="testuser21@mycompany.com", + max_budget=None, + spend=0.0042295, + model_max_budget={}, + model_spend={}, + user_email="testuser@mycompany.com", + models=[SpecialModelNames.no_default_models.value], + ), + } + + with pytest.raises(ProxyException) as e: + await can_user_call_model(**args) + + @pytest.mark.asyncio async def test_get_fuzzy_user_object(): from litellm.proxy.auth.auth_checks import _get_fuzzy_user_object diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index db4499f9ff..c505a954b8 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -93,20 +93,27 @@ const ChatUI: React.FC = ({ const [selectedModel, setSelectedModel] = useState( undefined ); + const [showCustomModelInput, setShowCustomModelInput] = useState(false); const [modelInfo, setModelInfo] = useState([]); + const customModelTimeout = useRef(null); const chatEndRef = useRef(null); useEffect(() => { - if (!accessToken || !token || !userRole || !userID) { + let useApiKey = apiKeySource === 'session' ? accessToken : apiKey; + console.log("useApiKey:", useApiKey); + if (!useApiKey || !token || !userRole || !userID) { + console.log("useApiKey or token or userRole or userID is missing = ", useApiKey, token, userRole, userID); return; } + + // Fetch model info and set the default selected model const fetchModelInfo = async () => { try { const fetchedAvailableModels = await modelAvailableCall( - accessToken, + useApiKey ?? '', // Use empty string if useApiKey is null, userID, userRole ); @@ -139,7 +146,7 @@ const ChatUI: React.FC = ({ }; fetchModelInfo(); - }, [accessToken, userID, userRole]); + }, [accessToken, userID, userRole, apiKeySource, apiKey]); useEffect(() => { @@ -234,6 +241,7 @@ const ChatUI: React.FC = ({ const onChange = (value: string) => { console.log(`selected ${value}`); setSelectedModel(value); + setShowCustomModelInput(value === 'custom'); }; return ( @@ -276,10 +284,29 @@ const ChatUI: React.FC = ({ {teams ? ( teams.map((team: any) => ( @@ -307,6 +314,39 @@ const Createuser: React.FC = ({ + + + Personal Key Creation + + + + Models{' '} + + + + + } name="models" help="Models user has access to, outside of team scope."> + + + All Proxy Models + + {userModels.map((model) => ( + + {getModelDisplayName(model)} + + ))} + + + +