Litellm dev 11 23 2024 (#6881)

* build(ui/create_key_button.tsx): support adding tags for cost tracking/routing when making key

* LiteLLM Minor Fixes & Improvements (11/23/2024)  (#6870)

* feat(pass_through_endpoints/): support logging anthropic/gemini pass through calls to langfuse/s3/etc.

* fix(utils.py): allow disabling end user cost tracking with new param

Allows proxy admin to disable cost tracking for end user - keeps prometheus metrics small

* docs(configs.md): add disable_end_user_cost_tracking reference to docs

* feat(key_management_endpoints.py): add support for restricting access to `/key/generate` by team/proxy level role

Enables admin to restrict key creation, and assign team admins to handle distributing keys

* test(test_key_management.py): add unit testing for personal / team key restriction checks

* docs: add docs on restricting key creation

* docs(finetuned_models.md): add new guide on calling finetuned models

* docs(input.md): cleanup anthropic supported params

Closes https://github.com/BerriAI/litellm/issues/6856

* test(test_embedding.py): add test for passing extra headers via embedding

* feat(cohere/embed): pass client to async embedding

* feat(rerank.py): add `/v1/rerank` if missing for cohere base url

Closes https://github.com/BerriAI/litellm/issues/6844

* fix(main.py): pass extra_headers param to openai

Fixes https://github.com/BerriAI/litellm/issues/6836

* fix(litellm_logging.py): don't disable global callbacks when dynamic callbacks are set

Fixes issue where global callbacks - e.g. prometheus were overriden when langfuse was set dynamically

* fix(handler.py): fix linting error

* fix: fix typing

* build: add conftest to proxy_admin_ui_tests/

* test: fix test

* fix: fix linting errors

* test: fix test

* fix: fix pass through testing

* feat(key_management_endpoints.py): allow proxy_admin to enforce params on key creation

allows admin to force team keys to have tags

* build(ui/): show teams in leftnav + allow team admin to add new members

* build(ui/): show created tags in dropdown

makes it easier for admin to add tags to keys

* test(test_key_management.py): fix test

* test: fix test

* fix playwright e2e ui test

* fix e2e ui testing deps

* fix: fix linting errors

* fix e2e ui testing

* fix e2e ui testing, only run e2e ui testing in playwright

---------

Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com>

docs/my-website/docs/proxy/virtual_keys.md

@@ -820,6 +820,7 @@ litellm_settings:
   key_generation_settings:
     team_key_generation:
       allowed_team_member_roles: ["admin"]
+      required_params: ["tags"] # require team admins to set tags for cost-tracking when generating a team key
     personal_key_generation: # maps to 'Default Team' on UI 
       allowed_user_roles: ["proxy_admin"]
 ```
@@ -829,10 +830,12 @@ litellm_settings:
 ```python
 class TeamUIKeyGenerationConfig(TypedDict):
     allowed_team_member_roles: List[str]
+    required_params: List[str] # require params on `/key/generate` to be set if a team key (team_id in request) is being generated
 
 
 class PersonalUIKeyGenerationConfig(TypedDict):
     allowed_user_roles: List[LitellmUserRoles] 
+    required_params: List[str] # require params on `/key/generate` to be set if a personal key (no team_id in request) is being generated
 
 
 class StandardKeyGenerationConfig(TypedDict, total=False):

litellm/proxy/_new_secret_config.yaml

@@ -11,28 +11,4 @@ model_list:
       model: vertex_ai/claude-3-5-sonnet-v2
       vertex_ai_project: "adroit-crow-413218"
       vertex_ai_location: "us-east5"  
-  - model_name: fake-openai-endpoint
-    litellm_params:
-      model: openai/fake
-      api_key: fake-key
-      api_base: https://exampleopenaiendpoint-production.up.railway.app/
 
-router_settings:
-  model_group_alias:
-    "gpt-4-turbo": # Aliased model name
-      model: "gpt-4"  # Actual model name in 'model_list'
-      hidden: true
-
-litellm_settings:
-  default_team_settings: 
-    - team_id: team-1
-      success_callback: ["langfuse"]
-      failure_callback: ["langfuse"]
-      langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1
-      langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1
-    - team_id: team-2
-      success_callback: ["langfuse"]
-      failure_callback: ["langfuse"]
-      langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2
-      langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2
-      langfuse_host: https://us.cloud.langfuse.com

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -39,16 +39,20 @@ from litellm.proxy.utils import (
     handle_exception_on_proxy,
 )
 from litellm.secret_managers.main import get_secret
+from litellm.types.utils import PersonalUIKeyGenerationConfig, TeamUIKeyGenerationConfig
 
 
 def _is_team_key(data: GenerateKeyRequest):
     return data.team_id is not None
 
 
-def _team_key_generation_check(user_api_key_dict: UserAPIKeyAuth):
+def _team_key_generation_team_member_check(
+    user_api_key_dict: UserAPIKeyAuth,
+    team_key_generation: Optional[TeamUIKeyGenerationConfig],
+):
     if (
-        litellm.key_generation_settings is None
-        or litellm.key_generation_settings.get("team_key_generation") is None
+        team_key_generation is None
+        or "allowed_team_member_roles" not in team_key_generation
     ):
         return True
 
@@ -59,12 +63,7 @@ def _team_key_generation_check(user_api_key_dict: UserAPIKeyAuth):
         )
 
     team_member_role = user_api_key_dict.team_member.role
-    if (
-        team_member_role
-        not in litellm.key_generation_settings["team_key_generation"][  # type: ignore
-            "allowed_team_member_roles"
-        ]
-    ):
+    if team_member_role not in team_key_generation["allowed_team_member_roles"]:
         raise HTTPException(
             status_code=400,
             detail=f"Team member role {team_member_role} not in allowed_team_member_roles={litellm.key_generation_settings['team_key_generation']['allowed_team_member_roles']}",  # type: ignore
@@ -72,7 +71,67 @@ def _team_key_generation_check(user_api_key_dict: UserAPIKeyAuth):
     return True
 
 
-def _personal_key_generation_check(user_api_key_dict: UserAPIKeyAuth):
+def _key_generation_required_param_check(
+    data: GenerateKeyRequest, required_params: Optional[List[str]]
+):
+    if required_params is None:
+        return True
+
+    data_dict = data.model_dump(exclude_unset=True)
+    for param in required_params:
+        if param not in data_dict:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Required param {param} not in data",
+            )
+    return True
+
+
+def _team_key_generation_check(
+    user_api_key_dict: UserAPIKeyAuth, data: GenerateKeyRequest
+):
+    if (
+        litellm.key_generation_settings is None
+        or litellm.key_generation_settings.get("team_key_generation") is None
+    ):
+        return True
+
+    _team_key_generation = litellm.key_generation_settings["team_key_generation"]  # type: ignore
+
+    _team_key_generation_team_member_check(
+        user_api_key_dict,
+        team_key_generation=_team_key_generation,
+    )
+    _key_generation_required_param_check(
+        data,
+        _team_key_generation.get("required_params"),
+    )
+
+    return True
+
+
+def _personal_key_membership_check(
+    user_api_key_dict: UserAPIKeyAuth,
+    personal_key_generation: Optional[PersonalUIKeyGenerationConfig],
+):
+    if (
+        personal_key_generation is None
+        or "allowed_user_roles" not in personal_key_generation
+    ):
+        return True
+
+    if user_api_key_dict.user_role not in personal_key_generation["allowed_user_roles"]:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Personal key creation has been restricted by admin. Allowed roles={litellm.key_generation_settings['personal_key_generation']['allowed_user_roles']}. Your role={user_api_key_dict.user_role}",  # type: ignore
+        )
+
+    return True
+
+
+def _personal_key_generation_check(
+    user_api_key_dict: UserAPIKeyAuth, data: GenerateKeyRequest
+):
 
     if (
         litellm.key_generation_settings is None
@@ -80,16 +139,18 @@ def _personal_key_generation_check(user_api_key_dict: UserAPIKeyAuth):
     ):
         return True
 
-    if (
-        user_api_key_dict.user_role
-        not in litellm.key_generation_settings["personal_key_generation"][  # type: ignore
-            "allowed_user_roles"
-        ]
-    ):
-        raise HTTPException(
-            status_code=400,
-            detail=f"Personal key creation has been restricted by admin. Allowed roles={litellm.key_generation_settings['personal_key_generation']['allowed_user_roles']}. Your role={user_api_key_dict.user_role}",  # type: ignore
+    _personal_key_generation = litellm.key_generation_settings["personal_key_generation"]  # type: ignore
+
+    _personal_key_membership_check(
+        user_api_key_dict,
+        personal_key_generation=_personal_key_generation,
     )
+
+    _key_generation_required_param_check(
+        data,
+        _personal_key_generation.get("required_params"),
+    )
+
     return True
 
 
@@ -99,16 +160,23 @@ def key_generation_check(
     """
     Check if admin has restricted key creation to certain roles for teams or individuals
     """
-    if litellm.key_generation_settings is None:
+    if (
+        litellm.key_generation_settings is None
+        or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
+    ):
         return True
 
     ## check if key is for team or individual
     is_team_key = _is_team_key(data=data)
 
     if is_team_key:
-        return _team_key_generation_check(user_api_key_dict)
+        return _team_key_generation_check(
+            user_api_key_dict=user_api_key_dict, data=data
+        )
     else:
-        return _personal_key_generation_check(user_api_key_dict=user_api_key_dict)
+        return _personal_key_generation_check(
+            user_api_key_dict=user_api_key_dict, data=data
+        )
 
 
 router = APIRouter()

litellm/types/utils.py

@@ -1604,11 +1604,17 @@ class StandardCallbackDynamicParams(TypedDict, total=False):
     langsmith_base_url: Optional[str]
 
 
-class TeamUIKeyGenerationConfig(TypedDict):
+class KeyGenerationConfig(TypedDict, total=False):
+    required_params: List[
+        str
+    ]  # specify params that must be present in the key generation request
+
+
+class TeamUIKeyGenerationConfig(KeyGenerationConfig):
     allowed_team_member_roles: List[str]
 
 
-class PersonalUIKeyGenerationConfig(TypedDict):
+class PersonalUIKeyGenerationConfig(KeyGenerationConfig):
     allowed_user_roles: List[str]
 
 

tests/proxy_admin_ui_tests/test_key_management.py

@@ -551,7 +551,7 @@ def test_is_team_key():
     assert not _is_team_key(GenerateKeyRequest(user_id="test_user_id"))
 
 
-def test_team_key_generation_check():
+def test_team_key_generation_team_member_check():
     from litellm.proxy.management_endpoints.key_management_endpoints import (
         _team_key_generation_check,
     )
@@ -562,22 +562,86 @@ def test_team_key_generation_check():
     }
 
     assert _team_key_generation_check(
-        UserAPIKeyAuth(
+        user_api_key_dict=UserAPIKeyAuth(
             user_role=LitellmUserRoles.INTERNAL_USER,
             api_key="sk-1234",
             team_member=Member(role="admin", user_id="test_user_id"),
-        )
+        ),
+        data=GenerateKeyRequest(),
     )
 
     with pytest.raises(HTTPException):
         _team_key_generation_check(
-            UserAPIKeyAuth(
+            user_api_key_dict=UserAPIKeyAuth(
                 user_role=LitellmUserRoles.INTERNAL_USER,
                 api_key="sk-1234",
                 user_id="test_user_id",
                 team_member=Member(role="user", user_id="test_user_id"),
+            ),
+            data=GenerateKeyRequest(),
+        )
+
+
+@pytest.mark.parametrize(
+    "team_key_generation_settings, input_data, expected_result",
+    [
+        ({"required_params": ["tags"]}, GenerateKeyRequest(tags=["test_tags"]), True),
+        ({}, GenerateKeyRequest(), True),
+        (
+            {"required_params": ["models"]},
+            GenerateKeyRequest(tags=["test_tags"]),
+            False,
+        ),
+    ],
+)
+@pytest.mark.parametrize("key_type", ["team_key", "personal_key"])
+def test_key_generation_required_params_check(
+    team_key_generation_settings, input_data, expected_result, key_type
+):
+    from litellm.proxy.management_endpoints.key_management_endpoints import (
+        _team_key_generation_check,
+        _personal_key_generation_check,
+    )
+    from litellm.types.utils import (
+        TeamUIKeyGenerationConfig,
+        StandardKeyGenerationConfig,
+        PersonalUIKeyGenerationConfig,
+    )
+    from fastapi import HTTPException
+
+    user_api_key_dict = UserAPIKeyAuth(
+        user_role=LitellmUserRoles.INTERNAL_USER,
+        api_key="sk-1234",
+        user_id="test_user_id",
+        team_id="test_team_id",
+        team_member=Member(role="admin", user_id="test_user_id"),
+    )
+
+    if key_type == "team_key":
+        litellm.key_generation_settings = StandardKeyGenerationConfig(
+            team_key_generation=TeamUIKeyGenerationConfig(
+                **team_key_generation_settings
             )
         )
+    elif key_type == "personal_key":
+        litellm.key_generation_settings = StandardKeyGenerationConfig(
+            personal_key_generation=PersonalUIKeyGenerationConfig(
+                **team_key_generation_settings
+            )
+        )
+
+    if expected_result:
+        if key_type == "team_key":
+            assert _team_key_generation_check(user_api_key_dict, input_data)
+        elif key_type == "personal_key":
+            assert _personal_key_generation_check(user_api_key_dict, input_data)
+    else:
+        if key_type == "team_key":
+            with pytest.raises(HTTPException):
+                _team_key_generation_check(user_api_key_dict, input_data)
+        elif key_type == "personal_key":
+            with pytest.raises(HTTPException):
+                _personal_key_generation_check(user_api_key_dict, input_data)
 
 
 def test_personal_key_generation_check():
@@ -591,16 +655,18 @@ def test_personal_key_generation_check():
     }
 
     assert _personal_key_generation_check(
-        UserAPIKeyAuth(
+        user_api_key_dict=UserAPIKeyAuth(
             user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234", user_id="admin"
-        )
+        ),
+        data=GenerateKeyRequest(),
     )
 
     with pytest.raises(HTTPException):
         _personal_key_generation_check(
-            UserAPIKeyAuth(
+            user_api_key_dict=UserAPIKeyAuth(
                 user_role=LitellmUserRoles.INTERNAL_USER,
                 api_key="sk-1234",
                 user_id="admin",
-            )
+            ),
+            data=GenerateKeyRequest(),
         )

ui/litellm-dashboard/src/components/create_key_button.tsx

@@ -40,6 +40,31 @@ interface CreateKeyProps {
   setData: React.Dispatch<React.SetStateAction<any[] | null>>;
 }
 
+const getPredefinedTags = (data: any[] | null) => {
+  let allTags = [];
+
+  console.log("data:", JSON.stringify(data));
+
+  if (data) {
+    for (let key of data) {
+      if (key["metadata"] && key["metadata"]["tags"]) {
+        allTags.push(...key["metadata"]["tags"]);
+      }
+    }
+  }
+
+  // Deduplicate using Set
+  const uniqueTags = Array.from(new Set(allTags)).map(tag => ({
+    value: tag,
+    label: tag,
+  }));
+
+
+  console.log("uniqueTags:", uniqueTags);
+  return uniqueTags;
+}
+
+
 const CreateKey: React.FC<CreateKeyProps> = ({
   userID,
   team,
@@ -55,6 +80,8 @@ const CreateKey: React.FC<CreateKeyProps> = ({
   const [userModels, setUserModels] = useState([]);
   const [modelsToPick, setModelsToPick] = useState([]);
   const [keyOwner, setKeyOwner] = useState("you");
+  const [predefinedTags, setPredefinedTags] = useState(getPredefinedTags(data));
+
 
   const handleOk = () => {
     setIsModalVisible(false);
@@ -355,6 +382,15 @@ const CreateKey: React.FC<CreateKeyProps> = ({
                     placeholder="Enter metadata as JSON"
                   />
                 </Form.Item>
+                <Form.Item label="Tags" name="tags" className="mt-8" help={`Tags for tracking spend and/or doing tag-based routing.`}>
+                <Select
+                    mode="tags"
+                    style={{ width: '100%' }}
+                    placeholder="Enter tags"
+                    tokenSeparators={[',']}
+                    options={predefinedTags}
+                  />
+                </Form.Item>
               </AccordionBody>
             </Accordion>
           </>

ui/litellm-dashboard/src/components/leftnav.tsx

@@ -32,7 +32,7 @@ const menuItems: MenuItem[] = [
   { key: "3", page: "llm-playground", label: "Test Key" }, // all roles
   { key: "2", page: "models", label: "Models", roles: all_admin_roles },
   { key: "4", page: "usage", label: "Usage"}, // all roles
-  { key: "6", page: "teams", label: "Teams", roles: all_admin_roles },
+  { key: "6", page: "teams", label: "Teams" },
   { key: "5", page: "users", label: "Internal Users", roles: all_admin_roles },
   { key: "8", page: "settings", label: "Logging & Alerts", roles: all_admin_roles },
   { key: "9", page: "caching", label: "Caching", roles: all_admin_roles },

ui/litellm-dashboard/src/components/networking.tsx

@@ -672,6 +672,7 @@ export const teamInfoCall = async (
 
 export const teamListCall = async (
   accessToken: String, 
+  userID: String | null = null
 ) => {
   /**
    * Get all available teams on proxy
@@ -679,6 +680,9 @@ export const teamListCall = async (
   try {
     let url = proxyBaseUrl ? `${proxyBaseUrl}/team/list` : `/team/list`;
     console.log("in teamInfoCall");
+    if (userID) {
+      url += `?user_id=${userID}`;
+    }
     const response = await fetch(url, {
       method: "GET",
       headers: {

ui/litellm-dashboard/src/components/teams.tsx

@@ -80,7 +80,13 @@ const Team: React.FC<TeamProps> = ({
     if (teams === null && accessToken) {
       // Call your function here
       const fetchData = async () => {
-        const givenTeams = await teamListCall(accessToken)
+        let givenTeams;
+        if (userRole != "Admin" && userRole != "Admin Viewer") {
+          givenTeams = await teamListCall(accessToken, userID)
+        } else {
+          givenTeams = await teamListCall(accessToken)
+        }
+        
         console.log(`givenTeams: ${givenTeams}`)
 
         setTeams(givenTeams)
@@ -98,6 +104,7 @@ const Team: React.FC<TeamProps> = ({
   const [selectedTeam, setSelectedTeam] = useState<null | any>(
     teams ? teams[0] : null
   );
+
   const [isTeamModalVisible, setIsTeamModalVisible] = useState(false);
   const [isAddMemberModalVisible, setIsAddMemberModalVisible] = useState(false);
   const [userModels, setUserModels] = useState([]);
@@ -327,7 +334,13 @@ const Team: React.FC<TeamProps> = ({
         }
 
         let _team_id_to_info: Record<string, any> = {};
-        const teamList = await teamListCall(accessToken)
+        let teamList;
+        if (userRole != "Admin" && userRole != "Admin Viewer") {
+          teamList = await teamListCall(accessToken, userID)
+        } else {
+          teamList = await teamListCall(accessToken)
+        }
+        
         for (let i = 0; i < teamList.length; i++) {
           let team = teamList[i];
           let _team_id = team.team_id;
@@ -376,6 +389,16 @@ const Team: React.FC<TeamProps> = ({
     }
   };
 
+  const is_team_admin = (team: any) => {
+    for (let i = 0; i < team.members_with_roles.length; i++) {
+      let member = team.members_with_roles[i];
+      if (member.user_id == userID && member.role == "admin") {
+        return true;
+      }
+    }
+    return false;
+  }
+
   const handleMemberCreate = async (formValues: Record<string, any>) => {
     try {
       if (accessToken != null && teams != null) {
@@ -557,6 +580,8 @@ const Team: React.FC<TeamProps> = ({
                           </Text>
                         </TableCell>
                         <TableCell>
+                          {userRole == "Admin" ? (
+                            <>
                             <Icon
                               icon={PencilAltIcon}
                               size="sm"
@@ -567,6 +592,8 @@ const Team: React.FC<TeamProps> = ({
                               icon={TrashIcon}
                               size="sm"
                             />
+                            </>
+                          ) : null}
                         </TableCell>
                       </TableRow>
                     ))
@@ -623,6 +650,7 @@ const Team: React.FC<TeamProps> = ({
             )}
           </Card>
         </Col>
+        {userRole == "Admin"? (
           <Col numColSpan={1}>
             <Button
             className="mx-auto"
@@ -708,6 +736,7 @@ const Team: React.FC<TeamProps> = ({
             </Form>
           </Modal>
           </Col>
+        ) : null}
         <Col numColSpan={1}>
           <Title level={4}>Team Members</Title>
           <Paragraph>
@@ -774,12 +803,14 @@ const Team: React.FC<TeamProps> = ({
           )}
         </Col>
         <Col numColSpan={1}>
+          {userRole == "Admin" || (selectedTeam && is_team_admin(selectedTeam)) ? (
             <Button
               className="mx-auto mb-5"
               onClick={() => setIsAddMemberModalVisible(true)}
             >
               + Add member
             </Button>
+          ) : null}
           <Modal
             title="Add member"
             visible={isAddMemberModalVisible}