From 5b0eab222436c94331542ae07770d6addeaa6b1a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 24 Feb 2024 11:37:20 -0800 Subject: [PATCH 1/8] fix(proxy_server.py): allow user to set team tpm/rpm limits/budget/models --- litellm/proxy/_types.py | 17 +++--------- litellm/proxy/proxy_server.py | 52 +++++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7f453980f..0ee23e18e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -236,6 +236,10 @@ class NewTeamRequest(LiteLLMBase): members: list = [] members_with_roles: List[Member] = [] metadata: Optional[dict] = None + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + max_budget: Optional[float] = None + models: list = [] class UpdateTeamRequest(LiteLLMBase): @@ -252,25 +256,12 @@ class DeleteTeamRequest(LiteLLMBase): class LiteLLM_TeamTable(NewTeamRequest): - max_budget: Optional[float] = None spend: Optional[float] = None - models: list = [] max_parallel_requests: Optional[int] = None - tpm_limit: Optional[int] = None - rpm_limit: Optional[int] = None budget_duration: Optional[str] = None budget_reset_at: Optional[datetime] = None -class NewTeamResponse(LiteLLMBase): - team_id: str - admins: list - members: list - metadata: dict - created_at: datetime - updated_at: datetime - - class TeamRequest(LiteLLMBase): teams: List[str] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 14f8bc96c..25e982065 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4314,7 +4314,7 @@ async def user_get_requests(): "/team/new", tags=["team management"], dependencies=[Depends(user_api_key_auth)], - response_model=NewTeamResponse, + response_model=LiteLLM_TeamTable, ) async def new_team( data: NewTeamRequest, @@ -4360,13 +4360,55 @@ async def new_team( if data.team_id is None: data.team_id = str(uuid.uuid4()) + if ( + data.tpm_limit is not None + and user_api_key_dict.tpm_limit is not None + and data.tpm_limit > user_api_key_dict.tpm_limit + ): + raise HTTPException( + status_code=400, + detail={ + "error": f"tpm limit higher than user max. User tpm limit={user_api_key_dict.tpm_limit}" + }, + ) + + if ( + data.rpm_limit is not None + and user_api_key_dict.rpm_limit is not None + and data.rpm_limit > user_api_key_dict.rpm_limit + ): + raise HTTPException( + status_code=400, + detail={ + "error": f"rpm limit higher than user max. User rpm limit={user_api_key_dict.rpm_limit}" + }, + ) + + if ( + data.max_budget is not None + and user_api_key_dict.max_budget is not None + and data.max_budget > user_api_key_dict.max_budget + ): + raise HTTPException( + status_code=400, + detail={ + "error": f"max budget higher than user max. User max budget={user_api_key_dict.max_budget}" + }, + ) + + if data.models is not None: + for m in data.models: + if m not in user_api_key_dict.models: + raise HTTPException( + status_code=400, + detail={ + "error": f"Model not in allowed user models. User allowed models={user_api_key_dict.models}" + }, + ) + complete_team_data = LiteLLM_TeamTable( **data.json(), - max_budget=user_api_key_dict.max_budget, - models=user_api_key_dict.models, max_parallel_requests=user_api_key_dict.max_parallel_requests, - tpm_limit=user_api_key_dict.tpm_limit, - rpm_limit=user_api_key_dict.rpm_limit, budget_duration=user_api_key_dict.budget_duration, budget_reset_at=user_api_key_dict.budget_reset_at, ) From 0b774200874a5f48a2e9a7554b2353b9607fcad5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 24 Feb 2024 11:41:42 -0800 Subject: [PATCH 2/8] docs(users.md): add team budgets to docs --- docs/my-website/docs/proxy/users.md | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/my-website/docs/proxy/users.md b/docs/my-website/docs/proxy/users.md index 159b311a9..9c8927caf 100644 --- a/docs/my-website/docs/proxy/users.md +++ b/docs/my-website/docs/proxy/users.md @@ -119,6 +119,51 @@ curl --location 'http://0.0.0.0:8000/key/generate' \ --data '{"models": ["azure-models"], "user_id": "krrish3@berri.ai"}' ``` + + +You can: +- Add budgets to Teams + + +#### **Add budgets to users** +```shell +curl --location 'http://localhost:8000/team/new' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "team_alias": "my-new-team_4", + "members_with_roles": [{"role": "admin", "user_id": "5c4a0aa3-a1e1-43dc-bd87-3c2da8382a3a"}], + "rpm_limit": 99 +}' +``` + +[**See Swagger**](https://litellm-api.up.railway.app/#/team%20management/new_team_team_new_post) + +**Sample Response** + +```shell +{ + "team_alias": "my-new-team_4", + "team_id": "13e83b19-f851-43fe-8e93-f96e21033100", + "admins": [], + "members": [], + "members_with_roles": [ + { + "role": "admin", + "user_id": "5c4a0aa3-a1e1-43dc-bd87-3c2da8382a3a" + } + ], + "metadata": {}, + "tpm_limit": null, + "rpm_limit": 99, + "max_budget": null, + "models": [], + "spend": 0.0, + "max_parallel_requests": null, + "budget_duration": null, + "budget_reset_at": null +} +``` From 785a4a2ac2a041854b9ee9eb67f38a8f0339263d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 24 Feb 2024 12:42:01 -0800 Subject: [PATCH 3/8] build(ui): show users the teams they're in --- .../src/components/dashboard_default_team.tsx | 40 +++++++++++++++++++ .../src/components/leftnav.tsx | 15 +++---- .../src/components/user_dashboard.tsx | 9 +++++ 3 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/dashboard_default_team.tsx diff --git a/ui/litellm-dashboard/src/components/dashboard_default_team.tsx b/ui/litellm-dashboard/src/components/dashboard_default_team.tsx new file mode 100644 index 000000000..8520096df --- /dev/null +++ b/ui/litellm-dashboard/src/components/dashboard_default_team.tsx @@ -0,0 +1,40 @@ +import React, { useState, useEffect } from "react"; +import { Typography } from "antd"; +import { Select, SelectItem } from "@tremor/react"; + +interface DashboardTeamProps { + teams: string[] | null; +} + +const DashboardTeam: React.FC = ({ teams }) => { + const { Title, Paragraph } = Typography; + const [value, setValue] = useState(""); + console.log(`received teams ${teams}`); + return ( +
+ Default Team + + If you belong to multiple teams, this setting controls which + organization is used by default when creating new API Keys. + + {teams && teams.length > 0 ? ( + + ) : ( + + No team created. Defaulting to personal account. + + )} +
+ ); +}; + +export default DashboardTeam; diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 8a13d2213..da7e9f432 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -30,13 +30,14 @@ const Sidebar: React.FC = ({ setPage, userRole }) => { setPage("usage")}> Usage - { - userRole == "Admin" ? - setPage("users")}> - Users - - : null - } + {userRole == "Admin" ? ( + setPage("users")}> + Users + + ) : null} + setPage("teams")}> + Teams + diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx index ad0070864..74ab3890c 100644 --- a/ui/litellm-dashboard/src/components/user_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx @@ -5,6 +5,7 @@ import { Grid, Col, Card, Text } from "@tremor/react"; import CreateKey from "./create_key_button"; import ViewKeyTable from "./view_key_table"; import ViewUserSpend from "./view_user_spend"; +import DashboardTeam from "./dashboard_default_team"; import EnterProxyUrl from "./enter_proxy_url"; import { message } from "antd"; import Navbar from "./navbar"; @@ -36,6 +37,7 @@ const UserDashboard: React.FC = ({ setUserEmail, }) => { const [data, setData] = useState(null); // Keep the initialization of state here + const [teams, setTeams] = useState(null); // Keep the initialization of state here const [userSpendData, setUserSpendData] = useState( null ); @@ -111,8 +113,14 @@ const UserDashboard: React.FC = ({ const fetchData = async () => { try { const response = await userInfoCall(accessToken, userID, userRole); + console.log( + `received teams in user dashboard: ${Object.keys( + response + )}; team type: ${Array.isArray(response.teams)}` + ); setUserSpendData(response["user_info"]); setData(response["keys"]); // Assuming this is the correct path to your data + setTeams(response["teams"]); sessionStorage.setItem( "userData" + userID, JSON.stringify(response["keys"]) @@ -191,6 +199,7 @@ const UserDashboard: React.FC = ({ data={data} setData={setData} /> + From ad9169a4da914921bc618a9f6db1b40e244cc237 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 24 Feb 2024 16:14:21 -0800 Subject: [PATCH 4/8] build(ui): add stubbed teams page --- ui/litellm-dashboard/src/app/page.tsx | 20 +- ui/litellm-dashboard/src/app/team/page.tsx | 149 ++++++++++ .../src/components/leftnav.tsx | 10 +- .../src/components/navbar.tsx | 13 +- ui/litellm-dashboard/src/components/teams.tsx | 274 ++++++++++++++++++ .../src/components/user_dashboard.tsx | 5 +- 6 files changed, 458 insertions(+), 13 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/team/page.tsx create mode 100644 ui/litellm-dashboard/src/components/teams.tsx diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 424ddfcec..cc3df26f9 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -5,14 +5,16 @@ import Navbar from "../components/navbar"; import UserDashboard from "../components/user_dashboard"; import ModelDashboard from "@/components/model_dashboard"; import ViewUserDashboard from "@/components/view_users"; +import Teams from "@/components/teams"; import ChatUI from "@/components/chat_ui"; import Sidebar from "../components/leftnav"; import Usage from "../components/usage"; import { jwtDecode } from "jwt-decode"; const CreateKeyPage = () => { - const [userRole, setUserRole] = useState(''); + const [userRole, setUserRole] = useState(""); const [userEmail, setUserEmail] = useState(null); + const [teams, setTeams] = useState(null); const searchParams = useSearchParams(); const userID = searchParams.get("userID"); @@ -74,14 +76,20 @@ const CreateKeyPage = () => {
- + {page == "api-keys" ? ( ) : page == "models" ? ( { token={token} accessToken={accessToken} /> - ) - : page == "users" ? ( + ) : page == "users" ? ( - ) - : ( + ) : page == "teams" ? ( + + ) : ( { + const [userRole, setUserRole] = useState(""); + const [userEmail, setUserEmail] = useState(null); + const [teams, setTeams] = useState(null); + const searchParams = useSearchParams(); + + const userID = searchParams.get("userID"); + const token = searchParams.get("token"); + + const [page, setPage] = useState("team"); + const [accessToken, setAccessToken] = useState(null); + + useEffect(() => { + if (token) { + const decoded = jwtDecode(token) as { [key: string]: any }; + if (decoded) { + // cast decoded to dictionary + console.log("Decoded token:", decoded); + + console.log("Decoded key:", decoded.key); + // set accessToken + setAccessToken(decoded.key); + + // check if userRole is defined + if (decoded.user_role) { + const formattedUserRole = formatUserRole(decoded.user_role); + console.log("Decoded user_role:", formattedUserRole); + setUserRole(formattedUserRole); + } else { + console.log("User role not defined"); + } + + if (decoded.user_email) { + setUserEmail(decoded.user_email); + } else { + console.log(`User Email is not set ${decoded}`); + } + } + } + }, [token]); + + function formatUserRole(userRole: string) { + if (!userRole) { + return "Undefined Role"; + } + console.log(`Received user role: ${userRole}`); + switch (userRole.toLowerCase()) { + case "app_owner": + return "App Owner"; + case "demo_app_owner": + return "App Owner"; + case "app_admin": + return "Admin"; + case "app_user": + return "App User"; + default: + return "Unknown Role"; + } + } + + return ( + Loading...
}> +
+ +
+ + + + + + + Team Name + Spend (USD) + Budget (USD) + TPM / RPM Limits + Settings + + + + + + Wilhelm Tell + 1 + Uri, Schwyz, Unterwalden + National Hero + + + + + + The Witcher + 129 + Kaedwen + Legend + + + + + + Mizutsune + 82 + Japan + N/A + + + + + +
+ + + + + + + + +
+
+ + ); +}; +export default TeamSettingsPage; diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index da7e9f432..59c0897bf 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -1,5 +1,6 @@ import { Layout, Menu } from "antd"; import Link from "next/link"; +import { List } from "postcss/lib/list"; const { Sider } = Layout; @@ -7,15 +8,20 @@ const { Sider } = Layout; interface SidebarProps { setPage: React.Dispatch>; userRole: string; + defaultSelectedKey: string[] | null; } -const Sidebar: React.FC = ({ setPage, userRole }) => { +const Sidebar: React.FC = ({ + setPage, + userRole, + defaultSelectedKey, +}) => { return ( setPage("api-keys")}> diff --git a/ui/litellm-dashboard/src/components/navbar.tsx b/ui/litellm-dashboard/src/components/navbar.tsx index 3546bcc5f..bb13d100f 100644 --- a/ui/litellm-dashboard/src/components/navbar.tsx +++ b/ui/litellm-dashboard/src/components/navbar.tsx @@ -29,14 +29,19 @@ const Navbar: React.FC = ({ userID, userRole, userEmail }) => { const isLocal = process.env.NODE_ENV === "development"; const imageUrl = isLocal ? "http://localhost:4000/get_image" : "/get_image"; - return (