From 7f0d85f12a94593dd6e0241617b500402eb56486 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 22 May 2024 16:20:06 -0700 Subject: [PATCH] feat(proxy_server.py): enable admin to create/delete budgets for end-users --- litellm/proxy/_types.py | 4 + litellm/proxy/proxy_server.py | 136 ++++++++++++ ui/litellm-dashboard/src/app/page.tsx | 32 +-- .../src/components/budgets/budget_modal.tsx | 140 ++++++++++++ .../src/components/budgets/budget_panel.tsx | 201 ++++++++++++++++++ .../components/budgets/budget_settings.tsx | 195 +++++++++++++++++ .../src/components/create_user_button.tsx | 31 ++- .../src/components/leftnav.tsx | 111 ++++------ .../src/components/networking.tsx | 141 ++++++++++++ .../src/components/view_users.tsx | 62 +++--- 10 files changed, 931 insertions(+), 122 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/budgets/budget_modal.tsx create mode 100644 ui/litellm-dashboard/src/components/budgets/budget_panel.tsx create mode 100644 ui/litellm-dashboard/src/components/budgets/budget_settings.tsx diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8bfa56004..8cdf6de72 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -705,6 +705,10 @@ class BudgetRequest(LiteLLMBase): budgets: List[str] +class BudgetDeleteRequest(LiteLLMBase): + id: str + + class KeyManagementSystem(enum.Enum): GOOGLE_KMS = "google_kms" AZURE_KEY_VAULT = "azure_key_vault" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f157f420c..e9edd3836 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8248,6 +8248,142 @@ async def info_budget(data: BudgetRequest): return response +@router.get( + "/budget/settings", + tags=["budget management"], + dependencies=[Depends(user_api_key_auth)], +) +async def budget_settings( + budget_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get list of configurable params + current value for a budget item + description of each field + + Used on Admin UI. + """ + if prisma_client is None: + raise HTTPException( + status_code=400, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + if user_api_key_dict.user_role != "proxy_admin": + raise HTTPException( + status_code=400, + detail={ + "error": "{}, your role={}".format( + CommonProxyErrors.not_allowed_access.value, + user_api_key_dict.user_role, + ) + }, + ) + + ## get budget item from db + db_budget_row = await prisma_client.db.litellm_budgettable.find_first( + where={"budget_id": budget_id} + ) + + if db_budget_row is not None: + db_budget_row_dict = db_budget_row.model_dump(exclude_none=True) + else: + db_budget_row_dict = {} + + allowed_args = { + "max_parallel_requests": {"type": "Integer"}, + "tpm_limit": {"type": "Integer"}, + "rpm_limit": {"type": "Integer"}, + "budget_duration": {"type": "String"}, + "max_budget": {"type": "Float"}, + "soft_budget": {"type": "Float"}, + } + + return_val = [] + + for field_name, field_info in BudgetNew.model_fields.items(): + if field_name in allowed_args: + + _stored_in_db = True + + _response_obj = ConfigList( + field_name=field_name, + field_type=allowed_args[field_name]["type"], + field_description=field_info.description or "", + field_value=db_budget_row_dict.get(field_name, None), + stored_in_db=_stored_in_db, + ) + return_val.append(_response_obj) + + return return_val + + +@router.get( + "/budget/list", + tags=["budget management"], + dependencies=[Depends(user_api_key_auth)], +) +async def list_budget( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """List all the created budgets in proxy db. Used on Admin UI.""" + if prisma_client is None: + raise HTTPException( + status_code=400, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + if user_api_key_dict.user_role != "proxy_admin": + raise HTTPException( + status_code=400, + detail={ + "error": "{}, your role={}".format( + CommonProxyErrors.not_allowed_access.value, + user_api_key_dict.user_role, + ) + }, + ) + + response = await prisma_client.db.litellm_budgettable.find_many() + + return response + + +@router.post( + "/budget/delete", + tags=["budget management"], + dependencies=[Depends(user_api_key_auth)], +) +async def delete_budget( + data: BudgetDeleteRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """Delete budget""" + global prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + if user_api_key_dict.user_role != "proxy_admin": + raise HTTPException( + status_code=400, + detail={ + "error": "{}, your role={}".format( + CommonProxyErrors.not_allowed_access.value, + user_api_key_dict.user_role, + ) + }, + ) + + response = await prisma_client.db.litellm_budgettable.delete( + where={"budget_id": data.id} + ) + + return response + + #### MODEL MANAGEMENT #### diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index ce6925081..7df196d9b 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -9,6 +9,7 @@ import Teams from "@/components/teams"; import AdminPanel from "@/components/admins"; import Settings from "@/components/settings"; import GeneralSettings from "@/components/general_settings"; +import BudgetPanel from "@/components/budgets/budget_panel"; import APIRef from "@/components/api_ref"; import ChatUI from "@/components/chat_ui"; import Sidebar from "../components/leftnav"; @@ -106,14 +107,13 @@ const CreateKeyPage = () => { />
- - +
- + {page == "api-keys" ? ( { showSSOBanner={showSSOBanner} /> ) : page == "api_ref" ? ( - + ) : page == "settings" ? ( - ) : page == "general-settings" ? ( - + ) : page == "budgets" ? ( + + ) : page == "general-settings" ? ( + ) : ( >; + setBudgetList: React.Dispatch>; +} +const BudgetModal: React.FC = ({ + isModalVisible, + accessToken, + setIsModalVisible, + setBudgetList, +}) => { + const [form] = Form.useForm(); + const handleOk = () => { + setIsModalVisible(false); + form.resetFields(); + }; + + const handleCancel = () => { + setIsModalVisible(false); + form.resetFields(); + }; + + const handleCreate = async (formValues: Record) => { + if (accessToken == null || accessToken == undefined) { + return; + } + try { + message.info("Making API Call"); + // setIsModalVisible(true); + const response = await budgetCreateCall(accessToken, formValues); + console.log("key create Response:", response); + setBudgetList((prevData) => + prevData ? [...prevData, response] : [response] + ); // Check if prevData is null + message.success("API Key Created"); + form.resetFields(); + } catch (error) { + console.error("Error creating the key:", error); + message.error(`Error creating the key: ${error}`, 20); + } + }; + + return ( + +
+ <> + + + + + + + + + + + + + Optional Settings + + + + + + + + + + + + +
+ Create Budget +
+
+ + ); +}; + +export default BudgetModal; diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx new file mode 100644 index 000000000..705597ab2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx @@ -0,0 +1,201 @@ +/** + * The parent pane, showing list of budgets + * + */ + +import React, { useState, useEffect } from "react"; +import BudgetSettings from "./budget_settings"; +import BudgetModal from "./budget_modal"; +import { + Table, + TableBody, + TableCell, + TableFoot, + TableHead, + TableHeaderCell, + TableRow, + Card, + Button, + Icon, + Text, + Tab, + TabGroup, + TabList, + TabPanel, + TabPanels, + Grid, +} from "@tremor/react"; +import { + InformationCircleIcon, + PencilAltIcon, + PencilIcon, + StatusOnlineIcon, + TrashIcon, + RefreshIcon, + CheckCircleIcon, + XCircleIcon, + QuestionMarkCircleIcon, +} from "@heroicons/react/outline"; +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { getBudgetList, budgetDeleteCall } from "../networking"; +import { message } from "antd"; +interface BudgetSettingsPageProps { + accessToken: string | null; +} + +interface budgetItem { + budget_id: string; + max_budget: string | null; + rpm_limit: number | null; + tpm_limit: number | null; +} + +const BudgetPanel: React.FC = ({ accessToken }) => { + const [isModalVisible, setIsModalVisible] = useState(false); + const [budgetList, setBudgetList] = useState([]); + useEffect(() => { + if (!accessToken) { + return; + } + getBudgetList(accessToken).then((data) => { + setBudgetList(data); + }); + }, [accessToken]); + + const handleDeleteCall = async (budget_id: string, index: number) => { + if (accessToken == null) { + return; + } + + message.info("Request made"); + + await budgetDeleteCall(accessToken, budget_id); + + const newBudgetList = [...budgetList]; + newBudgetList.splice(index, 1); + setBudgetList(newBudgetList); + + message.success("Budget Deleted."); + }; + + return ( +
+ + + + Create a budget to assign to customers. + + + + Budget ID + Max Budget + TPM + RPM + + + + + {budgetList.map((value: budgetItem, index: number) => ( + + {value.budget_id} + + {value.max_budget ? value.max_budget : "n/a"} + + + {value.tpm_limit ? value.tpm_limit : "n/a"} + + + {value.rpm_limit ? value.rpm_limit : "n/a"} + + handleDeleteCall(value.budget_id, index)} + /> + + ))} + +
+ +
+ How to use budget id + + + Assign Budget to Customer + Test it (Curl) + + Test it (OpenAI SDK) + + + + + {` +curl -X POST --location '/end_user/new' \ + +-H 'Authorization: Bearer ' \ + +-H 'Content-Type: application/json' \ + +-d '{"user_id": "my-customer-id', "budget_id": ""}' # 👈 KEY CHANGE + + `} + + + + + {` +curl -X POST --location '/chat/completions' \ + +-H 'Authorization: Bearer ' \ + +-H 'Content-Type: application/json' \ + +-d '{ + "model": "gpt-3.5-turbo', + "messages":[{"role": "user", "content": "Hey, how's it going?"}], + "user": "my-customer-id" +}' # 👈 KEY CHANGE + + `} + + + + + {`from openai import OpenAI +client = OpenAI( + base_url=" + + + +
+
+ ); +}; + +export default BudgetPanel; diff --git a/ui/litellm-dashboard/src/components/budgets/budget_settings.tsx b/ui/litellm-dashboard/src/components/budgets/budget_settings.tsx new file mode 100644 index 000000000..7f31ac248 --- /dev/null +++ b/ui/litellm-dashboard/src/components/budgets/budget_settings.tsx @@ -0,0 +1,195 @@ +import React, { useState, useEffect } from "react"; +import { + Card, + Title, + Subtitle, + Table, + TableHead, + TableRow, + Badge, + TableHeaderCell, + TableCell, + TableBody, + Metric, + Text, + Grid, + Button, + TextInput, + Select as Select2, + SelectItem, + Col, + Accordion, + AccordionBody, + AccordionHeader, + AccordionList, +} from "@tremor/react"; +import { + TabPanel, + TabPanels, + TabGroup, + TabList, + Tab, + Icon, +} from "@tremor/react"; +import { getBudgetSettings } from "../networking"; +import { + Modal, + Form, + Input, + Select, + Button as Button2, + message, + InputNumber, +} from "antd"; +import { + InformationCircleIcon, + PencilAltIcon, + PencilIcon, + StatusOnlineIcon, + TrashIcon, + RefreshIcon, + CheckCircleIcon, + XCircleIcon, + QuestionMarkCircleIcon, +} from "@heroicons/react/outline"; +import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider"; +import AddFallbacks from "../add_fallbacks"; +import openai from "openai"; +import Paragraph from "antd/es/skeleton/Paragraph"; + +interface BudgetSettingsPageProps { + accessToken: string | null; +} + +interface budgetSettingsItem { + field_name: string; + field_type: string; + field_value: any; + field_description: string; +} + +const BudgetSettings: React.FC = ({ accessToken }) => { + const [budgetSettings, setBudgetSettings] = useState( + [] + ); + useEffect(() => { + if (!accessToken) { + return; + } + getBudgetSettings(accessToken).then((data) => { + console.log("budget settings", data); + let budget_settings = data.budget_settings; + setBudgetSettings(budget_settings); + }); + }, [accessToken]); + + const handleInputChange = (fieldName: string, newValue: any) => { + // Update the value in the state + const updatedSettings = budgetSettings.map((setting) => + setting.field_name === fieldName + ? { ...setting, field_value: newValue } + : setting + ); + setBudgetSettings(updatedSettings); + }; + + const handleUpdateField = (fieldName: string, idx: number) => { + if (!accessToken) { + return; + } + + let fieldValue = budgetSettings[idx].field_value; + + if (fieldValue == null || fieldValue == undefined) { + return; + } + try { + const updatedSettings = budgetSettings.map((setting) => + setting.field_name === fieldName + ? { ...setting, stored_in_db: true } + : setting + ); + setBudgetSettings(updatedSettings); + } catch (error) { + // do something + } + }; + + const handleResetField = (fieldName: string, idx: number) => { + if (!accessToken) { + return; + } + + try { + const updatedSettings = budgetSettings.map((setting) => + setting.field_name === fieldName + ? { ...setting, stored_in_db: null, field_value: null } + : setting + ); + setBudgetSettings(updatedSettings); + } catch (error) { + // do something + } + }; + + return ( +
+ + + + + Setting + Value + + + + {budgetSettings.map((value, index) => ( + + + {value.field_name} +

+ {value.field_description} +

+ + + {value.field_type == "Integer" ? ( + + handleInputChange(value.field_name, newValue) + } // Handle value change + /> + ) : null} + + + + handleResetField(value.field_name, index)} + > + Reset + + + + ))} + +
+ +
+ ); +}; + +export default BudgetSettings; diff --git a/ui/litellm-dashboard/src/components/create_user_button.tsx b/ui/litellm-dashboard/src/components/create_user_button.tsx index c0332b35c..512ff23c7 100644 --- a/ui/litellm-dashboard/src/components/create_user_button.tsx +++ b/ui/litellm-dashboard/src/components/create_user_button.tsx @@ -10,7 +10,11 @@ interface CreateuserProps { teams: any[] | null; } -const Createuser: React.FC = ({ userID, accessToken, teams }) => { +const Createuser: React.FC = ({ + userID, + accessToken, + teams, +}) => { const [form] = Form.useForm(); const [isModalVisible, setIsModalVisible] = useState(false); const [apiuser, setApiuser] = useState(null); @@ -73,7 +77,7 @@ const Createuser: React.FC = ({ userID, accessToken, teams }) = return (
- setIsModalVisible(true)}> + setIsModalVisible(true)}> + Invite User = ({ userID, accessToken, teams }) = onOk={handleOk} onCancel={handleCancel} > - Invite a user to login to the Admin UI and create Keys - Note: SSO Setup Required for this + + Invite a user to login to the Admin UI and create Keys + + + Note: SSO Setup Required for this +
= ({ userID, accessToken, teams }) = - {teams ? ( teams.map((team: any) => (
diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index c4296f18e..ec2f6b478 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -1,7 +1,7 @@ import { Layout, Menu } from "antd"; import Link from "next/link"; import { List } from "postcss/lib/list"; -import { Text } from "@tremor/react" +import { Text } from "@tremor/react"; const { Sider } = Layout; @@ -54,88 +54,61 @@ const Sidebar: React.FC = ({ style={{ height: "100%", borderRight: 0 }} > setPage("api-keys")}> - - API Keys - + API Keys setPage("llm-playground")}> - - Test Key - + Test Key - - - { - userRole == "Admin" ? ( - setPage("models")}> - - Models - - - ) : null - } - { - userRole == "Admin" ? ( - setPage("usage")}> - - Usage - + {userRole == "Admin" ? ( + setPage("models")}> + Models - - ) : null - } + ) : null} + {userRole == "Admin" ? ( + setPage("usage")}> + Usage + + ) : null} {userRole == "Admin" ? ( setPage("teams")}> - - Teams - + Teams ) : null} - - - {userRole == "Admin" ? ( - setPage("users")}> - - Users - - - ) : null} - - { - userRole == "Admin" ? ( - setPage("settings")}> - - Logging & Alerts - - - ) : null - } - - { - userRole == "Admin" ? ( - setPage("general-settings")}> - - Router Settings - - - ) : null - } - {userRole == "Admin" ? ( - setPage("admin-panel")}> - - Admin - + setPage("users")}> + Users ) : null} - setPage("api_ref")}> - - API Reference - - + + {userRole == "Admin" ? ( + setPage("settings")}> + Logging & Alerts + + ) : null} + + {userRole == "Admin" ? ( + setPage("budgets")}> + Rate Limits + + ) : null} + + {userRole == "Admin" ? ( + setPage("general-settings")}> + Router Settings + + ) : null} + + {userRole == "Admin" ? ( + setPage("admin-panel")}> + Admin + + ) : null} + setPage("api_ref")}> + API Reference + diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index bb7899755..06e0a0692 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -97,6 +97,82 @@ export const modelDeleteCall = async ( } }; +export const budgetDeleteCall = async ( + accessToken: string | null, + budget_id: string +) => { + console.log(`budget_id in budget delete call: ${budget_id}`); + + if (accessToken == null) { + return; + } + + try { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/budget/delete` + : `/budget/delete`; + const response = await fetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: budget_id, + }), + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error("Failed to create key: " + errorData, 10); + console.error("Error response from the server:", errorData); + throw new Error("Network response was not ok"); + } + const data = await response.json(); + console.log("API Response:", data); + return data; + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + +export const budgetCreateCall = async ( + accessToken: string, + formValues: Record // Assuming formValues is an object +) => { + try { + console.log("Form Values in budgetCreateCall:", formValues); // Log the form values before making the API call + + console.log("Form Values after check:", formValues); + const url = proxyBaseUrl ? `${proxyBaseUrl}/budget/new` : `/budget/new`; + const response = await fetch(url, { + method: "POST", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + ...formValues, // Include formValues in the request body + }), + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error("Failed to create key: " + errorData, 10); + console.error("Error response from the server:", errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + console.log("API Response:", data); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; export const keyCreateCall = async ( accessToken: string, userID: string, @@ -1382,6 +1458,71 @@ export const serviceHealthCheck = async ( } }; +export const getBudgetList = async (accessToken: String) => { + /** + * Get all configurable params for setting a budget + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/budget/list` : `/budget/list`; + + //message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData, 10); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + //message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to get callbacks:", error); + throw error; + } +}; +export const getBudgetSettings = async (accessToken: String) => { + /** + * Get all configurable params for setting a budget + */ + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/budget/settings` + : `/budget/settings`; + + //message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData, 10); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + //message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to get callbacks:", error); + throw error; + } +}; + export const getCallbacksCall = async ( accessToken: String, userID: String, diff --git a/ui/litellm-dashboard/src/components/view_users.tsx b/ui/litellm-dashboard/src/components/view_users.tsx index 41c15be9c..3faf4c026 100644 --- a/ui/litellm-dashboard/src/components/view_users.tsx +++ b/ui/litellm-dashboard/src/components/view_users.tsx @@ -19,7 +19,7 @@ import { TabPanel, Select, SelectItem, - Dialog, + Dialog, DialogPanel, Icon, TextInput, @@ -82,8 +82,6 @@ const ViewUserDashboard: React.FC = ({ if (accessToken && token && userRole && userID) { fetchData(); } - - }, [accessToken, token, userRole, userID, currentPage]); if (!userData) { @@ -102,7 +100,7 @@ const ViewUserDashboard: React.FC = ({ return (
- Showing Page {currentPage+1} of {totalPages} + Showing Page {currentPage + 1} of {totalPages}