diff --git a/litellm/__init__.py b/litellm/__init__.py index e75f635f3..3ba337415 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -165,6 +165,7 @@ s3_callback_params: Optional[Dict] = None generic_logger_headers: Optional[Dict] = None default_key_generate_params: Optional[Dict] = None upperbound_key_generate_params: Optional[Dict] = None +default_user_params: Optional[Dict] = None default_team_settings: Optional[List] = None max_user_budget: Optional[float] = None #### RELIABILITY #### diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1a8e4dddc..4e425bd7f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -730,6 +730,8 @@ async def user_api_key_auth( "/user", "/model/info", "/v2/model/info", + "/models", + "/v1/models", ] # check if the current route startswith any of the allowed routes if ( @@ -1758,6 +1760,7 @@ async def generate_key_helper_fn( allowed_cache_controls: Optional[list] = [], permissions: Optional[dict] = {}, model_max_budget: Optional[dict] = {}, + table_name: Optional[Literal["key", "user"]] = None, ): global prisma_client, custom_db_client, user_api_key_cache @@ -1884,8 +1887,10 @@ async def generate_key_helper_fn( table_name="user", update_key_values=update_key_values, ) - if user_id == litellm_proxy_budget_name: - # do not create a key for litellm_proxy_budget_name + if user_id == litellm_proxy_budget_name or ( + table_name is not None and table_name == "user" + ): + # do not create a key for litellm_proxy_budget_name or if table name is set to just 'user' # we only need to ensure this exists in the user table # the LiteLLM_VerificationToken table will increase in size if we don't do this check return key_data @@ -5461,27 +5466,50 @@ async def auth_callback(request: Request): user_id_models: List = [] # User might not be already created on first generation of key - # But if it is, we want its models preferences + # But if it is, we want their models preferences + default_ui_key_values = { + "duration": "1hr", + "key_max_budget": 0.01, + "aliases": {}, + "config": {}, + "spend": 0, + "team_id": "litellm-dashboard", + } + user_defined_values = { + "models": user_id_models, + "user_id": user_id, + "user_email": user_email, + } try: if prisma_client is not None: user_info = await prisma_client.get_data(user_id=user_id, table_name="user") + verbose_proxy_logger.debug( + f"user_info: {user_info}; litellm.default_user_params: {litellm.default_user_params}" + ) if user_info is not None: - user_id_models = getattr(user_info, "models", []) + user_defined_values = { + "models": getattr(user_info, "models", []), + "user_id": getattr(user_info, "user_id", user_id), + "user_email": getattr(user_info, "user_id", user_email), + } + elif litellm.default_user_params is not None and isinstance( + litellm.default_user_params, dict + ): + user_defined_values = { + "models": litellm.default_user_params.get("models", user_id_models), + "user_id": litellm.default_user_params.get("user_id", user_id), + "user_email": litellm.default_user_params.get( + "user_email", user_email + ), + } except Exception as e: pass + verbose_proxy_logger.info( + f"user_defined_values for creating ui key: {user_defined_values}" + ) response = await generate_key_helper_fn( - **{ - "duration": "1hr", - "key_max_budget": 0.01, - "models": user_id_models, - "aliases": {}, - "config": {}, - "spend": 0, - "user_id": user_id, - "team_id": "litellm-dashboard", - "user_email": user_email, - } # type: ignore + **default_ui_key_values, **user_defined_values # type: ignore ) key = response["token"] # type: ignore user_id = response["user_id"] # type: ignore diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index edab67b2b..102354222 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -1,17 +1,26 @@ import React, { useState, useEffect } from "react"; import ReactMarkdown from "react-markdown"; -import { Card, Title, Table, TableHead, TableRow, TableCell, TableBody, Grid, Tab, - TabGroup, - TabList, - TabPanel, - Metric, - Select, - SelectItem, - TabPanels, } from "@tremor/react"; -import { modelInfoCall } from "./networking"; +import { + Card, + Title, + Table, + TableHead, + TableRow, + TableCell, + TableBody, + Grid, + Tab, + TabGroup, + TabList, + TabPanel, + Metric, + Select, + SelectItem, + TabPanels, +} from "@tremor/react"; +import { modelAvailableCall } from "./networking"; import openai from "openai"; -import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'; - +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; interface ChatUIProps { accessToken: string | null; @@ -20,12 +29,19 @@ interface ChatUIProps { userID: string | null; } -async function generateModelResponse(inputMessage: string, updateUI: (chunk: string) => void, selectedModel: string, accessToken: string) { - // base url should be the current base_url - const isLocal = process.env.NODE_ENV === "development"; - console.log("isLocal:", isLocal); - const proxyBaseUrl = isLocal ? "http://localhost:4000" : window.location.origin; - const client = new openai.OpenAI({ +async function generateModelResponse( + inputMessage: string, + updateUI: (chunk: string) => void, + selectedModel: string, + accessToken: string +) { + // base url should be the current base_url + const isLocal = process.env.NODE_ENV === "development"; + console.log("isLocal:", isLocal); + const proxyBaseUrl = isLocal + ? "http://localhost:4000" + : window.location.origin; + const client = new openai.OpenAI({ apiKey: accessToken, // Replace with your OpenAI API key baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL dangerouslyAllowBrowser: true, // using a temporary litellm proxy key @@ -36,7 +52,7 @@ async function generateModelResponse(inputMessage: string, updateUI: (chunk: str stream: true, messages: [ { - role: 'user', + role: "user", content: inputMessage, }, ], @@ -50,138 +66,166 @@ async function generateModelResponse(inputMessage: string, updateUI: (chunk: str } } -const ChatUI: React.FC = ({ accessToken, token, userRole, userID }) => { - const [inputMessage, setInputMessage] = useState(""); - const [chatHistory, setChatHistory] = useState([]); - const [selectedModel, setSelectedModel] = useState(undefined); - const [modelInfo, setModelInfo] = useState(null); // Declare modelInfo at the component level +const ChatUI: React.FC = ({ + accessToken, + token, + userRole, + userID, +}) => { + const [inputMessage, setInputMessage] = useState(""); + const [chatHistory, setChatHistory] = useState([]); + const [selectedModel, setSelectedModel] = useState( + undefined + ); + const [modelInfo, setModelInfo] = useState(null); // Declare modelInfo at the component level - useEffect(() => { - if (!accessToken || !token || !userRole || !userID) { - return; - } - // Fetch model info and set the default selected model - const fetchModelInfo = async () => { - const fetchedModelInfo = await modelInfoCall(accessToken, userID, userRole); - console.log("model_info:", fetchedModelInfo); - - if (fetchedModelInfo?.data.length > 0) { - setModelInfo(fetchedModelInfo); - setSelectedModel(fetchedModelInfo.data[0].model_name); - } - }; - - fetchModelInfo(); - }, [accessToken, userID, userRole]); - - const updateUI = (role: string, chunk: string) => { - setChatHistory((prevHistory) => { - const lastMessage = prevHistory[prevHistory.length - 1]; - - if (lastMessage && lastMessage.role === role) { - return [ - ...prevHistory.slice(0, prevHistory.length - 1), - { role, content: lastMessage.content + chunk }, - ]; - } else { - return [...prevHistory, { role, content: chunk }]; - } - }); - }; - - const handleSendMessage = async () => { - if (inputMessage.trim() === "") return; - - if (!accessToken || !token || !userRole || !userID) { - return; + useEffect(() => { + if (!accessToken || !token || !userRole || !userID) { + return; + } + // Fetch model info and set the default selected model + const fetchModelInfo = async () => { + const fetchedAvailableModels = await modelAvailableCall( + accessToken, + userID, + userRole + ); + console.log("model_info:", fetchedAvailableModels); + + if (fetchedAvailableModels?.data.length > 0) { + setModelInfo(fetchedAvailableModels.data); + setSelectedModel(fetchedAvailableModels.data[0].id); } - - setChatHistory((prevHistory) => [ - ...prevHistory, - { role: "user", content: inputMessage }, - ]); - - try { - if (selectedModel) { - await generateModelResponse(inputMessage, (chunk) => updateUI("assistant", chunk), selectedModel, accessToken); - } - } catch (error) { - console.error("Error fetching model response", error); - updateUI("assistant", "Error fetching model response"); - } - - setInputMessage(""); }; - - return ( -
- - + + fetchModelInfo(); + }, [accessToken, userID, userRole]); + + const updateUI = (role: string, chunk: string) => { + setChatHistory((prevHistory) => { + const lastMessage = prevHistory[prevHistory.length - 1]; + + if (lastMessage && lastMessage.role === role) { + return [ + ...prevHistory.slice(0, prevHistory.length - 1), + { role, content: lastMessage.content + chunk }, + ]; + } else { + return [...prevHistory, { role, content: chunk }]; + } + }); + }; + + const handleSendMessage = async () => { + if (inputMessage.trim() === "") return; + + if (!accessToken || !token || !userRole || !userID) { + return; + } + + setChatHistory((prevHistory) => [ + ...prevHistory, + { role: "user", content: inputMessage }, + ]); + + try { + if (selectedModel) { + await generateModelResponse( + inputMessage, + (chunk) => updateUI("assistant", chunk), + selectedModel, + accessToken + ); + } + } catch (error) { + console.error("Error fetching model response", error); + updateUI("assistant", "Error fetching model response"); + } + + setInputMessage(""); + }; + + return ( +
+ + - Chat - API Reference + Chat + API Reference - -
- - setSelectedModel(e.target.value)} + > + {/* Populate dropdown options from available models */} + {modelInfo?.map((element: { id: string }) => ( + + ))} + +
+ + + + + Chat + + + + + {chatHistory.map((message, index) => ( + + {`${message.role}: ${message.content}`} + + ))} + +
+
+
+ setInputMessage(e.target.value)} + className="flex-1 p-2 border rounded-md mr-2" + placeholder="Type your message..." + /> +
- - - - - Chat - - - - - {chatHistory.map((message, index) => ( - - {`${message.role}: ${message.content}`} - - ))} - -
-
-
- setInputMessage(e.target.value)} - className="flex-1 p-2 border rounded-md mr-2" - placeholder="Type your message..." - /> - -
-
- - - - - OpenAI Python SDK - LlamaIndex - Langchain Py - - - - - - {` + Send + +
+
+ + + + + OpenAI Python SDK + LlamaIndex + Langchain Py + + + + + {` import openai client = openai.OpenAI( api_key="your_api_key", @@ -208,12 +252,11 @@ response = client.chat.completions.create( print(response) `} - - - - - - {` + + + + + {` import os, dotenv from llama_index.llms import AzureOpenAI @@ -245,12 +288,11 @@ response = query_engine.query("What did the author do growing up?") print(response) `} - - - - - - {` + + + + + {` from langchain.chat_models import ChatOpenAI from langchain.prompts.chat import ( ChatPromptTemplate, @@ -286,20 +328,17 @@ response = chat(messages) print(response) `} - - - - - - + + + + + - - - -
- ); - }; + + + + + ); +}; - - - export default ChatUI; \ No newline at end of file +export default ChatUI; diff --git a/ui/litellm-dashboard/src/components/create_key_button.tsx b/ui/litellm-dashboard/src/components/create_key_button.tsx index 113952ea8..03ef9a89e 100644 --- a/ui/litellm-dashboard/src/components/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/create_key_button.tsx @@ -1,10 +1,17 @@ - "use client"; import React, { useState, useEffect, useRef } from "react"; import { Button, TextInput, Grid, Col } from "@tremor/react"; import { Card, Metric, Text } from "@tremor/react"; -import { Button as Button2, Modal, Form, Input, InputNumber, Select, message } from "antd"; +import { + Button as Button2, + Modal, + Form, + Input, + InputNumber, + Select, + message, +} from "antd"; import { keyCreateCall } from "./networking"; const { Option } = Select; @@ -50,12 +57,11 @@ const CreateKey: React.FC = ({ setApiKey(response["key"]); message.success("API Key Created"); form.resetFields(); - localStorage.removeItem("userData" + userID) + localStorage.removeItem("userData" + userID); } catch (error) { console.error("Error creating the key:", error); } }; - return (
@@ -70,98 +76,70 @@ const CreateKey: React.FC = ({ onOk={handleOk} onCancel={handleCancel} > -
- {userRole === 'App Owner' || userRole === 'Admin' ? ( + + {userRole === "App Owner" || userRole === "Admin" ? ( <> - - - - - - - - - - - - - - - - - - - - - - - - - - - ) : ( - <> + + + + + + + + + + + + + + + - - - - - + label="Requests per minute Limit (RPM)" + name="rpm_limit" + > + + + + + + + + + + ) : ( + <> + + + + + + - - - - - ) - } -
- - Create Key - -
+ + + + + )} +
+ Create Key +
{apiKey && ( @@ -173,22 +151,22 @@ const CreateKey: React.FC = ({ footer={null} > - -

- Please save this secret key somewhere safe and accessible. For - security reasons, you will not be able to view it again{" "} - through your LiteLLM account. If you lose this secret key, you will - need to generate a new one. -

- - - {apiKey != null ? ( - API Key: {apiKey} - ) : ( - Key being created, this might take 30s - )} - - + +

+ Please save this secret key somewhere safe and accessible. For + security reasons, you will not be able to view it again{" "} + through your LiteLLM account. If you lose this secret key, you + will need to generate a new one. +

+ + + {apiKey != null ? ( + API Key: {apiKey} + ) : ( + Key being created, this might take 30s + )} + + )}
diff --git a/ui/litellm-dashboard/src/components/create_user_button.tsx b/ui/litellm-dashboard/src/components/create_user_button.tsx index b58674d12..94ddcabb0 100644 --- a/ui/litellm-dashboard/src/components/create_user_button.tsx +++ b/ui/litellm-dashboard/src/components/create_user_button.tsx @@ -1,7 +1,7 @@ import React, { useState, useEffect } from "react"; import { Button, Modal, Form, Input, message, Select, InputNumber } from "antd"; import { Button as Button2 } from "@tremor/react"; -import { userCreateCall, modelInfoCall } from "./networking"; +import { userCreateCall, modelAvailableCall } from "./networking"; const { Option } = Select; interface CreateuserProps { @@ -14,18 +14,22 @@ const Createuser: React.FC = ({ userID, accessToken }) => { const [isModalVisible, setIsModalVisible] = useState(false); const [apiuser, setApiuser] = useState(null); const [userModels, setUserModels] = useState([]); - + // get all models useEffect(() => { const fetchData = async () => { try { const userRole = "any"; // You may need to get the user role dynamically - const modelDataResponse = await modelInfoCall(accessToken, userID, userRole); + const modelDataResponse = await modelAvailableCall( + accessToken, + userID, + userRole + ); // Assuming modelDataResponse.data contains an array of model objects with a 'model_name' property const availableModels = []; for (let i = 0; i < modelDataResponse.data.length; i++) { - const model = modelDataResponse.data[i]; - availableModels.push(model.model_name); + const model = modelDataResponse.data[i]; + availableModels.push(model.id); } console.log("Model data response:", modelDataResponse.data); console.log("Available models:", availableModels); @@ -79,77 +83,51 @@ const Createuser: React.FC = ({ userID, accessToken }) => { onOk={handleOk} onCancel={handleCancel} > -
- + + - - - - + + + + - - + - - - - - - - - - - - - - - - - - - - - -
- -
+ + + + + + + + + + + + + + + +
+ +
{apiuser && ( @@ -162,11 +140,15 @@ const Createuser: React.FC = ({ userID, accessToken }) => { >

Please save this secret user somewhere safe and accessible. For - security reasons, you will not be able to view it again through - your LiteLLM account. If you lose this secret user, you will need to - generate a new one. + security reasons, you will not be able to view it again{" "} + through your LiteLLM account. If you lose this secret user, you will + need to generate a new one. +

+

+ {apiuser != null + ? `API user: ${apiuser}` + : "User being created, this might take 30s"}

-

{apiuser != null ? `API user: ${apiuser}` : "User being created, this might take 30s"}

)} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 250515eeb..92b285848 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -69,7 +69,6 @@ export const keyCreateCall = async ( } }; - export const userCreateCall = async ( accessToken: string, userID: string, @@ -133,7 +132,6 @@ export const userCreateCall = async ( } }; - export const keyDeleteCall = async (accessToken: String, user_key: String) => { try { const url = proxyBaseUrl ? `${proxyBaseUrl}/key/delete` : `/key/delete`; @@ -207,13 +205,14 @@ export const userInfoCall = async ( } }; - - export const modelInfoCall = async ( accessToken: String, userID: String, userRole: String ) => { + /** + * Get all models on proxy + */ try { let url = proxyBaseUrl ? `${proxyBaseUrl}/v2/model/info` : `/v2/model/info`; @@ -242,6 +241,42 @@ export const modelInfoCall = async ( } }; +export const modelAvailableCall = async ( + accessToken: String, + userID: String, + userRole: String +) => { + /** + * Get all the models user has access to + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/models` : `/models`; + + message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + export const keySpendLogsCall = async (accessToken: String, token: String) => { try { const url = proxyBaseUrl ? `${proxyBaseUrl}/spend/logs` : `/spend/logs`; @@ -363,12 +398,16 @@ export const spendUsersCall = async (accessToken: String, userID: String) => { } }; - - - -export const userRequestModelCall = async (accessToken: String, model: String, UserID: String, justification: String) => { +export const userRequestModelCall = async ( + accessToken: String, + model: String, + UserID: String, + justification: String +) => { try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/user/request_model` : `/user/request_model`; + const url = proxyBaseUrl + ? `${proxyBaseUrl}/user/request_model` + : `/user/request_model`; const response = await fetch(url, { method: "POST", headers: { @@ -398,10 +437,11 @@ export const userRequestModelCall = async (accessToken: String, model: String, U } }; - export const userGetRequesedtModelsCall = async (accessToken: String) => { try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/user/get_requests` : `/user/get_requests`; + const url = proxyBaseUrl + ? `${proxyBaseUrl}/user/get_requests` + : `/user/get_requests`; console.log("in userGetRequesedtModelsCall:", url); const response = await fetch(url, { method: "GET", @@ -425,4 +465,4 @@ export const userGetRequesedtModelsCall = async (accessToken: String) => { console.error("Failed to get requested models:", error); throw error; } -}; \ No newline at end of file +}; diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx index a85d70a83..ad0070864 100644 --- a/ui/litellm-dashboard/src/components/user_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx @@ -1,6 +1,6 @@ "use client"; import React, { useState, useEffect } from "react"; -import { userInfoCall, modelInfoCall } from "./networking"; +import { userInfoCall, modelAvailableCall } from "./networking"; import { Grid, Col, Card, Text } from "@tremor/react"; import CreateKey from "./create_key_button"; import ViewKeyTable from "./view_key_table"; @@ -48,10 +48,9 @@ const UserDashboard: React.FC = ({ const token = searchParams.get("token"); const [accessToken, setAccessToken] = useState(null); const [userModels, setUserModels] = useState([]); - // check if window is not undefined if (typeof window !== "undefined") { - window.addEventListener('beforeunload', function() { + window.addEventListener("beforeunload", function () { // Clear session storage sessionStorage.clear(); }); @@ -78,7 +77,6 @@ const UserDashboard: React.FC = ({ // Moved useEffect inside the component and used a condition to run fetch only if the params are available useEffect(() => { - if (token) { const decoded = jwtDecode(token) as { [key: string]: any }; if (decoded) { @@ -109,32 +107,39 @@ const UserDashboard: React.FC = ({ const cachedUserModels = sessionStorage.getItem("userModels" + userID); if (cachedUserModels) { setUserModels(JSON.parse(cachedUserModels)); - } else { const fetchData = async () => { try { const response = await userInfoCall(accessToken, userID, userRole); setUserSpendData(response["user_info"]); setData(response["keys"]); // Assuming this is the correct path to your data - sessionStorage.setItem("userData" + userID, JSON.stringify(response["keys"])); + sessionStorage.setItem( + "userData" + userID, + JSON.stringify(response["keys"]) + ); sessionStorage.setItem( "userSpendData" + userID, JSON.stringify(response["user_info"]) ); - const model_info = await modelInfoCall(accessToken, userID, userRole); - console.log("model_info:", model_info); + const model_available = await modelAvailableCall( + accessToken, + userID, + userRole + ); // loop through model_info["data"] and create an array of element.model_name - let available_model_names = model_info["data"].filter((element: { model_name: string; user_access: boolean }) => element.user_access === true).map((element: { model_name: string; }) => element.model_name); + let available_model_names = model_available["data"].map( + (element: { id: string }) => element.id + ); console.log("available_model_names:", available_model_names); setUserModels(available_model_names); console.log("userModels:", userModels); - sessionStorage.setItem("userModels" + userID, JSON.stringify(available_model_names)); - - - + sessionStorage.setItem( + "userModels" + userID, + JSON.stringify(available_model_names) + ); } catch (error) { console.error("There was an error fetching the data", error); // Optionally, update your UI to reflect the error state here as well