diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 7ffeab9ba..3a07e78b1 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -176,6 +176,8 @@ model LiteLLM_SpendLogs { endTime DateTime // Assuming end_time is a DateTime field completionStartTime DateTime? // Assuming completionStartTime is a DateTime field model String @default("") + model_id String? @default("") // the model id stored in proxy model db + model_group String? @default("") // public model_name / model_group api_base String @default("") user String @default("") metadata Json @default("{}") diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ffff87c51..fdea28c2d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1874,6 +1874,9 @@ def get_logging_payload( # hash the api_key api_key = hash_token(api_key) + _model_id = metadata.get("model_info", {}).get("id", "") + _model_group = metadata.get("model_group", "") + # clean up litellm metadata if isinstance(metadata, dict): clean_metadata = {} @@ -1928,6 +1931,8 @@ def get_logging_payload( "request_tags": metadata.get("tags", []), "end_user": end_user_id or "", "api_base": litellm_params.get("api_base", ""), + "model_group": _model_group, + "model_id": _model_id, } verbose_proxy_logger.debug("SpendTable: created payload - payload: %s\n\n", payload) diff --git a/schema.prisma b/schema.prisma index 7ffeab9ba..3a07e78b1 100644 --- a/schema.prisma +++ b/schema.prisma @@ -176,6 +176,8 @@ model LiteLLM_SpendLogs { endTime DateTime // Assuming end_time is a DateTime field completionStartTime DateTime? // Assuming completionStartTime is a DateTime field model String @default("") + model_id String? @default("") // the model id stored in proxy model db + model_group String? @default("") // public model_name / model_group api_base String @default("") user String @default("") metadata Json @default("{}") diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 6fc6df07b..8c08da6cb 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -16,9 +16,38 @@ import { AccordionHeader, AccordionBody, } from "@tremor/react"; -import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon, DateRangePicker } from "@tremor/react"; -import { Select, SelectItem, MultiSelect, MultiSelectItem, DateRangePickerValue } from "@tremor/react"; -import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall, getCallbacksCall, setCallbacksCall } from "./networking"; +import { + TabPanel, + TabPanels, + TabGroup, + TabList, + Tab, + TextInput, + Icon, + DateRangePicker, +} from "@tremor/react"; +import { + Select, + SelectItem, + MultiSelect, + MultiSelectItem, + DateRangePickerValue, +} from "@tremor/react"; +import { + modelInfoCall, + userGetRequesedtModelsCall, + modelCreateCall, + Model, + modelCostMap, + modelDeleteCall, + healthCheckCall, + modelUpdateCall, + modelMetricsCall, + modelExceptionsCall, + modelMetricsSlowResponsesCall, + getCallbacksCall, + setCallbacksCall, +} from "./networking"; import { BarChart, AreaChart } from "@tremor/react"; import { Button as Button2, @@ -31,26 +60,36 @@ import { Descriptions, Tooltip, Space, - Row, Col, + Row, + Col, } from "antd"; import { Badge, BadgeDelta, Button } from "@tremor/react"; import RequestAccess from "./request_model_access"; import { Typography } from "antd"; import TextArea from "antd/es/input/TextArea"; -import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon, RefreshIcon, CheckCircleIcon, XCircleIcon } from "@heroicons/react/outline"; +import { + InformationCircleIcon, + PencilAltIcon, + PencilIcon, + StatusOnlineIcon, + TrashIcon, + RefreshIcon, + CheckCircleIcon, + XCircleIcon, +} from "@heroicons/react/outline"; import DeleteModelButton from "./delete_model_button"; const { Title: Title2, Link } = Typography; -import { UploadOutlined } from '@ant-design/icons'; -import type { UploadProps } from 'antd'; -import { Upload } from 'antd'; - +import { UploadOutlined } from "@ant-design/icons"; +import type { UploadProps } from "antd"; +import { Upload } from "antd"; +import TimeToFirstToken from "./model_metrics/time_to_first_token"; interface ModelDashboardProps { accessToken: string | null; token: string | null; userRole: string | null; userID: string | null; - modelData: any, - setModelData: any + modelData: any; + setModelData: any; } interface EditModelModalProps { @@ -73,87 +112,87 @@ enum Providers { Google_AI_Studio = "Gemini (Google AI Studio)", Bedrock = "Amazon Bedrock", OpenAI_Compatible = "OpenAI-Compatible Endpoints (Groq, Together AI, Mistral AI, etc.)", - Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)" + Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)", } -const provider_map: Record = { - "OpenAI": "openai", - "Azure": "azure", - "Anthropic": "anthropic", - "Google_AI_Studio": "gemini", - "Bedrock": "bedrock", - "OpenAI_Compatible": "openai", - "Vertex_AI": "vertex_ai" +const provider_map: Record = { + OpenAI: "openai", + Azure: "azure", + Anthropic: "anthropic", + Google_AI_Studio: "gemini", + Bedrock: "bedrock", + OpenAI_Compatible: "openai", + Vertex_AI: "vertex_ai", }; - - -const retry_policy_map: Record = { - "BadRequestError (400)": "BadRequestErrorRetries", +const retry_policy_map: Record = { + "BadRequestError (400)": "BadRequestErrorRetries", "AuthenticationError (401)": "AuthenticationErrorRetries", "TimeoutError (408)": "TimeoutErrorRetries", "RateLimitError (429)": "RateLimitErrorRetries", "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", - "InternalServerError (500)": "InternalServerErrorRetries" + "InternalServerError (500)": "InternalServerErrorRetries", }; - - -const handleSubmit = async (formValues: Record, accessToken: string, form: any) => { +const handleSubmit = async ( + formValues: Record, + accessToken: string, + form: any +) => { try { /** - * For multiple litellm model names - create a separate deployment for each + * For multiple litellm model names - create a separate deployment for each * - get the list - * - iterate through it + * - iterate through it * - create a new deployment for each - * + * * For single model name -> make it a 1 item list */ // get the list of deployments - let deployments: Array = Array.isArray(formValues["model"]) ? formValues["model"] : [formValues["model"]]; - console.log(`received deployments: ${deployments}`) - console.log(`received type of deployments: ${typeof deployments}`) - deployments.forEach(async (litellm_model) => { - console.log(`litellm_model: ${litellm_model}`) - const litellmParamsObj: Record = {}; - const modelInfoObj: Record = {}; + let deployments: Array = Array.isArray(formValues["model"]) + ? formValues["model"] + : [formValues["model"]]; + console.log(`received deployments: ${deployments}`); + console.log(`received type of deployments: ${typeof deployments}`); + deployments.forEach(async (litellm_model) => { + console.log(`litellm_model: ${litellm_model}`); + const litellmParamsObj: Record = {}; + const modelInfoObj: Record = {}; // Iterate through the key-value pairs in formValues - litellmParamsObj["model"] = litellm_model - let modelName: string = ""; + litellmParamsObj["model"] = litellm_model; + let modelName: string = ""; console.log("formValues add deployment:", formValues); for (const [key, value] of Object.entries(formValues)) { - if (value === '') { + if (value === "") { continue; } if (key == "model_name") { - modelName = modelName + value - } - else if (key == "custom_llm_provider") { + modelName = modelName + value; + } else if (key == "custom_llm_provider") { // const providerEnumValue = Providers[value as keyof typeof Providers]; // const mappingResult = provider_map[providerEnumValue]; // Get the corresponding value from the mapping // modelName = mappingResult + "/" + modelName - continue - } - else if (key == "model") { - continue + continue; + } else if (key == "model") { + continue; } // Check if key is "base_model" else if (key === "base_model") { // Add key-value pair to model_info dictionary modelInfoObj[key] = value; - } - - else if (key == "litellm_extra_params") { + } else if (key == "litellm_extra_params") { console.log("litellm_extra_params:", value); let litellmExtraParams = {}; if (value && value != undefined) { try { litellmExtraParams = JSON.parse(value); - } - catch (error) { - message.error("Failed to parse LiteLLM Extra Params: " + error, 10); + } catch (error) { + message.error( + "Failed to parse LiteLLM Extra Params: " + error, + 10 + ); throw new Error("Failed to parse litellm_extra_params: " + error); } for (const [key, value] of Object.entries(litellmExtraParams)) { @@ -169,29 +208,22 @@ const handleSubmit = async (formValues: Record, accessToken: string } } - const new_model: Model = { - "model_name": modelName, - "litellm_params": litellmParamsObj, - "model_info": modelInfoObj - } + const new_model: Model = { + model_name: modelName, + litellm_params: litellmParamsObj, + model_info: modelInfoObj, + }; - - - const response: any = await modelCreateCall( - accessToken, - new_model - ); + const response: any = await modelCreateCall(accessToken, new_model); console.log(`response for model create call: ${response["data"]}`); - }); - - form.resetFields(); + }); - - } catch (error) { - message.error("Failed to create model: " + error, 10); - } -} + form.resetFields(); + } catch (error) { + message.error("Failed to create model: " + error, 10); + } +}; const ModelDashboard: React.FC = ({ accessToken, @@ -199,56 +231,68 @@ const ModelDashboard: React.FC = ({ userRole, userID, modelData = { data: [] }, - setModelData + setModelData, }) => { const [pendingRequests, setPendingRequests] = useState([]); const [form] = Form.useForm(); const [modelMap, setModelMap] = useState(null); - const [lastRefreshed, setLastRefreshed] = useState(''); + const [lastRefreshed, setLastRefreshed] = useState(""); const [providerModels, setProviderModels] = useState>([]); // Explicitly typing providerModels as a string array - const providers = Object.values(Providers).filter(key => isNaN(Number(key))); - + const providers = Object.values(Providers).filter((key) => + isNaN(Number(key)) + ); + const [selectedProvider, setSelectedProvider] = useState("OpenAI"); - const [healthCheckResponse, setHealthCheckResponse] = useState(''); + const [healthCheckResponse, setHealthCheckResponse] = useState(""); const [editModalVisible, setEditModalVisible] = useState(false); const [selectedModel, setSelectedModel] = useState(null); - const [availableModelGroups, setAvailableModelGroups] = useState>([]); - const [selectedModelGroup, setSelectedModelGroup] = useState(null); + const [availableModelGroups, setAvailableModelGroups] = useState< + Array + >([]); + const [selectedModelGroup, setSelectedModelGroup] = useState( + null + ); const [modelLatencyMetrics, setModelLatencyMetrics] = useState([]); const [modelMetrics, setModelMetrics] = useState([]); - const [modelMetricsCategories, setModelMetricsCategories] = useState([]); + const [modelMetricsCategories, setModelMetricsCategories] = useState( + [] + ); const [modelExceptions, setModelExceptions] = useState([]); const [allExceptions, setAllExceptions] = useState([]); const [failureTableData, setFailureTableData] = useState([]); const [slowResponsesData, setSlowResponsesData] = useState([]); const [dateValue, setDateValue] = useState({ - from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), + from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), to: new Date(), }); - const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState(null); + const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = + useState(null); const [defaultRetry, setDefaultRetry] = useState(0); - - const EditModelModal: React.FC = ({ visible, onCancel, model, onSubmit }) => { + const EditModelModal: React.FC = ({ + visible, + onCancel, + model, + onSubmit, + }) => { const [form] = Form.useForm(); - let litellm_params_to_edit: Record = {} + let litellm_params_to_edit: Record = {}; let model_name = ""; let model_id = ""; if (model) { - litellm_params_to_edit = model.litellm_params + litellm_params_to_edit = model.litellm_params; model_name = model.model_name; let model_info = model.model_info; - if (model_info ) { + if (model_info) { model_id = model_info.id; - console.log(`model_id: ${model_id}`) + console.log(`model_id: ${model_id}`); litellm_params_to_edit.model_id = model_id; } } - - + const handleOk = () => { form .validateFields() @@ -259,17 +303,17 @@ const ModelDashboard: React.FC = ({ .catch((error) => { console.error("Validation failed:", error); }); - }; - + }; + return ( - +
= ({ wrapperCol={{ span: 16 }} labelAlign="left" > - <> + <> + + + - - + + + - - - - + + + - + + + - - - + + + - + + + - - + + + - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Save -
-
+ + +
+ Save +
+ ); }; - const handleEditClick = (model: any) => { setSelectedModel(model); @@ -383,56 +403,52 @@ const ModelDashboard: React.FC = ({ setSelectedModel(null); }; + const handleEditSubmit = async (formValues: Record) => { + // Call API to update team with teamId and values -const handleEditSubmit = async (formValues: Record) => { - // Call API to update team with teamId and values - - console.log("handleEditSubmit:", formValues); - if (accessToken == null) { - return; - } - - let newLiteLLMParams: Record = {} - let model_info_model_id = null; - - for (const [key, value] of Object.entries(formValues)) { - if (key !== "model_id") { - newLiteLLMParams[key] = value; - } else { - model_info_model_id = value; + console.log("handleEditSubmit:", formValues); + if (accessToken == null) { + return; } - } - let payload = { - litellm_params: newLiteLLMParams, - model_info: { - "id": model_info_model_id + let newLiteLLMParams: Record = {}; + let model_info_model_id = null; + + for (const [key, value] of Object.entries(formValues)) { + if (key !== "model_id") { + newLiteLLMParams[key] = value; + } else { + model_info_model_id = value; + } } - } - console.log("handleEditSubmit payload:", payload); + let payload = { + litellm_params: newLiteLLMParams, + model_info: { + id: model_info_model_id, + }, + }; - try { - let newModelValue = await modelUpdateCall(accessToken, payload); - message.success("Model updated successfully, restart server to see updates"); + console.log("handleEditSubmit payload:", payload); - setEditModalVisible(false); - setSelectedModel(null); - } catch (error) { - console.log(`Error occurred`) - } - -}; - - - + try { + let newModelValue = await modelUpdateCall(accessToken, payload); + message.success( + "Model updated successfully, restart server to see updates" + ); + setEditModalVisible(false); + setSelectedModel(null); + } catch (error) { + console.log(`Error occurred`); + } + }; const props: UploadProps = { - name: 'file', - accept: '.json', - beforeUpload: file => { - if (file.type === 'application/json') { + name: "file", + accept: ".json", + beforeUpload: (file) => { + if (file.type === "application/json") { const reader = new FileReader(); reader.onload = (e) => { if (e.target) { @@ -446,12 +462,12 @@ const handleEditSubmit = async (formValues: Record) => { return false; }, onChange(info) { - if (info.file.status !== 'uploading') { + if (info.file.status !== "uploading") { console.log(info.file, info.fileList); } - if (info.file.status === 'done') { + if (info.file.status === "done") { message.success(`${info.file.name} file uploaded successfully`); - } else if (info.file.status === 'error') { + } else if (info.file.status === "error") { message.error(`${info.file.name} file upload failed.`); } }, @@ -470,14 +486,14 @@ const handleEditSubmit = async (formValues: Record) => { } console.log("new modelGroupRetryPolicy:", modelGroupRetryPolicy); - + try { const payload = { - router_settings: { - model_group_retry_policy: modelGroupRetryPolicy - } + router_settings: { + model_group_retry_policy: modelGroupRetryPolicy, + }, }; - + await setCallbacksCall(accessToken, payload); message.success("Retry settings saved successfully"); } catch (error) { @@ -486,7 +502,6 @@ const handleEditSubmit = async (formValues: Record) => { } }; - useEffect(() => { if (!accessToken || !token || !userRole || !userID) { return; @@ -502,31 +517,31 @@ const handleEditSubmit = async (formValues: Record) => { console.log("Model data response:", modelDataResponse.data); setModelData(modelDataResponse); - // loop through modelDataResponse and get all`model_name` values + // loop through modelDataResponse and get all`model_name` values let all_model_groups: Set = new Set(); for (let i = 0; i < modelDataResponse.data.length; i++) { const model = modelDataResponse.data[i]; - all_model_groups.add(model.model_name) + all_model_groups.add(model.model_name); } - console.log("all_model_groups:", all_model_groups) - let _array_model_groups = Array.from(all_model_groups) + console.log("all_model_groups:", all_model_groups); + let _array_model_groups = Array.from(all_model_groups); // sort _array_model_groups alphabetically _array_model_groups = _array_model_groups.sort(); setAvailableModelGroups(_array_model_groups); - console.log("array_model_groups:", _array_model_groups) - let _initial_model_group = "all" + console.log("array_model_groups:", _array_model_groups); + let _initial_model_group = "all"; if (_array_model_groups.length > 0) { // set selectedModelGroup to the last model group - _initial_model_group = _array_model_groups[_array_model_groups.length - 1]; - console.log("_initial_model_group:", _initial_model_group) + _initial_model_group = + _array_model_groups[_array_model_groups.length - 1]; + console.log("_initial_model_group:", _initial_model_group); setSelectedModelGroup(_initial_model_group); } - console.log("selectedModelGroup:", selectedModelGroup) - + console.log("selectedModelGroup:", selectedModelGroup); const modelMetricsResponse = await modelMetricsCall( accessToken, @@ -540,11 +555,9 @@ const handleEditSubmit = async (formValues: Record) => { console.log("Model metrics response:", modelMetricsResponse); // Sort by latency (avg_latency_per_token) - setModelMetrics(modelMetricsResponse.data); setModelMetricsCategories(modelMetricsResponse.all_api_bases); - const modelExceptionsResponse = await modelExceptionsCall( accessToken, userID, @@ -552,12 +565,11 @@ const handleEditSubmit = async (formValues: Record) => { _initial_model_group, dateValue.from?.toISOString(), dateValue.to?.toISOString() - ) + ); console.log("Model exceptions response:", modelExceptionsResponse); setModelExceptions(modelExceptionsResponse.data); setAllExceptions(modelExceptionsResponse.exception_types); - const slowResponses = await modelMetricsSlowResponsesCall( accessToken, userID, @@ -565,29 +577,29 @@ const handleEditSubmit = async (formValues: Record) => { _initial_model_group, dateValue.from?.toISOString(), dateValue.to?.toISOString() - ) + ); - console.log("slowResponses:", slowResponses) + console.log("slowResponses:", slowResponses); setSlowResponsesData(slowResponses); - - const routerSettingsInfo = await getCallbacksCall(accessToken, userID, userRole); + const routerSettingsInfo = await getCallbacksCall( + accessToken, + userID, + userRole + ); let router_settings = routerSettingsInfo.router_settings; - console.log("routerSettingsInfo:", router_settings) + console.log("routerSettingsInfo:", router_settings); let model_group_retry_policy = router_settings.model_group_retry_policy; let default_retries = router_settings.num_retries; - console.log("model_group_retry_policy:", model_group_retry_policy) - console.log("default_retries:", default_retries) + console.log("model_group_retry_policy:", model_group_retry_policy); + console.log("default_retries:", default_retries); setModelGroupRetryPolicy(model_group_retry_policy); setDefaultRetry(default_retries); - - - } catch (error) { console.error("There was an error fetching the model data", error); } @@ -598,15 +610,15 @@ const handleEditSubmit = async (formValues: Record) => { } const fetchModelMap = async () => { - const data = await modelCostMap() - console.log(`received model cost map data: ${Object.keys(data)}`) - setModelMap(data) - } + const data = await modelCostMap(); + console.log(`received model cost map data: ${Object.keys(data)}`); + setModelMap(data); + }; if (modelMap == null) { - fetchModelMap() + fetchModelMap(); } - handleRefreshClick() + handleRefreshClick(); }, [accessToken, token, userRole, userID, modelMap, lastRefreshed]); if (!modelData) { @@ -621,7 +633,7 @@ const handleEditSubmit = async (formValues: Record) => { // loop through model data and edit each row for (let i = 0; i < modelData.data.length; i++) { let curr_model = modelData.data[i]; - let litellm_model_name = curr_model?.litellm_params?.model + let litellm_model_name = curr_model?.litellm_params?.model; let model_info = curr_model?.model_info; let defaultProvider = "openai"; @@ -636,16 +648,16 @@ const handleEditSubmit = async (formValues: Record) => { /** * Use model map * - check if model in model map - * - return it's litellm_provider, if so + * - return it's litellm_provider, if so */ - console.log(`GET PROVIDER CALLED! - ${modelMap}`) + console.log(`GET PROVIDER CALLED! - ${modelMap}`); if (modelMap !== null && modelMap !== undefined) { if (typeof modelMap == "object" && model in modelMap) { - return modelMap[model]["litellm_provider"] + return modelMap[model]["litellm_provider"]; } } - return "openai" - } + return "openai"; + }; // Check if litellm_model_name is null or undefined if (litellm_model_name) { @@ -656,10 +668,13 @@ const handleEditSubmit = async (formValues: Record) => { let firstElement = splitModel[0]; // If there is only one element, default provider to openai - provider = splitModel.length === 1 ? getProviderFromModel(litellm_model_name) : firstElement; + provider = + splitModel.length === 1 + ? getProviderFromModel(litellm_model_name) + : firstElement; } else { // litellm_model_name is null or undefined, default provider to openai - provider = "openai" + provider = "openai"; } if (model_info) { @@ -675,22 +690,25 @@ const handleEditSubmit = async (formValues: Record) => { ([key]) => key !== "model" && key !== "api_base" ) ); - } + } modelData.data[i].provider = provider; modelData.data[i].input_cost = input_cost; modelData.data[i].output_cost = output_cost; - - // Convert Cost in terms of Cost per 1M tokens + // Convert Cost in terms of Cost per 1M tokens if (modelData.data[i].input_cost) { - modelData.data[i].input_cost = (Number(modelData.data[i].input_cost) * 1000000).toFixed(2); + modelData.data[i].input_cost = ( + Number(modelData.data[i].input_cost) * 1000000 + ).toFixed(2); } if (modelData.data[i].output_cost) { - modelData.data[i].output_cost = (Number(modelData.data[i].output_cost) * 1000000).toFixed(2); + modelData.data[i].output_cost = ( + Number(modelData.data[i].output_cost) * 1000000 + ).toFixed(2); } - + modelData.data[i].max_tokens = max_tokens; modelData.data[i].max_input_tokens = max_input_tokens; modelData.data[i].api_base = curr_model?.litellm_params?.api_base; @@ -715,58 +733,73 @@ const handleEditSubmit = async (formValues: Record) => { } const setProviderModelsFn = (provider: string) => { - console.log(`received provider string: ${provider}`) - const providerKey = Object.keys(Providers).find(key => (Providers as {[index: string]: any})[key] === provider); + console.log(`received provider string: ${provider}`); + const providerKey = Object.keys(Providers).find( + (key) => (Providers as { [index: string]: any })[key] === provider + ); if (providerKey) { const mappingResult = provider_map[providerKey]; // Get the corresponding value from the mapping - console.log(`mappingResult: ${mappingResult}`) - let _providerModels: Array = [] - if (typeof modelMap === 'object') { + console.log(`mappingResult: ${mappingResult}`); + let _providerModels: Array = []; + if (typeof modelMap === "object") { Object.entries(modelMap).forEach(([key, value]) => { if ( - value !== null - && typeof value === 'object' - && "litellm_provider" in (value as object) - && ( - (value as any)["litellm_provider"] === mappingResult - || (value as any)["litellm_provider"].includes(mappingResult) - )) { + value !== null && + typeof value === "object" && + "litellm_provider" in (value as object) && + ((value as any)["litellm_provider"] === mappingResult || + (value as any)["litellm_provider"].includes(mappingResult)) + ) { _providerModels.push(key); } }); } - setProviderModels(_providerModels) + setProviderModels(_providerModels); console.log(`providerModels: ${providerModels}`); } - } - - const runHealthCheck = async () => { - try { - message.info('Running health check...'); - setHealthCheckResponse(''); - const response = await healthCheckCall(accessToken); - setHealthCheckResponse(response); - } catch (error) { - console.error('Error running health check:', error); - setHealthCheckResponse('Error running health check'); - } }; + const runHealthCheck = async () => { + try { + message.info("Running health check..."); + setHealthCheckResponse(""); + const response = await healthCheckCall(accessToken); + setHealthCheckResponse(response); + } catch (error) { + console.error("Error running health check:", error); + setHealthCheckResponse("Error running health check"); + } + }; - const updateModelMetrics = async (modelGroup: string | null, startTime: Date | undefined, endTime: Date | undefined) => { + const updateModelMetrics = async ( + modelGroup: string | null, + startTime: Date | undefined, + endTime: Date | undefined + ) => { console.log("Updating model metrics for group:", modelGroup); if (!accessToken || !userID || !userRole || !startTime || !endTime) { - return + return; } - console.log("inside updateModelMetrics - startTime:", startTime, "endTime:", endTime) - setSelectedModelGroup(modelGroup); // If you want to store the selected model group in state + console.log( + "inside updateModelMetrics - startTime:", + startTime, + "endTime:", + endTime + ); + setSelectedModelGroup(modelGroup); // If you want to store the selected model group in state - try { - const modelMetricsResponse = await modelMetricsCall(accessToken, userID, userRole, modelGroup, startTime.toISOString(), endTime.toISOString()); + const modelMetricsResponse = await modelMetricsCall( + accessToken, + userID, + userRole, + modelGroup, + startTime.toISOString(), + endTime.toISOString() + ); console.log("Model metrics response:", modelMetricsResponse); - + // Assuming modelMetricsResponse now contains the metric data for the specified model group setModelMetrics(modelMetricsResponse.data); setModelMetricsCategories(modelMetricsResponse.all_api_bases); @@ -778,12 +811,11 @@ const handleEditSubmit = async (formValues: Record) => { modelGroup, startTime.toISOString(), endTime.toISOString() - ) + ); console.log("Model exceptions response:", modelExceptionsResponse); setModelExceptions(modelExceptionsResponse.data); setAllExceptions(modelExceptionsResponse.exception_types); - const slowResponses = await modelMetricsSlowResponsesCall( accessToken, userID, @@ -791,52 +823,61 @@ const handleEditSubmit = async (formValues: Record) => { modelGroup, startTime.toISOString(), endTime.toISOString() - ) + ); - console.log("slowResponses:", slowResponses) + console.log("slowResponses:", slowResponses); setSlowResponsesData(slowResponses); - } catch (error) { console.error("Failed to fetch model metrics", error); } - } + }; const customTooltip = (props: any) => { const { payload, active } = props; if (!active || !payload) return null; - + // Extract the date from the first item in the payload array const date = payload[0]?.payload?.date; - + // Sort the payload array by category.value in descending order let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value); - + // Only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown if (sortedPayload.length > 5) { let remainingItems = sortedPayload.length - 5; sortedPayload = sortedPayload.slice(0, 5); sortedPayload.push({ dataKey: `${remainingItems} other deployments`, - value: payload.slice(5).reduce((acc: number, curr: any) => acc + curr.value, 0), + value: payload + .slice(5) + .reduce((acc: number, curr: any) => acc + curr.value, 0), color: "gray", }); } - + return (
- {date &&

Date: {date}

} + {date && ( +

Date: {date}

+ )} {sortedPayload.map((category: any, idx: number) => { const roundedValue = parseFloat(category.value.toFixed(5)); const displayValue = - roundedValue === 0 && category.value > 0 ? "<0.00001" : roundedValue.toFixed(5); + roundedValue === 0 && category.value > 0 + ? "<0.00001" + : roundedValue.toFixed(5); return (
-
+

{category.dataKey}

-

{displayValue}

+

+ {displayValue} +

); })} @@ -844,207 +885,394 @@ const handleEditSubmit = async (formValues: Record) => { ); }; - - const getPlaceholder = (selectedProvider: string): string => { if (selectedProvider === Providers.Vertex_AI) { - return 'gemini-pro'; + return "gemini-pro"; } else if (selectedProvider == Providers.Anthropic) { - return 'claude-3-opus' + return "claude-3-opus"; } else if (selectedProvider == Providers.Bedrock) { - return 'claude-3-opus' + return "claude-3-opus"; } else if (selectedProvider == Providers.Google_AI_Studio) { - return 'gemini-pro' + return "gemini-pro"; } else { - return 'gpt-3.5-turbo'; + return "gpt-3.5-turbo"; } }; const handleOk = () => { form - .validateFields() - .then((values) => { - handleSubmit(values, accessToken, form); - // form.resetFields(); - }) - .catch((error) => { - console.error("Validation failed:", error); - }); + .validateFields() + .then((values) => { + handleSubmit(values, accessToken, form); + // form.resetFields(); + }) + .catch((error) => { + console.error("Validation failed:", error); + }); }; - console.log(`selectedProvider: ${selectedProvider}`) - console.log(`providerModels.length: ${providerModels.length}`) + console.log(`selectedProvider: ${selectedProvider}`); + console.log(`providerModels.length: ${providerModels.length}`); return ( -
+
- -
- All Models - Add Model - 
/health Models
 + +
+ All Models + Add Model + + 
/health Models
+ Model Analytics Model Retry Settings -
- -
- {lastRefreshed && ( - - Last Refreshed: {lastRefreshed} - - )} - -
- - - - -
- Filter by Public Model Name -
- - - - - Public Model Name - Provider - {userRole === "Admin" && ( - API Base - )} - Extra litellm Params - Input Price

/1M Tokens ($)

 - Output Price

/1M Tokens ($)

 - Max Tokens - Status - - - - {modelData.data - .filter((model: any) => - selectedModelGroup === "all" || - model.model_name === selectedModelGroup || - selectedModelGroup === null || - selectedModelGroup === undefined || - selectedModelGroup === "" - ) - .map((model: any, index: number) => ( - - - {model.model_name} - - {model.provider} - {userRole === "Admin" && ( - {model.api_base} - )} - - - - Litellm params - - - 
{JSON.stringify(model.cleanedLitellmParams, null, 2)}
- - - - {model.input_cost || model.litellm_params.input_cost_per_token || null} - {model.output_cost || model.litellm_params.output_cost_per_token || null} - -

- Max Tokens: {model.max_tokens}

- Max Input Tokens: {model.max_input_tokens} -

- - - {model.model_info.db_model ? ( - -

DB Model

- - ) : ( - -

Config Model

- - )} - - - handleEditClick(model)} /> - - - - ))} - -
- - - - - - Add new model - -
- <> - - + setSelectedModelGroup(value === "all" ? "all" : value) + } + value={ + selectedModelGroup + ? selectedModelGroup + : availableModelGroups[0] + } + > + All Models + {availableModelGroups.map((group, idx) => ( { - setProviderModelsFn(provider); - setSelectedProvider(provider); - }} + key={idx} + value={group} + onClick={() => setSelectedModelGroup(group)} > - {provider} + {group} ))} - - - - - - - - Model name your users will pass in. - - - {selectedProvider === Providers.Azure ? ( + +
+ + + + + + Public Model Name + + + Provider + + {userRole === "Admin" && ( + + API Base + + )} + + Extra litellm Params + + + Input Price{" "} +

+ /1M Tokens ($) +

+ + + Output Price{" "} +

+ /1M Tokens ($) +

+ + + Max Tokens + + + Status + + + + + {modelData.data + .filter( + (model: any) => + selectedModelGroup === "all" || + model.model_name === selectedModelGroup || + selectedModelGroup === null || + selectedModelGroup === undefined || + selectedModelGroup === "" + ) + .map((model: any, index: number) => ( + + + {model.model_name} + + + {model.provider} + + {userRole === "Admin" && ( + + {model.api_base} + + )} + + + + Litellm params + + + 
+                                  {JSON.stringify(
+                                    model.cleanedLitellmParams,
+                                    null,
+                                    2
+                                  )}
+
+ + + + + {model.input_cost || + model.litellm_params.input_cost_per_token || + null} + + + {model.output_cost || + model.litellm_params.output_cost_per_token || + null} + + +

+ Max Tokens: {model.max_tokens}

+ Max Input Tokens: {model.max_input_tokens} +

+ + + {model.model_info.db_model ? ( + +

DB Model

+ + ) : ( + +

Config Model

+ + )} + + + handleEditClick(model)} + /> + + + + ))} + +
+ + + + + + Add new model + + + <> + + + + + + + + + + + Model name your users will pass in. + + + + + {selectedProvider === Providers.Azure ? ( ) : providerModels.length > 0 ? ( @@ -1057,339 +1285,433 @@ const handleEditSubmit = async (formValues: Record) => { ) : ( )} - - - - Actual model name used for making litellm.completion() call. We'll loadbalance models with the same 'public name' - { - selectedProvider != Providers.Bedrock && selectedProvider != Providers.Vertex_AI && - - } - { - selectedProvider == Providers.OpenAI && - - - } - { - selectedProvider == Providers.Vertex_AI && - } - { - selectedProvider == Providers.Vertex_AI && - } - { - selectedProvider == Providers.Vertex_AI && - - }>Click to Upload - - - } - { - selectedProvider == Providers.Vertex_AI && - - Give litellm a gcp service account(.json file), so it can make the relevant calls - - } - { - (selectedProvider == Providers.Azure || selectedProvider == Providers.OpenAI_Compatible) && - - - } - { - selectedProvider == Providers.Azure && - - - } - { - selectedProvider == Providers.Azure && - -
+ + + + + Actual model name used for making{" "} + + litellm.completion() call + + . We'll{" "} + + loadbalance + {" "} + models with the same 'public name' + + + + {selectedProvider != Providers.Bedrock && + selectedProvider != Providers.Vertex_AI && ( + + + + )} + {selectedProvider == Providers.OpenAI && ( + + + + )} + {selectedProvider == Providers.Vertex_AI && ( + + + )} + {selectedProvider == Providers.Vertex_AI && ( + + + + )} + {selectedProvider == Providers.Vertex_AI && ( + - + + }> + Click to Upload + + - - - The actual model your azure deployment uses. Used for accurate cost tracking. Select name from here - - -
- } - { - selectedProvider == Providers.Bedrock && - - - } - { - selectedProvider == Providers.Bedrock && - - - } - { - selectedProvider == Providers.Bedrock && - - - } - -