phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 03:34:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

ui - show slow responses on admin UI

Browse source
This commit is contained in:
Ishaan Jaff 2024-05-01 17:23:37 -07:00
parent 26eda88b26
commit 5041b924ac
2 changed files with 93 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

96
ui/litellm-dashboard/src/components/model_dashboard.tsx
View file

@ -18,7 +18,7 @@ import {
} from "@tremor/react"; } from "@tremor/react";
import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react";
import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react";
import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall } from "./networking"; import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall } from "./networking";
import { BarChart, AreaChart } from "@tremor/react"; import { BarChart, AreaChart } from "@tremor/react";
import { import {
Button as Button2, Button as Button2,
@ -205,6 +205,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const [modelExceptions, setModelExceptions] = useState<any[]>([]); const [modelExceptions, setModelExceptions] = useState<any[]>([]);
const [allExceptions, setAllExceptions] = useState<any[]>([]); const [allExceptions, setAllExceptions] = useState<any[]>([]);
const [failureTableData, setFailureTableData] = useState<any[]>([]); const [failureTableData, setFailureTableData] = useState<any[]>([]);
const [slowResponsesData, setSlowResponsesData] = useState<any[]>([]);
const EditModelModal: React.FC<EditModelModalProps> = ({ visible, onCancel, model, onSubmit }) => { const EditModelModal: React.FC<EditModelModalProps> = ({ visible, onCancel, model, onSubmit }) => {
const [form] = Form.useForm(); const [form] = Form.useForm();
@ -479,39 +480,51 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
setAllExceptions(modelExceptionsResponse.exception_types); setAllExceptions(modelExceptionsResponse.exception_types);
let modelMetricsData = modelMetricsResponse.data; const slowResponses = await modelMetricsSlowResponsesCall(
let successdeploymentToSuccess: Record<string, number> = {}; accessToken,
for (let i = 0; i < modelMetricsData.length; i++) { userID,
let element = modelMetricsData[i]; userRole,
let _model_name = element.model; null
let _num_requests = element.num_requests; )
successdeploymentToSuccess[_model_name] = _num_requests
} console.log("slowResponses:", slowResponses)
console.log("successdeploymentToSuccess:", successdeploymentToSuccess)
setSlowResponsesData(slowResponses);
// let modelMetricsData = modelMetricsResponse.data;
// let successdeploymentToSuccess: Record<string, number> = {};
// for (let i = 0; i < modelMetricsData.length; i++) {
// let element = modelMetricsData[i];
// let _model_name = element.model;
// let _num_requests = element.num_requests;
// successdeploymentToSuccess[_model_name] = _num_requests
// }
// console.log("successdeploymentToSuccess:", successdeploymentToSuccess)
let failureTableData = []; // let failureTableData = [];
let _failureData = modelExceptionsResponse.data; // let _failureData = modelExceptionsResponse.data;
for (let i = 0; i < _failureData.length; i++) { // for (let i = 0; i < _failureData.length; i++) {
const model = _failureData[i]; // const model = _failureData[i];
let _model_name = model.model; // let _model_name = model.model;
let total_exceptions = model.total_exceptions; // let total_exceptions = model.total_exceptions;
let total_Requests = successdeploymentToSuccess[_model_name]; // let total_Requests = successdeploymentToSuccess[_model_name];
if (total_Requests == null) { // if (total_Requests == null) {
total_Requests = 0 // total_Requests = 0
} // }
let _data = { // let _data = {
model: _model_name, // model: _model_name,
total_exceptions: total_exceptions, // total_exceptions: total_exceptions,
total_Requests: total_Requests, // total_Requests: total_Requests,
failure_rate: total_Requests / total_exceptions // failure_rate: total_Requests / total_exceptions
} // }
failureTableData.push(_data); // failureTableData.push(_data);
// sort failureTableData by failure_rate // // sort failureTableData by failure_rate
failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); // failureTableData.sort((a, b) => b.failure_rate - a.failure_rate);
setFailureTableData(failureTableData); // setFailureTableData(failureTableData);
console.log("failureTableData:", failureTableData); // console.log("failureTableData:", failureTableData);
} // }
} catch (error) { } catch (error) {
console.error("There was an error fetching the model data", error); console.error("There was an error fetching the model data", error);
@ -1126,8 +1139,6 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
/> />
)} )}
</Card> </Card>
</Col> </Col>
<Col> <Col>
@ -1135,20 +1146,19 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
<Table> <Table>
<TableHead> <TableHead>
<TableRow> <TableRow>
<TableHeaderCell>Model</TableHeaderCell> <TableHeaderCell>Deployment</TableHeaderCell>
<TableHeaderCell>Success Requests</TableHeaderCell> <TableHeaderCell>Success Responses</TableHeaderCell>
<TableHeaderCell>Error Requests</TableHeaderCell> <TableHeaderCell>Slow Responses</TableHeaderCell>
<TableHeaderCell>Failure %</TableHeaderCell>
</TableRow> </TableRow>
</TableHead> </TableHead>
<TableBody> <TableBody>
{failureTableData.map((metric, idx) => ( {slowResponsesData.map((metric, idx) => (
<TableRow key={idx}> <TableRow key={idx}>
<TableCell>{metric.model}</TableCell> <TableCell>{metric.api_base}</TableCell>
<TableCell>{metric.total_Requests}</TableCell> <TableCell>{metric.total_count}</TableCell>
<TableCell>{metric.total_exceptions}</TableCell> <TableCell>{metric.slow_count}</TableCell>
<TableCell>{metric.failure_rate}%</TableCell>
</TableRow> </TableRow>
))} ))}

40
ui/litellm-dashboard/src/components/networking.tsx
View file

@ -475,6 +475,46 @@ export const modelMetricsCall = async (
}; };
export const modelMetricsSlowResponsesCall = async (
accessToken: String,
userID: String,
userRole: String,
modelGroup: String | null,
) => {
/**
* Get all models on proxy
*/
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/slow_responses` : `/model/metrics/slow_responses`;
if (modelGroup) {
url = `${url}?_selected_model_group=${modelGroup}`
}
// message.info("Requesting model data");
const response = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
const errorData = await response.text();
message.error(errorData, 20);
throw new Error("Network response was not ok");
}
const data = await response.json();
// message.info("Received model data");
return data;
// Handle success - you might want to update some state or UI based on the created key
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const modelExceptionsCall = async ( export const modelExceptionsCall = async (
accessToken: String, accessToken: String,
userID: String, userID: String,