phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 03:34:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

ui - show slow responses on admin UI

Browse source
This commit is contained in:
Ishaan Jaff 2024-05-01 17:23:37 -07:00
parent 26eda88b26
commit 5041b924ac
2 changed files with 93 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

96
ui/litellm-dashboard/src/components/model_dashboard.tsx
View file

@ -18,7 +18,7 @@ import {
} from "@tremor/react";
import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react";
import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react";
import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall } from "./networking";
import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall } from "./networking";
import { BarChart, AreaChart } from "@tremor/react";
import {
Button as Button2,
@ -205,6 +205,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const [modelExceptions, setModelExceptions] = useState<any[]>([]);
const [allExceptions, setAllExceptions] = useState<any[]>([]);
const [failureTableData, setFailureTableData] = useState<any[]>([]);
const [slowResponsesData, setSlowResponsesData] = useState<any[]>([]);
const EditModelModal: React.FC<EditModelModalProps> = ({ visible, onCancel, model, onSubmit }) => {
const [form] = Form.useForm();
@ -479,39 +480,51 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
setAllExceptions(modelExceptionsResponse.exception_types);
let modelMetricsData = modelMetricsResponse.data;
let successdeploymentToSuccess: Record<string, number> = {};
for (let i = 0; i < modelMetricsData.length; i++) {
let element = modelMetricsData[i];
let _model_name = element.model;
let _num_requests = element.num_requests;
successdeploymentToSuccess[_model_name] = _num_requests
}
console.log("successdeploymentToSuccess:", successdeploymentToSuccess)
const slowResponses = await modelMetricsSlowResponsesCall(
accessToken,
userID,
userRole,
null
)
let failureTableData = [];
let _failureData = modelExceptionsResponse.data;
for (let i = 0; i < _failureData.length; i++) {
const model = _failureData[i];
let _model_name = model.model;
let total_exceptions = model.total_exceptions;
let total_Requests = successdeploymentToSuccess[_model_name];
if (total_Requests == null) {
total_Requests = 0
}
let _data = {
model: _model_name,
total_exceptions: total_exceptions,
total_Requests: total_Requests,
failure_rate: total_Requests / total_exceptions
}
failureTableData.push(_data);
// sort failureTableData by failure_rate
failureTableData.sort((a, b) => b.failure_rate - a.failure_rate);
console.log("slowResponses:", slowResponses)
setFailureTableData(failureTableData);
console.log("failureTableData:", failureTableData);
}
setSlowResponsesData(slowResponses);
// let modelMetricsData = modelMetricsResponse.data;
// let successdeploymentToSuccess: Record<string, number> = {};
// for (let i = 0; i < modelMetricsData.length; i++) {
// let element = modelMetricsData[i];
// let _model_name = element.model;
// let _num_requests = element.num_requests;
// successdeploymentToSuccess[_model_name] = _num_requests
// }
// console.log("successdeploymentToSuccess:", successdeploymentToSuccess)
// let failureTableData = [];
// let _failureData = modelExceptionsResponse.data;
// for (let i = 0; i < _failureData.length; i++) {
// const model = _failureData[i];
// let _model_name = model.model;
// let total_exceptions = model.total_exceptions;
// let total_Requests = successdeploymentToSuccess[_model_name];
// if (total_Requests == null) {
// total_Requests = 0
// }
// let _data = {
// model: _model_name,
// total_exceptions: total_exceptions,
// total_Requests: total_Requests,
// failure_rate: total_Requests / total_exceptions
// }
// failureTableData.push(_data);
// // sort failureTableData by failure_rate
// failureTableData.sort((a, b) => b.failure_rate - a.failure_rate);
// setFailureTableData(failureTableData);
// console.log("failureTableData:", failureTableData);
// }
} catch (error) {
console.error("There was an error fetching the model data", error);
@ -1126,8 +1139,6 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
/>
)}
</Card>
</Col>
<Col>
@ -1135,20 +1146,19 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
<Table>
<TableHead>
<TableRow>
<TableHeaderCell>Model</TableHeaderCell>
<TableHeaderCell>Success Requests</TableHeaderCell>
<TableHeaderCell>Error Requests</TableHeaderCell>
<TableHeaderCell>Failure %</TableHeaderCell>
<TableHeaderCell>Deployment</TableHeaderCell>
<TableHeaderCell>Success Responses</TableHeaderCell>
<TableHeaderCell>Slow Responses</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
{failureTableData.map((metric, idx) => (
{slowResponsesData.map((metric, idx) => (
<TableRow key={idx}>
<TableCell>{metric.model}</TableCell>
<TableCell>{metric.total_Requests}</TableCell>
<TableCell>{metric.total_exceptions}</TableCell>
<TableCell>{metric.failure_rate}%</TableCell>
<TableCell>{metric.api_base}</TableCell>
<TableCell>{metric.total_count}</TableCell>
<TableCell>{metric.slow_count}</TableCell>
</TableRow>
))}

40
ui/litellm-dashboard/src/components/networking.tsx
View file

@ -475,6 +475,46 @@ export const modelMetricsCall = async (
};
export const modelMetricsSlowResponsesCall = async (
accessToken: String,
userID: String,
userRole: String,
modelGroup: String | null,
) => {
/**
* Get all models on proxy
*/
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/slow_responses` : `/model/metrics/slow_responses`;
if (modelGroup) {
url = `${url}?_selected_model_group=${modelGroup}`
}
// message.info("Requesting model data");
const response = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
const errorData = await response.text();
message.error(errorData, 20);
throw new Error("Network response was not ok");
}
const data = await response.json();
// message.info("Received model data");
return data;
// Handle success - you might want to update some state or UI based on the created key
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const modelExceptionsCall = async (
accessToken: String,
userID: String,