ui - show slow responses on admin UI

ui/litellm-dashboard/src/components/model_dashboard.tsx

@@ -18,7 +18,7 @@ import {
 } from "@tremor/react";
 import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react";
 import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react";
-import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall } from "./networking";
+import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall } from "./networking";
 import { BarChart, AreaChart } from "@tremor/react";
 import {
   Button as Button2,
@@ -205,6 +205,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
   const [modelExceptions, setModelExceptions] = useState<any[]>([]);
   const [allExceptions, setAllExceptions] = useState<any[]>([]);
   const [failureTableData, setFailureTableData] = useState<any[]>([]);
+  const [slowResponsesData, setSlowResponsesData] = useState<any[]>([]);
 
   const EditModelModal: React.FC<EditModelModalProps> = ({ visible, onCancel, model, onSubmit }) => {
     const [form] = Form.useForm();
@@ -479,39 +480,51 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
         setAllExceptions(modelExceptionsResponse.exception_types);
 
 
-        let modelMetricsData = modelMetricsResponse.data;
+        const slowResponses = await modelMetricsSlowResponsesCall(
-        let successdeploymentToSuccess: Record<string, number> = {};
+          accessToken,
-        for  (let i = 0; i < modelMetricsData.length; i++) {
+          userID,
-          let element = modelMetricsData[i];
+          userRole,
-          let _model_name = element.model;
+          null
-          let _num_requests = element.num_requests;
+        )
-          successdeploymentToSuccess[_model_name] = _num_requests
-        }
-        console.log("successdeploymentToSuccess:", successdeploymentToSuccess)
 
-        let failureTableData = [];
+        console.log("slowResponses:", slowResponses)
-        let _failureData = modelExceptionsResponse.data;
-        for (let i = 0; i < _failureData.length; i++) {
-          const model = _failureData[i];
-          let _model_name = model.model;
-          let total_exceptions = model.total_exceptions;
-          let total_Requests = successdeploymentToSuccess[_model_name];
-          if (total_Requests == null) {
-            total_Requests = 0
-          }
-          let _data = {
-            model: _model_name,
-            total_exceptions: total_exceptions,
-            total_Requests: total_Requests,
-            failure_rate: total_Requests / total_exceptions
-          }
-          failureTableData.push(_data);
-          // sort failureTableData by failure_rate
-          failureTableData.sort((a, b) => b.failure_rate - a.failure_rate);
 
-          setFailureTableData(failureTableData);
+        setSlowResponsesData(slowResponses);
-          console.log("failureTableData:", failureTableData);
+
-        }
+
+        // let modelMetricsData = modelMetricsResponse.data;
+        // let successdeploymentToSuccess: Record<string, number> = {};
+        // for  (let i = 0; i < modelMetricsData.length; i++) {
+        //   let element = modelMetricsData[i];
+        //   let _model_name = element.model;
+        //   let _num_requests = element.num_requests;
+        //   successdeploymentToSuccess[_model_name] = _num_requests
+        // }
+        // console.log("successdeploymentToSuccess:", successdeploymentToSuccess)
+        
+        // let failureTableData = [];
+        // let _failureData = modelExceptionsResponse.data;
+        // for (let i = 0; i < _failureData.length; i++) {
+        //   const model = _failureData[i];
+        //   let _model_name = model.model;
+        //   let total_exceptions = model.total_exceptions;
+        //   let total_Requests = successdeploymentToSuccess[_model_name];
+        //   if (total_Requests == null) {
+        //     total_Requests = 0
+        //   }
+        //   let _data = {
+        //     model: _model_name,
+        //     total_exceptions: total_exceptions,
+        //     total_Requests: total_Requests,
+        //     failure_rate: total_Requests / total_exceptions
+        //   }
+        //   failureTableData.push(_data);
+        //   // sort failureTableData by failure_rate
+        //   failureTableData.sort((a, b) => b.failure_rate - a.failure_rate);
+        
+        //   setFailureTableData(failureTableData);
+        //   console.log("failureTableData:", failureTableData);
+        // }
 
       } catch (error) {
         console.error("There was an error fetching the model data", error);
@@ -1126,8 +1139,6 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
                 />
               )}
 
-                  
-              
               </Card>
               </Col>
             <Col>
@@ -1135,20 +1146,19 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
               <Table>
               <TableHead>
                 <TableRow>
-                  <TableHeaderCell>Model</TableHeaderCell>
+                  <TableHeaderCell>Deployment</TableHeaderCell>
-                  <TableHeaderCell>Success Requests</TableHeaderCell>
+                  <TableHeaderCell>Success Responses</TableHeaderCell>
-                  <TableHeaderCell>Error Requests</TableHeaderCell>
+                  <TableHeaderCell>Slow Responses</TableHeaderCell>
-                  <TableHeaderCell>Failure %</TableHeaderCell>
 
                 </TableRow>
               </TableHead>
               <TableBody>
-                {failureTableData.map((metric, idx) => (
+                {slowResponsesData.map((metric, idx) => (
                   <TableRow key={idx}>
-                    <TableCell>{metric.model}</TableCell>
+                    <TableCell>{metric.api_base}</TableCell>
-                    <TableCell>{metric.total_Requests}</TableCell>
+                    <TableCell>{metric.total_count}</TableCell>
-                    <TableCell>{metric.total_exceptions}</TableCell>
+                    <TableCell>{metric.slow_count}</TableCell>
-                    <TableCell>{metric.failure_rate}%</TableCell>
+
                   </TableRow>
 
                 ))}

ui/litellm-dashboard/src/components/networking.tsx

@@ -475,6 +475,46 @@ export const modelMetricsCall = async (
 };
 
 
+
+export const modelMetricsSlowResponsesCall = async (
+  accessToken: String,
+  userID: String,
+  userRole: String, 
+  modelGroup: String | null,
+) => {
+  /**
+   * Get all models on proxy
+   */
+  try {
+    let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/slow_responses` : `/model/metrics/slow_responses`;
+    if (modelGroup) {
+      url = `${url}?_selected_model_group=${modelGroup}`
+    }
+    // message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      message.error(errorData, 20);
+      throw new Error("Network response was not ok");
+    }
+    const data = await response.json();
+    // message.info("Received model data");
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to create key:", error);
+    throw error;
+  }
+};
+
+
 export const modelExceptionsCall = async (
   accessToken: String,
   userID: String,