forked from phoenix/litellm-mirror
build(add-fallbacks-on-UI): allows admin to add fallbacks on the UI
This commit is contained in:
Krrish Dholakia
parent
b1e2728906
commit
bae6f41017
9 changed files with 248 additions and 20 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -50,3 +50,4 @@ kub.yaml
|
|||
loadtest_kub.yaml
|
||||
litellm/proxy/_new_secret_config.yaml
|
||||
litellm/proxy/_new_secret_config.yaml
|
||||
litellm/proxy/_super_secret_config.yaml
|
||||
|
|
|
|||
|
|
@ -2533,6 +2533,8 @@ class Router:
|
|||
"timeout",
|
||||
"max_retries",
|
||||
"retry_after",
|
||||
"fallbacks",
|
||||
"context_window_fallbacks",
|
||||
]
|
||||
|
||||
for var in vars_to_include:
|
||||
|
|
|
|||
16
ui/litellm-dashboard/package-lock.json
generated
16
ui/litellm-dashboard/package-lock.json
generated
|
|
@ -36,6 +36,7 @@
|
|||
"eslint": "^8",
|
||||
"eslint-config-next": "14.1.0",
|
||||
"postcss": "^8.4.33",
|
||||
"prettier": "3.2.5",
|
||||
"tailwindcss": "^3.4.1",
|
||||
"typescript": "5.3.3"
|
||||
}
|
||||
|
|
@ -5515,6 +5516,21 @@
|
|||
"node": ">= 0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/prettier": {
|
||||
"version": "3.2.5",
|
||||
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz",
|
||||
"integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==",
|
||||
"dev": true,
|
||||
"bin": {
|
||||
"prettier": "bin/prettier.cjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/prettier/prettier?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/prismjs": {
|
||||
"version": "1.29.0",
|
||||
"resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz",
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
"eslint": "^8",
|
||||
"eslint-config-next": "14.1.0",
|
||||
"postcss": "^8.4.33",
|
||||
"prettier": "3.2.5",
|
||||
"tailwindcss": "^3.4.1",
|
||||
"typescript": "5.3.3"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ const CreateKeyPage = () => {
|
|||
const [keys, setKeys] = useState<null | any[]>(null);
|
||||
const [showSSOBanner, setShowSSOBanner] = useState<boolean>(true);
|
||||
const searchParams = useSearchParams();
|
||||
|
||||
const [modelData, setModelData] = useState<any>({ data: [] });
|
||||
const userID = searchParams.get("userID");
|
||||
const token = searchParams.get("token");
|
||||
|
||||
|
|
@ -132,6 +132,8 @@ const CreateKeyPage = () => {
|
|||
userRole={userRole}
|
||||
token={token}
|
||||
accessToken={accessToken}
|
||||
modelData={modelData}
|
||||
setModelData={setModelData}
|
||||
/>
|
||||
) : page == "llm-playground" ? (
|
||||
<ChatUI
|
||||
|
|
@ -179,6 +181,7 @@ const CreateKeyPage = () => {
|
|||
userID={userID}
|
||||
userRole={userRole}
|
||||
accessToken={accessToken}
|
||||
modelData={modelData}
|
||||
/>
|
||||
) : (
|
||||
<Usage
|
||||
|
|
|
|||
164
ui/litellm-dashboard/src/components/add_fallbacks.tsx
Normal file
164
ui/litellm-dashboard/src/components/add_fallbacks.tsx
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
/**
|
||||
* Modal to add fallbacks to the proxy router config
|
||||
*/
|
||||
|
||||
|
||||
|
||||
import React, { useState, useEffect, useRef } from "react";
|
||||
import { Button, TextInput, Grid, Col } from "@tremor/react";
|
||||
import { Select, SelectItem, MultiSelect, MultiSelectItem, Card, Metric, Text, Title, Subtitle, Accordion, AccordionHeader, AccordionBody, } from "@tremor/react";
|
||||
import { CopyToClipboard } from 'react-copy-to-clipboard';
|
||||
import { setCallbacksCall } from "./networking";
|
||||
import {
|
||||
Button as Button2,
|
||||
Modal,
|
||||
Form,
|
||||
Input,
|
||||
InputNumber,
|
||||
Select as Select2,
|
||||
message,
|
||||
} from "antd";
|
||||
import { keyCreateCall, slackBudgetAlertsHealthCheck, modelAvailableCall } from "./networking";
|
||||
import { list } from "postcss";
|
||||
|
||||
const { Option } = Select2;
|
||||
|
||||
interface AddFallbacksProps {
|
||||
models: string[] | undefined;
|
||||
accessToken: string;
|
||||
routerSettings: { [key: string]: any; }
|
||||
setRouterSettings: React.Dispatch<React.SetStateAction<{ [key: string]: any }>>;
|
||||
}
|
||||
|
||||
const AddFallbacks: React.FC<AddFallbacksProps> = ({
|
||||
models,
|
||||
accessToken,
|
||||
routerSettings,
|
||||
setRouterSettings
|
||||
}) => {
|
||||
const [form] = Form.useForm();
|
||||
const [isModalVisible, setIsModalVisible] = useState(false);
|
||||
const [selectedModel, setSelectedModel] = useState("");
|
||||
const handleOk = () => {
|
||||
setIsModalVisible(false);
|
||||
form.resetFields();
|
||||
};
|
||||
|
||||
const handleCancel = () => {
|
||||
setIsModalVisible(false);
|
||||
form.resetFields();
|
||||
};
|
||||
|
||||
const updateFallbacks = (formValues: Record<string, any>) => {
|
||||
// Print the received value
|
||||
console.log(formValues);
|
||||
|
||||
// Extract model_name and models from formValues
|
||||
const { model_name, models } = formValues;
|
||||
|
||||
// Create new fallback
|
||||
const newFallback = { [model_name]: models };
|
||||
|
||||
// Get current fallbacks, or an empty array if it's null
|
||||
const currentFallbacks = routerSettings.fallbacks || [];
|
||||
|
||||
// Add new fallback to the current fallbacks
|
||||
const updatedFallbacks = [...currentFallbacks, newFallback];
|
||||
|
||||
// Create a new routerSettings object with updated fallbacks
|
||||
const updatedRouterSettings = { ...routerSettings, fallbacks: updatedFallbacks };
|
||||
|
||||
// Print updated routerSettings
|
||||
console.log(updatedRouterSettings);
|
||||
|
||||
const payload = {
|
||||
router_settings: updatedRouterSettings
|
||||
};
|
||||
|
||||
try {
|
||||
setCallbacksCall(accessToken, payload);
|
||||
// Update routerSettings state
|
||||
setRouterSettings(updatedRouterSettings);
|
||||
} catch (error) {
|
||||
message.error("Failed to update router settings: " + error, 20);
|
||||
}
|
||||
|
||||
message.success("router settings updated successfully");
|
||||
|
||||
setIsModalVisible(false)
|
||||
form.resetFields();
|
||||
};
|
||||
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Button className="mx-auto" onClick={() => setIsModalVisible(true)}>
|
||||
+ Add Fallbacks
|
||||
</Button>
|
||||
<Modal
|
||||
title="Add Fallbacks"
|
||||
visible={isModalVisible}
|
||||
width={800}
|
||||
footer={null}
|
||||
onOk={handleOk}
|
||||
onCancel={handleCancel}
|
||||
>
|
||||
<Form
|
||||
form={form}
|
||||
onFinish={updateFallbacks}
|
||||
labelCol={{ span: 8 }}
|
||||
wrapperCol={{ span: 16 }}
|
||||
labelAlign="left"
|
||||
>
|
||||
<>
|
||||
<Form.Item
|
||||
label="Public Model Name"
|
||||
name="model_name"
|
||||
rules={[{ required: true, message: 'Set the model to fallback for' }]}
|
||||
help="required"
|
||||
>
|
||||
<Select defaultValue={selectedModel}>
|
||||
{models && models.map((model: string, index) => (
|
||||
<SelectItem
|
||||
key={index}
|
||||
value={model}
|
||||
onClick={() => setSelectedModel(model)}
|
||||
>
|
||||
{model}
|
||||
</SelectItem>
|
||||
))
|
||||
}
|
||||
</Select>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
label="Fallback Models"
|
||||
name="models"
|
||||
rules={[{ required: true, message: 'Please select a model' }]}
|
||||
help="required"
|
||||
>
|
||||
<MultiSelect value={models}>
|
||||
{models &&
|
||||
models.filter(data => data != selectedModel).map((model: string) => (
|
||||
(
|
||||
<MultiSelectItem key={model} value={model}>
|
||||
{model}
|
||||
</MultiSelectItem>
|
||||
)
|
||||
))}
|
||||
|
||||
</MultiSelect>
|
||||
</Form.Item>
|
||||
</>
|
||||
|
||||
<div style={{ textAlign: "right", marginTop: "10px" }}>
|
||||
<Button2 htmlType="submit">Add Fallbacks</Button2>
|
||||
</div>
|
||||
</Form>
|
||||
</Modal>
|
||||
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default AddFallbacks;
|
||||
|
|
@ -17,20 +17,24 @@ import {
|
|||
TextInput,
|
||||
Col,
|
||||
} from "@tremor/react";
|
||||
import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react";
|
||||
import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
|
||||
import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
|
||||
import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
|
||||
import AddFallbacks from "./add_fallbacks"
|
||||
|
||||
interface GeneralSettingsPageProps {
|
||||
accessToken: string | null;
|
||||
userRole: string | null;
|
||||
userID: string | null;
|
||||
modelData: any
|
||||
}
|
||||
|
||||
const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
|
||||
accessToken,
|
||||
userRole,
|
||||
userID,
|
||||
modelData
|
||||
}) => {
|
||||
const [routerSettings, setRouterSettings] = useState<{ [key: string]: any }>({});
|
||||
const [isModalVisible, setIsModalVisible] = useState(false);
|
||||
|
|
@ -103,6 +107,13 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
|
|||
|
||||
return (
|
||||
<div className="w-full mx-4">
|
||||
<TabGroup className="gap-2 p-8 h-[75vh] w-full mt-2">
|
||||
<TabList variant="line" defaultValue="1">
|
||||
<Tab value="1">General Settings</Tab>
|
||||
<Tab value="2">Fallbacks</Tab>
|
||||
</TabList>
|
||||
<TabPanels>
|
||||
<TabPanel>
|
||||
<Grid numItems={1} className="gap-2 p-8 w-full mt-2">
|
||||
<Title>Router Settings</Title>
|
||||
<Card >
|
||||
|
|
@ -129,8 +140,8 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
|
|||
/>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</Card>
|
||||
<Col>
|
||||
|
|
@ -139,7 +150,34 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
|
|||
</Button>
|
||||
</Col>
|
||||
</Grid>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<Table>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableHeaderCell>Model Name</TableHeaderCell>
|
||||
<TableHeaderCell>Fallbacks</TableHeaderCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
|
||||
<TableBody>
|
||||
{
|
||||
routerSettings["fallbacks"] &&
|
||||
routerSettings["fallbacks"].map((item: Object, index: number) =>
|
||||
Object.entries(item).map(([key, value]) => (
|
||||
<TableRow key={index.toString() + key}>
|
||||
<TableCell>{key}</TableCell>
|
||||
<TableCell>{Array.isArray(value) ? value.join(', ') : value}</TableCell>
|
||||
</TableRow>
|
||||
))
|
||||
)
|
||||
}
|
||||
</TableBody>
|
||||
</Table>
|
||||
<AddFallbacks models={modelData?.data ? modelData.data.map((data: any) => data.model_name) : []} accessToken={accessToken} routerSettings={routerSettings} setRouterSettings={setRouterSettings}/>
|
||||
</TabPanel>
|
||||
</TabPanels>
|
||||
</TabGroup>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ const Sidebar: React.FC<SidebarProps> = ({
|
|||
userRole == "Admin" ? (
|
||||
<Menu.Item key="9" onClick={() => setPage("general-settings")}>
|
||||
<Text>
|
||||
Settings
|
||||
Router Settings
|
||||
</Text>
|
||||
</Menu.Item>
|
||||
) : null
|
||||
|
|
|
|||
|
|
@ -49,6 +49,8 @@ interface ModelDashboardProps {
|
|||
token: string | null;
|
||||
userRole: string | null;
|
||||
userID: string | null;
|
||||
modelData: any,
|
||||
setModelData: any
|
||||
}
|
||||
|
||||
interface EditModelModalProps {
|
||||
|
|
@ -176,8 +178,9 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
token,
|
||||
userRole,
|
||||
userID,
|
||||
modelData = { data: [] },
|
||||
setModelData
|
||||
}) => {
|
||||
const [modelData, setModelData] = useState<any>({ data: [] });
|
||||
const [pendingRequests, setPendingRequests] = useState<any[]>([]);
|
||||
const [form] = Form.useForm();
|
||||
const [modelMap, setModelMap] = useState<any>(null);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue