phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

build(add-fallbacks-on-UI): allows admin to add fallbacks on the UI

Browse source
This commit is contained in:
Krrish Dholakia 2024-04-24 15:39:48 -07:00
parent b1e2728906
commit bae6f41017
9 changed files with 248 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -50,3 +50,4 @@ kub.yaml
loadtest_kub.yaml
litellm/proxy/_new_secret_config.yaml
litellm/proxy/_new_secret_config.yaml
litellm/proxy/_super_secret_config.yaml

2
litellm/router.py
View file

@ -2533,6 +2533,8 @@ class Router:
"timeout",
"max_retries",
"retry_after",
"fallbacks",
"context_window_fallbacks",
]
for var in vars_to_include:

16
ui/litellm-dashboard/package-lock.json generated
View file

@ -36,6 +36,7 @@
"eslint": "^8",
"eslint-config-next": "14.1.0",
"postcss": "^8.4.33",
"prettier": "3.2.5",
"tailwindcss": "^3.4.1",
"typescript": "5.3.3"
}
@ -5515,6 +5516,21 @@
"node": ">= 0.8.0"
}
},
"node_modules/prettier": {
"version": "3.2.5",
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz",
"integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==",
"dev": true,
"bin": {
"prettier": "bin/prettier.cjs"
},
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/prettier/prettier?sponsor=1"
}
},
"node_modules/prismjs": {
"version": "1.29.0",
"resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz",

1
ui/litellm-dashboard/package.json
View file

@ -37,6 +37,7 @@
"eslint": "^8",
"eslint-config-next": "14.1.0",
"postcss": "^8.4.33",
"prettier": "3.2.5",
"tailwindcss": "^3.4.1",
"typescript": "5.3.3"
}

5
ui/litellm-dashboard/src/app/page.tsx
View file

@ -24,7 +24,7 @@ const CreateKeyPage = () => {
const [keys, setKeys] = useState<null | any[]>(null);
const [showSSOBanner, setShowSSOBanner] = useState<boolean>(true);
const searchParams = useSearchParams();
const [modelData, setModelData] = useState<any>({ data: [] });
const userID = searchParams.get("userID");
const token = searchParams.get("token");
@ -132,6 +132,8 @@ const CreateKeyPage = () => {
userRole={userRole}
token={token}
accessToken={accessToken}
modelData={modelData}
setModelData={setModelData}
/>
) : page == "llm-playground" ? (
<ChatUI
@ -179,6 +181,7 @@ const CreateKeyPage = () => {
userID={userID}
userRole={userRole}
accessToken={accessToken}
modelData={modelData}
/>
) : (
<Usage

164
ui/litellm-dashboard/src/components/add_fallbacks.tsx Normal file
View file

@ -0,0 +1,164 @@
/**
* Modal to add fallbacks to the proxy router config
*/
import React, { useState, useEffect, useRef } from "react";
import { Button, TextInput, Grid, Col } from "@tremor/react";
import { Select, SelectItem, MultiSelect, MultiSelectItem, Card, Metric, Text, Title, Subtitle, Accordion, AccordionHeader, AccordionBody, } from "@tremor/react";
import { CopyToClipboard } from 'react-copy-to-clipboard';
import { setCallbacksCall } from "./networking";
import {
Button as Button2,
Modal,
Form,
Input,
InputNumber,
Select as Select2,
message,
} from "antd";
import { keyCreateCall, slackBudgetAlertsHealthCheck, modelAvailableCall } from "./networking";
import { list } from "postcss";
const { Option } = Select2;
interface AddFallbacksProps {
models: string[] | undefined;
accessToken: string;
routerSettings: { [key: string]: any; }
setRouterSettings: React.Dispatch<React.SetStateAction<{ [key: string]: any }>>;
}
const AddFallbacks: React.FC<AddFallbacksProps> = ({
models,
accessToken,
routerSettings,
setRouterSettings
}) => {
const [form] = Form.useForm();
const [isModalVisible, setIsModalVisible] = useState(false);
const [selectedModel, setSelectedModel] = useState("");
const handleOk = () => {
setIsModalVisible(false);
form.resetFields();
};
const handleCancel = () => {
setIsModalVisible(false);
form.resetFields();
};
const updateFallbacks = (formValues: Record<string, any>) => {
// Print the received value
console.log(formValues);
// Extract model_name and models from formValues
const { model_name, models } = formValues;
// Create new fallback
const newFallback = { [model_name]: models };
// Get current fallbacks, or an empty array if it's null
const currentFallbacks = routerSettings.fallbacks || [];
// Add new fallback to the current fallbacks
const updatedFallbacks = [...currentFallbacks, newFallback];
// Create a new routerSettings object with updated fallbacks
const updatedRouterSettings = { ...routerSettings, fallbacks: updatedFallbacks };
// Print updated routerSettings
console.log(updatedRouterSettings);
const payload = {
router_settings: updatedRouterSettings
};
try {
setCallbacksCall(accessToken, payload);
// Update routerSettings state
setRouterSettings(updatedRouterSettings);
} catch (error) {
message.error("Failed to update router settings: " + error, 20);
}
message.success("router settings updated successfully");
setIsModalVisible(false)
form.resetFields();
};
return (
<div>
<Button className="mx-auto" onClick={() => setIsModalVisible(true)}>
+ Add Fallbacks
</Button>
<Modal
title="Add Fallbacks"
visible={isModalVisible}
width={800}
footer={null}
onOk={handleOk}
onCancel={handleCancel}
>
<Form
form={form}
onFinish={updateFallbacks}
labelCol={{ span: 8 }}
wrapperCol={{ span: 16 }}
labelAlign="left"
>
<>
<Form.Item
label="Public Model Name"
name="model_name"
rules={[{ required: true, message: 'Set the model to fallback for' }]}
help="required"
>
<Select defaultValue={selectedModel}>
{models && models.map((model: string, index) => (
<SelectItem
key={index}
value={model}
onClick={() => setSelectedModel(model)}
>
{model}
</SelectItem>
))
}
</Select>
</Form.Item>
<Form.Item
label="Fallback Models"
name="models"
rules={[{ required: true, message: 'Please select a model' }]}
help="required"
>
<MultiSelect value={models}>
{models &&
models.filter(data => data != selectedModel).map((model: string) => (
(
<MultiSelectItem key={model} value={model}>
{model}
</MultiSelectItem>
)
))}
</MultiSelect>
</Form.Item>
</>
<div style={{ textAlign: "right", marginTop: "10px" }}>
<Button2 htmlType="submit">Add Fallbacks</Button2>
</div>
</Form>
</Modal>
</div>
);
};
export default AddFallbacks;

72
ui/litellm-dashboard/src/components/general_settings.tsx
View file

@ -17,20 +17,24 @@ import {
TextInput,
Col,
} from "@tremor/react";
import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react";
import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
import AddFallbacks from "./add_fallbacks"
interface GeneralSettingsPageProps {
accessToken: string | null;
userRole: string | null;
userID: string | null;
modelData: any
}
const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
accessToken,
userRole,
userID,
modelData
}) => {
const [routerSettings, setRouterSettings] = useState<{ [key: string]: any }>({});
const [isModalVisible, setIsModalVisible] = useState(false);
@ -103,6 +107,13 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
return (
<div className="w-full mx-4">
<TabGroup className="gap-2 p-8 h-[75vh] w-full mt-2">
<TabList variant="line" defaultValue="1">
<Tab value="1">General Settings</Tab>
<Tab value="2">Fallbacks</Tab>
</TabList>
<TabPanels>
<TabPanel>
<Grid numItems={1} className="gap-2 p-8 w-full mt-2">
<Title>Router Settings</Title>
<Card >
@ -114,23 +125,23 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
</TableRow>
</TableHead>
<TableBody>
{Object.entries(routerSettings).map(([param, value]) => (
<TableRow key={param}>
<TableCell>
<Text>{param}</Text>
<p style={{fontSize: '0.65rem', color: '#808080', fontStyle: 'italic'}} className="mt-1">{paramExplanation[param]}</p>
</TableCell>
<TableCell>
<TextInput
name={param}
defaultValue={
typeof value === 'object' ? JSON.stringify(value, null, 2) : value.toString()
}
/>
</TableCell>
</TableRow>
))}
</TableBody>
{Object.entries(routerSettings).map(([param, value]) => (
<TableRow key={param}>
<TableCell>
<Text>{param}</Text>
<p style={{fontSize: '0.65rem', color: '#808080', fontStyle: 'italic'}} className="mt-1">{paramExplanation[param]}</p>
</TableCell>
<TableCell>
<TextInput
name={param}
defaultValue={
typeof value === 'object' ? JSON.stringify(value, null, 2) : value.toString()
}
/>
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</Card>
<Col>
@ -139,7 +150,34 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
</Button>
</Col>
</Grid>
</TabPanel>
<TabPanel>
<Table>
<TableHead>
<TableRow>
<TableHeaderCell>Model Name</TableHeaderCell>
<TableHeaderCell>Fallbacks</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
{
routerSettings["fallbacks"] &&
routerSettings["fallbacks"].map((item: Object, index: number) =>
Object.entries(item).map(([key, value]) => (
<TableRow key={index.toString() + key}>
<TableCell>{key}</TableCell>
<TableCell>{Array.isArray(value) ? value.join(', ') : value}</TableCell>
</TableRow>
))
)
}
</TableBody>
</Table>
<AddFallbacks models={modelData?.data ? modelData.data.map((data: any) => data.model_name) : []} accessToken={accessToken} routerSettings={routerSettings} setRouterSettings={setRouterSettings}/>
</TabPanel>
</TabPanels>
</TabGroup>
</div>
);
};

2
ui/litellm-dashboard/src/components/leftnav.tsx
View file

@ -118,7 +118,7 @@ const Sidebar: React.FC<SidebarProps> = ({
userRole == "Admin" ? (
<Menu.Item key="9" onClick={() => setPage("general-settings")}>
<Text>
Settings
Router Settings
</Text>
</Menu.Item>
) : null

5
ui/litellm-dashboard/src/components/model_dashboard.tsx
View file

@ -49,6 +49,8 @@ interface ModelDashboardProps {
token: string | null;
userRole: string | null;
userID: string | null;
modelData: any,
setModelData: any
}
interface EditModelModalProps {
@ -176,8 +178,9 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
token,
userRole,
userID,
modelData = { data: [] },
setModelData
}) => {
const [modelData, setModelData] = useState<any>({ data: [] });
const [pendingRequests, setPendingRequests] = useState<any[]>([]);
const [form] = Form.useForm();
const [modelMap, setModelMap] = useState<any>(null);