phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 02:34:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

initial commit for cache control

Browse source
This commit is contained in:
Ishaan Jaff 2025-04-14 18:21:26 -07:00
parent cb521a8b3e
commit 39e567416c
2 changed files with 159 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

26
ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx
View file

@ -5,6 +5,7 @@ import { Row, Col, Typography, Card } from "antd";
import TextArea from "antd/es/input/TextArea";
import { Team } from "../key_team_helpers/key_list";
import TeamDropdown from "../common_components/team_dropdown";
import CacheControlSettings from "./cache_control_settings";
const { Link } = Typography;
interface AdvancedSettingsProps {
@ -21,6 +22,7 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({
const [form] = Form.useForm();
const [customPricing, setCustomPricing] = React.useState(false);
const [pricingModel, setPricingModel] = React.useState<'per_token' | 'per_second'>('per_token');
const [showCacheControl, setShowCacheControl] = React.useState(false);
// Add validation function for numbers
const validateNumber = (_: any, value: string) => {
@ -83,6 +85,24 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({
}
};
const handleCacheControlChange = (checked: boolean) => {
setShowCacheControl(checked);
if (!checked) {
const currentParams = form.getFieldValue('litellm_extra_params');
try {
let paramsObj = currentParams ? JSON.parse(currentParams) : {};
delete paramsObj.cache_control_injection_points;
if (Object.keys(paramsObj).length > 0) {
form.setFieldValue('litellm_extra_params', JSON.stringify(paramsObj, null, 2));
} else {
form.setFieldValue('litellm_extra_params', '');
}
} catch (error) {
form.setFieldValue('litellm_extra_params', '');
}
}
};
return (
<>
<Accordion className="mt-2 mb-4">
@ -150,6 +170,12 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({
</div>
)}
<CacheControlSettings
form={form}
showCacheControl={showCacheControl}
onCacheControlChange={handleCacheControlChange}
/>
<Form.Item
label="Use in pass through routes"
name="use_in_pass_through"

133
ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx Normal file
View file

@ -0,0 +1,133 @@
import React from "react";
import { Form, Switch, Select, Input, Button } from "antd";
import { MinusCircleOutlined, PlusOutlined } from '@ant-design/icons';
interface CacheControlInjectionPoint {
location: "message";
role?: "user" | "system" | "assistant";
index?: number;
}
interface CacheControlSettingsProps {
form: any; // Form instance from parent
showCacheControl: boolean;
onCacheControlChange: (checked: boolean) => void;
}
const CacheControlSettings: React.FC<CacheControlSettingsProps> = ({
form,
showCacheControl,
onCacheControlChange,
}) => {
const updateCacheControlPoints = (injectionPoints: CacheControlInjectionPoint[]) => {
const currentParams = form.getFieldValue('litellm_extra_params');
try {
let paramsObj = currentParams ? JSON.parse(currentParams) : {};
if (injectionPoints.length > 0) {
paramsObj.cache_control_injection_points = injectionPoints;
} else {
delete paramsObj.cache_control_injection_points;
}
if (Object.keys(paramsObj).length > 0) {
form.setFieldValue('litellm_extra_params', JSON.stringify(paramsObj, null, 2));
} else {
form.setFieldValue('litellm_extra_params', '');
}
} catch (error) {
console.error('Error updating cache control points:', error);
}
};
return (
<>
<Form.Item
label="Cache Control Injection Points"
name="cache_control"
valuePropName="checked"
className="mb-4"
tooltip="Tell litellm where to inject cache control checkpoints. This is useful for reducing costs by caching responses."
>
<Switch onChange={onCacheControlChange} className="bg-gray-600" />
</Form.Item>
{showCacheControl && (
<div className="ml-6 pl-4 border-l-2 border-gray-200">
<Form.List
name="cache_control_points"
initialValue={[{ location: "message" }]}
>
{(fields, { add, remove }) => (
<>
{fields.map((field, index) => (
<div key={field.key} style={{ display: 'flex', marginBottom: 8, gap: 8, alignItems: 'baseline' }}>
<Form.Item
{...field}
name={[field.name, 'location']}
initialValue="message"
style={{ marginBottom: 0, width: '120px' }}
>
<Select disabled options={[{ value: 'message', label: 'Message' }]} />
</Form.Item>
<Form.Item
{...field}
name={[field.name, 'role']}
style={{ marginBottom: 0, width: '120px' }}
>
<Select
placeholder="Select role"
allowClear
options={[
{ value: 'user', label: 'User' },
{ value: 'system', label: 'System' },
{ value: 'assistant', label: 'Assistant' },
]}
onChange={() => {
const values = form.getFieldValue('cache_control_points');
updateCacheControlPoints(values);
}}
/>
</Form.Item>
<Form.Item
{...field}
name={[field.name, 'index']}
style={{ marginBottom: 0, width: '120px' }}
>
<Input
type="number"
placeholder="Index (optional)"
onChange={() => {
const values = form.getFieldValue('cache_control_points');
updateCacheControlPoints(values);
}}
/>
</Form.Item>
{fields.length > 1 && (
<MinusCircleOutlined onClick={() => {
remove(field.name);
setTimeout(() => {
const values = form.getFieldValue('cache_control_points');
updateCacheControlPoints(values);
}, 0);
}} />
)}
</div>
))}
<Form.Item style={{ marginTop: 8 }}>
<Button
type="dashed"
onClick={() => add()}
icon={<PlusOutlined />}
>
Add Injection Point
</Button>
</Form.Item>
</>
)}
</Form.List>
</div>
)}
</>
);
};
export default CacheControlSettings;