phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Merge pull request #3995 from BerriAI/litellm_filter_model_analytics_key_alias

Browse source 
[Feat] UI - Filter model latency by API Key Alias
This commit is contained in:
Ishaan Jaff 2024-06-03 19:50:01 -07:00 • committed by GitHub
commit 9f29dbaed6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 147 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

40
litellm/proxy/proxy_server.py
View file

@ -11615,6 +11615,7 @@ async def model_metrics(
_selected_model_group: Optional[str] = "gpt-4-32k",
startTime: Optional[datetime] = None,
endTime: Optional[datetime] = None,
api_key: Optional[str] = None,
):
global prisma_client, llm_router
if prisma_client is None:
@ -11627,6 +11628,9 @@ async def model_metrics(
startTime = startTime or datetime.now() - timedelta(days=30)
endTime = endTime or datetime.now()
if api_key is None or api_key == "undefined":
api_key = "null"
sql_query = """
SELECT
api_base,
@ -11639,6 +11643,12 @@ async def model_metrics(
WHERE
"startTime" BETWEEN $2::timestamp AND $3::timestamp
AND "model_group" = $1 AND "cache_hit" != 'True'
AND (
CASE
WHEN $4 != 'null' THEN "api_key" = $4
ELSE TRUE
END
)
GROUP BY
api_base,
model_group,
@ -11651,7 +11661,7 @@ async def model_metrics(
"""
_all_api_bases = set()
db_response = await prisma_client.db.query_raw(
sql_query, _selected_model_group, startTime, endTime
sql_query, _selected_model_group, startTime, endTime, api_key
)
_daily_entries: dict = {} # {"Jun 23": {"model1": 0.002, "model2": 0.003}}
@ -11710,6 +11720,7 @@ async def model_metrics_slow_responses(
_selected_model_group: Optional[str] = "gpt-4-32k",
startTime: Optional[datetime] = None,
endTime: Optional[datetime] = None,
api_key: Optional[str] = None,
):
global prisma_client, llm_router, proxy_logging_obj
if prisma_client is None:
@ -11719,6 +11730,9 @@ async def model_metrics_slow_responses(
param="None",
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
if api_key is None or api_key == "undefined":
api_key = "null"
startTime = startTime or datetime.now() - timedelta(days=30)
endTime = endTime or datetime.now()
@ -11742,6 +11756,12 @@ WHERE
AND "cache_hit" != 'True'
AND "startTime" >= $3::timestamp
AND "startTime" <= $4::timestamp
AND (
CASE
WHEN $5 != 'null' THEN "api_key" = $5
ELSE TRUE
END
)
GROUP BY
api_base
ORDER BY
@ -11749,7 +11769,12 @@ ORDER BY
"""
db_response = await prisma_client.db.query_raw(
sql_query, alerting_threshold, _selected_model_group, startTime, endTime
sql_query,
alerting_threshold,
_selected_model_group,
startTime,
endTime,
api_key,
)
if db_response is not None:
@ -11773,6 +11798,7 @@ async def model_metrics_exceptions(
_selected_model_group: Optional[str] = None,
startTime: Optional[datetime] = None,
endTime: Optional[datetime] = None,
api_key: Optional[str] = None,
):
global prisma_client, llm_router
if prisma_client is None:
@ -11786,6 +11812,9 @@ async def model_metrics_exceptions(
startTime = startTime or datetime.now() - timedelta(days=30)
endTime = endTime or datetime.now()
if api_key is None or api_key == "undefined":
api_key = "null"
"""
"""
sql_query = """
@ -11795,7 +11824,10 @@ async def model_metrics_exceptions(
exception_type,
COUNT(*) AS num_rate_limit_exceptions
FROM "LiteLLM_ErrorLogs"
WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp AND model_group = $3
WHERE
"startTime" >= $1::timestamp
AND "endTime" <= $2::timestamp
AND model_group = $3
GROUP BY combined_model_api_base, exception_type
)
SELECT
@ -11808,7 +11840,7 @@ async def model_metrics_exceptions(
LIMIT 200;
"""
db_response = await prisma_client.db.query_raw(
sql_query, startTime, endTime, _selected_model_group
sql_query, startTime, endTime, _selected_model_group, api_key
)
response: List[dict] = []
exception_types = set()

1
ui/litellm-dashboard/src/app/page.tsx
View file

@ -138,6 +138,7 @@ const CreateKeyPage = () => {
userID={userID}
userRole={userRole}
token={token}
keys={keys}
accessToken={accessToken}
modelData={modelData}
setModelData={setModelData}

114
ui/litellm-dashboard/src/components/model_dashboard.tsx
View file

@ -56,6 +56,7 @@ import { BarChart, AreaChart } from "@tremor/react";
import {
Button as Button2,
Modal,
Popover,
Form,
Input,
Select as Select2,
@ -80,6 +81,7 @@ import {
RefreshIcon,
CheckCircleIcon,
XCircleIcon,
FilterIcon,
} from "@heroicons/react/outline";
import DeleteModelButton from "./delete_model_button";
const { Title: Title2, Link } = Typography;
@ -96,6 +98,7 @@ interface ModelDashboardProps {
userRole: string | null;
userID: string | null;
modelData: any;
keys: any[] | null;
setModelData: any;
premiumUser: boolean;
}
@ -260,6 +263,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
userRole,
userID,
modelData = { data: [] },
keys,
setModelData,
premiumUser,
}) => {
@ -313,6 +317,17 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const [globalExceptionData, setGlobalExceptionData] = useState<GlobalExceptionActivityData>({} as GlobalExceptionActivityData);
const [globalExceptionPerDeployment, setGlobalExceptionPerDeployment] = useState<any[]>([]);
const [showAdvancedFilters, setShowAdvancedFilters] = useState<boolean>(false);
const [selectedAPIKey, setSelectedAPIKey] = useState<any | null>(null);
useEffect(() => {
updateModelMetrics(
selectedModelGroup,
dateValue.from,
dateValue.to
);
}, [selectedAPIKey]);
function formatCreatedAt(createdAt: string | null) {
if (createdAt) {
const date = new Date(createdAt);
@ -612,7 +627,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
userRole,
_initial_model_group,
dateValue.from?.toISOString(),
dateValue.to?.toISOString()
dateValue.to?.toISOString(),
selectedAPIKey?.token
);
console.log("Model metrics response:", modelMetricsResponse);
@ -640,7 +656,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
userRole,
_initial_model_group,
dateValue.from?.toISOString(),
dateValue.to?.toISOString()
dateValue.to?.toISOString(),
selectedAPIKey?.token
);
console.log("Model exceptions response:", modelExceptionsResponse);
setModelExceptions(modelExceptionsResponse.data);
@ -652,7 +669,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
userRole,
_initial_model_group,
dateValue.from?.toISOString(),
dateValue.to?.toISOString()
dateValue.to?.toISOString(),
selectedAPIKey?.token
);
const dailyExceptions = await adminGlobalActivityExceptions(
@ -874,7 +892,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const updateModelMetrics = async (
modelGroup: string | null,
startTime: Date | undefined,
endTime: Date | undefined
endTime: Date | undefined,
) => {
console.log("Updating model metrics for group:", modelGroup);
if (!accessToken || !userID || !userRole || !startTime || !endTime) {
@ -888,6 +906,11 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
);
setSelectedModelGroup(modelGroup); // If you want to store the selected model group in state
let selected_token = selectedAPIKey?.token;
if (selected_token === undefined) {
selected_token = null;
}
try {
const modelMetricsResponse = await modelMetricsCall(
accessToken,
@ -895,7 +918,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
userRole,
modelGroup,
startTime.toISOString(),
endTime.toISOString()
endTime.toISOString(),
selected_token
);
console.log("Model metrics response:", modelMetricsResponse);
@ -922,7 +946,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
userRole,
modelGroup,
startTime.toISOString(),
endTime.toISOString()
endTime.toISOString(),
selected_token
);
console.log("Model exceptions response:", modelExceptionsResponse);
setModelExceptions(modelExceptionsResponse.data);
@ -934,7 +959,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
userRole,
modelGroup,
startTime.toISOString(),
endTime.toISOString()
endTime.toISOString(),
selected_token
);
console.log("slowResponses:", slowResponses);
@ -969,6 +995,51 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
}
};
const FilterByContent = (
<div >
<Text className="mb-1">Select API Key Name</Text>
<Select defaultValue="all-keys">
<SelectItem
key="all-keys"
value="all-keys"
onClick={() => {
setSelectedAPIKey(null);
}}
>
All Keys
</SelectItem>
{keys?.map((key: any, index: number) => {
if (
key &&
key["key_alias"] !== null &&
key["key_alias"].length > 0
) {
return (
<SelectItem
key={index}
value={String(index)}
onClick={() => {
setSelectedAPIKey(key);
}}
>
{key["key_alias"]}
</SelectItem>
);
}
return null; // Add this line to handle the case when the condition is not met
})}
</Select>
</div>
);
const customTooltip = (props: any) => {
const { payload, active } = props;
if (!active || !payload) return null;
@ -1722,9 +1793,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
</Card>
</TabPanel>
<TabPanel>
{/* <p style={{fontSize: '0.85rem', color: '#808080'}}>View how requests were load balanced within a model group</p> */}
<Grid numItems={2} className="mt-2">
<Grid numItems={3} className="mt-2 mb-2">
<Col>
<Text>Select Time Range</Text>
<DateRangePicker
@ -1743,7 +1812,6 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
<Col>
<Text>Select Model Group</Text>
<Select
className="mb-4 mt-2"
defaultValue={
selectedModelGroup
? selectedModelGroup
@ -1768,7 +1836,27 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
))}
</Select>
</Col>
</Grid>
<Col>
<Popover
trigger="click" content={FilterByContent}
>
<Button
icon={FilterIcon}
size="md"
variant="secondary"
className="mt-4 ml-2"
style={{
border: "none",
}}
onClick={() => setShowAdvancedFilters(true)}
>
</Button>
</Popover>
</Col>
</Grid>
<Grid numItems={2}>
<Col>
@ -2035,7 +2123,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
</TabPanel>
</TabPanels>
</TabGroup>
</div>
</div>
);
};

15
ui/litellm-dashboard/src/components/networking.tsx
View file

@ -726,7 +726,8 @@ export const modelMetricsCall = async (
userRole: String,
modelGroup: String | null,
startTime: String | undefined,
endTime: String | undefined
endTime: String | undefined,
apiKey: String | null,
) => {
/**
* Get all models on proxy
@ -734,7 +735,7 @@ export const modelMetricsCall = async (
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics` : `/model/metrics`;
if (modelGroup) {
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}`;
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}`;
}
// message.info("Requesting model data");
const response = await fetch(url, {
@ -805,7 +806,8 @@ export const modelMetricsSlowResponsesCall = async (
userRole: String,
modelGroup: String | null,
startTime: String | undefined,
endTime: String | undefined
endTime: String | undefined,
apiKey: String | null
) => {
/**
* Get all models on proxy
@ -815,7 +817,7 @@ export const modelMetricsSlowResponsesCall = async (
? `${proxyBaseUrl}/model/metrics/slow_responses`
: `/model/metrics/slow_responses`;
if (modelGroup) {
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}`;
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}`;
}
// message.info("Requesting model data");
@ -848,7 +850,8 @@ export const modelExceptionsCall = async (
userRole: String,
modelGroup: String | null,
startTime: String | undefined,
endTime: String | undefined
endTime: String | undefined,
apiKey: String | null
) => {
/**
* Get all models on proxy
@ -859,7 +862,7 @@ export const modelExceptionsCall = async (
: `/model/metrics/exceptions`;
if (modelGroup) {
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}`;
url = `${url}?_selected_model_group=${modelGroup}&startTime=${startTime}&endTime=${endTime}&api_key=${apiKey}`;
}
const response = await fetch(url, {
method: "GET",

1
ui/litellm-dashboard/ui_colors.json
View file

@ -1,7 +1,6 @@
{
"brand": {
"DEFAULT": "#6366f1",
"faint": "#6c6fed",
"muted": "#8688ef",
"subtle": "#8e91eb",
"emphasis": "#5558eb",