phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 19:24:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Litellm dev 12 26 2024 p3 (#7434)

Browse source 
* build(model_prices_and_context_window.json): update groq models to specify 'supports_vision' parameter

Closes https://github.com/BerriAI/litellm/issues/7433

* docs(groq.md): add groq vision example to docs

Closes https://github.com/BerriAI/litellm/issues/7433

* fix(prometheus.py): refactor self.litellm_proxy_failed_requests_metric to use label factory

* feat(prometheus.py): new 'litellm_proxy_failed_requests_by_tag_metric'

allows tracking failed requests by tag on proxy

* fix(prometheus.py): fix exception logging

* feat(prometheus.py): add new 'litellm_request_total_latency_by_tag_metric'

enables tracking latency by use-case

* feat(prometheus.py): add new llm api latency by tag metric

* feat(prometheus.py): new litellm_deployment_latency_per_output_token_by_tag metric

allows tracking deployment latency by tag

* fix(prometheus.py): refactor 'litellm_requests_metric' to use enum values + label factory

* feat(prometheus.py): new litellm_proxy_total_requests_by_tag metric

allows tracking total requests by tag

* feat(prometheus.py): new metric litellm_deployment_successful_fallbacks_by_tag

allows tracking deployment fallbacks by tag

* fix(prometheus.py): new 'litellm_deployment_failed_fallbacks_by_tag' metric

allows tracking failed fallbacks on deployment by custom tag

* test: fix test

* test: rename test to run earlier

* test: skip flaky test
This commit is contained in:
Krish Dholakia 2024-12-26 21:21:16 -08:00 • committed by GitHub
parent c72f1aeeda
commit f30260343b
7 changed files with 389 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

96
docs/my-website/docs/providers/groq.md
View file

@ -259,6 +259,99 @@ if tool_calls:
print("second response\n", second_response) print("second response\n", second_response)
``` ```
## Groq - Vision Example
Select Groq models support vision. Check out their [model list](https://console.groq.com/docs/vision) for more details.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
from litellm import completion
os.environ["GROQ_API_KEY"] = "your-api-key"
# openai call
response = completion(
model = "groq/llama-3.2-11b-vision-preview",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What’s in this image?"
},
{
"type": "image_url",
"image_url": {
"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
}
}
]
}
],
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Add Groq models to config.yaml
```yaml
model_list:
- model_name: groq-llama3-8b-8192 # Model Alias to use for requests
litellm_params:
model: groq/llama3-8b-8192
api_key: "os.environ/GROQ_API_KEY" # ensure you have `GROQ_API_KEY` in your .env
```
2. Start Proxy
```bash
litellm --config config.yaml
```
3. Test it
```python
import os
from openai import OpenAI
client = OpenAI(
api_key="sk-1234", # your litellm proxy api key
)
response = client.chat.completions.create(
model = "gpt-4-vision-preview", # use model="llava-hf" to test your custom OpenAI endpoint
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What’s in this image?"
},
{
"type": "image_url",
"image_url": {
"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
}
}
]
}
],
)
```
</TabItem>
</Tabs>
## Speech to Text - Whisper ## Speech to Text - Whisper
```python ```python
@ -274,4 +367,5 @@ transcript = litellm.transcription(
) )
print("response=", transcript) print("response=", transcript)
``` ```

231
litellm/integrations/prometheus.py
View file

@ -3,7 +3,7 @@
# On success, log events to Prometheus # On success, log events to Prometheus
import sys import sys
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import List, Optional from typing import List, Optional, cast
from litellm._logging import print_verbose, verbose_logger from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_logger import CustomLogger
@ -37,16 +37,12 @@ class PrometheusLogger(CustomLogger):
self.litellm_proxy_failed_requests_metric = Counter( self.litellm_proxy_failed_requests_metric = Counter(
name="litellm_proxy_failed_requests_metric", name="litellm_proxy_failed_requests_metric",
documentation="Total number of failed responses from proxy - the client did not get a success response from litellm proxy", documentation="Total number of failed responses from proxy - the client did not get a success response from litellm proxy",
labelnames=[ labelnames=PrometheusMetricLabels.litellm_proxy_failed_requests_metric.value,
"end_user", )
"hashed_api_key", self.litellm_proxy_failed_requests_by_tag_metric = Counter(
"api_key_alias", name="litellm_proxy_failed_requests_by_tag_metric",
REQUESTED_MODEL, documentation="Total number of failed responses from proxy - the client did not get a success response from litellm proxy",
"team", labelnames=PrometheusMetricLabels.litellm_proxy_failed_requests_by_tag_metric.value,
"team_alias",
"user",
]
+ EXCEPTION_LABELS,
) )
self.litellm_proxy_total_requests_metric = Counter( self.litellm_proxy_total_requests_metric = Counter(
@ -55,6 +51,12 @@ class PrometheusLogger(CustomLogger):
labelnames=PrometheusMetricLabels.litellm_proxy_total_requests_metric.value, labelnames=PrometheusMetricLabels.litellm_proxy_total_requests_metric.value,
) )
self.litellm_proxy_total_requests_by_tag_metric = Counter(
name="litellm_proxy_total_requests_by_tag_metric",
documentation="Total number of requests made to the proxy server - track number of client side requests by custom metadata tags",
labelnames=PrometheusMetricLabels.litellm_proxy_total_requests_by_tag_metric.value,
)
# request latency metrics # request latency metrics
self.litellm_request_total_latency_metric = Histogram( self.litellm_request_total_latency_metric = Histogram(
"litellm_request_total_latency_metric", "litellm_request_total_latency_metric",
@ -63,12 +65,25 @@ class PrometheusLogger(CustomLogger):
buckets=LATENCY_BUCKETS, buckets=LATENCY_BUCKETS,
) )
self.litellm_request_total_latency_by_tag_metric = Histogram(
"litellm_request_total_latency_by_tag_metric",
"Total latency (seconds) for a request to LiteLLM by custom metadata tags",
labelnames=PrometheusMetricLabels.litellm_request_total_latency_by_tag_metric.value,
buckets=LATENCY_BUCKETS,
)
self.litellm_llm_api_latency_metric = Histogram( self.litellm_llm_api_latency_metric = Histogram(
"litellm_llm_api_latency_metric", "litellm_llm_api_latency_metric",
"Total latency (seconds) for a models LLM API call", "Total latency (seconds) for a models LLM API call",
labelnames=PrometheusMetricLabels.litellm_llm_api_latency_metric.value, labelnames=PrometheusMetricLabels.litellm_llm_api_latency_metric.value,
buckets=LATENCY_BUCKETS, buckets=LATENCY_BUCKETS,
) )
self.litellm_llm_api_latency_by_tag_metric = Histogram(
"litellm_llm_api_latency_by_tag_metric",
"Total latency (seconds) for a models LLM API call by custom metadata tags",
labelnames=PrometheusMetricLabels.litellm_llm_api_latency_by_tag_metric.value,
buckets=LATENCY_BUCKETS,
)
self.litellm_llm_api_time_to_first_token_metric = Histogram( self.litellm_llm_api_time_to_first_token_metric = Histogram(
"litellm_llm_api_time_to_first_token_metric", "litellm_llm_api_time_to_first_token_metric",
@ -301,22 +316,36 @@ class PrometheusLogger(CustomLogger):
self.litellm_deployment_latency_per_output_token = Histogram( self.litellm_deployment_latency_per_output_token = Histogram(
name="litellm_deployment_latency_per_output_token", name="litellm_deployment_latency_per_output_token",
documentation="LLM Deployment Analytics - Latency per output token", documentation="LLM Deployment Analytics - Latency per output token",
labelnames=_logged_llm_labels + team_and_key_labels, labelnames=PrometheusMetricLabels.litellm_deployment_latency_per_output_token.value,
)
self.litellm_deployment_latency_per_output_token_by_tag = Histogram(
name="litellm_deployment_latency_per_output_token_by_tag",
documentation="LLM Deployment Analytics - Latency per output token by custom metadata tags",
labelnames=PrometheusMetricLabels.litellm_deployment_latency_per_output_token_by_tag.value,
) )
self.litellm_deployment_successful_fallbacks = Counter( self.litellm_deployment_successful_fallbacks = Counter(
"litellm_deployment_successful_fallbacks", "litellm_deployment_successful_fallbacks",
"LLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model", "LLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model",
[REQUESTED_MODEL, "fallback_model"] PrometheusMetricLabels.litellm_deployment_successful_fallbacks.value,
+ team_and_key_labels
+ EXCEPTION_LABELS,
) )
self.litellm_deployment_successful_fallbacks_by_tag = Counter(
"litellm_deployment_successful_fallbacks_by_tag",
"LLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model by custom metadata tags",
PrometheusMetricLabels.litellm_deployment_successful_fallbacks_by_tag.value,
)
self.litellm_deployment_failed_fallbacks = Counter( self.litellm_deployment_failed_fallbacks = Counter(
"litellm_deployment_failed_fallbacks", "litellm_deployment_failed_fallbacks",
"LLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model", "LLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model",
[REQUESTED_MODEL, "fallback_model"] PrometheusMetricLabels.litellm_deployment_failed_fallbacks.value,
+ team_and_key_labels )
+ EXCEPTION_LABELS,
self.litellm_deployment_failed_fallbacks_by_tag = Counter(
"litellm_deployment_failed_fallbacks_by_tag",
"LLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model by custom metadata tags",
PrometheusMetricLabels.litellm_deployment_failed_fallbacks_by_tag.value,
) )
self.litellm_llm_api_failed_requests_metric = Counter( self.litellm_llm_api_failed_requests_metric = Counter(
@ -336,15 +365,7 @@ class PrometheusLogger(CustomLogger):
self.litellm_requests_metric = Counter( self.litellm_requests_metric = Counter(
name="litellm_requests_metric", name="litellm_requests_metric",
documentation="deprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, user", documentation="deprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, user",
labelnames=[ labelnames=PrometheusMetricLabels.litellm_requests_metric.value,
"end_user",
"hashed_api_key",
"api_key_alias",
"model",
"team",
"team_alias",
"user",
],
) )
except Exception as e: except Exception as e:
@ -422,6 +443,7 @@ class PrometheusLogger(CustomLogger):
user_api_team_alias=user_api_team_alias, user_api_team_alias=user_api_team_alias,
user_id=user_id, user_id=user_id,
response_cost=response_cost, response_cost=response_cost,
enum_values=enum_values,
) )
# input, output, total token metrics # input, output, total token metrics
@ -472,7 +494,7 @@ class PrometheusLogger(CustomLogger):
# set x-ratelimit headers # set x-ratelimit headers
self.set_llm_deployment_success_metrics( self.set_llm_deployment_success_metrics(
kwargs, start_time, end_time, output_tokens kwargs, start_time, end_time, enum_values, output_tokens
) )
if ( if (
@ -484,6 +506,14 @@ class PrometheusLogger(CustomLogger):
) )
self.litellm_proxy_total_requests_metric.labels(**_labels).inc() self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
for tag in enum_values.tags:
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_proxy_total_requests_by_tag_metric.value,
enum_values=enum_values,
tag=tag,
)
self.litellm_proxy_total_requests_by_tag_metric.labels(**_labels).inc()
def _increment_token_metrics( def _increment_token_metrics(
self, self,
standard_logging_payload: StandardLoggingPayload, standard_logging_payload: StandardLoggingPayload,
@ -594,16 +624,14 @@ class PrometheusLogger(CustomLogger):
user_api_team_alias: Optional[str], user_api_team_alias: Optional[str],
user_id: Optional[str], user_id: Optional[str],
response_cost: float, response_cost: float,
enum_values: UserAPIKeyLabelValues,
): ):
self.litellm_requests_metric.labels( _labels = prometheus_label_factory(
end_user_id, supported_enum_labels=PrometheusMetricLabels.litellm_requests_metric.value,
user_api_key, enum_values=enum_values,
user_api_key_alias, )
model, self.litellm_requests_metric.labels(**_labels).inc()
user_api_team,
user_api_team_alias,
user_id,
).inc()
self.litellm_spend_metric.labels( self.litellm_spend_metric.labels(
end_user_id, end_user_id,
user_api_key, user_api_key,
@ -716,6 +744,15 @@ class PrometheusLogger(CustomLogger):
self.litellm_llm_api_latency_metric.labels(**_labels).observe( self.litellm_llm_api_latency_metric.labels(**_labels).observe(
api_call_total_time_seconds api_call_total_time_seconds
) )
for tag in enum_values.tags:
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_llm_api_latency_by_tag_metric.value,
enum_values=enum_values,
tag=tag,
)
self.litellm_llm_api_latency_by_tag_metric.labels(**_labels).observe(
api_call_total_time_seconds
)
# total request latency # total request latency
if start_time is not None and isinstance(start_time, datetime): if start_time is not None and isinstance(start_time, datetime):
@ -729,6 +766,16 @@ class PrometheusLogger(CustomLogger):
total_time_seconds total_time_seconds
) )
for tag in enum_values.tags:
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_request_total_latency_by_tag_metric.value,
enum_values=enum_values,
tag=tag,
)
self.litellm_request_total_latency_by_tag_metric.labels(
**_labels
).observe(total_time_seconds)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
from litellm.types.utils import StandardLoggingPayload from litellm.types.utils import StandardLoggingPayload
@ -793,6 +840,7 @@ class PrometheusLogger(CustomLogger):
] + EXCEPTION_LABELS, ] + EXCEPTION_LABELS,
""" """
try: try:
_tags = cast(List[str], request_data.get("tags") or [])
enum_values = UserAPIKeyLabelValues( enum_values = UserAPIKeyLabelValues(
end_user=user_api_key_dict.end_user_id, end_user=user_api_key_dict.end_user_id,
user=user_api_key_dict.user_id, user=user_api_key_dict.user_id,
@ -802,27 +850,37 @@ class PrometheusLogger(CustomLogger):
team_alias=user_api_key_dict.team_alias, team_alias=user_api_key_dict.team_alias,
requested_model=request_data.get("model", ""), requested_model=request_data.get("model", ""),
status_code=str(getattr(original_exception, "status_code", None)), status_code=str(getattr(original_exception, "status_code", None)),
exception_status=str(getattr(original_exception, "status_code", None)),
exception_class=str(original_exception.__class__.__name__), exception_class=str(original_exception.__class__.__name__),
tags=_tags,
) )
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_proxy_failed_requests_metric.value,
enum_values=enum_values,
)
self.litellm_proxy_failed_requests_metric.labels(**_labels).inc()
self.litellm_proxy_failed_requests_metric.labels( for tag in _tags:
end_user=user_api_key_dict.end_user_id, _labels = prometheus_label_factory(
hashed_api_key=user_api_key_dict.api_key, supported_enum_labels=PrometheusMetricLabels.litellm_proxy_failed_requests_by_tag_metric.value,
api_key_alias=user_api_key_dict.key_alias, enum_values=enum_values,
requested_model=request_data.get("model", ""), tag=tag,
team=user_api_key_dict.team_id, )
team_alias=user_api_key_dict.team_alias, self.litellm_proxy_failed_requests_by_tag_metric.labels(**_labels).inc()
user=user_api_key_dict.user_id,
exception_status=getattr(original_exception, "status_code", None),
exception_class=str(original_exception.__class__.__name__),
).inc()
_labels = prometheus_label_factory( _labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_proxy_total_requests_metric.value, supported_enum_labels=PrometheusMetricLabels.litellm_proxy_total_requests_metric.value,
enum_values=enum_values, enum_values=enum_values,
) )
self.litellm_proxy_total_requests_metric.labels(**_labels).inc() self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
pass
for tag in enum_values.tags:
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_proxy_total_requests_by_tag_metric.value,
enum_values=enum_values,
tag=tag,
)
self.litellm_proxy_total_requests_by_tag_metric.labels(**_labels).inc()
except Exception as e: except Exception as e:
verbose_logger.exception( verbose_logger.exception(
"prometheus Layer Error(): Exception occured - {}".format(str(e)) "prometheus Layer Error(): Exception occured - {}".format(str(e))
@ -851,6 +909,14 @@ class PrometheusLogger(CustomLogger):
enum_values=enum_values, enum_values=enum_values,
) )
self.litellm_proxy_total_requests_metric.labels(**_labels).inc() self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
for tag in enum_values.tags:
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_proxy_total_requests_by_tag_metric.value,
enum_values=enum_values,
tag=tag,
)
self.litellm_proxy_total_requests_by_tag_metric.labels(**_labels).inc()
except Exception as e: except Exception as e:
verbose_logger.exception( verbose_logger.exception(
"prometheus Layer Error(): Exception occured - {}".format(str(e)) "prometheus Layer Error(): Exception occured - {}".format(str(e))
@ -962,6 +1028,7 @@ class PrometheusLogger(CustomLogger):
request_kwargs: dict, request_kwargs: dict,
start_time, start_time,
end_time, end_time,
enum_values: UserAPIKeyLabelValues,
output_tokens: float = 1.0, output_tokens: float = 1.0,
): ):
try: try:
@ -1092,21 +1159,12 @@ class PrometheusLogger(CustomLogger):
latency_per_token = None latency_per_token = None
if output_tokens is not None and output_tokens > 0: if output_tokens is not None and output_tokens > 0:
latency_per_token = _latency_seconds / output_tokens latency_per_token = _latency_seconds / output_tokens
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_deployment_latency_per_output_token.value,
enum_values=enum_values,
)
self.litellm_deployment_latency_per_output_token.labels( self.litellm_deployment_latency_per_output_token.labels(
litellm_model_name=litellm_model_name, **_labels
model_id=model_id,
api_base=api_base,
api_provider=llm_provider,
hashed_api_key=standard_logging_payload["metadata"][
"user_api_key_hash"
],
api_key_alias=standard_logging_payload["metadata"][
"user_api_key_alias"
],
team=standard_logging_payload["metadata"]["user_api_key_team_id"],
team_alias=standard_logging_payload["metadata"][
"user_api_key_team_alias"
],
).observe(latency_per_token) ).observe(latency_per_token)
except Exception as e: except Exception as e:
@ -1142,7 +1200,8 @@ class PrometheusLogger(CustomLogger):
) )
) )
_new_model = kwargs.get("model") _new_model = kwargs.get("model")
self.litellm_deployment_successful_fallbacks.labels( _tags = cast(List[str], kwargs.get("tags") or [])
enum_values = UserAPIKeyLabelValues(
requested_model=original_model_group, requested_model=original_model_group,
fallback_model=_new_model, fallback_model=_new_model,
hashed_api_key=standard_metadata["user_api_key_hash"], hashed_api_key=standard_metadata["user_api_key_hash"],
@ -1151,7 +1210,21 @@ class PrometheusLogger(CustomLogger):
team_alias=standard_metadata["user_api_key_team_alias"], team_alias=standard_metadata["user_api_key_team_alias"],
exception_status=str(getattr(original_exception, "status_code", None)), exception_status=str(getattr(original_exception, "status_code", None)),
exception_class=str(original_exception.__class__.__name__), exception_class=str(original_exception.__class__.__name__),
).inc() tags=_tags,
)
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_deployment_successful_fallbacks.value,
enum_values=enum_values,
)
self.litellm_deployment_successful_fallbacks.labels(**_labels).inc()
for tag in _tags:
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_deployment_successful_fallbacks_by_tag.value,
enum_values=enum_values,
tag=tag,
)
self.litellm_deployment_successful_fallbacks_by_tag.labels(**_labels).inc()
async def log_failure_fallback_event( async def log_failure_fallback_event(
self, original_model_group: str, kwargs: dict, original_exception: Exception self, original_model_group: str, kwargs: dict, original_exception: Exception
@ -1171,12 +1244,14 @@ class PrometheusLogger(CustomLogger):
) )
_new_model = kwargs.get("model") _new_model = kwargs.get("model")
_metadata = kwargs.get("metadata", {}) _metadata = kwargs.get("metadata", {})
_tags = cast(List[str], kwargs.get("tags") or [])
standard_metadata: StandardLoggingMetadata = ( standard_metadata: StandardLoggingMetadata = (
StandardLoggingPayloadSetup.get_standard_logging_metadata( StandardLoggingPayloadSetup.get_standard_logging_metadata(
metadata=_metadata metadata=_metadata
) )
) )
self.litellm_deployment_failed_fallbacks.labels(
enum_values = UserAPIKeyLabelValues(
requested_model=original_model_group, requested_model=original_model_group,
fallback_model=_new_model, fallback_model=_new_model,
hashed_api_key=standard_metadata["user_api_key_hash"], hashed_api_key=standard_metadata["user_api_key_hash"],
@ -1185,7 +1260,22 @@ class PrometheusLogger(CustomLogger):
team_alias=standard_metadata["user_api_key_team_alias"], team_alias=standard_metadata["user_api_key_team_alias"],
exception_status=str(getattr(original_exception, "status_code", None)), exception_status=str(getattr(original_exception, "status_code", None)),
exception_class=str(original_exception.__class__.__name__), exception_class=str(original_exception.__class__.__name__),
).inc() tags=_tags,
)
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_deployment_failed_fallbacks.value,
enum_values=enum_values,
)
self.litellm_deployment_failed_fallbacks.labels(**_labels).inc()
for tag in _tags:
_labels = prometheus_label_factory(
supported_enum_labels=PrometheusMetricLabels.litellm_deployment_failed_fallbacks_by_tag.value,
enum_values=enum_values,
tag=tag,
)
self.litellm_deployment_failed_fallbacks_by_tag.labels(**_labels).inc()
def set_litellm_deployment_state( def set_litellm_deployment_state(
self, self,
@ -1273,7 +1363,9 @@ class PrometheusLogger(CustomLogger):
def prometheus_label_factory( def prometheus_label_factory(
supported_enum_labels: List[str], enum_values: UserAPIKeyLabelValues supported_enum_labels: List[str],
enum_values: UserAPIKeyLabelValues,
tag: Optional[str] = None,
) -> dict: ) -> dict:
""" """
Returns a dictionary of label + values for prometheus. Returns a dictionary of label + values for prometheus.
@ -1290,6 +1382,9 @@ def prometheus_label_factory(
if label in supported_enum_labels if label in supported_enum_labels
} }
if tag and "tag" in supported_enum_labels:
filtered_labels["tag"] = tag
if UserAPIKeyLabelNames.END_USER.value in filtered_labels: if UserAPIKeyLabelNames.END_USER.value in filtered_labels:
filtered_labels["end_user"] = get_end_user_id_for_cost_tracking( filtered_labels["end_user"] = get_end_user_id_for_cost_tracking(
litellm_params={"user_api_key_end_user_id": enum_values.end_user}, litellm_params={"user_api_key_end_user_id": enum_values.end_user},

6
litellm/model_prices_and_context_window_backup.json
View file

@ -2119,7 +2119,8 @@
"litellm_provider": "groq", "litellm_provider": "groq",
"mode": "chat", "mode": "chat",
"supports_function_calling": true, "supports_function_calling": true,
"supports_response_schema": true "supports_response_schema": true,
"supports_vision": true
}, },
"groq/llama-3.2-90b-text-preview": { "groq/llama-3.2-90b-text-preview": {
"max_tokens": 8192, "max_tokens": 8192,
@ -2141,7 +2142,8 @@
"litellm_provider": "groq", "litellm_provider": "groq",
"mode": "chat", "mode": "chat",
"supports_function_calling": true, "supports_function_calling": true,
"supports_response_schema": true "supports_response_schema": true,
"supports_vision": true
}, },
"groq/llama3-70b-8192": { "groq/llama3-70b-8192": {
"max_tokens": 8192, "max_tokens": 8192,

89
litellm/types/integrations/prometheus.py
View file

@ -65,6 +65,7 @@ class UserAPIKeyLabelNames(Enum):
EXCEPTION_STATUS = EXCEPTION_STATUS EXCEPTION_STATUS = EXCEPTION_STATUS
EXCEPTION_CLASS = EXCEPTION_CLASS EXCEPTION_CLASS = EXCEPTION_CLASS
STATUS_CODE = "status_code" STATUS_CODE = "status_code"
FALLBACK_MODEL = "fallback_model"
class PrometheusMetricLabels(Enum): class PrometheusMetricLabels(Enum):
@ -101,6 +102,92 @@ class PrometheusMetricLabels(Enum):
UserAPIKeyLabelNames.STATUS_CODE.value, UserAPIKeyLabelNames.STATUS_CODE.value,
] ]
litellm_proxy_failed_requests_metric = [
UserAPIKeyLabelNames.END_USER.value,
UserAPIKeyLabelNames.API_KEY_HASH.value,
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.EXCEPTION_STATUS.value,
UserAPIKeyLabelNames.EXCEPTION_CLASS.value,
]
litellm_proxy_failed_requests_by_tag_metric = (
litellm_proxy_failed_requests_metric
+ [
UserAPIKeyLabelNames.TAG.value,
]
)
litellm_request_total_latency_by_tag_metric = (
litellm_request_total_latency_metric
+ [
UserAPIKeyLabelNames.TAG.value,
]
)
litellm_llm_api_latency_by_tag_metric = litellm_llm_api_latency_metric + [
UserAPIKeyLabelNames.TAG.value,
]
litellm_deployment_latency_per_output_token = [
UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.MODEL_ID.value,
UserAPIKeyLabelNames.API_BASE.value,
UserAPIKeyLabelNames.API_PROVIDER.value,
UserAPIKeyLabelNames.API_KEY_HASH.value,
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
]
litellm_deployment_latency_per_output_token_by_tag = (
litellm_deployment_latency_per_output_token
+ [
UserAPIKeyLabelNames.TAG.value,
]
)
litellm_requests_metric = [
UserAPIKeyLabelNames.END_USER.value,
UserAPIKeyLabelNames.API_KEY_HASH.value,
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.USER.value,
]
litellm_proxy_total_requests_by_tag_metric = litellm_proxy_total_requests_metric + [
UserAPIKeyLabelNames.TAG.value,
]
litellm_deployment_successful_fallbacks = [
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
UserAPIKeyLabelNames.FALLBACK_MODEL.value,
UserAPIKeyLabelNames.API_KEY_HASH.value,
UserAPIKeyLabelNames.API_KEY_ALIAS.value,
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.EXCEPTION_STATUS.value,
UserAPIKeyLabelNames.EXCEPTION_CLASS.value,
]
litellm_deployment_successful_fallbacks_by_tag = (
litellm_deployment_successful_fallbacks
+ [
UserAPIKeyLabelNames.TAG.value,
]
)
litellm_deployment_failed_fallbacks = litellm_deployment_successful_fallbacks
litellm_deployment_failed_fallbacks_by_tag = (
litellm_deployment_successful_fallbacks_by_tag
)
from typing import List, Optional from typing import List, Optional
@ -124,6 +211,7 @@ class UserAPIKeyLabelValues(BaseModel):
exception_status: Optional[str] = None exception_status: Optional[str] = None
exception_class: Optional[str] = None exception_class: Optional[str] = None
status_code: Optional[str] = None status_code: Optional[str] = None
fallback_model: Optional[str] = None
class Config: class Config:
fields = { fields = {
@ -142,4 +230,5 @@ class UserAPIKeyLabelValues(BaseModel):
"exception_status": {"alias": UserAPIKeyLabelNames.EXCEPTION_STATUS}, "exception_status": {"alias": UserAPIKeyLabelNames.EXCEPTION_STATUS},
"exception_class": {"alias": UserAPIKeyLabelNames.EXCEPTION_CLASS}, "exception_class": {"alias": UserAPIKeyLabelNames.EXCEPTION_CLASS},
"status_code": {"alias": UserAPIKeyLabelNames.STATUS_CODE}, "status_code": {"alias": UserAPIKeyLabelNames.STATUS_CODE},
"fallback_model": {"alias": UserAPIKeyLabelNames.FALLBACK_MODEL},
} }

6
model_prices_and_context_window.json
View file

@ -2119,7 +2119,8 @@
"litellm_provider": "groq", "litellm_provider": "groq",
"mode": "chat", "mode": "chat",
"supports_function_calling": true, "supports_function_calling": true,
"supports_response_schema": true "supports_response_schema": true,
"supports_vision": true
}, },
"groq/llama-3.2-90b-text-preview": { "groq/llama-3.2-90b-text-preview": {
"max_tokens": 8192, "max_tokens": 8192,
@ -2141,7 +2142,8 @@
"litellm_provider": "groq", "litellm_provider": "groq",
"mode": "chat", "mode": "chat",
"supports_function_calling": true, "supports_function_calling": true,
"supports_response_schema": true "supports_response_schema": true,
"supports_vision": true
}, },
"groq/llama3-70b-8192": { "groq/llama3-70b-8192": {
"max_tokens": 8192, "max_tokens": 8192,

4
tests/local_testing/test_gcs_bucket.py
View file

@ -522,9 +522,9 @@ async def test_basic_gcs_logging_per_request_with_no_litellm_callback_set():
) )
@pytest.mark.flaky(retries=5, delay=3) @pytest.mark.skip(reason="This test is flaky")
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_get_gcs_logging_config_without_service_account(): async def test_aaaget_gcs_logging_config_without_service_account():
""" """
Test the get_gcs_logging_config works for IAM auth on GCS Test the get_gcs_logging_config works for IAM auth on GCS
1. Key based logging without a service account 1. Key based logging without a service account

35
tests/logging_callback_tests/test_prometheus_unit_tests.py
View file

@ -14,7 +14,7 @@ from prometheus_client import REGISTRY, CollectorRegistry
import litellm import litellm
from litellm import completion from litellm import completion
from litellm._logging import verbose_logger from litellm._logging import verbose_logger
from litellm.integrations.prometheus import PrometheusLogger from litellm.integrations.prometheus import PrometheusLogger, UserAPIKeyLabelValues
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import ( from litellm.types.utils import (
StandardLoggingPayload, StandardLoggingPayload,
@ -339,6 +339,16 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger):
- litellm_requests_metric is incremented by 1 - litellm_requests_metric is incremented by 1
- litellm_spend_metric is incremented by the response cost in the standard logging payload - litellm_spend_metric is incremented by the response cost in the standard logging payload
""" """
standard_logging_payload = create_standard_logging_payload()
enum_values = UserAPIKeyLabelValues(
litellm_model_name=standard_logging_payload["model"],
api_provider=standard_logging_payload["custom_llm_provider"],
hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
team=standard_logging_payload["metadata"]["user_api_key_team_id"],
team_alias=standard_logging_payload["metadata"]["user_api_key_team_alias"],
**standard_logging_payload,
)
prometheus_logger.litellm_requests_metric = MagicMock() prometheus_logger.litellm_requests_metric = MagicMock()
prometheus_logger.litellm_spend_metric = MagicMock() prometheus_logger.litellm_spend_metric = MagicMock()
@ -351,10 +361,17 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger):
user_api_team_alias="team_alias1", user_api_team_alias="team_alias1",
user_id="user1", user_id="user1",
response_cost=0.1, response_cost=0.1,
enum_values=enum_values,
) )
prometheus_logger.litellm_requests_metric.labels.assert_called_once_with( prometheus_logger.litellm_requests_metric.labels.assert_called_once_with(
"user1", "key1", "alias1", "gpt-3.5-turbo", "team1", "team_alias1", "user1" end_user=None,
user=None,
hashed_api_key="test_hash",
api_key_alias="test_alias",
team="test_team",
team_alias="test_team_alias",
model="gpt-3.5-turbo",
) )
prometheus_logger.litellm_requests_metric.labels().inc.assert_called_once() prometheus_logger.litellm_requests_metric.labels().inc.assert_called_once()
@ -496,7 +513,7 @@ async def test_async_post_call_failure_hook(prometheus_logger):
team="test_team", team="test_team",
team_alias="test_team_alias", team_alias="test_team_alias",
user="test_user", user="test_user",
exception_status=429, exception_status="429",
exception_class="RateLimitError", exception_class="RateLimitError",
) )
prometheus_logger.litellm_proxy_failed_requests_metric.labels().inc.assert_called_once() prometheus_logger.litellm_proxy_failed_requests_metric.labels().inc.assert_called_once()
@ -584,6 +601,16 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
"standard_logging_object": standard_logging_payload, "standard_logging_object": standard_logging_payload,
} }
enum_values = UserAPIKeyLabelValues(
litellm_model_name=standard_logging_payload["model"],
api_provider=standard_logging_payload["custom_llm_provider"],
hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"],
api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"],
team=standard_logging_payload["metadata"]["user_api_key_team_id"],
team_alias=standard_logging_payload["metadata"]["user_api_key_team_alias"],
**standard_logging_payload,
)
start_time = datetime.now() start_time = datetime.now()
end_time = start_time + timedelta(seconds=1) end_time = start_time + timedelta(seconds=1)
output_tokens = 10 output_tokens = 10
@ -594,6 +621,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
start_time=start_time, start_time=start_time,
end_time=end_time, end_time=end_time,
output_tokens=output_tokens, output_tokens=output_tokens,
enum_values=enum_values,
) )
# Verify remaining requests metric # Verify remaining requests metric
@ -780,6 +808,7 @@ def test_deployment_state_management(prometheus_logger):
def test_increment_deployment_cooled_down(prometheus_logger): def test_increment_deployment_cooled_down(prometheus_logger):
prometheus_logger.litellm_deployment_cooled_down = MagicMock() prometheus_logger.litellm_deployment_cooled_down = MagicMock()
prometheus_logger.increment_deployment_cooled_down( prometheus_logger.increment_deployment_cooled_down(