phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat - use api to get prometheus api metrics

Browse source
This commit is contained in:
Ishaan Jaff 2024-08-10 16:36:06 -07:00
parent 35de40e6eb
commit 28b930c0b7
1 changed files with 30 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

47
litellm/integrations/prometheus_helpers/prometheus_api.py
View file

@ -7,6 +7,7 @@ import os
import time import time
import litellm import litellm
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
PROMETHEUS_URL = litellm.get_secret("PROMETHEUS_URL") PROMETHEUS_URL = litellm.get_secret("PROMETHEUS_URL")
@ -23,31 +24,43 @@ async def get_metric_from_prometheus(
"PROMETHEUS_URL not set please set 'PROMETHEUS_URL=<>' in .env" "PROMETHEUS_URL not set please set 'PROMETHEUS_URL=<>' in .env"
) )
start_of_day = int(time.time()) - (int(time.time()) % 86400) query = f"{metric_name}[24h]"
query = metric_name now = int(time.time())
response = await async_http_handler.get( response = await async_http_handler.get(
f"{PROMETHEUS_URL}/api/v1/query", params={"query": query} f"{PROMETHEUS_URL}/api/v1/query", params={"query": query, "time": now}
) # End of the day ) # End of the day
_json_response = response.json() _json_response = response.json()
verbose_logger.debug("json response from prometheus /query api %s", _json_response)
results = response.json()["data"]["result"] results = response.json()["data"]["result"]
return results return results
async def get_fallback_metric_from_prometheus(): async def get_fallback_metric_from_prometheus():
response_json = await get_metric_from_prometheus( """
metric_name="llm_deployment_successful_fallbacks_total" Gets fallback metrics from prometheus for the last 24 hours
) """
if not response_json:
return "No fallback data available."
response_message = "" response_message = ""
for result in response_json: relevant_metrics = [
result = response_json "llm_deployment_successful_fallbacks_total",
metric = result["metric"] "llm_deployment_failed_fallbacks_total",
value = int(float(result["value"][1])) # Convert value to integer ]
for metric in relevant_metrics:
response_json = await get_metric_from_prometheus(
metric_name=metric,
)
primary_model = metric.get("primary_model", "Unknown") if response_json:
fallback_model = metric.get("fallback_model", "Unknown") verbose_logger.debug("response json %s", response_json)
response_message += f"`{value} successful fallback requests` with `primary model={primary_model} -> fallback model={fallback_model}`" for result in response_json:
verbose_logger.debug("result= %s", result)
metric = result["metric"]
metric_values = result["values"]
most_recent_value = metric_values[0]
value = int(float(most_recent_value[1])) # Convert value to integer
primary_model = metric.get("primary_model", "Unknown")
fallback_model = metric.get("fallback_model", "Unknown")
response_message += f"`{value} successful fallback requests` with primary model=`{primary_model}` -> fallback model=`{fallback_model}`"
response_message += "\n"
verbose_logger.debug("response message %s", response_message)
return response_message return response_message