phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 19:24:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(feat proxy prometheus) track virtual key, key alias, error code, error code class on prometheus (#5968)

Browse source 
* track api key and team in prom latency metric

* add test for latency metric

* test prometheus success metrics for latency

* track team and key labels for deployment failures

* add test for litellm_deployment_failure_responses_total

* fix checks for premium user on prometheus

* log_success_fallback_event and log_failure_fallback_event

* log original_exception in log_success_fallback_event

* track key, team and exception status and class on fallback metrics

* use get_standard_logging_metadata

* fix import error

* track litellm_deployment_successful_fallbacks

* add test test_proxy_fallback_metrics

* add log log_success_fallback_event

* fix test prometheus
This commit is contained in:
Ishaan Jaff 2024-09-28 19:00:21 -07:00 • committed by GitHub
parent 1dba353b1e
commit 97aeacc1fa
5 changed files with 426 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

121
litellm/router_utils/fallback_event_handlers.py
View file

@ -1,8 +1,9 @@
from typing import TYPE_CHECKING, Any, Dict, List, Tuple
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
import litellm
from litellm._logging import verbose_router_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.main import verbose_logger
if TYPE_CHECKING:
from litellm.router import Router as _Router
@ -41,13 +42,17 @@ async def run_async_fallback(
verbose_router_logger.info("Successful fallback b/w models.")
# callback for successfull_fallback_event():
await log_success_fallback_event(
original_model_group=original_model_group, kwargs=kwargs
original_model_group=original_model_group,
kwargs=kwargs,
original_exception=original_exception,
)
return response
except Exception as e:
error_from_fallbacks = e
await log_failure_fallback_event(
original_model_group=original_model_group, kwargs=kwargs
original_model_group=original_model_group,
kwargs=kwargs,
original_exception=original_exception,
)
raise error_from_fallbacks
@ -83,29 +88,115 @@ def run_sync_fallback(
raise error_from_fallbacks
async def log_success_fallback_event(original_model_group: str, kwargs: dict):
async def log_success_fallback_event(
original_model_group: str, kwargs: dict, original_exception: Exception
):
"""
Log a successful fallback event to all registered callbacks.
This function iterates through all callbacks, initializing _known_custom_logger_compatible_callbacks if needed,
and calls the log_success_fallback_event method on CustomLogger instances.
Args:
original_model_group (str): The original model group before fallback.
kwargs (dict): kwargs for the request
Note:
Errors during logging are caught and reported but do not interrupt the process.
"""
from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,
)
for _callback in litellm.callbacks:
if isinstance(_callback, CustomLogger):
if isinstance(_callback, CustomLogger) or (
_callback in litellm._known_custom_logger_compatible_callbacks
):
try:
await _callback.log_success_fallback_event(
original_model_group=original_model_group, kwargs=kwargs
_callback_custom_logger: Optional[CustomLogger] = None
if _callback in litellm._known_custom_logger_compatible_callbacks:
_callback_custom_logger = _init_custom_logger_compatible_class(
logging_integration=_callback, # type: ignore
llm_router=None,
internal_usage_cache=None,
)
elif isinstance(_callback, CustomLogger):
_callback_custom_logger = _callback
else:
verbose_router_logger.exception(
f"{_callback} logger not found / initialized properly"
)
continue
if _callback_custom_logger is None:
verbose_router_logger.exception(
f"{_callback} logger not found / initialized properly, callback is None"
)
continue
await _callback_custom_logger.log_success_fallback_event(
original_model_group=original_model_group,
kwargs=kwargs,
original_exception=original_exception,
)
except Exception as e:
verbose_router_logger.error(
f"Error in log_success_fallback_event: {(str(e))}"
f"Error in log_success_fallback_event: {str(e)}"
)
pass
async def log_failure_fallback_event(original_model_group: str, kwargs: dict):
async def log_failure_fallback_event(
original_model_group: str, kwargs: dict, original_exception: Exception
):
"""
Log a failed fallback event to all registered callbacks.
This function iterates through all callbacks, initializing _known_custom_logger_compatible_callbacks if needed,
and calls the log_failure_fallback_event method on CustomLogger instances.
Args:
original_model_group (str): The original model group before fallback.
kwargs (dict): kwargs for the request
Note:
Errors during logging are caught and reported but do not interrupt the process.
"""
from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,
)
for _callback in litellm.callbacks:
if isinstance(_callback, CustomLogger):
if isinstance(_callback, CustomLogger) or (
_callback in litellm._known_custom_logger_compatible_callbacks
):
try:
await _callback.log_failure_fallback_event(
original_model_group=original_model_group, kwargs=kwargs
_callback_custom_logger: Optional[CustomLogger] = None
if _callback in litellm._known_custom_logger_compatible_callbacks:
_callback_custom_logger = _init_custom_logger_compatible_class(
logging_integration=_callback, # type: ignore
llm_router=None,
internal_usage_cache=None,
)
elif isinstance(_callback, CustomLogger):
_callback_custom_logger = _callback
else:
verbose_router_logger.exception(
f"{_callback} logger not found / initialized properly"
)
continue
if _callback_custom_logger is None:
verbose_router_logger.exception(
f"{_callback} logger not found / initialized properly"
)
continue
await _callback_custom_logger.log_failure_fallback_event(
original_model_group=original_model_group,
kwargs=kwargs,
original_exception=original_exception,
)
except Exception as e:
verbose_router_logger.error(
f"Error in log_failure_fallback_event: {(str(e))}"
f"Error in log_failure_fallback_event: {str(e)}"
)
pass