phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

v0 add types of alerts to slack alerting

Browse source
This commit is contained in:
Ishaan Jaff 2024-04-17 18:16:19 -07:00
parent e5a91f35c8
commit 52d7fc22bb
1 changed files with 25 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

25
litellm/proxy/utils.py
View file

@ -64,6 +64,21 @@ class ProxyLogging:
self.cache_control_check = _PROXY_CacheControlCheck() self.cache_control_check = _PROXY_CacheControlCheck()
self.alerting: Optional[List] = None self.alerting: Optional[List] = None
self.alerting_threshold: float = 300 # default to 5 min. threshold self.alerting_threshold: float = 300 # default to 5 min. threshold
self.alert_types: List[
Literal[
"llm_exceptions",
"llm_too_slow",
"llm_requests_hanging",
"budget_alerts",
"db_exceptions",
]
] = [
"llm_exceptions",
"llm_too_slow",
"llm_requests_hanging",
"budget_alerts",
"db_exceptions",
]
def update_values( def update_values(
self, self,
@ -210,6 +225,8 @@ class ProxyLogging:
): ):
if self.alerting is None: if self.alerting is None:
return return
if "llm_too_slow" not in self.alert_types:
return
time_difference_float, model, api_base, messages = ( time_difference_float, model, api_base, messages = (
self._response_taking_too_long_callback( self._response_taking_too_long_callback(
kwargs=kwargs, kwargs=kwargs,
@ -256,6 +273,8 @@ class ProxyLogging:
if type == "hanging_request": if type == "hanging_request":
# Simulate a long-running operation that could take more than 5 minutes # Simulate a long-running operation that could take more than 5 minutes
if "llm_requests_hanging" not in self.alert_types:
return
await asyncio.sleep( await asyncio.sleep(
self.alerting_threshold self.alerting_threshold
) # Set it to 5 minutes - i'd imagine this might be different for streaming, non-streaming, non-completion (embedding + img) requests ) # Set it to 5 minutes - i'd imagine this might be different for streaming, non-streaming, non-completion (embedding + img) requests
@ -304,6 +323,8 @@ class ProxyLogging:
if self.alerting is None: if self.alerting is None:
# do nothing if alerting is not switched on # do nothing if alerting is not switched on
return return
if "budget_alerts" not in self.alert_types:
return
_id: str = "default_id" # used for caching _id: str = "default_id" # used for caching
if type == "user_and_proxy_budget": if type == "user_and_proxy_budget":
user_info = dict(user_info) user_info = dict(user_info)
@ -460,6 +481,8 @@ class ProxyLogging:
Currently only logs exceptions to sentry Currently only logs exceptions to sentry
""" """
### ALERTING ### ### ALERTING ###
if "db_exceptions" not in self.alert_types:
return
if isinstance(original_exception, HTTPException): if isinstance(original_exception, HTTPException):
if isinstance(original_exception.detail, str): if isinstance(original_exception.detail, str):
error_message = original_exception.detail error_message = original_exception.detail
@ -494,6 +517,8 @@ class ProxyLogging:
""" """
### ALERTING ### ### ALERTING ###
if "llm_exceptions" not in self.alert_types:
return
asyncio.create_task( asyncio.create_task(
self.alerting_handler( self.alerting_handler(
message=f"LLM API call failed: {str(original_exception)}", level="High" message=f"LLM API call failed: {str(original_exception)}", level="High"