phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 19:54:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) only alert users when requests are hanging

Browse source
This commit is contained in:
ishaan-jaff 2024-01-24 15:58:07 -08:00
parent 0f51cd0bab
commit 2686d1f087
2 changed files with 15 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/proxy/proxy_server.py
View file

@ -1863,6 +1863,8 @@ async def chat_completion(
else: # router is not set else: # router is not set
response = await litellm.acompletion(**data) response = await litellm.acompletion(**data)
# Post Call Processing
data["litellm_status"] = "success" # used for alerting
if hasattr(response, "_hidden_params"): if hasattr(response, "_hidden_params"):
model_id = response._hidden_params.get("model_id", None) or "" model_id = response._hidden_params.get("model_id", None) or ""
else: else:
@ -2048,6 +2050,7 @@ async def embeddings(
response = await litellm.aembedding(**data) response = await litellm.aembedding(**data)
### ALERTING ### ### ALERTING ###
data["litellm_status"] = "success" # used for alerting
end_time = time.time() end_time = time.time()
asyncio.create_task( asyncio.create_task(
proxy_logging_obj.response_taking_too_long( proxy_logging_obj.response_taking_too_long(
@ -2163,6 +2166,7 @@ async def image_generation(
response = await litellm.aimage_generation(**data) response = await litellm.aimage_generation(**data)
### ALERTING ### ### ALERTING ###
data["litellm_status"] = "success" # used for alerting
end_time = time.time() end_time = time.time()
asyncio.create_task( asyncio.create_task(
proxy_logging_obj.response_taking_too_long( proxy_logging_obj.response_taking_too_long(

5
litellm/proxy/utils.py
View file

@ -158,7 +158,10 @@ class ProxyLogging:
await asyncio.sleep( await asyncio.sleep(
self.alerting_threshold self.alerting_threshold
) # Set it to 5 minutes - i'd imagine this might be different for streaming, non-streaming, non-completion (embedding + img) requests ) # Set it to 5 minutes - i'd imagine this might be different for streaming, non-streaming, non-completion (embedding + img) requests
if (
request_data is not None
and request_data.get("litellm_status", "") != "success"
):
alerting_message = ( alerting_message = (
f"Requests are hanging - {self.alerting_threshold}s+ request time" f"Requests are hanging - {self.alerting_threshold}s+ request time"
) )