phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Merge pull request #3152 from BerriAI/litellm_show_api_base_hanging_req

Browse source 
[Fix] show api base hanging request alerts
This commit is contained in:
Ishaan Jaff 2024-04-18 21:02:30 -07:00 • committed by GitHub
commit c720c8ffe9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 14 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

10
litellm/proxy/utils.py
View file

@ -327,6 +327,16 @@ class ProxyLogging:
_api_base = ""
request_info += f"\nAPI Base: {_api_base}"
elif request_data.get("metadata", None) is not None and isinstance(
request_data["metadata"], dict
):
# In hanging requests sometime it has not made it to the point where the deployment is passed to the `request_data``
# in that case we fallback to the api base set in the request metadata
_metadata = request_data["metadata"]
_api_base = _metadata.get("api_base", "")
if _api_base is None:
_api_base = ""
request_info += f"\nAPI Base: `{_api_base}`"
# only alert hanging responses if they have not been marked as success
alerting_message = (
f"`Requests are hanging - {self.alerting_threshold}s+ request time`"

4
litellm/router.py
View file

@ -351,6 +351,7 @@ class Router:
kwargs.setdefault("metadata", {}).update(
{
"deployment": deployment["litellm_params"]["model"],
"api_base": deployment.get("litellm_params", {}).get("api_base"),
"model_info": deployment.get("model_info", {}),
}
)
@ -438,6 +439,7 @@ class Router:
{
"deployment": deployment["litellm_params"]["model"],
"model_info": deployment.get("model_info", {}),
"api_base": deployment.get("litellm_params", {}).get("api_base"),
}
)
kwargs["model_info"] = deployment.get("model_info", {})
@ -951,6 +953,7 @@ class Router:
{
"deployment": deployment["litellm_params"]["model"],
"model_info": deployment.get("model_info", {}),
"api_base": deployment.get("litellm_params", {}).get("api_base"),
}
)
kwargs["model_info"] = deployment.get("model_info", {})
@ -1118,6 +1121,7 @@ class Router:
{
"deployment": deployment["litellm_params"]["model"],
"model_info": deployment.get("model_info", {}),
"api_base": deployment.get("litellm_params", {}).get("api_base"),
}
)
kwargs["model_info"] = deployment.get("model_info", {})