phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix - set latency stats in kwargs

Browse source
This commit is contained in:
Ishaan Jaff 2024-04-24 20:13:45 -07:00
parent 1c109086ac
commit 5dae1cf303
2 changed files with 19 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

8
litellm/router.py
View file

@ -454,6 +454,7 @@ class Router:
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
# debug how often this deployment picked
@ -2818,6 +2819,7 @@ class Router:
messages: Optional[List[Dict[str, str]]] = None,
input: Optional[Union[str, List]] = None,
specific_deployment: Optional[bool] = False,
request_kwargs: Optional[Dict] = None,
):
"""
Async implementation of 'get_available_deployments'.
@ -2833,6 +2835,7 @@ class Router:
messages=messages,
input=input,
specific_deployment=specific_deployment,
request_kwargs=request_kwargs,
)
model, healthy_deployments = self._common_checks_available_deployment(
@ -2936,6 +2939,7 @@ class Router:
messages: Optional[List[Dict[str, str]]] = None,
input: Optional[Union[str, List]] = None,
specific_deployment: Optional[bool] = False,
request_kwargs: Optional[Dict] = None,
):
"""
Returns the deployment based on routing strategy
@ -3022,7 +3026,9 @@ class Router:
and self.lowestlatency_logger is not None
):
deployment = self.lowestlatency_logger.get_available_deployments(
model_group=model, healthy_deployments=healthy_deployments
model_group=model,
healthy_deployments=healthy_deployments,
request_kwargs=request_kwargs,
)
elif (
self.routing_strategy == "usage-based-routing"