feat(router.py): enable settting 'order' for a deployment in model list

Allows user to control which model gets called first in model group
This commit is contained in:
Krrish Dholakia 2024-06-06 09:46:51 -07:00
parent 58bd2b4ea6
commit a7dcf25722
3 changed files with 69 additions and 0 deletions

View file

@ -6196,6 +6196,27 @@ def calculate_max_parallel_requests(
return None
def _get_order_filtered_deployments(healthy_deployments: List[Dict]) -> List:
min_order = min(
(
deployment["litellm_params"]["order"]
for deployment in healthy_deployments
if "order" in deployment["litellm_params"]
),
default=None,
)
if min_order is not None:
filtered_deployments = [
deployment
for deployment in healthy_deployments
if deployment["litellm_params"].get("order") == min_order
]
return filtered_deployments
return healthy_deployments
def _get_model_region(
custom_llm_provider: str, litellm_params: LiteLLM_Params
) -> Optional[str]: