phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

router - use free paid tier routing

Browse source
This commit is contained in:
Ishaan Jaff 2024-07-18 17:09:42 -07:00 • committed by Krrish Dholakia
parent b0f0898f2f
commit d42963a0ae
1 changed files with 9 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

13
litellm/router_strategy/free_paid_tiers.py
View file

@ -17,14 +17,19 @@ class Deployment(TypedDict):
async def get_deployments_for_tier( async def get_deployments_for_tier(
request_kwargs: dict, request_kwargs: Optional[Dict[Any, Any]] = None,
healthy_deployments: Optional[ healthy_deployments: Optional[Union[List[Any], Dict[Any, Any]]] = None,
Union[List[DeploymentTypedDict], List[Dict[str, Any]]]
] = None,
): ):
""" """
if request_kwargs contains {"metadata": {"tier": "free"}} or {"metadata": {"tier": "paid"}}, then routes the request to free/paid tier models if request_kwargs contains {"metadata": {"tier": "free"}} or {"metadata": {"tier": "paid"}}, then routes the request to free/paid tier models
""" """
if request_kwargs is None:
verbose_logger.debug(
"get_deployments_for_tier: request_kwargs is None returning healthy_deployments: %s",
healthy_deployments,
)
return healthy_deployments
verbose_logger.debug("request metadata: %s", request_kwargs.get("metadata")) verbose_logger.debug("request metadata: %s", request_kwargs.get("metadata"))
if "metadata" in request_kwargs: if "metadata" in request_kwargs:
metadata = request_kwargs["metadata"] metadata = request_kwargs["metadata"]