phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

router - use free paid tier routing

Browse source
This commit is contained in:
Ishaan Jaff 2024-07-18 17:09:42 -07:00
parent 957b8a1b8d
commit d4cad75d34
3 changed files with 106 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

7
litellm/router.py
View file

@ -47,6 +47,7 @@ from litellm.assistants.main import AssistantDeleted
from litellm.caching import DualCache, InMemoryCache, RedisCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.azure import get_azure_ad_token_from_oidc
from litellm.router_strategy.free_paid_tiers import get_deployments_for_tier
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
@ -4472,6 +4473,12 @@ class Router:
request_kwargs=request_kwargs,
)
# check free / paid tier for each deployment
healthy_deployments = await get_deployments_for_tier(
request_kwargs=request_kwargs,
healthy_deployments=healthy_deployments,
)
if len(healthy_deployments) == 0:
if _allowed_model_region is None:
_allowed_model_region = "n/a"