phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #4786 from BerriAI/litellm_use_model_tier_keys

Browse source 
[Feat-Enterprise] Use free/paid tiers for Virtual Keys
This commit is contained in:
Ishaan Jaff 2024-07-18 18:07:09 -07:00 • committed by GitHub
commit 4b96cd46b2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 359 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

7
litellm/router.py
View file

@ -47,6 +47,7 @@ from litellm.assistants.main import AssistantDeleted
from litellm.caching import DualCache, InMemoryCache, RedisCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.azure import get_azure_ad_token_from_oidc
from litellm.router_strategy.free_paid_tiers import get_deployments_for_tier
from litellm.router_strategy.least_busy import LeastBusyLoggingHandler
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler
@ -4481,6 +4482,12 @@ class Router:
request_kwargs=request_kwargs,
)
# check free / paid tier for each deployment
healthy_deployments = await get_deployments_for_tier(
request_kwargs=request_kwargs,
healthy_deployments=healthy_deployments,
)
if len(healthy_deployments) == 0:
if _allowed_model_region is None:
_allowed_model_region = "n/a"