phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'litellm_dev_03_10_2025_p3' into litellm_router_client_init_migration

Browse source
This commit is contained in:
Krish Dholakia 2025-03-11 18:27:56 -07:00 • committed by GitHub
commit 8dea6e91a6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 21 additions and 94 deletions
Download patch file Download diff file Expand all files Collapse all files

23
litellm/router_utils/client_initalization_utils.py
View file

@ -45,18 +45,11 @@ class InitalizeOpenAISDKClient:
return True
@staticmethod
def set_client( # noqa: PLR0915
def set_max_parallel_requests_client(
litellm_router_instance: LitellmRouter, model: dict
):
"""
- Initializes Azure/OpenAI clients. Stores them in cache, b/c of this - https://github.com/BerriAI/litellm/issues/1278
- Initializes Semaphore for client w/ rpm. Stores them in cache. b/c of this - https://github.com/BerriAI/litellm/issues/2994
"""
client_ttl = litellm_router_instance.client_ttl
litellm_params = model.get("litellm_params", {})
model_name = litellm_params.get("model")
model_id = model["model_info"]["id"]
# ### IF RPM SET - initialize a semaphore ###
rpm = litellm_params.get("rpm", None)
tpm = litellm_params.get("tpm", None)
max_parallel_requests = litellm_params.get("max_parallel_requests", None)
@ -75,6 +68,19 @@ class InitalizeOpenAISDKClient:
local_only=True,
)
@staticmethod
def set_client( # noqa: PLR0915
litellm_router_instance: LitellmRouter, model: dict
):
"""
- Initializes Azure/OpenAI clients. Stores them in cache, b/c of this - https://github.com/BerriAI/litellm/issues/1278
- Initializes Semaphore for client w/ rpm. Stores them in cache. b/c of this - https://github.com/BerriAI/litellm/issues/2994
"""
client_ttl = litellm_router_instance.client_ttl
litellm_params = model.get("litellm_params", {})
model_name = litellm_params.get("model")
model_id = model["model_info"]["id"]
#### for OpenAI / Azure we need to initalize the Client for High Traffic ########
custom_llm_provider = litellm_params.get("custom_llm_provider")
custom_llm_provider = custom_llm_provider or model_name.split("/", 1)[0] or ""
@ -185,7 +191,6 @@ class InitalizeOpenAISDKClient:
organization_env_name = organization.replace("os.environ/", "")
organization = get_secret_str(organization_env_name)
litellm_params["organization"] = organization
else:
_api_key = api_key # type: ignore
if _api_key is not None and isinstance(_api_key, str):