mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-27 11:43:54 +00:00
fix(client_initialization_utils.py): fix merge conflicts
This commit is contained in:
Krrish Dholakia
parent
8661da1980
commit
14ab1e5d2d
1 changed files with 38 additions and 105 deletions
|
|
@ -1,7 +1,9 @@
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import os
|
||||||
import traceback
|
import traceback
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
import openai
|
import openai
|
||||||
|
|
||||||
import litellm
|
import litellm
|
||||||
|
|
@ -169,39 +171,6 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
max_retries = litellm.get_secret(max_retries_env_name)
|
max_retries = litellm.get_secret(max_retries_env_name)
|
||||||
litellm_params["max_retries"] = max_retries
|
litellm_params["max_retries"] = max_retries
|
||||||
|
|
||||||
# proxy support
|
|
||||||
import os
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
# Check if the HTTP_PROXY and HTTPS_PROXY environment variables are set and use them accordingly.
|
|
||||||
http_proxy = os.getenv("HTTP_PROXY", None)
|
|
||||||
https_proxy = os.getenv("HTTPS_PROXY", None)
|
|
||||||
no_proxy = os.getenv("NO_PROXY", None)
|
|
||||||
|
|
||||||
# Create the proxies dictionary only if the environment variables are set.
|
|
||||||
sync_proxy_mounts = None
|
|
||||||
async_proxy_mounts = None
|
|
||||||
if http_proxy is not None and https_proxy is not None:
|
|
||||||
sync_proxy_mounts = {
|
|
||||||
"http://": httpx.HTTPTransport(proxy=httpx.Proxy(url=http_proxy)),
|
|
||||||
"https://": httpx.HTTPTransport(proxy=httpx.Proxy(url=https_proxy)),
|
|
||||||
}
|
|
||||||
async_proxy_mounts = {
|
|
||||||
"http://": httpx.AsyncHTTPTransport(proxy=httpx.Proxy(url=http_proxy)),
|
|
||||||
"https://": httpx.AsyncHTTPTransport(
|
|
||||||
proxy=httpx.Proxy(url=https_proxy)
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
# assume no_proxy is a list of comma separated urls
|
|
||||||
if no_proxy is not None and isinstance(no_proxy, str):
|
|
||||||
no_proxy_urls = no_proxy.split(",")
|
|
||||||
|
|
||||||
for url in no_proxy_urls: # set no-proxy support for specific urls
|
|
||||||
sync_proxy_mounts[url] = None # type: ignore
|
|
||||||
async_proxy_mounts[url] = None # type: ignore
|
|
||||||
|
|
||||||
organization = litellm_params.get("organization", None)
|
organization = litellm_params.get("organization", None)
|
||||||
if isinstance(organization, str) and organization.startswith("os.environ/"):
|
if isinstance(organization, str) and organization.startswith("os.environ/"):
|
||||||
organization_env_name = organization.replace("os.environ/", "")
|
organization_env_name = organization.replace("os.environ/", "")
|
||||||
|
|
@ -241,13 +210,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
http_client=httpx.AsyncClient(
|
http_client=httpx.AsyncClient(
|
||||||
transport=AsyncCustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
|
||||||
mounts=async_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
@ -269,13 +235,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
http_client=httpx.Client(
|
http_client=httpx.Client(
|
||||||
transport=CustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
|
||||||
mounts=sync_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
@ -294,13 +257,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
timeout=stream_timeout,
|
timeout=stream_timeout,
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
http_client=httpx.AsyncClient(
|
http_client=httpx.AsyncClient(
|
||||||
transport=AsyncCustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
|
||||||
mounts=async_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
@ -322,13 +282,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
timeout=stream_timeout,
|
timeout=stream_timeout,
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
http_client=httpx.Client(
|
http_client=httpx.Client(
|
||||||
transport=CustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
|
||||||
mounts=sync_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
@ -365,13 +322,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
http_client=httpx.AsyncClient(
|
http_client=httpx.AsyncClient(
|
||||||
transport=AsyncCustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
|
||||||
mounts=async_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
@ -389,13 +343,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
http_client=httpx.Client(
|
http_client=httpx.Client(
|
||||||
transport=CustomHTTPTransport(
|
|
||||||
verify=litellm.ssl_verify,
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
),
|
verify=litellm.ssl_verify,
|
||||||
mounts=sync_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
@ -412,14 +363,11 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
timeout=stream_timeout,
|
timeout=stream_timeout,
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
http_client=httpx.AsyncClient(
|
http_client=httpx.AsyncClient(
|
||||||
transport=AsyncCustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
),
|
||||||
mounts=async_proxy_mounts,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
key=cache_key,
|
key=cache_key,
|
||||||
|
|
@ -437,14 +385,11 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
timeout=stream_timeout,
|
timeout=stream_timeout,
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
http_client=httpx.Client(
|
http_client=httpx.Client(
|
||||||
transport=CustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
),
|
||||||
mounts=sync_proxy_mounts,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
key=cache_key,
|
key=cache_key,
|
||||||
|
|
@ -469,13 +414,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
organization=organization,
|
organization=organization,
|
||||||
http_client=httpx.AsyncClient(
|
http_client=httpx.AsyncClient(
|
||||||
transport=AsyncCustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
|
||||||
mounts=async_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
@ -496,13 +438,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
organization=organization,
|
organization=organization,
|
||||||
http_client=httpx.Client(
|
http_client=httpx.Client(
|
||||||
transport=CustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
|
||||||
mounts=sync_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
@ -521,13 +460,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
organization=organization,
|
organization=organization,
|
||||||
http_client=httpx.AsyncClient(
|
http_client=httpx.AsyncClient(
|
||||||
transport=AsyncCustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
|
||||||
mounts=async_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
@ -549,13 +485,10 @@ def set_client(litellm_router_instance: LitellmRouter, model: dict):
|
||||||
max_retries=max_retries,
|
max_retries=max_retries,
|
||||||
organization=organization,
|
organization=organization,
|
||||||
http_client=httpx.Client(
|
http_client=httpx.Client(
|
||||||
transport=CustomHTTPTransport(
|
|
||||||
limits=httpx.Limits(
|
limits=httpx.Limits(
|
||||||
max_connections=1000, max_keepalive_connections=100
|
max_connections=1000, max_keepalive_connections=100
|
||||||
),
|
),
|
||||||
verify=litellm.ssl_verify,
|
verify=litellm.ssl_verify,
|
||||||
),
|
|
||||||
mounts=sync_proxy_mounts,
|
|
||||||
), # type: ignore
|
), # type: ignore
|
||||||
)
|
)
|
||||||
litellm_router_instance.cache.set_cache(
|
litellm_router_instance.cache.set_cache(
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue