phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(router.py): bump httpx pool limits

Browse source
This commit is contained in:
Krrish Dholakia 2024-01-11 12:51:29 +05:30
parent 8394315173
commit 40c7400894
1 changed files with 52 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

54
litellm/router.py
View file

@ -1363,6 +1363,12 @@ class Router:
api_version=api_version,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1378,6 +1384,12 @@ class Router:
api_version=api_version,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1393,6 +1405,12 @@ class Router:
api_version=api_version,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1408,6 +1426,12 @@ class Router:
api_version=api_version,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1471,9 +1495,10 @@ class Router:
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
)
),
),
)
self.cache.set_cache(
@ -1491,9 +1516,10 @@ class Router:
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
)
),
),
)
self.cache.set_cache(
@ -1513,6 +1539,12 @@ class Router:
base_url=api_base,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1527,6 +1559,12 @@ class Router:
base_url=api_base,
timeout=timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1542,6 +1580,12 @@ class Router:
base_url=api_base,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,
@ -1557,6 +1601,12 @@ class Router:
base_url=api_base,
timeout=stream_timeout,
max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
)
self.cache.set_cache(
key=cache_key,