From 40c740089474b74baf8bdb68a31e4aa3dc00753a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 11 Jan 2024 12:51:29 +0530 Subject: [PATCH] fix(router.py): bump httpx pool limits --- litellm/router.py | 54 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index f63555509..d5b42343c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1363,6 +1363,12 @@ class Router: api_version=api_version, timeout=timeout, max_retries=max_retries, + http_client=httpx.AsyncClient( + transport=AsyncCustomHTTPTransport(), + limits=httpx.Limits( + max_connections=1000, max_keepalive_connections=100 + ), + ), # type: ignore ) self.cache.set_cache( key=cache_key, @@ -1378,6 +1384,12 @@ class Router: api_version=api_version, timeout=timeout, max_retries=max_retries, + http_client=httpx.Client( + transport=CustomHTTPTransport(), + limits=httpx.Limits( + max_connections=1000, max_keepalive_connections=100 + ), + ), # type: ignore ) self.cache.set_cache( key=cache_key, @@ -1393,6 +1405,12 @@ class Router: api_version=api_version, timeout=stream_timeout, max_retries=max_retries, + http_client=httpx.AsyncClient( + transport=AsyncCustomHTTPTransport(), + limits=httpx.Limits( + max_connections=1000, max_keepalive_connections=100 + ), + ), # type: ignore ) self.cache.set_cache( key=cache_key, @@ -1408,6 +1426,12 @@ class Router: api_version=api_version, timeout=stream_timeout, max_retries=max_retries, + http_client=httpx.Client( + transport=CustomHTTPTransport(), + limits=httpx.Limits( + max_connections=1000, max_keepalive_connections=100 + ), + ), # type: ignore ) self.cache.set_cache( key=cache_key, @@ -1471,9 +1495,10 @@ class Router: timeout=stream_timeout, max_retries=max_retries, http_client=httpx.AsyncClient( + transport=AsyncCustomHTTPTransport(), limits=httpx.Limits( max_connections=1000, max_keepalive_connections=100 - ) + ), ), ) self.cache.set_cache( @@ -1491,9 +1516,10 @@ class Router: timeout=stream_timeout, max_retries=max_retries, http_client=httpx.Client( + transport=CustomHTTPTransport(), limits=httpx.Limits( max_connections=1000, max_keepalive_connections=100 - ) + ), ), ) self.cache.set_cache( @@ -1513,6 +1539,12 @@ class Router: base_url=api_base, timeout=timeout, max_retries=max_retries, + http_client=httpx.AsyncClient( + transport=AsyncCustomHTTPTransport(), + limits=httpx.Limits( + max_connections=1000, max_keepalive_connections=100 + ), + ), # type: ignore ) self.cache.set_cache( key=cache_key, @@ -1527,6 +1559,12 @@ class Router: base_url=api_base, timeout=timeout, max_retries=max_retries, + http_client=httpx.Client( + transport=CustomHTTPTransport(), + limits=httpx.Limits( + max_connections=1000, max_keepalive_connections=100 + ), + ), # type: ignore ) self.cache.set_cache( key=cache_key, @@ -1542,6 +1580,12 @@ class Router: base_url=api_base, timeout=stream_timeout, max_retries=max_retries, + http_client=httpx.AsyncClient( + transport=AsyncCustomHTTPTransport(), + limits=httpx.Limits( + max_connections=1000, max_keepalive_connections=100 + ), + ), # type: ignore ) self.cache.set_cache( key=cache_key, @@ -1557,6 +1601,12 @@ class Router: base_url=api_base, timeout=stream_timeout, max_retries=max_retries, + http_client=httpx.Client( + transport=CustomHTTPTransport(), + limits=httpx.Limits( + max_connections=1000, max_keepalive_connections=100 + ), + ), # type: ignore ) self.cache.set_cache( key=cache_key,