phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(router.py): bump httpx pool limits

Browse source
This commit is contained in:
Krrish Dholakia 2024-01-11 12:51:29 +05:30
parent 8394315173
commit 40c7400894
1 changed files with 52 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

54
litellm/router.py
View file

@ -1363,6 +1363,12 @@ class Router:
api_version=api_version, api_version=api_version,
timeout=timeout, timeout=timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,
@ -1378,6 +1384,12 @@ class Router:
api_version=api_version, api_version=api_version,
timeout=timeout, timeout=timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,
@ -1393,6 +1405,12 @@ class Router:
api_version=api_version, api_version=api_version,
timeout=stream_timeout, timeout=stream_timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,
@ -1408,6 +1426,12 @@ class Router:
api_version=api_version, api_version=api_version,
timeout=stream_timeout, timeout=stream_timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,
@ -1471,9 +1495,10 @@ class Router:
timeout=stream_timeout, timeout=stream_timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.AsyncClient( http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits( limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100 max_connections=1000, max_keepalive_connections=100
) ),
), ),
) )
self.cache.set_cache( self.cache.set_cache(
@ -1491,9 +1516,10 @@ class Router:
timeout=stream_timeout, timeout=stream_timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.Client( http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits( limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100 max_connections=1000, max_keepalive_connections=100
) ),
), ),
) )
self.cache.set_cache( self.cache.set_cache(
@ -1513,6 +1539,12 @@ class Router:
base_url=api_base, base_url=api_base,
timeout=timeout, timeout=timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,
@ -1527,6 +1559,12 @@ class Router:
base_url=api_base, base_url=api_base,
timeout=timeout, timeout=timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,
@ -1542,6 +1580,12 @@ class Router:
base_url=api_base, base_url=api_base,
timeout=stream_timeout, timeout=stream_timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,
@ -1557,6 +1601,12 @@ class Router:
base_url=api_base, base_url=api_base,
timeout=stream_timeout, timeout=stream_timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.Client(
transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000, max_keepalive_connections=100
),
), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,