phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(router.py): increasing connection pool limits for azure router

Browse source
This commit is contained in:
Krrish Dholakia 2024-01-08 14:39:49 +05:30
parent f07daa3780
commit ec83243521
1 changed files with 20 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

20
litellm/router.py
View file

@ -1384,6 +1384,10 @@ class Router:
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.AsyncClient( http_client=httpx.AsyncClient(
transport=AsyncCustomHTTPTransport(), transport=AsyncCustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000,
max_keepalive_connections=100
)
), # type: ignore ), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
@ -1402,6 +1406,10 @@ class Router:
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.Client( http_client=httpx.Client(
transport=CustomHTTPTransport(), transport=CustomHTTPTransport(),
limits=httpx.Limits(
max_connections=1000,
max_keepalive_connections=100
)
), # type: ignore ), # type: ignore
) )
self.cache.set_cache( self.cache.set_cache(
@ -1419,6 +1427,12 @@ class Router:
api_version=api_version, api_version=api_version,
timeout=stream_timeout, timeout=stream_timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.Client(
limits=httpx.Limits(
max_connections=1000,
max_keepalive_connections=100
)
)
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,
@ -1434,6 +1448,12 @@ class Router:
api_version=api_version, api_version=api_version,
timeout=stream_timeout, timeout=stream_timeout,
max_retries=max_retries, max_retries=max_retries,
http_client=httpx.Client(
limits=httpx.Limits(
max_connections=1000,
max_keepalive_connections=100
)
)
) )
self.cache.set_cache( self.cache.set_cache(
key=cache_key, key=cache_key,