phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(fix) httpx handler - bind to ipv4 for httpx handler (#6785)

Browse source 
* bind to ipv4 on httpx handler

* add force_ipv4

* use helper for _create_async_transport

* fix circular import

* document force_ipv4

* test_async_http_handler_force_ipv4
This commit is contained in:
Ishaan Jaff 2024-11-18 12:22:51 -08:00 • committed by GitHub
parent b854f6c07b
commit f43768d617
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 87 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

31
litellm/llms/custom_httpx/http_handler.py
View file

@ -4,7 +4,7 @@ import traceback
from typing import TYPE_CHECKING, Any, Callable, List, Mapping, Optional, Union
import httpx
from httpx import USE_CLIENT_DEFAULT
from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport
import litellm
@ -60,8 +60,10 @@ class AsyncHTTPHandler:
if timeout is None:
timeout = _DEFAULT_TIMEOUT
# Create a client with a connection pool
transport = self._create_async_transport()
return httpx.AsyncClient(
transport=transport,
event_hooks=event_hooks,
timeout=timeout,
limits=httpx.Limits(
@ -297,6 +299,18 @@ class AsyncHTTPHandler:
except Exception:
pass
def _create_async_transport(self) -> Optional[AsyncHTTPTransport]:
"""
Create an async transport with IPv4 only if litellm.force_ipv4 is True.
Otherwise, return None.
Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them
"""
if litellm.force_ipv4:
return AsyncHTTPTransport(local_address="0.0.0.0")
else:
return None
class HTTPHandler:
def __init__(
@ -316,8 +330,11 @@ class HTTPHandler:
cert = os.getenv("SSL_CERTIFICATE", litellm.ssl_certificate)
if client is None:
transport = self._create_sync_transport()
# Create a client with a connection pool
self.client = httpx.Client(
transport=transport,
timeout=timeout,
limits=httpx.Limits(
max_connections=concurrent_limit,
@ -427,6 +444,18 @@ class HTTPHandler:
except Exception:
pass
def _create_sync_transport(self) -> Optional[HTTPTransport]:
"""
Create an HTTP transport with IPv4 only if litellm.force_ipv4 is True.
Otherwise, return None.
Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them
"""
if litellm.force_ipv4:
return HTTPTransport(local_address="0.0.0.0")
else:
return None
def get_async_httpx_client(
llm_provider: Union[LlmProviders, httpxSpecialProvider],