phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix get_async_httpx_client

Browse source
This commit is contained in:
Ishaan Jaff 2024-11-21 10:24:18 -08:00
parent 0a10b1ef1c
commit 398e6d0ac6
2 changed files with 12 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

11
litellm/llms/vertex_ai_and_google_ai_studio/context_caching/vertex_ai_context_caching.py
View file

@ -6,7 +6,11 @@ import httpx
import litellm import litellm
from litellm.caching.caching import Cache, LiteLLMCacheType from litellm.caching.caching import Cache, LiteLLMCacheType
from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
get_async_httpx_client,
)
from litellm.llms.OpenAI.openai import AllMessageValues from litellm.llms.OpenAI.openai import AllMessageValues
from litellm.types.llms.vertex_ai import ( from litellm.types.llms.vertex_ai import (
CachedContentListAllResponseBody, CachedContentListAllResponseBody,
@ -352,7 +356,10 @@ class ContextCachingEndpoints(VertexBase):
if isinstance(timeout, float) or isinstance(timeout, int): if isinstance(timeout, float) or isinstance(timeout, int):
timeout = httpx.Timeout(timeout) timeout = httpx.Timeout(timeout)
_params["timeout"] = timeout _params["timeout"] = timeout
client = AsyncHTTPHandler(**_params) # type: ignore client = get_async_httpx_client(
llm_provider=litellm.LlmProviders.VERTEX_AI,
params={"timeout": timeout},
)
else: else:
client = client client = client

4
litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -1026,7 +1026,9 @@ async def make_call(
logging_obj, logging_obj,
): ):
if client is None: if client is None:
client = AsyncHTTPHandler() # Create a new client if none provided client = get_async_httpx_client(
llm_provider=litellm.LlmProviders.VERTEX_AI,
)
try: try:
response = await client.post(api_base, headers=headers, data=data, stream=True) response = await client.post(api_base, headers=headers, data=data, stream=True)