phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(openai.py): fix client caching logic

Browse source
This commit is contained in:
Krrish Dholakia 2024-06-01 16:45:34 -07:00
parent 63fb3a95be
commit 93c9ea160d
2 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/llms/openai.py
View file

@ -515,6 +515,7 @@ class OpenAIChatCompletion(BaseLLM):
organization: Optional[str] = None, organization: Optional[str] = None,
client: Optional[Union[OpenAI, AsyncOpenAI]] = None, client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
): ):
args = locals()
if client is None: if client is None:
if not isinstance(max_retries, int): if not isinstance(max_retries, int):
raise OpenAIError( raise OpenAIError(
@ -532,7 +533,7 @@ class OpenAIChatCompletion(BaseLLM):
# Hexadecimal representation of the hash # Hexadecimal representation of the hash
hashed_api_key = hash_object.hexdigest() hashed_api_key = hash_object.hexdigest()
_cache_key = f"hashed_api_key={hashed_api_key},api_base={api_base},timeout={timeout},max_retries={max_retries},organization={organization}" _cache_key = f"hashed_api_key={hashed_api_key},api_base={api_base},timeout={timeout},max_retries={max_retries},organization={organization},is_async={is_async}"
if _cache_key in litellm.in_memory_llm_clients_cache: if _cache_key in litellm.in_memory_llm_clients_cache:
return litellm.in_memory_llm_clients_cache[_cache_key] return litellm.in_memory_llm_clients_cache[_cache_key]
@ -555,6 +556,7 @@ class OpenAIChatCompletion(BaseLLM):
organization=organization, organization=organization,
) )
## SAVE CACHE KEY
litellm.in_memory_llm_clients_cache[_cache_key] = _new_client litellm.in_memory_llm_clients_cache[_cache_key] = _new_client
return _new_client return _new_client

1
litellm/main.py
View file

@ -14,6 +14,7 @@ from functools import partial
import dotenv, traceback, random, asyncio, time, contextvars import dotenv, traceback, random, asyncio, time, contextvars
from copy import deepcopy from copy import deepcopy
import httpx import httpx
import litellm import litellm
from ._logging import verbose_logger from ._logging import verbose_logger
from litellm import ( # type: ignore from litellm import ( # type: ignore