phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

feat(batch_redis_get.py): batch redis GET requests for a given key + call type

Browse source 
reduces the number of GET requests we're making in high-throughput scenarios
This commit is contained in:
Krrish Dholakia 2024-03-15 14:40:11 -07:00
parent e033e84720
commit 226953e1d8
5 changed files with 189 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
litellm/utils.py
View file

@ -72,7 +72,7 @@ from .integrations.litedebugger import LiteDebugger
from .proxy._types import KeyManagementSystem
from openai import OpenAIError as OriginalError
from openai._models import BaseModel as OpenAIObject
from .caching import S3Cache, RedisSemanticCache
from .caching import S3Cache, RedisSemanticCache, RedisCache
from .exceptions import (
AuthenticationError,
BadRequestError,
@ -2806,7 +2806,9 @@ def client(original_function):
):
if len(cached_result) == 1 and cached_result[0] is None:
cached_result = None
elif isinstance(litellm.cache.cache, RedisSemanticCache):
elif isinstance(
litellm.cache.cache, RedisSemanticCache
) or isinstance(litellm.cache.cache, RedisCache):
preset_cache_key = litellm.cache.get_cache_key(*args, **kwargs)
kwargs["preset_cache_key"] = (
preset_cache_key # for streaming calls, we need to pass the preset_cache_key