phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1829 from BerriAI/litellm_add_semantic_cache

Browse source 
[Feat] Add Semantic Caching to litellm💰
This commit is contained in:
Ishaan Jaff 2024-02-06 13:18:59 -08:00 • committed by GitHub
commit 8a8f538329
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 569 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

10
litellm/utils.py
View file

@ -55,7 +55,7 @@ from .integrations.litedebugger import LiteDebugger
from .proxy._types import KeyManagementSystem
from openai import OpenAIError as OriginalError
from openai._models import BaseModel as OpenAIObject
from .caching import S3Cache
from .caching import S3Cache, RedisSemanticCache
from .exceptions import (
AuthenticationError,
BadRequestError,
@ -2533,6 +2533,14 @@ def client(original_function):
):
if len(cached_result) == 1 and cached_result[0] is None:
cached_result = None
elif isinstance(litellm.cache.cache, RedisSemanticCache):
preset_cache_key = litellm.cache.get_cache_key(*args, **kwargs)
kwargs[
"preset_cache_key"
] = preset_cache_key # for streaming calls, we need to pass the preset_cache_key
cached_result = await litellm.cache.async_get_cache(
*args, **kwargs
)
else:
preset_cache_key = litellm.cache.get_cache_key(*args, **kwargs)
kwargs[