phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix should_use_cache

Browse source
This commit is contained in:
Ishaan Jaff 2024-08-24 09:37:41 -07:00
parent 0d03b807b9
commit cad77c5969
2 changed files with 16 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

24
litellm/caching.py
View file

@ -2584,18 +2584,22 @@ class Cache:
verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}") verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}")
def should_use_cache(self, *args, **kwargs): def should_use_cache(self, *args, **kwargs):
"""
Returns true if we should use the cache for LLM API calls
If cache is default_on then this is True
If cache is default_off then this is only true when user has opted in to use cache
"""
if self.mode == CacheMode.default_on: if self.mode == CacheMode.default_on:
return True return True
else:
# when mode == default_off -> Cache is opt in only # when mode == default_off -> Cache is opt in only
_cache = kwargs.get("cache", None) _cache = kwargs.get("cache", None)
verbose_logger.debug( verbose_logger.debug("should_use_cache: kwargs: %s; _cache: %s", kwargs, _cache)
f"should_use_cache: kwargs: {kwargs}; _cache: {_cache}" if _cache and isinstance(_cache, dict):
) if _cache.get("use-cache", False) is True:
if _cache and isinstance(_cache, dict): return True
if _cache.get("use-cache", True) is False: return False
return True
return True
async def batch_cache_write(self, result, *args, **kwargs): async def batch_cache_write(self, result, *args, **kwargs):
cache_key, cached_data, kwargs = self._add_cache_logic( cache_key, cached_data, kwargs = self._add_cache_logic(

2
litellm/tests/test_caching.py
View file

@ -1941,6 +1941,8 @@ async def test_cache_default_off_acompletion():
) )
print(f"Response3: {response3}") print(f"Response3: {response3}")
await asyncio.sleep(2)
response4 = await litellm.acompletion( response4 = await litellm.acompletion(
model="gpt-3.5-turbo", model="gpt-3.5-turbo",
messages=[ messages=[