perf: remove 'always_read_redis' - adding +830ms on each llm call (#6414)

* perf: remove 'always_read_redis' - adding +830ms on each llm call

* test: cleanup codestral tests - backend api unavailable
This commit is contained in:
Krish Dholakia 2024-10-24 17:48:36 -07:00 committed by GitHub
parent 0f0470f574
commit d59f8f952d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 49 additions and 57 deletions

View file

@ -2066,46 +2066,6 @@ async def test_cache_default_off_acompletion():
assert response3.id == response4.id
@pytest.mark.asyncio()
@pytest.mark.skip(reason="dual caching should first prioritze local cache")
async def test_dual_cache_uses_redis():
"""
- Store diff values in redis and in memory cache
- call get cache
- Assert that value from redis is used
"""
litellm.set_verbose = True
from litellm.caching.caching import DualCache, RedisCache
current_usage = uuid.uuid4()
_cache_obj = DualCache(redis_cache=RedisCache(), always_read_redis=True)
# set cache
await _cache_obj.async_set_cache(key=f"current_usage: {current_usage}", value=10)
# modify value of in memory cache
_cache_obj.in_memory_cache.cache_dict[f"current_usage: {current_usage}"] = 1
# get cache
value = await _cache_obj.async_get_cache(key=f"current_usage: {current_usage}")
print("value from dual cache", value)
assert value == 10
@pytest.mark.asyncio()
async def test_proxy_logging_setup():
"""
Assert always_read_redis is True when used by internal usage cache
"""
from litellm.caching.caching import DualCache
from litellm.proxy.utils import ProxyLogging
pl_obj = ProxyLogging(user_api_key_cache=DualCache())
assert pl_obj.internal_usage_cache.dual_cache.always_read_redis is True
@pytest.mark.skip(reason="local test. Requires sentinel setup.")
@pytest.mark.asyncio
async def test_redis_sentinel_caching():