forked from phoenix/litellm-mirror
Merge pull request #2785 from BerriAI/litellm_high_traffic_redis_caching_fixes
[Feat] Proxy - high traffic redis caching - when using `url`
This commit is contained in:
commit
b14b6083f5
2 changed files with 52 additions and 0 deletions
|
@ -109,6 +109,7 @@ class RedisCache(BaseCache):
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
from ._redis import get_redis_client, get_redis_connection_pool
|
from ._redis import get_redis_client, get_redis_connection_pool
|
||||||
|
import redis
|
||||||
|
|
||||||
redis_kwargs = {}
|
redis_kwargs = {}
|
||||||
if host is not None:
|
if host is not None:
|
||||||
|
@ -122,6 +123,14 @@ class RedisCache(BaseCache):
|
||||||
self.redis_client = get_redis_client(**redis_kwargs)
|
self.redis_client = get_redis_client(**redis_kwargs)
|
||||||
self.redis_kwargs = redis_kwargs
|
self.redis_kwargs = redis_kwargs
|
||||||
self.async_redis_conn_pool = get_redis_connection_pool(**redis_kwargs)
|
self.async_redis_conn_pool = get_redis_connection_pool(**redis_kwargs)
|
||||||
|
|
||||||
|
if "url" in redis_kwargs and redis_kwargs["url"] is not None:
|
||||||
|
parsed_kwargs = redis.connection.parse_url(redis_kwargs["url"])
|
||||||
|
redis_kwargs.update(parsed_kwargs)
|
||||||
|
self.redis_kwargs.update(parsed_kwargs)
|
||||||
|
# pop url
|
||||||
|
self.redis_kwargs.pop("url")
|
||||||
|
|
||||||
# redis namespaces
|
# redis namespaces
|
||||||
self.namespace = namespace
|
self.namespace = namespace
|
||||||
# for high traffic, we store the redis results in memory and then batch write to redis
|
# for high traffic, we store the redis results in memory and then batch write to redis
|
||||||
|
@ -338,6 +347,14 @@ class RedisCache(BaseCache):
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
|
def client_list(self):
|
||||||
|
client_list = self.redis_client.client_list()
|
||||||
|
return client_list
|
||||||
|
|
||||||
|
def info(self):
|
||||||
|
info = self.redis_client.info()
|
||||||
|
return info
|
||||||
|
|
||||||
def flush_cache(self):
|
def flush_cache(self):
|
||||||
self.redis_client.flushall()
|
self.redis_client.flushall()
|
||||||
|
|
||||||
|
|
|
@ -8050,6 +8050,41 @@ async def cache_ping():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/cache/redis/info",
|
||||||
|
tags=["caching"],
|
||||||
|
dependencies=[Depends(user_api_key_auth)],
|
||||||
|
)
|
||||||
|
async def cache_redis_info():
|
||||||
|
"""
|
||||||
|
Endpoint for getting /redis/info
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if litellm.cache is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503, detail="Cache not initialized. litellm.cache is None"
|
||||||
|
)
|
||||||
|
if litellm.cache.type == "redis":
|
||||||
|
client_list = litellm.cache.cache.client_list()
|
||||||
|
redis_info = litellm.cache.cache.info()
|
||||||
|
num_clients = len(client_list)
|
||||||
|
return {
|
||||||
|
"num_clients": num_clients,
|
||||||
|
"clients": client_list,
|
||||||
|
"info": redis_info,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=500,
|
||||||
|
detail=f"Cache type {litellm.cache.type} does not support flushing",
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail=f"Service Unhealthy ({str(e)})",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post(
|
@router.post(
|
||||||
"/cache/flushall",
|
"/cache/flushall",
|
||||||
tags=["caching"],
|
tags=["caching"],
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue