From 31783196c00c13e116242dc99a80668bc6ea9eab Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 13 Jul 2024 18:22:44 -0700 Subject: [PATCH] feat - return size of in memory cache --- litellm/proxy/common_utils/debug_utils.py | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index 9d8d35e19..c6968bc6f 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -26,6 +26,37 @@ if os.environ.get("LITELLM_PROFILE", "false").lower() == "true": return {"top_50_memory_usage": result} + @router.get("/memory-usage-in-mem-cache", include_in_schema=False) + async def memory_usage_in_mem_cache(): + # returns the size of all in-memory caches on the proxy server + """ + 1. user_api_key_cache + 2. router_cache + 3. proxy_logging_cache + 4. internal_usage_cache + """ + from litellm.proxy.proxy_server import ( + llm_router, + proxy_logging_obj, + user_api_key_cache, + ) + + num_items_in_user_api_key_cache = len( + user_api_key_cache.in_memory_cache.cache_dict + ) + len(user_api_key_cache.in_memory_cache.ttl_dict) + num_items_in_llm_router_cache = len( + llm_router.cache.in_memory_cache.cache_dict + ) + len(llm_router.cache.in_memory_cache.ttl_dict) + num_items_in_proxy_logging_obj_cache = len( + proxy_logging_obj.internal_usage_cache.in_memory_cache.cache_dict + ) + len(proxy_logging_obj.internal_usage_cache.in_memory_cache.ttl_dict) + + return { + "num_items_in_user_api_key_cache": num_items_in_user_api_key_cache, + "num_items_in_llm_router_cache": num_items_in_llm_router_cache, + "num_items_in_proxy_logging_obj_cache": num_items_in_proxy_logging_obj_cache, + } + @router.get("/otel-spans", include_in_schema=False) async def get_otel_spans():