phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

feat - return size of in memory cache

Browse source
This commit is contained in:
Ishaan Jaff 2024-07-13 18:22:44 -07:00
parent 5c6e24370e
commit 31783196c0
1 changed files with 31 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

31
litellm/proxy/common_utils/debug_utils.py
View file

@ -26,6 +26,37 @@ if os.environ.get("LITELLM_PROFILE", "false").lower() == "true":
return {"top_50_memory_usage": result}
@router.get("/memory-usage-in-mem-cache", include_in_schema=False)
async def memory_usage_in_mem_cache():
# returns the size of all in-memory caches on the proxy server
"""
1. user_api_key_cache
2. router_cache
3. proxy_logging_cache
4. internal_usage_cache
"""
from litellm.proxy.proxy_server import (
llm_router,
proxy_logging_obj,
user_api_key_cache,
)
num_items_in_user_api_key_cache = len(
user_api_key_cache.in_memory_cache.cache_dict
) + len(user_api_key_cache.in_memory_cache.ttl_dict)
num_items_in_llm_router_cache = len(
llm_router.cache.in_memory_cache.cache_dict
) + len(llm_router.cache.in_memory_cache.ttl_dict)
num_items_in_proxy_logging_obj_cache = len(
proxy_logging_obj.internal_usage_cache.in_memory_cache.cache_dict
) + len(proxy_logging_obj.internal_usage_cache.in_memory_cache.ttl_dict)
return {
"num_items_in_user_api_key_cache": num_items_in_user_api_key_cache,
"num_items_in_llm_router_cache": num_items_in_llm_router_cache,
"num_items_in_proxy_logging_obj_cache": num_items_in_proxy_logging_obj_cache,
}
@router.get("/otel-spans", include_in_schema=False)
async def get_otel_spans():