phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix show debugging utils on in mem usage

Browse source
This commit is contained in:
Ishaan Jaff 2024-07-15 10:05:57 -07:00
parent c6abfcf595
commit a9f77c2af3
1 changed files with 54 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

82
litellm/proxy/common_utils/debug_utils.py
View file

@ -39,36 +39,62 @@ if os.environ.get("LITELLM_PROFILE", "false").lower() == "true":
return {"top_50_memory_usage": result} return {"top_50_memory_usage": result}
@router.get("/memory-usage-in-mem-cache", include_in_schema=False)
async def memory_usage_in_mem_cache():
# returns the size of all in-memory caches on the proxy server
"""
1. user_api_key_cache
2. router_cache
3. proxy_logging_cache
4. internal_usage_cache
"""
from litellm.proxy.proxy_server import (
llm_router,
proxy_logging_obj,
user_api_key_cache,
)
num_items_in_user_api_key_cache = len( @router.get("/memory-usage-in-mem-cache", include_in_schema=False)
user_api_key_cache.in_memory_cache.cache_dict async def memory_usage_in_mem_cache():
) + len(user_api_key_cache.in_memory_cache.ttl_dict) # returns the size of all in-memory caches on the proxy server
num_items_in_llm_router_cache = len( """
llm_router.cache.in_memory_cache.cache_dict 1. user_api_key_cache
) + len(llm_router.cache.in_memory_cache.ttl_dict) 2. router_cache
num_items_in_proxy_logging_obj_cache = len( 3. proxy_logging_cache
proxy_logging_obj.internal_usage_cache.in_memory_cache.cache_dict 4. internal_usage_cache
) + len(proxy_logging_obj.internal_usage_cache.in_memory_cache.ttl_dict) """
from litellm.proxy.proxy_server import (
llm_router,
proxy_logging_obj,
user_api_key_cache,
)
return { num_items_in_user_api_key_cache = len(
"num_items_in_user_api_key_cache": num_items_in_user_api_key_cache, user_api_key_cache.in_memory_cache.cache_dict
"num_items_in_llm_router_cache": num_items_in_llm_router_cache, ) + len(user_api_key_cache.in_memory_cache.ttl_dict)
"num_items_in_proxy_logging_obj_cache": num_items_in_proxy_logging_obj_cache, num_items_in_llm_router_cache = len(
} llm_router.cache.in_memory_cache.cache_dict
) + len(llm_router.cache.in_memory_cache.ttl_dict)
num_items_in_proxy_logging_obj_cache = len(
proxy_logging_obj.internal_usage_cache.in_memory_cache.cache_dict
) + len(proxy_logging_obj.internal_usage_cache.in_memory_cache.ttl_dict)
return {
"num_items_in_user_api_key_cache": num_items_in_user_api_key_cache,
"num_items_in_llm_router_cache": num_items_in_llm_router_cache,
"num_items_in_proxy_logging_obj_cache": num_items_in_proxy_logging_obj_cache,
}
@router.get("/memory-usage-in-mem-cache-items", include_in_schema=False)
async def memory_usage_in_mem_cache_items():
# returns the size of all in-memory caches on the proxy server
"""
1. user_api_key_cache
2. router_cache
3. proxy_logging_cache
4. internal_usage_cache
"""
from litellm.proxy.proxy_server import (
llm_router,
proxy_logging_obj,
user_api_key_cache,
)
return {
"user_api_key_cache": user_api_key_cache.in_memory_cache.cache_dict,
"user_api_key_ttl": user_api_key_cache.in_memory_cache.ttl_dict,
"llm_router_cache": llm_router.cache.in_memory_cache.cache_dict,
"llm_router_ttl": llm_router.cache.in_memory_cache.ttl_dict,
"proxy_logging_obj_cache": proxy_logging_obj.internal_usage_cache.in_memory_cache.cache_dict,
"proxy_logging_obj_ttl": proxy_logging_obj.internal_usage_cache.in_memory_cache.ttl_dict,
}
@router.get("/otel-spans", include_in_schema=False) @router.get("/otel-spans", include_in_schema=False)