phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix show debugging utils on in mem usage

Browse source
This commit is contained in:
Ishaan Jaff 2024-07-15 10:05:57 -07:00
parent c6abfcf595
commit a9f77c2af3
1 changed files with 54 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

30
litellm/proxy/common_utils/debug_utils.py
View file

@ -39,8 +39,9 @@ if os.environ.get("LITELLM_PROFILE", "false").lower() == "true":
return {"top_50_memory_usage": result} return {"top_50_memory_usage": result}
@router.get("/memory-usage-in-mem-cache", include_in_schema=False)
async def memory_usage_in_mem_cache(): @router.get("/memory-usage-in-mem-cache", include_in_schema=False)
async def memory_usage_in_mem_cache():
# returns the size of all in-memory caches on the proxy server # returns the size of all in-memory caches on the proxy server
""" """
1. user_api_key_cache 1. user_api_key_cache
@ -71,6 +72,31 @@ if os.environ.get("LITELLM_PROFILE", "false").lower() == "true":
} }
@router.get("/memory-usage-in-mem-cache-items", include_in_schema=False)
async def memory_usage_in_mem_cache_items():
# returns the size of all in-memory caches on the proxy server
"""
1. user_api_key_cache
2. router_cache
3. proxy_logging_cache
4. internal_usage_cache
"""
from litellm.proxy.proxy_server import (
llm_router,
proxy_logging_obj,
user_api_key_cache,
)
return {
"user_api_key_cache": user_api_key_cache.in_memory_cache.cache_dict,
"user_api_key_ttl": user_api_key_cache.in_memory_cache.ttl_dict,
"llm_router_cache": llm_router.cache.in_memory_cache.cache_dict,
"llm_router_ttl": llm_router.cache.in_memory_cache.ttl_dict,
"proxy_logging_obj_cache": proxy_logging_obj.internal_usage_cache.in_memory_cache.cache_dict,
"proxy_logging_obj_ttl": proxy_logging_obj.internal_usage_cache.in_memory_cache.ttl_dict,
}
@router.get("/otel-spans", include_in_schema=False) @router.get("/otel-spans", include_in_schema=False)
async def get_otel_spans(): async def get_otel_spans():
from litellm.integrations.opentelemetry import OpenTelemetry from litellm.integrations.opentelemetry import OpenTelemetry