diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py new file mode 100644 index 0000000000..dc77958a62 --- /dev/null +++ b/litellm/proxy/common_utils/debug_utils.py @@ -0,0 +1,27 @@ +# Start tracing memory allocations +import os +import tracemalloc + +from fastapi import APIRouter + +from litellm._logging import verbose_proxy_logger + +router = APIRouter() + +if os.environ.get("LITELLM_PROFILE", "false").lower() == "true": + tracemalloc.start() + + @router.get("/memory-usage", include_in_schema=False) + async def memory_usage(): + # Take a snapshot of the current memory usage + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics("lineno") + verbose_proxy_logger.debug("TOP STATS: %s", top_stats) + + # Get the top 50 memory usage lines + top_50 = top_stats[:50] + result = [] + for stat in top_50: + result.append(f"{stat.traceback.format()}: {stat.size / 1024} KiB") + + return {"top_50_memory_usage": result} diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 630aa3f3e1..9b78ecf8ab 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -140,6 +140,7 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth ## Import All Misc routes here ## from litellm.proxy.caching_routes import router as caching_router +from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.health_check import perform_health_check from litellm.proxy.health_endpoints._health_endpoints import router as health_router @@ -9173,3 +9174,4 @@ app.include_router(team_router) app.include_router(spend_management_router) app.include_router(caching_router) app.include_router(analytics_router) +app.include_router(debugging_endpoints_router)