From 05f810922c71fdb9494901370a931cb222c3ae3a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 27 Nov 2024 19:34:51 -0800 Subject: [PATCH] (feat) Allow disabling ErrorLogs written to the DB (#6940) * fix - allow disabling logging error logs * docs on disabling error logs * doc string for _PROXY_failure_handler * test_disable_error_logs * rename file * fix rename file * increase test coverage for test_enable_error_logs --- docs/my-website/docs/proxy/db_info.md | 14 ++- docs/my-website/docs/proxy/prod.md | 14 ++- litellm/proxy/hooks/proxy_failure_handler.py | 87 ++++++++++++++ litellm/proxy/proxy_server.py | 71 +---------- .../test_unit_test_proxy_hooks.py | 111 ++++++++++++++++++ 5 files changed, 218 insertions(+), 79 deletions(-) create mode 100644 litellm/proxy/hooks/proxy_failure_handler.py create mode 100644 tests/proxy_unit_tests/test_unit_test_proxy_hooks.py diff --git a/docs/my-website/docs/proxy/db_info.md b/docs/my-website/docs/proxy/db_info.md index 6e6a48bd1..8429f6360 100644 --- a/docs/my-website/docs/proxy/db_info.md +++ b/docs/my-website/docs/proxy/db_info.md @@ -50,18 +50,22 @@ You can see the full DB Schema [here](https://github.com/BerriAI/litellm/blob/ma | LiteLLM_ErrorLogs | Captures failed requests and errors. Stores exception details and request information. Helps with debugging and monitoring. | **Medium - on errors only** | | LiteLLM_AuditLog | Tracks changes to system configuration. Records who made changes and what was modified. Maintains history of updates to teams, users, and models. | **Off by default**, **High - when enabled** | -## How to Disable `LiteLLM_SpendLogs` +## Disable `LiteLLM_SpendLogs` & `LiteLLM_ErrorLogs` -You can disable spend_logs by setting `disable_spend_logs` to `True` on the `general_settings` section of your proxy_config.yaml file. +You can disable spend_logs and error_logs by setting `disable_spend_logs` and `disable_error_logs` to `True` on the `general_settings` section of your proxy_config.yaml file. ```yaml general_settings: - disable_spend_logs: True + disable_spend_logs: True # Disable writing spend logs to DB + disable_error_logs: True # Disable writing error logs to DB ``` +### What is the impact of disabling these logs? -### What is the impact of disabling `LiteLLM_SpendLogs`? - +When disabling spend logs (`disable_spend_logs: True`): - You **will not** be able to view Usage on the LiteLLM UI - You **will** continue seeing cost metrics on s3, Prometheus, Langfuse (any other Logging integration you are using) +When disabling error logs (`disable_error_logs: True`): +- You **will not** be able to view Errors on the LiteLLM UI +- You **will** continue seeing error logs in your application logs and any other logging integrations you are using diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 32a6fceee..9dacedaab 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -23,6 +23,7 @@ general_settings: # OPTIONAL Best Practices disable_spend_logs: True # turn off writing each transaction to the db. We recommend doing this is you don't need to see Usage on the LiteLLM UI and are tracking metrics via Prometheus + disable_error_logs: True # turn off writing LLM Exceptions to DB allow_requests_on_db_unavailable: True # Only USE when running LiteLLM on your VPC. Allow requests to still be processed even if the DB is unavailable. We recommend doing this if you're running LiteLLM on VPC that cannot be accessed from the public internet. litellm_settings: @@ -102,17 +103,22 @@ general_settings: allow_requests_on_db_unavailable: True ``` -## 6. Disable spend_logs if you're not using the LiteLLM UI +## 6. Disable spend_logs & error_logs if not using the LiteLLM UI -By default LiteLLM will write every request to the `LiteLLM_SpendLogs` table. This is used for viewing Usage on the LiteLLM UI. +By default, LiteLLM writes several types of logs to the database: +- Every LLM API request to the `LiteLLM_SpendLogs` table +- LLM Exceptions to the `LiteLLM_LogsErrors` table -If you're not viewing Usage on the LiteLLM UI (most users use Prometheus when this is disabled), you can disable spend_logs by setting `disable_spend_logs` to `True`. +If you're not viewing these logs on the LiteLLM UI (most users use Prometheus for monitoring), you can disable them by setting the following flags to `True`: ```yaml general_settings: - disable_spend_logs: True + disable_spend_logs: True # Disable writing spend logs to DB + disable_error_logs: True # Disable writing error logs to DB ``` +[More information about what the Database is used for here](db_info) + ## 7. Use Helm PreSync Hook for Database Migrations [BETA] To ensure only one service manages database migrations, use our [Helm PreSync hook for Database Migrations](https://github.com/BerriAI/litellm/blob/main/deploy/charts/litellm-helm/templates/migrations-job.yaml). This ensures migrations are handled during `helm upgrade` or `helm install`, while LiteLLM pods explicitly disable migrations. diff --git a/litellm/proxy/hooks/proxy_failure_handler.py b/litellm/proxy/hooks/proxy_failure_handler.py new file mode 100644 index 000000000..d316eab13 --- /dev/null +++ b/litellm/proxy/hooks/proxy_failure_handler.py @@ -0,0 +1,87 @@ +""" +Runs when LLM Exceptions occur on LiteLLM Proxy +""" + +import copy +import json +import uuid + +import litellm +from litellm.proxy._types import LiteLLM_ErrorLogs + + +async def _PROXY_failure_handler( + kwargs, # kwargs to completion + completion_response: litellm.ModelResponse, # response from completion + start_time=None, + end_time=None, # start/end time for completion +): + """ + Async Failure Handler - runs when LLM Exceptions occur on LiteLLM Proxy. + This function logs the errors to the Prisma DB + + Can be disabled by setting the following on proxy_config.yaml: + ```yaml + general_settings: + disable_error_logs: True + ``` + + """ + from litellm._logging import verbose_proxy_logger + from litellm.proxy.proxy_server import general_settings, prisma_client + + if general_settings.get("disable_error_logs") is True: + return + + if prisma_client is not None: + verbose_proxy_logger.debug( + "inside _PROXY_failure_handler kwargs=", extra=kwargs + ) + + _exception = kwargs.get("exception") + _exception_type = _exception.__class__.__name__ + _model = kwargs.get("model", None) + + _optional_params = kwargs.get("optional_params", {}) + _optional_params = copy.deepcopy(_optional_params) + + for k, v in _optional_params.items(): + v = str(v) + v = v[:100] + + _status_code = "500" + try: + _status_code = str(_exception.status_code) + except Exception: + # Don't let this fail logging the exception to the dB + pass + + _litellm_params = kwargs.get("litellm_params", {}) or {} + _metadata = _litellm_params.get("metadata", {}) or {} + _model_id = _metadata.get("model_info", {}).get("id", "") + _model_group = _metadata.get("model_group", "") + api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) + _exception_string = str(_exception) + + error_log = LiteLLM_ErrorLogs( + request_id=str(uuid.uuid4()), + model_group=_model_group, + model_id=_model_id, + litellm_model_name=kwargs.get("model"), + request_kwargs=_optional_params, + api_base=api_base, + exception_type=_exception_type, + status_code=_status_code, + exception_string=_exception_string, + startTime=kwargs.get("start_time"), + endTime=kwargs.get("end_time"), + ) + + error_log_dict = error_log.model_dump() + error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"]) + + await prisma_client.db.litellm_errorlogs.create( + data=error_log_dict # type: ignore + ) + + pass diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8e04951ad..ca6d4f363 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -176,6 +176,7 @@ from litellm.proxy.health_endpoints._health_endpoints import router as health_ro from litellm.proxy.hooks.prompt_injection_detection import ( _OPTIONAL_PromptInjectionDetection, ) +from litellm.proxy.hooks.proxy_failure_handler import _PROXY_failure_handler from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.management_endpoints.customer_endpoints import ( router as customer_router, @@ -529,14 +530,6 @@ db_writer_client: Optional[HTTPHandler] = None ### logger ### -def _get_pydantic_json_dict(pydantic_obj: BaseModel) -> dict: - try: - return pydantic_obj.model_dump() # type: ignore - except Exception: - # if using pydantic v1 - return pydantic_obj.dict() - - def get_custom_headers( *, user_api_key_dict: UserAPIKeyAuth, @@ -690,68 +683,6 @@ def cost_tracking(): litellm._async_success_callback.append(_PROXY_track_cost_callback) # type: ignore -async def _PROXY_failure_handler( - kwargs, # kwargs to completion - completion_response: litellm.ModelResponse, # response from completion - start_time=None, - end_time=None, # start/end time for completion -): - global prisma_client - if prisma_client is not None: - verbose_proxy_logger.debug( - "inside _PROXY_failure_handler kwargs=", extra=kwargs - ) - - _exception = kwargs.get("exception") - _exception_type = _exception.__class__.__name__ - _model = kwargs.get("model", None) - - _optional_params = kwargs.get("optional_params", {}) - _optional_params = copy.deepcopy(_optional_params) - - for k, v in _optional_params.items(): - v = str(v) - v = v[:100] - - _status_code = "500" - try: - _status_code = str(_exception.status_code) - except Exception: - # Don't let this fail logging the exception to the dB - pass - - _litellm_params = kwargs.get("litellm_params", {}) or {} - _metadata = _litellm_params.get("metadata", {}) or {} - _model_id = _metadata.get("model_info", {}).get("id", "") - _model_group = _metadata.get("model_group", "") - api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) - _exception_string = str(_exception) - - error_log = LiteLLM_ErrorLogs( - request_id=str(uuid.uuid4()), - model_group=_model_group, - model_id=_model_id, - litellm_model_name=kwargs.get("model"), - request_kwargs=_optional_params, - api_base=api_base, - exception_type=_exception_type, - status_code=_status_code, - exception_string=_exception_string, - startTime=kwargs.get("start_time"), - endTime=kwargs.get("end_time"), - ) - - # helper function to convert to dict on pydantic v2 & v1 - error_log_dict = _get_pydantic_json_dict(error_log) - error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"]) - - await prisma_client.db.litellm_errorlogs.create( - data=error_log_dict # type: ignore - ) - - pass - - @log_db_metrics async def _PROXY_track_cost_callback( kwargs, # kwargs to completion diff --git a/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py new file mode 100644 index 000000000..095b15368 --- /dev/null +++ b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py @@ -0,0 +1,111 @@ +import asyncio +import os +import sys +from unittest.mock import Mock, patch, AsyncMock +import pytest +from fastapi import Request +from litellm.proxy.utils import _get_redoc_url, _get_docs_url + +sys.path.insert(0, os.path.abspath("../..")) +import litellm + + +@pytest.mark.asyncio +async def test_disable_error_logs(): + """ + Test that the error logs are not written to the database when disable_error_logs is True + """ + # Mock the necessary components + mock_prisma_client = AsyncMock() + mock_general_settings = {"disable_error_logs": True} + + with patch( + "litellm.proxy.proxy_server.general_settings", mock_general_settings + ), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + + # Create a test exception + test_exception = Exception("Test error") + test_kwargs = { + "model": "gpt-4", + "exception": test_exception, + "optional_params": {}, + "litellm_params": {"metadata": {}}, + } + + # Call the failure handler + from litellm.proxy.proxy_server import _PROXY_failure_handler + + await _PROXY_failure_handler( + kwargs=test_kwargs, + completion_response=None, + start_time="2024-01-01", + end_time="2024-01-01", + ) + + # Verify prisma client was not called to create error logs + if hasattr(mock_prisma_client, "db"): + assert not mock_prisma_client.db.litellm_errorlogs.create.called + + +@pytest.mark.asyncio +async def test_disable_spend_logs(): + """ + Test that the spend logs are not written to the database when disable_spend_logs is True + """ + # Mock the necessary components + mock_prisma_client = Mock() + mock_prisma_client.spend_log_transactions = [] + + with patch("litellm.proxy.proxy_server.disable_spend_logs", True), patch( + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ): + from litellm.proxy.proxy_server import update_database + + # Call update_database with disable_spend_logs=True + await update_database( + token="fake-token", + response_cost=0.1, + user_id="user123", + completion_response=None, + start_time="2024-01-01", + end_time="2024-01-01", + ) + # Verify no spend logs were added + assert len(mock_prisma_client.spend_log_transactions) == 0 + + +@pytest.mark.asyncio +async def test_enable_error_logs(): + """ + Test that the error logs are written to the database when disable_error_logs is False + """ + # Mock the necessary components + mock_prisma_client = AsyncMock() + mock_general_settings = {"disable_error_logs": False} + + with patch( + "litellm.proxy.proxy_server.general_settings", mock_general_settings + ), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + + # Create a test exception + test_exception = Exception("Test error") + test_kwargs = { + "model": "gpt-4", + "exception": test_exception, + "optional_params": {}, + "litellm_params": {"metadata": {}}, + } + + # Call the failure handler + from litellm.proxy.proxy_server import _PROXY_failure_handler + + await _PROXY_failure_handler( + kwargs=test_kwargs, + completion_response=None, + start_time="2024-01-01", + end_time="2024-01-01", + ) + + # Verify prisma client was called to create error logs + if hasattr(mock_prisma_client, "db"): + assert mock_prisma_client.db.litellm_errorlogs.create.called