increase test coverage for test_enable_error_logs

docs/my-website/docs/proxy/db_info.md

@@ -50,18 +50,22 @@ You can see the full DB Schema [here](https://github.com/BerriAI/litellm/blob/ma
 | LiteLLM_ErrorLogs | Captures failed requests and errors. Stores exception details and request information. Helps with debugging and monitoring. | **Medium - on errors only** |
 | LiteLLM_AuditLog | Tracks changes to system configuration. Records who made changes and what was modified. Maintains history of updates to teams, users, and models. | **Off by default**, **High - when enabled** |
 
-## How to Disable `LiteLLM_SpendLogs`
+## Disable `LiteLLM_SpendLogs` & `LiteLLM_ErrorLogs`
 
-You can disable spend_logs by setting `disable_spend_logs` to `True` on the `general_settings` section of your proxy_config.yaml file.
+You can disable spend_logs and error_logs by setting `disable_spend_logs` and `disable_error_logs` to `True` on the `general_settings` section of your proxy_config.yaml file.
 
 ```yaml
 general_settings:
-  disable_spend_logs: True
+  disable_spend_logs: True   # Disable writing spend logs to DB
+  disable_error_logs: True   # Disable writing error logs to DB
 ```
 
+### What is the impact of disabling these logs?
 
-### What is the impact of disabling `LiteLLM_SpendLogs`?
+When disabling spend logs (`disable_spend_logs: True`):
-
 - You **will not** be able to view Usage on the LiteLLM UI
 - You **will** continue seeing cost metrics on s3, Prometheus, Langfuse (any other Logging integration you are using)
 
+When disabling error logs (`disable_error_logs: True`):
+- You **will not** be able to view Errors on the LiteLLM UI
+- You **will** continue seeing error logs in your application logs and any other logging integrations you are using

docs/my-website/docs/proxy/prod.md

@@ -23,6 +23,7 @@ general_settings:
 
   # OPTIONAL Best Practices
   disable_spend_logs: True # turn off writing each transaction to the db. We recommend doing this is you don't need to see Usage on the LiteLLM UI and are tracking metrics via Prometheus
+  disable_error_logs: True # turn off writing LLM Exceptions to DB
   allow_requests_on_db_unavailable: True # Only USE when running LiteLLM on your VPC. Allow requests to still be processed even if the DB is unavailable. We recommend doing this if you're running LiteLLM on VPC that cannot be accessed from the public internet.
 
 litellm_settings:
@@ -102,17 +103,22 @@ general_settings:
   allow_requests_on_db_unavailable: True
 ```
 
-## 6. Disable spend_logs if you're not using the LiteLLM UI
+## 6. Disable spend_logs & error_logs if not using the LiteLLM UI
 
-By default LiteLLM will write every request to the `LiteLLM_SpendLogs` table. This is used for viewing Usage on the LiteLLM UI. 
+By default, LiteLLM writes several types of logs to the database:
+- Every LLM API request to the `LiteLLM_SpendLogs` table
+- LLM Exceptions to the `LiteLLM_LogsErrors` table
 
-If you're not viewing Usage on the LiteLLM UI (most users use Prometheus when this is disabled), you can disable spend_logs by setting `disable_spend_logs` to `True`.
+If you're not viewing these logs on the LiteLLM UI (most users use Prometheus for monitoring), you can disable them by setting the following flags to `True`:
 
 ```yaml
 general_settings:
-  disable_spend_logs: True
+  disable_spend_logs: True    # Disable writing spend logs to DB
+  disable_error_logs: True    # Disable writing error logs to DB
 ```
 
+[More information about what the Database is used for here](db_info)
+
 ## 7. Use Helm PreSync Hook for Database Migrations [BETA]
 
 To ensure only one service manages database migrations, use our [Helm PreSync hook for Database Migrations](https://github.com/BerriAI/litellm/blob/main/deploy/charts/litellm-helm/templates/migrations-job.yaml). This ensures migrations are handled during `helm upgrade` or `helm install`, while LiteLLM pods explicitly disable migrations.

litellm/proxy/hooks/proxy_failure_handler.py

@@ -0,0 +1,87 @@
+"""
+Runs when LLM Exceptions occur on LiteLLM Proxy
+"""
+
+import copy
+import json
+import uuid
+
+import litellm
+from litellm.proxy._types import LiteLLM_ErrorLogs
+
+
+async def _PROXY_failure_handler(
+    kwargs,  # kwargs to completion
+    completion_response: litellm.ModelResponse,  # response from completion
+    start_time=None,
+    end_time=None,  # start/end time for completion
+):
+    """
+    Async Failure Handler - runs when LLM Exceptions occur on LiteLLM Proxy.
+    This function logs the errors to the Prisma DB
+
+    Can be disabled by setting the following on proxy_config.yaml:
+    ```yaml
+    general_settings:
+      disable_error_logs: True
+    ```
+
+    """
+    from litellm._logging import verbose_proxy_logger
+    from litellm.proxy.proxy_server import general_settings, prisma_client
+
+    if general_settings.get("disable_error_logs") is True:
+        return
+
+    if prisma_client is not None:
+        verbose_proxy_logger.debug(
+            "inside _PROXY_failure_handler kwargs=", extra=kwargs
+        )
+
+        _exception = kwargs.get("exception")
+        _exception_type = _exception.__class__.__name__
+        _model = kwargs.get("model", None)
+
+        _optional_params = kwargs.get("optional_params", {})
+        _optional_params = copy.deepcopy(_optional_params)
+
+        for k, v in _optional_params.items():
+            v = str(v)
+            v = v[:100]
+
+        _status_code = "500"
+        try:
+            _status_code = str(_exception.status_code)
+        except Exception:
+            # Don't let this fail logging the exception to the dB
+            pass
+
+        _litellm_params = kwargs.get("litellm_params", {}) or {}
+        _metadata = _litellm_params.get("metadata", {}) or {}
+        _model_id = _metadata.get("model_info", {}).get("id", "")
+        _model_group = _metadata.get("model_group", "")
+        api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params)
+        _exception_string = str(_exception)
+
+        error_log = LiteLLM_ErrorLogs(
+            request_id=str(uuid.uuid4()),
+            model_group=_model_group,
+            model_id=_model_id,
+            litellm_model_name=kwargs.get("model"),
+            request_kwargs=_optional_params,
+            api_base=api_base,
+            exception_type=_exception_type,
+            status_code=_status_code,
+            exception_string=_exception_string,
+            startTime=kwargs.get("start_time"),
+            endTime=kwargs.get("end_time"),
+        )
+
+        error_log_dict = error_log.model_dump()
+        error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"])
+
+        await prisma_client.db.litellm_errorlogs.create(
+            data=error_log_dict  # type: ignore
+        )
+
+    pass

litellm/proxy/proxy_server.py

@@ -173,6 +173,7 @@ from litellm.proxy.health_endpoints._health_endpoints import router as health_ro
 from litellm.proxy.hooks.prompt_injection_detection import (
     _OPTIONAL_PromptInjectionDetection,
 )
+from litellm.proxy.hooks.proxy_failure_handler import _PROXY_failure_handler
 from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
 from litellm.proxy.management_endpoints.customer_endpoints import (
     router as customer_router,
@@ -526,14 +527,6 @@ db_writer_client: Optional[HTTPHandler] = None
 ### logger ###
 
 
-def _get_pydantic_json_dict(pydantic_obj: BaseModel) -> dict:
-    try:
-        return pydantic_obj.model_dump()  # type: ignore
-    except Exception:
-        # if using pydantic v1
-        return pydantic_obj.dict()
-
-
 def get_custom_headers(
     *,
     user_api_key_dict: UserAPIKeyAuth,
@@ -687,68 +680,6 @@ def cost_tracking():
                 litellm._async_success_callback.append(_PROXY_track_cost_callback)  # type: ignore
 
 
-async def _PROXY_failure_handler(
-    kwargs,  # kwargs to completion
-    completion_response: litellm.ModelResponse,  # response from completion
-    start_time=None,
-    end_time=None,  # start/end time for completion
-):
-    global prisma_client
-    if prisma_client is not None:
-        verbose_proxy_logger.debug(
-            "inside _PROXY_failure_handler kwargs=", extra=kwargs
-        )
-
-        _exception = kwargs.get("exception")
-        _exception_type = _exception.__class__.__name__
-        _model = kwargs.get("model", None)
-
-        _optional_params = kwargs.get("optional_params", {})
-        _optional_params = copy.deepcopy(_optional_params)
-
-        for k, v in _optional_params.items():
-            v = str(v)
-            v = v[:100]
-
-        _status_code = "500"
-        try:
-            _status_code = str(_exception.status_code)
-        except Exception:
-            # Don't let this fail logging the exception to the dB
-            pass
-
-        _litellm_params = kwargs.get("litellm_params", {}) or {}
-        _metadata = _litellm_params.get("metadata", {}) or {}
-        _model_id = _metadata.get("model_info", {}).get("id", "")
-        _model_group = _metadata.get("model_group", "")
-        api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params)
-        _exception_string = str(_exception)
-
-        error_log = LiteLLM_ErrorLogs(
-            request_id=str(uuid.uuid4()),
-            model_group=_model_group,
-            model_id=_model_id,
-            litellm_model_name=kwargs.get("model"),
-            request_kwargs=_optional_params,
-            api_base=api_base,
-            exception_type=_exception_type,
-            status_code=_status_code,
-            exception_string=_exception_string,
-            startTime=kwargs.get("start_time"),
-            endTime=kwargs.get("end_time"),
-        )
-
-        # helper function to convert to dict on pydantic v2 & v1
-        error_log_dict = _get_pydantic_json_dict(error_log)
-        error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"])
-
-        await prisma_client.db.litellm_errorlogs.create(
-            data=error_log_dict  # type: ignore
-        )
-
-    pass
-
-
 @log_db_metrics
 async def _PROXY_track_cost_callback(
     kwargs,  # kwargs to completion

tests/proxy_unit_tests/test_unit_test_proxy_hooks.py

@@ -0,0 +1,111 @@
+import asyncio
+import os
+import sys
+from unittest.mock import Mock, patch, AsyncMock
+import pytest
+from fastapi import Request
+from litellm.proxy.utils import _get_redoc_url, _get_docs_url
+
+sys.path.insert(0, os.path.abspath("../.."))
+import litellm
+
+
+@pytest.mark.asyncio
+async def test_disable_error_logs():
+    """
+    Test that the error logs are not written to the database when disable_error_logs is True
+    """
+    # Mock the necessary components
+    mock_prisma_client = AsyncMock()
+    mock_general_settings = {"disable_error_logs": True}
+
+    with patch(
+        "litellm.proxy.proxy_server.general_settings", mock_general_settings
+    ), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client):
+
+        # Create a test exception
+        test_exception = Exception("Test error")
+        test_kwargs = {
+            "model": "gpt-4",
+            "exception": test_exception,
+            "optional_params": {},
+            "litellm_params": {"metadata": {}},
+        }
+
+        # Call the failure handler
+        from litellm.proxy.proxy_server import _PROXY_failure_handler
+
+        await _PROXY_failure_handler(
+            kwargs=test_kwargs,
+            completion_response=None,
+            start_time="2024-01-01",
+            end_time="2024-01-01",
+        )
+
+        # Verify prisma client was not called to create error logs
+        if hasattr(mock_prisma_client, "db"):
+            assert not mock_prisma_client.db.litellm_errorlogs.create.called
+
+
+@pytest.mark.asyncio
+async def test_disable_spend_logs():
+    """
+    Test that the spend logs are not written to the database when disable_spend_logs is True
+    """
+    # Mock the necessary components
+    mock_prisma_client = Mock()
+    mock_prisma_client.spend_log_transactions = []
+
+    with patch("litellm.proxy.proxy_server.disable_spend_logs", True), patch(
+        "litellm.proxy.proxy_server.prisma_client", mock_prisma_client
+    ):
+        from litellm.proxy.proxy_server import update_database
+
+        # Call update_database with disable_spend_logs=True
+        await update_database(
+            token="fake-token",
+            response_cost=0.1,
+            user_id="user123",
+            completion_response=None,
+            start_time="2024-01-01",
+            end_time="2024-01-01",
+        )
+        # Verify no spend logs were added
+        assert len(mock_prisma_client.spend_log_transactions) == 0
+
+
+@pytest.mark.asyncio
+async def test_enable_error_logs():
+    """
+    Test that the error logs are written to the database when disable_error_logs is False
+    """
+    # Mock the necessary components
+    mock_prisma_client = AsyncMock()
+    mock_general_settings = {"disable_error_logs": False}
+
+    with patch(
+        "litellm.proxy.proxy_server.general_settings", mock_general_settings
+    ), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client):
+
+        # Create a test exception
+        test_exception = Exception("Test error")
+        test_kwargs = {
+            "model": "gpt-4",
+            "exception": test_exception,
+            "optional_params": {},
+            "litellm_params": {"metadata": {}},
+        }
+
+        # Call the failure handler
+        from litellm.proxy.proxy_server import _PROXY_failure_handler
+
+        await _PROXY_failure_handler(
+            kwargs=test_kwargs,
+            completion_response=None,
+            start_time="2024-01-01",
+            end_time="2024-01-01",
+        )
+
+        # Verify prisma client was called to create error logs
+        if hasattr(mock_prisma_client, "db"):
+            assert mock_prisma_client.db.litellm_errorlogs.create.called