From 778dbf1c2f1b99b26d73ee3f8352eb1cc9c62c33 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 27 Nov 2024 09:20:14 -0800 Subject: [PATCH 1/7] fix - allow disabling logging error logs --- litellm/proxy/hooks/failure_handler.py | 81 ++++++++++++++++++++++++++ litellm/proxy/proxy_server.py | 71 +--------------------- 2 files changed, 82 insertions(+), 70 deletions(-) create mode 100644 litellm/proxy/hooks/failure_handler.py diff --git a/litellm/proxy/hooks/failure_handler.py b/litellm/proxy/hooks/failure_handler.py new file mode 100644 index 000000000..36e0fb0e6 --- /dev/null +++ b/litellm/proxy/hooks/failure_handler.py @@ -0,0 +1,81 @@ +""" +Runs when LLM Exceptions occur on LiteLLM Proxy +""" + +import copy +import json +import uuid + +import litellm +from litellm.proxy._types import LiteLLM_ErrorLogs + + +async def _PROXY_failure_handler( + kwargs, # kwargs to completion + completion_response: litellm.ModelResponse, # response from completion + start_time=None, + end_time=None, # start/end time for completion +): + """ + Async Failure Handler - runs when LLM Exceptions occur on LiteLLM Proxy. + + This function logs the errors to the Prisma DB + """ + from litellm._logging import verbose_proxy_logger + from litellm.proxy.proxy_server import general_settings, prisma_client + + if general_settings.get("disable_error_logs") is True: + return + + if prisma_client is not None: + verbose_proxy_logger.debug( + "inside _PROXY_failure_handler kwargs=", extra=kwargs + ) + + _exception = kwargs.get("exception") + _exception_type = _exception.__class__.__name__ + _model = kwargs.get("model", None) + + _optional_params = kwargs.get("optional_params", {}) + _optional_params = copy.deepcopy(_optional_params) + + for k, v in _optional_params.items(): + v = str(v) + v = v[:100] + + _status_code = "500" + try: + _status_code = str(_exception.status_code) + except Exception: + # Don't let this fail logging the exception to the dB + pass + + _litellm_params = kwargs.get("litellm_params", {}) or {} + _metadata = _litellm_params.get("metadata", {}) or {} + _model_id = _metadata.get("model_info", {}).get("id", "") + _model_group = _metadata.get("model_group", "") + api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) + _exception_string = str(_exception) + + error_log = LiteLLM_ErrorLogs( + request_id=str(uuid.uuid4()), + model_group=_model_group, + model_id=_model_id, + litellm_model_name=kwargs.get("model"), + request_kwargs=_optional_params, + api_base=api_base, + exception_type=_exception_type, + status_code=_status_code, + exception_string=_exception_string, + startTime=kwargs.get("start_time"), + endTime=kwargs.get("end_time"), + ) + + error_log_dict = error_log.model_dump() + error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"]) + + await prisma_client.db.litellm_errorlogs.create( + data=error_log_dict # type: ignore + ) + + pass diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 15971263a..011ed04de 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -170,6 +170,7 @@ from litellm.proxy.guardrails.init_guardrails import ( ) from litellm.proxy.health_check import perform_health_check from litellm.proxy.health_endpoints._health_endpoints import router as health_router +from litellm.proxy.hooks.failure_handler import _PROXY_failure_handler from litellm.proxy.hooks.prompt_injection_detection import ( _OPTIONAL_PromptInjectionDetection, ) @@ -526,14 +527,6 @@ db_writer_client: Optional[HTTPHandler] = None ### logger ### -def _get_pydantic_json_dict(pydantic_obj: BaseModel) -> dict: - try: - return pydantic_obj.model_dump() # type: ignore - except Exception: - # if using pydantic v1 - return pydantic_obj.dict() - - def get_custom_headers( *, user_api_key_dict: UserAPIKeyAuth, @@ -687,68 +680,6 @@ def cost_tracking(): litellm._async_success_callback.append(_PROXY_track_cost_callback) # type: ignore -async def _PROXY_failure_handler( - kwargs, # kwargs to completion - completion_response: litellm.ModelResponse, # response from completion - start_time=None, - end_time=None, # start/end time for completion -): - global prisma_client - if prisma_client is not None: - verbose_proxy_logger.debug( - "inside _PROXY_failure_handler kwargs=", extra=kwargs - ) - - _exception = kwargs.get("exception") - _exception_type = _exception.__class__.__name__ - _model = kwargs.get("model", None) - - _optional_params = kwargs.get("optional_params", {}) - _optional_params = copy.deepcopy(_optional_params) - - for k, v in _optional_params.items(): - v = str(v) - v = v[:100] - - _status_code = "500" - try: - _status_code = str(_exception.status_code) - except Exception: - # Don't let this fail logging the exception to the dB - pass - - _litellm_params = kwargs.get("litellm_params", {}) or {} - _metadata = _litellm_params.get("metadata", {}) or {} - _model_id = _metadata.get("model_info", {}).get("id", "") - _model_group = _metadata.get("model_group", "") - api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) - _exception_string = str(_exception) - - error_log = LiteLLM_ErrorLogs( - request_id=str(uuid.uuid4()), - model_group=_model_group, - model_id=_model_id, - litellm_model_name=kwargs.get("model"), - request_kwargs=_optional_params, - api_base=api_base, - exception_type=_exception_type, - status_code=_status_code, - exception_string=_exception_string, - startTime=kwargs.get("start_time"), - endTime=kwargs.get("end_time"), - ) - - # helper function to convert to dict on pydantic v2 & v1 - error_log_dict = _get_pydantic_json_dict(error_log) - error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"]) - - await prisma_client.db.litellm_errorlogs.create( - data=error_log_dict # type: ignore - ) - - pass - - @log_db_metrics async def _PROXY_track_cost_callback( kwargs, # kwargs to completion From 0869a1c13fd2b13c3870356d0415f2afc379b12b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 27 Nov 2024 09:30:41 -0800 Subject: [PATCH 2/7] docs on disabling error logs --- docs/my-website/docs/proxy/db_info.md | 14 +++++++++----- docs/my-website/docs/proxy/prod.md | 14 ++++++++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/my-website/docs/proxy/db_info.md b/docs/my-website/docs/proxy/db_info.md index 6e6a48bd1..8429f6360 100644 --- a/docs/my-website/docs/proxy/db_info.md +++ b/docs/my-website/docs/proxy/db_info.md @@ -50,18 +50,22 @@ You can see the full DB Schema [here](https://github.com/BerriAI/litellm/blob/ma | LiteLLM_ErrorLogs | Captures failed requests and errors. Stores exception details and request information. Helps with debugging and monitoring. | **Medium - on errors only** | | LiteLLM_AuditLog | Tracks changes to system configuration. Records who made changes and what was modified. Maintains history of updates to teams, users, and models. | **Off by default**, **High - when enabled** | -## How to Disable `LiteLLM_SpendLogs` +## Disable `LiteLLM_SpendLogs` & `LiteLLM_ErrorLogs` -You can disable spend_logs by setting `disable_spend_logs` to `True` on the `general_settings` section of your proxy_config.yaml file. +You can disable spend_logs and error_logs by setting `disable_spend_logs` and `disable_error_logs` to `True` on the `general_settings` section of your proxy_config.yaml file. ```yaml general_settings: - disable_spend_logs: True + disable_spend_logs: True # Disable writing spend logs to DB + disable_error_logs: True # Disable writing error logs to DB ``` +### What is the impact of disabling these logs? -### What is the impact of disabling `LiteLLM_SpendLogs`? - +When disabling spend logs (`disable_spend_logs: True`): - You **will not** be able to view Usage on the LiteLLM UI - You **will** continue seeing cost metrics on s3, Prometheus, Langfuse (any other Logging integration you are using) +When disabling error logs (`disable_error_logs: True`): +- You **will not** be able to view Errors on the LiteLLM UI +- You **will** continue seeing error logs in your application logs and any other logging integrations you are using diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 32a6fceee..9dacedaab 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -23,6 +23,7 @@ general_settings: # OPTIONAL Best Practices disable_spend_logs: True # turn off writing each transaction to the db. We recommend doing this is you don't need to see Usage on the LiteLLM UI and are tracking metrics via Prometheus + disable_error_logs: True # turn off writing LLM Exceptions to DB allow_requests_on_db_unavailable: True # Only USE when running LiteLLM on your VPC. Allow requests to still be processed even if the DB is unavailable. We recommend doing this if you're running LiteLLM on VPC that cannot be accessed from the public internet. litellm_settings: @@ -102,17 +103,22 @@ general_settings: allow_requests_on_db_unavailable: True ``` -## 6. Disable spend_logs if you're not using the LiteLLM UI +## 6. Disable spend_logs & error_logs if not using the LiteLLM UI -By default LiteLLM will write every request to the `LiteLLM_SpendLogs` table. This is used for viewing Usage on the LiteLLM UI. +By default, LiteLLM writes several types of logs to the database: +- Every LLM API request to the `LiteLLM_SpendLogs` table +- LLM Exceptions to the `LiteLLM_LogsErrors` table -If you're not viewing Usage on the LiteLLM UI (most users use Prometheus when this is disabled), you can disable spend_logs by setting `disable_spend_logs` to `True`. +If you're not viewing these logs on the LiteLLM UI (most users use Prometheus for monitoring), you can disable them by setting the following flags to `True`: ```yaml general_settings: - disable_spend_logs: True + disable_spend_logs: True # Disable writing spend logs to DB + disable_error_logs: True # Disable writing error logs to DB ``` +[More information about what the Database is used for here](db_info) + ## 7. Use Helm PreSync Hook for Database Migrations [BETA] To ensure only one service manages database migrations, use our [Helm PreSync hook for Database Migrations](https://github.com/BerriAI/litellm/blob/main/deploy/charts/litellm-helm/templates/migrations-job.yaml). This ensures migrations are handled during `helm upgrade` or `helm install`, while LiteLLM pods explicitly disable migrations. From 5c6d9200c405b179228fc88d2e10b69c87eb4ef5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 27 Nov 2024 09:32:57 -0800 Subject: [PATCH 3/7] doc string for _PROXY_failure_handler --- litellm/proxy/hooks/failure_handler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/hooks/failure_handler.py b/litellm/proxy/hooks/failure_handler.py index 36e0fb0e6..d316eab13 100644 --- a/litellm/proxy/hooks/failure_handler.py +++ b/litellm/proxy/hooks/failure_handler.py @@ -18,8 +18,14 @@ async def _PROXY_failure_handler( ): """ Async Failure Handler - runs when LLM Exceptions occur on LiteLLM Proxy. - This function logs the errors to the Prisma DB + + Can be disabled by setting the following on proxy_config.yaml: + ```yaml + general_settings: + disable_error_logs: True + ``` + """ from litellm._logging import verbose_proxy_logger from litellm.proxy.proxy_server import general_settings, prisma_client From 459ba986074d5f2e12e9933ecb9d9fd35c0b47c6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 27 Nov 2024 09:40:35 -0800 Subject: [PATCH 4/7] test_disable_error_logs --- .../test_unit_test_proxy_hooks.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/proxy_unit_tests/test_unit_test_proxy_hooks.py diff --git a/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py new file mode 100644 index 000000000..5573ad096 --- /dev/null +++ b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py @@ -0,0 +1,74 @@ +import asyncio +import os +import sys +from unittest.mock import Mock, patch, AsyncMock +import pytest +from fastapi import Request +from litellm.proxy.utils import _get_redoc_url, _get_docs_url + +sys.path.insert(0, os.path.abspath("../..")) +import litellm + + +@pytest.mark.asyncio +async def test_disable_error_logs(): + """ + Test that the error logs are not written to the database when disable_error_logs is True + """ + # Mock the necessary components + mock_prisma_client = AsyncMock() + mock_general_settings = {"disable_error_logs": True} + + with patch( + "litellm.proxy.proxy_server.general_settings", mock_general_settings + ), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + + # Create a test exception + test_exception = Exception("Test error") + test_kwargs = { + "model": "gpt-4", + "exception": test_exception, + "optional_params": {}, + "litellm_params": {"metadata": {}}, + } + + # Call the failure handler + from litellm.proxy.proxy_server import _PROXY_failure_handler + + await _PROXY_failure_handler( + kwargs=test_kwargs, + completion_response=None, + start_time="2024-01-01", + end_time="2024-01-01", + ) + + # Verify prisma client was not called to create error logs + if hasattr(mock_prisma_client, "db"): + assert not mock_prisma_client.db.litellm_errorlogs.create.called + + +@pytest.mark.asyncio +async def test_disable_spend_logs(): + """ + Test that the spend logs are not written to the database when disable_spend_logs is True + """ + # Mock the necessary components + mock_prisma_client = Mock() + mock_prisma_client.spend_log_transactions = [] + + with patch("litellm.proxy.proxy_server.disable_spend_logs", True), patch( + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ): + from litellm.proxy.proxy_server import update_database + + # Call update_database with disable_spend_logs=True + await update_database( + token="fake-token", + response_cost=0.1, + user_id="user123", + completion_response=None, + start_time="2024-01-01", + end_time="2024-01-01", + ) + # Verify no spend logs were added + assert len(mock_prisma_client.spend_log_transactions) == 0 From 97b846cb08b25c91a8bf895feed77228f3fa59f9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 27 Nov 2024 11:35:59 -0800 Subject: [PATCH 5/7] rename file --- litellm/proxy/hooks/proxy_failure_handler.py | 87 ++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 litellm/proxy/hooks/proxy_failure_handler.py diff --git a/litellm/proxy/hooks/proxy_failure_handler.py b/litellm/proxy/hooks/proxy_failure_handler.py new file mode 100644 index 000000000..d316eab13 --- /dev/null +++ b/litellm/proxy/hooks/proxy_failure_handler.py @@ -0,0 +1,87 @@ +""" +Runs when LLM Exceptions occur on LiteLLM Proxy +""" + +import copy +import json +import uuid + +import litellm +from litellm.proxy._types import LiteLLM_ErrorLogs + + +async def _PROXY_failure_handler( + kwargs, # kwargs to completion + completion_response: litellm.ModelResponse, # response from completion + start_time=None, + end_time=None, # start/end time for completion +): + """ + Async Failure Handler - runs when LLM Exceptions occur on LiteLLM Proxy. + This function logs the errors to the Prisma DB + + Can be disabled by setting the following on proxy_config.yaml: + ```yaml + general_settings: + disable_error_logs: True + ``` + + """ + from litellm._logging import verbose_proxy_logger + from litellm.proxy.proxy_server import general_settings, prisma_client + + if general_settings.get("disable_error_logs") is True: + return + + if prisma_client is not None: + verbose_proxy_logger.debug( + "inside _PROXY_failure_handler kwargs=", extra=kwargs + ) + + _exception = kwargs.get("exception") + _exception_type = _exception.__class__.__name__ + _model = kwargs.get("model", None) + + _optional_params = kwargs.get("optional_params", {}) + _optional_params = copy.deepcopy(_optional_params) + + for k, v in _optional_params.items(): + v = str(v) + v = v[:100] + + _status_code = "500" + try: + _status_code = str(_exception.status_code) + except Exception: + # Don't let this fail logging the exception to the dB + pass + + _litellm_params = kwargs.get("litellm_params", {}) or {} + _metadata = _litellm_params.get("metadata", {}) or {} + _model_id = _metadata.get("model_info", {}).get("id", "") + _model_group = _metadata.get("model_group", "") + api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) + _exception_string = str(_exception) + + error_log = LiteLLM_ErrorLogs( + request_id=str(uuid.uuid4()), + model_group=_model_group, + model_id=_model_id, + litellm_model_name=kwargs.get("model"), + request_kwargs=_optional_params, + api_base=api_base, + exception_type=_exception_type, + status_code=_status_code, + exception_string=_exception_string, + startTime=kwargs.get("start_time"), + endTime=kwargs.get("end_time"), + ) + + error_log_dict = error_log.model_dump() + error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"]) + + await prisma_client.db.litellm_errorlogs.create( + data=error_log_dict # type: ignore + ) + + pass From ca2f0680dfe78576849b973983e11f005e84460b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 27 Nov 2024 11:36:07 -0800 Subject: [PATCH 6/7] fix rename file --- litellm/proxy/hooks/failure_handler.py | 87 -------------------------- 1 file changed, 87 deletions(-) delete mode 100644 litellm/proxy/hooks/failure_handler.py diff --git a/litellm/proxy/hooks/failure_handler.py b/litellm/proxy/hooks/failure_handler.py deleted file mode 100644 index d316eab13..000000000 --- a/litellm/proxy/hooks/failure_handler.py +++ /dev/null @@ -1,87 +0,0 @@ -""" -Runs when LLM Exceptions occur on LiteLLM Proxy -""" - -import copy -import json -import uuid - -import litellm -from litellm.proxy._types import LiteLLM_ErrorLogs - - -async def _PROXY_failure_handler( - kwargs, # kwargs to completion - completion_response: litellm.ModelResponse, # response from completion - start_time=None, - end_time=None, # start/end time for completion -): - """ - Async Failure Handler - runs when LLM Exceptions occur on LiteLLM Proxy. - This function logs the errors to the Prisma DB - - Can be disabled by setting the following on proxy_config.yaml: - ```yaml - general_settings: - disable_error_logs: True - ``` - - """ - from litellm._logging import verbose_proxy_logger - from litellm.proxy.proxy_server import general_settings, prisma_client - - if general_settings.get("disable_error_logs") is True: - return - - if prisma_client is not None: - verbose_proxy_logger.debug( - "inside _PROXY_failure_handler kwargs=", extra=kwargs - ) - - _exception = kwargs.get("exception") - _exception_type = _exception.__class__.__name__ - _model = kwargs.get("model", None) - - _optional_params = kwargs.get("optional_params", {}) - _optional_params = copy.deepcopy(_optional_params) - - for k, v in _optional_params.items(): - v = str(v) - v = v[:100] - - _status_code = "500" - try: - _status_code = str(_exception.status_code) - except Exception: - # Don't let this fail logging the exception to the dB - pass - - _litellm_params = kwargs.get("litellm_params", {}) or {} - _metadata = _litellm_params.get("metadata", {}) or {} - _model_id = _metadata.get("model_info", {}).get("id", "") - _model_group = _metadata.get("model_group", "") - api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) - _exception_string = str(_exception) - - error_log = LiteLLM_ErrorLogs( - request_id=str(uuid.uuid4()), - model_group=_model_group, - model_id=_model_id, - litellm_model_name=kwargs.get("model"), - request_kwargs=_optional_params, - api_base=api_base, - exception_type=_exception_type, - status_code=_status_code, - exception_string=_exception_string, - startTime=kwargs.get("start_time"), - endTime=kwargs.get("end_time"), - ) - - error_log_dict = error_log.model_dump() - error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"]) - - await prisma_client.db.litellm_errorlogs.create( - data=error_log_dict # type: ignore - ) - - pass From ea54fba9f3d6aa986df14e92f2fcdf339ca2ceaa Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 27 Nov 2024 11:39:20 -0800 Subject: [PATCH 7/7] increase test coverage for test_enable_error_logs --- litellm/proxy/proxy_server.py | 2 +- .../test_unit_test_proxy_hooks.py | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 011ed04de..3281fe4b4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -170,10 +170,10 @@ from litellm.proxy.guardrails.init_guardrails import ( ) from litellm.proxy.health_check import perform_health_check from litellm.proxy.health_endpoints._health_endpoints import router as health_router -from litellm.proxy.hooks.failure_handler import _PROXY_failure_handler from litellm.proxy.hooks.prompt_injection_detection import ( _OPTIONAL_PromptInjectionDetection, ) +from litellm.proxy.hooks.proxy_failure_handler import _PROXY_failure_handler from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.management_endpoints.customer_endpoints import ( router as customer_router, diff --git a/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py index 5573ad096..095b15368 100644 --- a/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py +++ b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py @@ -72,3 +72,40 @@ async def test_disable_spend_logs(): ) # Verify no spend logs were added assert len(mock_prisma_client.spend_log_transactions) == 0 + + +@pytest.mark.asyncio +async def test_enable_error_logs(): + """ + Test that the error logs are written to the database when disable_error_logs is False + """ + # Mock the necessary components + mock_prisma_client = AsyncMock() + mock_general_settings = {"disable_error_logs": False} + + with patch( + "litellm.proxy.proxy_server.general_settings", mock_general_settings + ), patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + + # Create a test exception + test_exception = Exception("Test error") + test_kwargs = { + "model": "gpt-4", + "exception": test_exception, + "optional_params": {}, + "litellm_params": {"metadata": {}}, + } + + # Call the failure handler + from litellm.proxy.proxy_server import _PROXY_failure_handler + + await _PROXY_failure_handler( + kwargs=test_kwargs, + completion_response=None, + start_time="2024-01-01", + end_time="2024-01-01", + ) + + # Verify prisma client was called to create error logs + if hasattr(mock_prisma_client, "db"): + assert mock_prisma_client.db.litellm_errorlogs.create.called