@@ -390,7 +395,7 @@ def _gemini_convert_messages_with_history(messages: list) -> List[ContentType]:
assistant_content.extend(_parts)
elif messages[msg_i].get(
"tool_calls", []
- ): # support assistant tool invoke convertion
+ ): # support assistant tool invoke conversion
assistant_content.extend(
convert_to_gemini_tool_call_invoke(messages[msg_i]["tool_calls"])
)
@@ -642,9 +647,9 @@ def completion(
prompt = " ".join(
[
- message["content"]
+ message.get("content")
for message in messages
- if isinstance(message["content"], str)
+ if isinstance(message.get("content", None), str)
]
)
diff --git a/litellm/main.py b/litellm/main.py
index 15334d0414..5012ef1662 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -365,7 +365,10 @@ async def acompletion(
) # sets the logging event loop if the user does sync streaming (e.g. on proxy for sagemaker calls)
return response
except Exception as e:
- traceback.print_exc()
+ verbose_logger.error(
+ "litellm.acompletion(): Exception occured - {}".format(str(e))
+ )
+ verbose_logger.debug(traceback.format_exc())
custom_llm_provider = custom_llm_provider or "openai"
raise exception_type(
model=model,
@@ -478,7 +481,10 @@ def mock_completion(
except Exception as e:
if isinstance(e, openai.APIError):
raise e
- traceback.print_exc()
+ verbose_logger.error(
+ "litellm.mock_completion(): Exception occured - {}".format(str(e))
+ )
+ verbose_logger.debug(traceback.format_exc())
raise Exception("Mock completion response failed")
@@ -4449,7 +4455,10 @@ async def ahealth_check(
response = {} # args like remaining ratelimit etc.
return response
except Exception as e:
- traceback.print_exc()
+ verbose_logger.error(
+ "litellm.ahealth_check(): Exception occured - {}".format(str(e))
+ )
+ verbose_logger.debug(traceback.format_exc())
stack_trace = traceback.format_exc()
if isinstance(stack_trace, str):
stack_trace = stack_trace[:1000]
diff --git a/litellm/proxy/_logging.py b/litellm/proxy/_logging.py
index 22cbd88cb7..f453cef395 100644
--- a/litellm/proxy/_logging.py
+++ b/litellm/proxy/_logging.py
@@ -1,6 +1,7 @@
import json
import logging
from logging import Formatter
+import sys
class JsonFormatter(Formatter):
diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml
index 72479bd5d5..7fa1bbc19c 100644
--- a/litellm/proxy/_super_secret_config.yaml
+++ b/litellm/proxy/_super_secret_config.yaml
@@ -56,8 +56,10 @@ router_settings:
litellm_settings:
success_callback: ["langfuse"]
- json_logs: true
general_settings:
alerting: ["email"]
+ key_management_system: "aws_kms"
+ key_management_settings:
+ hosted_keys: ["LITELLM_MASTER_KEY"]
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index f54bee399b..c19fd7e69e 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -76,6 +76,17 @@ class LitellmUserRoles(str, enum.Enum):
return ui_labels.get(self.value, "")
+class LitellmTableNames(str, enum.Enum):
+ """
+ Enum for Table Names used by LiteLLM
+ """
+
+ TEAM_TABLE_NAME: str = "LiteLLM_TeamTable"
+ USER_TABLE_NAME: str = "LiteLLM_UserTable"
+ KEY_TABLE_NAME: str = "LiteLLM_VerificationToken"
+ PROXY_MODEL_TABLE_NAME: str = "LiteLLM_ModelTable"
+
+
AlertType = Literal[
"llm_exceptions",
"llm_too_slow",
@@ -935,6 +946,7 @@ class KeyManagementSystem(enum.Enum):
AZURE_KEY_VAULT = "azure_key_vault"
AWS_SECRET_MANAGER = "aws_secret_manager"
LOCAL = "local"
+ AWS_KMS = "aws_kms"
class KeyManagementSettings(LiteLLMBase):
@@ -1276,6 +1288,22 @@ class LiteLLM_ErrorLogs(LiteLLMBase):
endTime: Union[str, datetime, None]
+class LiteLLM_AuditLogs(LiteLLMBase):
+ id: str
+ updated_at: datetime
+ changed_by: str
+ action: Literal["created", "updated", "deleted"]
+ table_name: Literal[
+ LitellmTableNames.TEAM_TABLE_NAME,
+ LitellmTableNames.USER_TABLE_NAME,
+ LitellmTableNames.KEY_TABLE_NAME,
+ LitellmTableNames.PROXY_MODEL_TABLE_NAME,
+ ]
+ object_id: str
+ before_value: Optional[Json] = None
+ updated_values: Optional[Json] = None
+
+
class LiteLLM_SpendLogs_ResponseObject(LiteLLMBase):
response: Optional[List[Union[LiteLLM_SpendLogs, Any]]] = None
diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py
index 5b5139f8c7..47ba36a683 100644
--- a/litellm/proxy/hooks/azure_content_safety.py
+++ b/litellm/proxy/hooks/azure_content_safety.py
@@ -88,7 +88,7 @@ class _PROXY_AzureContentSafety(
verbose_proxy_logger.debug(
"Error in Azure Content-Safety: %s", traceback.format_exc()
)
- traceback.print_exc()
+ verbose_proxy_logger.debug(traceback.format_exc())
raise
result = self._compute_result(response)
@@ -123,7 +123,12 @@ class _PROXY_AzureContentSafety(
except HTTPException as e:
raise e
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.hooks.azure_content_safety.py::async_pre_call_hook(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
async def async_post_call_success_hook(
self,
diff --git a/litellm/proxy/hooks/batch_redis_get.py b/litellm/proxy/hooks/batch_redis_get.py
index 64541c1bff..d506109b81 100644
--- a/litellm/proxy/hooks/batch_redis_get.py
+++ b/litellm/proxy/hooks/batch_redis_get.py
@@ -94,7 +94,12 @@ class _PROXY_BatchRedisRequests(CustomLogger):
except HTTPException as e:
raise e
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.hooks.batch_redis_get.py::async_pre_call_hook(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
async def async_get_cache(self, *args, **kwargs):
"""
diff --git a/litellm/proxy/hooks/cache_control_check.py b/litellm/proxy/hooks/cache_control_check.py
index 3160fe97ad..89971a0bf7 100644
--- a/litellm/proxy/hooks/cache_control_check.py
+++ b/litellm/proxy/hooks/cache_control_check.py
@@ -1,13 +1,13 @@
# What this does?
## Checks if key is allowed to use the cache controls passed in to the completion() call
-from typing import Optional
import litellm
+from litellm import verbose_logger
from litellm.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.integrations.custom_logger import CustomLogger
from fastapi import HTTPException
-import json, traceback
+import traceback
class _PROXY_CacheControlCheck(CustomLogger):
@@ -54,4 +54,9 @@ class _PROXY_CacheControlCheck(CustomLogger):
except HTTPException as e:
raise e
except Exception as e:
- traceback.print_exc()
+ verbose_logger.error(
+ "litellm.proxy.hooks.cache_control_check.py::async_pre_call_hook(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_logger.debug(traceback.format_exc())
diff --git a/litellm/proxy/hooks/max_budget_limiter.py b/litellm/proxy/hooks/max_budget_limiter.py
index 442cc53e37..c4b328bab0 100644
--- a/litellm/proxy/hooks/max_budget_limiter.py
+++ b/litellm/proxy/hooks/max_budget_limiter.py
@@ -1,10 +1,10 @@
-from typing import Optional
+from litellm import verbose_logger
import litellm
from litellm.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.integrations.custom_logger import CustomLogger
from fastapi import HTTPException
-import json, traceback
+import traceback
class _PROXY_MaxBudgetLimiter(CustomLogger):
@@ -44,4 +44,9 @@ class _PROXY_MaxBudgetLimiter(CustomLogger):
except HTTPException as e:
raise e
except Exception as e:
- traceback.print_exc()
+ verbose_logger.error(
+ "litellm.proxy.hooks.max_budget_limiter.py::async_pre_call_hook(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_logger.debug(traceback.format_exc())
diff --git a/litellm/proxy/hooks/presidio_pii_masking.py b/litellm/proxy/hooks/presidio_pii_masking.py
index 95a6e9c3c8..e64e69c457 100644
--- a/litellm/proxy/hooks/presidio_pii_masking.py
+++ b/litellm/proxy/hooks/presidio_pii_masking.py
@@ -8,8 +8,8 @@
# Tell us how we can improve! - Krrish & Ishaan
-from typing import Optional, Literal, Union
-import litellm, traceback, sys, uuid, json
+from typing import Optional, Union
+import litellm, traceback, uuid, json # noqa: E401
from litellm.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.integrations.custom_logger import CustomLogger
@@ -21,8 +21,8 @@ from litellm.utils import (
ImageResponse,
StreamingChoices,
)
-from datetime import datetime
-import aiohttp, asyncio
+import aiohttp
+import asyncio
class _OPTIONAL_PresidioPIIMasking(CustomLogger):
@@ -138,7 +138,12 @@ class _OPTIONAL_PresidioPIIMasking(CustomLogger):
else:
raise Exception(f"Invalid anonymizer response: {redacted_text}")
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.hooks.presidio_pii_masking.py::async_pre_call_hook(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
raise e
async def async_pre_call_hook(
diff --git a/litellm/proxy/hooks/prompt_injection_detection.py b/litellm/proxy/hooks/prompt_injection_detection.py
index 08dbedd8c8..ed33e3b519 100644
--- a/litellm/proxy/hooks/prompt_injection_detection.py
+++ b/litellm/proxy/hooks/prompt_injection_detection.py
@@ -204,7 +204,12 @@ class _OPTIONAL_PromptInjectionDetection(CustomLogger):
return e.detail["error"]
raise e
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
async def async_moderation_hook(
self,
diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml
index e3d4effe80..88fc0e9136 100644
--- a/litellm/proxy/proxy_config.yaml
+++ b/litellm/proxy/proxy_config.yaml
@@ -23,4 +23,5 @@ general_settings:
master_key: sk-1234
litellm_settings:
- callbacks: ["otel"]
\ No newline at end of file
+ callbacks: ["otel"]
+ store_audit_logs: true
\ No newline at end of file
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 25035a016f..9d1688815c 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -103,6 +103,7 @@ from litellm.proxy.utils import (
update_spend,
encrypt_value,
decrypt_value,
+ get_error_message_str,
)
from litellm import (
CreateBatchRequest,
@@ -112,7 +113,10 @@ from litellm import (
CreateFileRequest,
)
from litellm.proxy.secret_managers.google_kms import load_google_kms
-from litellm.proxy.secret_managers.aws_secret_manager import load_aws_secret_manager
+from litellm.proxy.secret_managers.aws_secret_manager import (
+ load_aws_secret_manager,
+ load_aws_kms,
+)
import pydantic
from litellm.proxy._types import *
from litellm.caching import DualCache, RedisCache
@@ -125,7 +129,10 @@ from litellm.router import (
AssistantsTypedDict,
)
from litellm.router import ModelInfo as RouterModelInfo
-from litellm._logging import verbose_router_logger, verbose_proxy_logger
+from litellm._logging import (
+ verbose_router_logger,
+ verbose_proxy_logger,
+)
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.litellm_license import LicenseCheck
from litellm.proxy.auth.model_checks import (
@@ -1471,7 +1478,12 @@ async def user_api_key_auth(
else:
raise Exception()
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, litellm.BudgetExceededError):
raise ProxyException(
message=e.message, type="auth_error", param=None, code=400
@@ -2736,10 +2748,12 @@ class ProxyConfig:
load_google_kms(use_google_kms=True)
elif (
key_management_system
- == KeyManagementSystem.AWS_SECRET_MANAGER.value
+ == KeyManagementSystem.AWS_SECRET_MANAGER.value # noqa: F405
):
### LOAD FROM AWS SECRET MANAGER ###
load_aws_secret_manager(use_aws_secret_manager=True)
+ elif key_management_system == KeyManagementSystem.AWS_KMS.value:
+ load_aws_kms(use_aws_kms=True)
else:
raise ValueError("Invalid Key Management System selected")
key_management_settings = general_settings.get(
@@ -2773,6 +2787,7 @@ class ProxyConfig:
master_key = general_settings.get(
"master_key", litellm.get_secret("LITELLM_MASTER_KEY", None)
)
+
if master_key and master_key.startswith("os.environ/"):
master_key = litellm.get_secret(master_key)
if not isinstance(master_key, str):
@@ -3476,7 +3491,12 @@ async def generate_key_helper_fn(
)
key_data["token_id"] = getattr(create_key_response, "token", None)
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.generate_key_helper_fn(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise e
raise HTTPException(
@@ -3515,7 +3535,12 @@ async def delete_verification_token(tokens: List, user_id: Optional[str] = None)
else:
raise Exception("DB not connected. prisma_client is None")
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.delete_verification_token(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
raise e
return deleted_tokens
@@ -3676,7 +3701,12 @@ async def async_assistants_data_generator(
done_message = "[DONE]"
yield f"data: {done_message}\n\n"
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
@@ -3686,9 +3716,6 @@ async def async_assistants_data_generator(
f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`"
)
router_model_names = llm_router.model_names if llm_router is not None else []
- if user_debug:
- traceback.print_exc()
-
if isinstance(e, HTTPException):
raise e
else:
@@ -3728,7 +3755,12 @@ async def async_data_generator(
done_message = "[DONE]"
yield f"data: {done_message}\n\n"
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.async_data_generator(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
@@ -3738,8 +3770,6 @@ async def async_data_generator(
f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`"
)
router_model_names = llm_router.model_names if llm_router is not None else []
- if user_debug:
- traceback.print_exc()
if isinstance(e, HTTPException):
raise e
@@ -3800,6 +3830,18 @@ def on_backoff(details):
verbose_proxy_logger.debug("Backing off... this was attempt # %s", details["tries"])
+def giveup(e):
+ result = not (
+ isinstance(e, ProxyException)
+ and getattr(e, "message", None) is not None
+ and isinstance(e.message, str)
+ and "Max parallel request limit reached" in e.message
+ )
+ if result:
+ verbose_proxy_logger.info(json.dumps({"event": "giveup", "exception": str(e)}))
+ return result
+
+
@router.on_event("startup")
async def startup_event():
global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client, store_model_in_db
@@ -4084,12 +4126,8 @@ def model_list(
max_tries=litellm.num_retries or 3, # maximum number of retries
max_time=litellm.request_timeout or 60, # maximum total time to retry for
on_backoff=on_backoff, # specifying the function to call on backoff
- giveup=lambda e: not (
- isinstance(e, ProxyException)
- and getattr(e, "message", None) is not None
- and isinstance(e.message, str)
- and "Max parallel request limit reached" in e.message
- ), # the result of the logical expression is on the second position
+ giveup=giveup,
+ logger=verbose_proxy_logger,
)
async def chat_completion(
request: Request,
@@ -4098,6 +4136,7 @@ async def chat_completion(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
global general_settings, user_debug, proxy_logging_obj, llm_model_list
+
data = {}
try:
body = await request.body()
@@ -4386,7 +4425,12 @@ async def chat_completion(
return _chat_response
except Exception as e:
data["litellm_status"] = "fail" # used for alerting
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.chat_completion(): Exception occured - {}".format(
+ get_error_message_str(e=e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
@@ -4397,8 +4441,6 @@ async def chat_completion(
litellm_debug_info,
)
router_model_names = llm_router.model_names if llm_router is not None else []
- if user_debug:
- traceback.print_exc()
if isinstance(e, HTTPException):
raise ProxyException(
@@ -4630,15 +4672,12 @@ async def completion(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- verbose_proxy_logger.debug("EXCEPTION RAISED IN PROXY MAIN.PY")
- litellm_debug_info = getattr(e, "litellm_debug_info", "")
- verbose_proxy_logger.debug(
- "\033[1;31mAn error occurred: %s %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`",
- e,
- litellm_debug_info,
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.completion(): Exception occured - {}".format(
+ str(e)
+ )
)
- traceback.print_exc()
- error_traceback = traceback.format_exc()
+ verbose_proxy_logger.debug(traceback.format_exc())
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -4848,7 +4887,12 @@ async def embeddings(
e,
litellm_debug_info,
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.embeddings(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e)),
@@ -5027,7 +5071,12 @@ async def image_generation(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.image_generation(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e)),
@@ -5205,7 +5254,12 @@ async def audio_speech(
)
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.audio_speech(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
raise e
@@ -5394,7 +5448,12 @@ async def audio_transcriptions(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.audio_transcription(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -5403,7 +5462,6 @@ async def audio_transcriptions(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -5531,7 +5589,12 @@ async def get_assistants(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.get_assistants(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -5540,7 +5603,6 @@ async def get_assistants(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -5660,7 +5722,12 @@ async def create_threads(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.create_threads(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -5669,7 +5736,6 @@ async def create_threads(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -5788,7 +5854,12 @@ async def get_thread(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.get_thread(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -5797,7 +5868,6 @@ async def get_thread(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -5919,7 +5989,12 @@ async def add_messages(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.add_messages(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -5928,7 +6003,6 @@ async def add_messages(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -6046,7 +6120,12 @@ async def get_messages(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.get_messages(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -6055,7 +6134,6 @@ async def get_messages(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -6187,7 +6265,12 @@ async def run_thread(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.run_thread(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -6196,7 +6279,6 @@ async def run_thread(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -6335,7 +6417,12 @@ async def create_batch(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.create_batch(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -6344,7 +6431,6 @@ async def create_batch(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -6478,7 +6564,12 @@ async def retrieve_batch(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.retrieve_batch(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -6631,7 +6722,12 @@ async def create_file(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.create_file(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
@@ -6640,7 +6736,6 @@ async def create_file(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -6816,7 +6911,12 @@ async def moderations(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.moderations(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e)),
@@ -6825,7 +6925,6 @@ async def moderations(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
- error_traceback = traceback.format_exc()
error_msg = f"{str(e)}"
raise ProxyException(
message=getattr(e, "message", error_msg),
@@ -7115,9 +7214,33 @@ async def generate_key_fn(
)
)
+ # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
+ if litellm.store_audit_logs is True:
+ _updated_values = json.dumps(response)
+ asyncio.create_task(
+ create_audit_log_for_update(
+ request_data=LiteLLM_AuditLogs(
+ id=str(uuid.uuid4()),
+ updated_at=datetime.now(timezone.utc),
+ changed_by=user_api_key_dict.user_id
+ or litellm_proxy_admin_name,
+ table_name=LitellmTableNames.KEY_TABLE_NAME,
+ object_id=response.get("token_id", ""),
+ action="created",
+ updated_values=_updated_values,
+ before_value=None,
+ )
+ )
+ )
+
return GenerateKeyResponse(**response)
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.generate_key_fn(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -7138,7 +7261,11 @@ async def generate_key_fn(
@router.post(
"/key/update", tags=["key management"], dependencies=[Depends(user_api_key_auth)]
)
-async def update_key_fn(request: Request, data: UpdateKeyRequest):
+async def update_key_fn(
+ request: Request,
+ data: UpdateKeyRequest,
+ user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
"""
Update an existing key
"""
@@ -7150,6 +7277,16 @@ async def update_key_fn(request: Request, data: UpdateKeyRequest):
if prisma_client is None:
raise Exception("Not connected to DB!")
+ existing_key_row = await prisma_client.get_data(
+ token=data.key, table_name="key", query_type="find_unique"
+ )
+
+ if existing_key_row is None:
+ raise HTTPException(
+ status_code=404,
+ detail={"error": f"Team not found, passed team_id={data.team_id}"},
+ )
+
# get non default values for key
non_default_values = {}
for k, v in data_json.items():
@@ -7176,6 +7313,29 @@ async def update_key_fn(request: Request, data: UpdateKeyRequest):
hashed_token = hash_token(key)
user_api_key_cache.delete_cache(hashed_token)
+ # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
+ if litellm.store_audit_logs is True:
+ _updated_values = json.dumps(data_json)
+
+ _before_value = existing_key_row.json(exclude_none=True)
+ _before_value = json.dumps(_before_value)
+
+ asyncio.create_task(
+ create_audit_log_for_update(
+ request_data=LiteLLM_AuditLogs(
+ id=str(uuid.uuid4()),
+ updated_at=datetime.now(timezone.utc),
+ changed_by=user_api_key_dict.user_id
+ or litellm_proxy_admin_name,
+ table_name=LitellmTableNames.KEY_TABLE_NAME,
+ object_id=data.key,
+ action="updated",
+ updated_values=_updated_values,
+ before_value=_before_value,
+ )
+ )
+ )
+
return {"key": key, **response["data"]}
# update based on remaining passed in values
except Exception as e:
@@ -7238,6 +7398,34 @@ async def delete_key_fn(
):
user_id = None # unless they're admin
+ # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
+ # we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes
+ if litellm.store_audit_logs is True:
+ # make an audit log for each team deleted
+ for key in data.keys:
+ key_row = await prisma_client.get_data( # type: ignore
+ token=key, table_name="key", query_type="find_unique"
+ )
+
+ key_row = key_row.json(exclude_none=True)
+ _key_row = json.dumps(key_row)
+
+ asyncio.create_task(
+ create_audit_log_for_update(
+ request_data=LiteLLM_AuditLogs(
+ id=str(uuid.uuid4()),
+ updated_at=datetime.now(timezone.utc),
+ changed_by=user_api_key_dict.user_id
+ or litellm_proxy_admin_name,
+ table_name=LitellmTableNames.KEY_TABLE_NAME,
+ object_id=key,
+ action="deleted",
+ updated_values="{}",
+ before_value=_key_row,
+ )
+ )
+ )
+
number_deleted_keys = await delete_verification_token(
tokens=keys, user_id=user_id
)
@@ -9507,7 +9695,12 @@ async def user_info(
}
return response_data
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.user_info(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -9602,7 +9795,12 @@ async def user_update(data: UpdateUserRequest):
return response
# update based on remaining passed in values
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.user_update(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -9655,7 +9853,12 @@ async def user_request_model(request: Request):
return {"status": "success"}
# update based on remaining passed in values
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.user_request_model(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -9697,7 +9900,12 @@ async def user_get_requests():
return {"requests": response}
# update based on remaining passed in values
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.user_get_requests(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -10087,7 +10295,12 @@ async def update_end_user(
# update based on remaining passed in values
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.update_end_user(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Internal Server Error({str(e)})"),
@@ -10171,7 +10384,12 @@ async def delete_end_user(
# update based on remaining passed in values
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.delete_end_user(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Internal Server Error({str(e)})"),
@@ -10365,12 +10583,65 @@ async def new_team(
}
},
)
+
+ # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
+ if litellm.store_audit_logs is True:
+ _updated_values = complete_team_data.json(exclude_none=True)
+ _updated_values = json.dumps(_updated_values)
+
+ asyncio.create_task(
+ create_audit_log_for_update(
+ request_data=LiteLLM_AuditLogs(
+ id=str(uuid.uuid4()),
+ updated_at=datetime.now(timezone.utc),
+ changed_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
+ table_name=LitellmTableNames.TEAM_TABLE_NAME,
+ object_id=data.team_id,
+ action="created",
+ updated_values=_updated_values,
+ before_value=None,
+ )
+ )
+ )
+
try:
return team_row.model_dump()
except Exception as e:
return team_row.dict()
+async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs):
+ if premium_user is not True:
+ return
+
+ if litellm.store_audit_logs is not True:
+ return
+ if prisma_client is None:
+ raise Exception("prisma_client is None, no DB connected")
+
+ verbose_proxy_logger.debug("creating audit log for %s", request_data)
+
+ if isinstance(request_data.updated_values, dict):
+ request_data.updated_values = json.dumps(request_data.updated_values)
+
+ if isinstance(request_data.before_value, dict):
+ request_data.before_value = json.dumps(request_data.before_value)
+
+ _request_data = request_data.dict(exclude_none=True)
+
+ try:
+ await prisma_client.db.litellm_auditlog.create(
+ data={
+ **_request_data, # type: ignore
+ }
+ )
+ except Exception as e:
+ # [Non-Blocking Exception. Do not allow blocking LLM API call]
+ verbose_proxy_logger.error(f"Failed Creating audit log {e}")
+
+ return
+
+
@router.post(
"/team/update", tags=["team management"], dependencies=[Depends(user_api_key_auth)]
)
@@ -10443,6 +10714,27 @@ async def update_team(
team_id=data.team_id,
)
+ # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
+ if litellm.store_audit_logs is True:
+ _before_value = existing_team_row.json(exclude_none=True)
+ _before_value = json.dumps(_before_value)
+ _after_value: str = json.dumps(updated_kv)
+
+ asyncio.create_task(
+ create_audit_log_for_update(
+ request_data=LiteLLM_AuditLogs(
+ id=str(uuid.uuid4()),
+ updated_at=datetime.now(timezone.utc),
+ changed_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
+ table_name=LitellmTableNames.TEAM_TABLE_NAME,
+ object_id=data.team_id,
+ action="updated",
+ updated_values=_after_value,
+ before_value=_before_value,
+ )
+ )
+ )
+
return team_row
@@ -10714,6 +11006,35 @@ async def delete_team(
detail={"error": f"Team not found, passed team_id={team_id}"},
)
+ # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
+ # we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes
+ if litellm.store_audit_logs is True:
+ # make an audit log for each team deleted
+ for team_id in data.team_ids:
+ team_row = await prisma_client.get_data( # type: ignore
+ team_id=team_id, table_name="team", query_type="find_unique"
+ )
+
+ _team_row = team_row.json(exclude_none=True)
+
+ asyncio.create_task(
+ create_audit_log_for_update(
+ request_data=LiteLLM_AuditLogs(
+ id=str(uuid.uuid4()),
+ updated_at=datetime.now(timezone.utc),
+ changed_by=user_api_key_dict.user_id
+ or litellm_proxy_admin_name,
+ table_name=LitellmTableNames.TEAM_TABLE_NAME,
+ object_id=team_id,
+ action="deleted",
+ updated_values="{}",
+ before_value=_team_row,
+ )
+ )
+ )
+
+ # End of Audit logging
+
## DELETE ASSOCIATED KEYS
await prisma_client.delete_data(team_id_list=data.team_ids, table_name="key")
## DELETE TEAMS
@@ -11371,7 +11692,12 @@ async def add_new_model(
return model_response
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.add_new_model(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -11485,7 +11811,12 @@ async def update_model(
return model_response
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.update_model(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -13719,7 +14050,12 @@ async def update_config(config_info: ConfigYAML):
return {"message": "Config updated successfully"}
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.update_config(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -14192,7 +14528,12 @@ async def get_config():
"available_callbacks": all_available_callbacks,
}
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.get_config(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -14443,7 +14784,12 @@ async def health_services_endpoint(
}
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
@@ -14522,7 +14868,12 @@ async def health_endpoint(
"unhealthy_count": len(unhealthy_endpoints),
}
except Exception as e:
- traceback.print_exc()
+ verbose_proxy_logger.error(
+ "litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - {}".format(
+ str(e)
+ )
+ )
+ verbose_proxy_logger.debug(traceback.format_exc())
raise e
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index 243f063376..7cc688ee8e 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -243,4 +243,16 @@ model LiteLLM_InvitationLink {
liteLLM_user_table_user LiteLLM_UserTable @relation("UserId", fields: [user_id], references: [user_id])
liteLLM_user_table_created LiteLLM_UserTable @relation("CreatedBy", fields: [created_by], references: [user_id])
liteLLM_user_table_updated LiteLLM_UserTable @relation("UpdatedBy", fields: [updated_by], references: [user_id])
+}
+
+
+model LiteLLM_AuditLog {
+ id String @id @default(uuid())
+ updated_at DateTime @default(now())
+ changed_by String // user or system that performed the action
+ action String // create, update, delete
+ table_name String // on of LitellmTableNames.TEAM_TABLE_NAME, LitellmTableNames.USER_TABLE_NAME, LitellmTableNames.PROXY_MODEL_TABLE_NAME,
+ object_id String // id of the object being audited. This can be the key id, team id, user id, model id
+ before_value Json? // value of the row
+ updated_values Json? // value of the row after change
}
\ No newline at end of file
diff --git a/litellm/proxy/secret_managers/aws_secret_manager.py b/litellm/proxy/secret_managers/aws_secret_manager.py
index a40b1dffa2..8dd6772cf7 100644
--- a/litellm/proxy/secret_managers/aws_secret_manager.py
+++ b/litellm/proxy/secret_managers/aws_secret_manager.py
@@ -8,7 +8,8 @@ Requires:
* `pip install boto3>=1.28.57`
"""
-import litellm, os
+import litellm
+import os
from typing import Optional
from litellm.proxy._types import KeyManagementSystem
@@ -38,3 +39,21 @@ def load_aws_secret_manager(use_aws_secret_manager: Optional[bool]):
except Exception as e:
raise e
+
+
+def load_aws_kms(use_aws_kms: Optional[bool]):
+ if use_aws_kms is None or use_aws_kms is False:
+ return
+ try:
+ import boto3
+
+ validate_environment()
+
+ # Create a Secrets Manager client
+ kms_client = boto3.client("kms", region_name=os.getenv("AWS_REGION_NAME"))
+
+ litellm.secret_manager_client = kms_client
+ litellm._key_management_system = KeyManagementSystem.AWS_KMS
+
+ except Exception as e:
+ raise e
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index eed59664b1..b3d00e8e74 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -2709,13 +2709,15 @@ def decrypt_value(value: bytes, master_key: str) -> str:
# LiteLLM Admin UI - Non SSO Login
-html_form = """
+url_to_redirect_to = os.getenv("PROXY_BASE_URL", "")
+url_to_redirect_to += "/login"
+html_form = f"""
LiteLLM Login
-
-
-
"""
@@ -2837,3 +2837,17 @@ missing_keys_html_form = """