refactor location of proxy

README.md

@@ -402,7 +402,7 @@ If you have suggestions on how to improve the code quality feel free to open an
 1. (In root) create virtual environment `python -m venv .venv`
 2. Activate virtual environment `source .venv/bin/activate`
 3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `uvicorn litellm.proxy.proxy_server:app --host localhost --port 4000 --reload`
+4. Start proxy backend `uvicorn litellm_proxy.proxy_server:app --host localhost --port 4000 --reload`
 
 ### Frontend
 1. Navigate to `ui/litellm-dashboard`

docs/my-website/docs/proxy/call_hooks.md

@@ -17,7 +17,7 @@ This function is called just before a litellm completion call is made, and allow
 ```python
 from litellm.integrations.custom_logger import CustomLogger
 import litellm
-from litellm.proxy.proxy_server import UserAPIKeyAuth, DualCache
+from litellm_proxy.proxy_server import UserAPIKeyAuth, DualCache
 from typing import Optional, Literal
 
 # This file includes the custom callbacks for LiteLLM Proxy

docs/my-website/docs/proxy/custom_auth.md

@@ -9,7 +9,7 @@ Here's how:
 Make sure the response type follows the `UserAPIKeyAuth` pydantic object. This is used by for logging usage specific to that user key.
 
 ```python
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 
 async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth: 
     try: 

docs/my-website/docs/proxy/custom_sso.md

@@ -20,12 +20,12 @@ Make sure the response type follows the `SSOUserDefinedValues` pydantic object.
 from fastapi import Request
 from fastapi_sso.sso.base import OpenID
 
-from litellm.proxy._types import LitellmUserRoles, SSOUserDefinedValues
-from litellm.proxy.management_endpoints.internal_user_endpoints import (
+from litellm_proxy._types import LitellmUserRoles, SSOUserDefinedValues
+from litellm_proxy.management_endpoints.internal_user_endpoints import (
     new_user,
     user_info,
 )
-from litellm.proxy.management_endpoints.team_endpoints import add_new_member
+from litellm_proxy.management_endpoints.team_endpoints import add_new_member
 
 
 async def custom_sso_handler(userIDPInfo: OpenID) -> SSOUserDefinedValues:

docs/my-website/docs/proxy/guardrails/custom_guardrail.md

@@ -29,8 +29,8 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_guardrail import CustomGuardrail
-from litellm.proxy._types import UserAPIKeyAuth
-from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
+from litellm_proxy._types import UserAPIKeyAuth
+from litellm_proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
 from litellm.types.guardrails import GuardrailEventHooks
 
 
@@ -449,7 +449,7 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_guardrail import CustomGuardrail
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 
 class myCustomGuardrail(CustomGuardrail):
     def __init__(self, **kwargs):

enterprise/enterprise_hooks/aporia_ai.py

@@ -14,11 +14,11 @@ sys.path.insert(
 from typing import Optional, Literal, Any
 import litellm
 import sys
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from fastapi import HTTPException
 from litellm._logging import verbose_proxy_logger
-from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
+from litellm_proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
 from litellm.litellm_core_utils.logging_utils import (
     convert_litellm_response_object_to_str,
 )
@@ -140,7 +140,7 @@ class AporiaGuardrail(CustomGuardrail):
         user_api_key_dict: UserAPIKeyAuth,
         response,
     ):
-        from litellm.proxy.common_utils.callback_utils import (
+        from litellm_proxy.common_utils.callback_utils import (
             add_guardrail_to_applied_guardrails_header,
         )
 
@@ -176,7 +176,7 @@ class AporiaGuardrail(CustomGuardrail):
             "responses",
         ],
     ):
-        from litellm.proxy.common_utils.callback_utils import (
+        from litellm_proxy.common_utils.callback_utils import (
             add_guardrail_to_applied_guardrails_header,
         )
 

enterprise/enterprise_hooks/banned_keywords.py

@@ -10,7 +10,7 @@
 from typing import Literal
 import litellm
 from litellm.caching.caching import DualCache
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from litellm._logging import verbose_proxy_logger
 from fastapi import HTTPException

enterprise/enterprise_hooks/blocked_user_list.py

@@ -9,9 +9,9 @@
 
 from typing import Optional, Literal
 import litellm
-from litellm.proxy.utils import PrismaClient
+from litellm_proxy.utils import PrismaClient
 from litellm.caching.caching import DualCache
-from litellm.proxy._types import UserAPIKeyAuth, LiteLLM_EndUserTable
+from litellm_proxy._types import UserAPIKeyAuth, LiteLLM_EndUserTable
 from litellm.integrations.custom_logger import CustomLogger
 from litellm._logging import verbose_proxy_logger
 from fastapi import HTTPException

enterprise/enterprise_hooks/google_text_moderation.py

@@ -9,7 +9,7 @@
 
 from typing import Literal
 import litellm
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from fastapi import HTTPException
 from litellm._logging import verbose_proxy_logger

enterprise/enterprise_hooks/llama_guard.py

@@ -17,7 +17,7 @@ sys.path.insert(
 from typing import Optional, Literal
 import litellm
 import sys
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from fastapi import HTTPException
 from litellm._logging import verbose_proxy_logger

enterprise/enterprise_hooks/llm_guard.py

@@ -9,7 +9,7 @@
 
 from typing import Optional, Literal
 import litellm
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from fastapi import HTTPException
 from litellm._logging import verbose_proxy_logger

enterprise/enterprise_hooks/openai_moderation.py

@@ -14,7 +14,7 @@ sys.path.insert(
 from typing import Literal
 import litellm
 import sys
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from fastapi import HTTPException
 from litellm._logging import verbose_proxy_logger
@@ -50,7 +50,7 @@ class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
                 if "content" in m and isinstance(m["content"], str):
                     text += m["content"]
 
-        from litellm.proxy.proxy_server import llm_router
+        from litellm_proxy.proxy_server import llm_router
 
         if llm_router is None:
             return

enterprise/enterprise_hooks/secret_detection.py

@@ -13,7 +13,7 @@ sys.path.insert(
 )  # Adds the parent directory to the system path
 from typing import Optional
 from litellm.caching.caching import DualCache
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 from litellm._logging import verbose_proxy_logger
 import tempfile
 from litellm.integrations.custom_guardrail import CustomGuardrail

litellm/__init__.py

@@ -61,7 +61,7 @@ from litellm.constants import (
     DEFAULT_ALLOWED_FAILS,
 )
 from litellm.types.guardrails import GuardrailItem
-from litellm.proxy._types import (
+from litellm_proxy._types import (
     KeyManagementSystem,
     KeyManagementSettings,
     LiteLLM_UpperboundKeyGenerateParams,
@@ -1048,7 +1048,6 @@ from .exceptions import (
     MockException,
 )
 from .budget_manager import BudgetManager
-from .proxy.proxy_cli import run_server
 from .router import Router
 from .assistants.main import *
 from .batches.main import *

litellm/_service_logger.py

@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm_proxy._types import UserAPIKeyAuth
 
 from .integrations.custom_logger import CustomLogger
 from .integrations.datadog.datadog import DataDogLogger
@@ -143,7 +143,7 @@ class ServiceLogging(CustomLogger):
                     event_metadata=event_metadata,
                 )
             elif callback == "otel" or isinstance(callback, OpenTelemetry):
-                from litellm.proxy.proxy_server import open_telemetry_logger
+                from litellm_proxy.proxy_server import open_telemetry_logger
 
                 await self.init_otel_logger_if_none()
 
@@ -188,7 +188,7 @@ class ServiceLogging(CustomLogger):
         initializes otel_logger if it is None or no attribute exists on ServiceLogging Object
 
         """
-        from litellm.proxy.proxy_server import open_telemetry_logger
+        from litellm_proxy.proxy_server import open_telemetry_logger
 
         if not hasattr(self, "otel_logger"):
             if open_telemetry_logger is not None and isinstance(
@@ -251,7 +251,7 @@ class ServiceLogging(CustomLogger):
                     event_metadata=event_metadata,
                 )
             elif callback == "otel" or isinstance(callback, OpenTelemetry):
-                from litellm.proxy.proxy_server import open_telemetry_logger
+                from litellm_proxy.proxy_server import open_telemetry_logger
 
                 await self.init_otel_logger_if_none()
 

litellm/caching/qdrant_semantic_cache.py

@@ -281,7 +281,7 @@ class QdrantSemanticCache(BaseCache):
     async def async_set_cache(self, key, value, **kwargs):
         import uuid
 
-        from litellm.proxy.proxy_server import llm_model_list, llm_router
+        from litellm_proxy.proxy_server import llm_model_list, llm_router
 
         print_verbose(f"async qdrant semantic-cache set_cache, kwargs: {kwargs}")
 
@@ -344,7 +344,7 @@ class QdrantSemanticCache(BaseCache):
 
     async def async_get_cache(self, key, **kwargs):
         print_verbose(f"async qdrant semantic-cache get_cache, kwargs: {kwargs}")
-        from litellm.proxy.proxy_server import llm_model_list, llm_router
+        from litellm_proxy.proxy_server import llm_model_list, llm_router
 
         # get the messages
         messages = kwargs["messages"]

litellm/caching/redis_semantic_cache.py

@@ -279,7 +279,7 @@ class RedisSemanticCache(BaseCache):
         Returns:
             List[float]: The embedding vector
         """
-        from litellm.proxy.proxy_server import llm_model_list, llm_router
+        from litellm_proxy.proxy_server import llm_model_list, llm_router
 
         # Route the embedding request through the proxy if appropriate
         router_model_names = (

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -26,8 +26,8 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
-from litellm.proxy._types import AlertType, CallInfo, VirtualKeyEvent, WebhookEvent
 from litellm.types.integrations.slack_alerting import *
+from litellm_proxy._types import AlertType, CallInfo, VirtualKeyEvent, WebhookEvent
 
 from ..email_templates.templates import *
 from .batching_handler import send_to_webhook, squash_payloads
@@ -823,9 +823,9 @@ class SlackAlerting(CustomBatchLogger):
         ### UNIQUE CACHE KEY ###
         cache_key = provider + region_name
 
-        outage_value: Optional[
-            ProviderRegionOutageModel
-        ] = await self.internal_usage_cache.async_get_cache(key=cache_key)
+        outage_value: Optional[ProviderRegionOutageModel] = (
+            await self.internal_usage_cache.async_get_cache(key=cache_key)
+        )
 
         if (
             getattr(exception, "status_code", None) is None
@@ -1148,7 +1148,7 @@ Model Info:
         email_logo_url: Optional[str] = None,
         email_support_contact: Optional[str] = None,
     ):
-        from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
+        from litellm_proxy.proxy_server import CommonProxyErrors, premium_user
 
         if premium_user is not True:
             if email_logo_url is not None or email_support_contact is not None:
@@ -1161,7 +1161,7 @@ Model Info:
         self, webhook_event: WebhookEvent
     ) -> bool:
         try:
-            from litellm.proxy.utils import send_email
+            from litellm_proxy.utils import send_email
 
             if self.alerting is None or "email" not in self.alerting:
                 # do nothing if user does not want email alerts
@@ -1170,7 +1170,7 @@ Model Info:
                     self.alerting,
                 )
                 return False
-            from litellm.proxy.proxy_server import premium_user, prisma_client
+            from litellm_proxy.proxy_server import premium_user, prisma_client
 
             email_logo_url = os.getenv(
                 "SMTP_SENDER_LOGO", os.getenv("EMAIL_LOGO_URL", None)
@@ -1271,8 +1271,8 @@ Model Info:
 
         Returns -> True if sent, False if not.
         """
-        from litellm.proxy.proxy_server import premium_user
-        from litellm.proxy.utils import send_email
+        from litellm_proxy.proxy_server import premium_user
+        from litellm_proxy.utils import send_email
 
         email_logo_url = os.getenv(
             "SMTP_SENDER_LOGO", os.getenv("EMAIL_LOGO_URL", None)
@@ -1406,9 +1406,9 @@ Model Info:
             self.alert_to_webhook_url is not None
             and alert_type in self.alert_to_webhook_url
         ):
-            slack_webhook_url: Optional[
-                Union[str, List[str]]
-            ] = self.alert_to_webhook_url[alert_type]
+            slack_webhook_url: Optional[Union[str, List[str]]] = (
+                self.alert_to_webhook_url[alert_type]
+            )
         elif self.default_webhook_url is not None:
             slack_webhook_url = self.default_webhook_url
         else:
@@ -1598,7 +1598,7 @@ Model Info:
             return
 
         try:
-            from litellm.proxy.spend_tracking.spend_management_endpoints import (
+            from litellm_proxy.spend_tracking.spend_management_endpoints import (
                 _get_spend_report_for_time_range,
             )
 
@@ -1662,7 +1662,7 @@ Model Info:
         try:
             from calendar import monthrange
 
-            from litellm.proxy.spend_tracking.spend_management_endpoints import (
+            from litellm_proxy.spend_tracking.spend_management_endpoints import (
                 _get_spend_report_for_time_range,
             )
 

litellm/integrations/SlackAlerting/utils.py

@@ -5,8 +5,8 @@ Utils used for slack alerting
 import asyncio
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
-from litellm.proxy._types import AlertType
 from litellm.secret_managers.main import get_secret
+from litellm_proxy._types import AlertType
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _Logging
@@ -17,7 +17,7 @@ else:
 
 
 def process_slack_alerting_variables(
-    alert_to_webhook_url: Optional[Dict[AlertType, Union[List[str], str]]]
+    alert_to_webhook_url: Optional[Dict[AlertType, Union[List[str], str]]],
 ) -> Optional[Dict[AlertType, Union[List[str], str]]]:
     """
     process alert_to_webhook_url

litellm/integrations/azure_storage/azure_storage.py

@@ -321,7 +321,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
         """
         Checks if the user is a premium user, raises an error if not
         """
-        from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
+        from litellm_proxy.proxy_server import CommonProxyErrors, premium_user
 
         if premium_user is not True:
             raise ValueError(

litellm/integrations/custom_guardrail.py

@@ -165,7 +165,7 @@ class CustomGuardrail(CustomLogger):
         """
         Returns True if the user is a premium user
         """
-        from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
+        from litellm_proxy.proxy_server import CommonProxyErrors, premium_user
 
         if premium_user is not True:
             verbose_logger.warning(
@@ -183,7 +183,7 @@ class CustomGuardrail(CustomLogger):
         """
         Builds `StandardLoggingGuardrailInformation` and adds it to the request metadata so it can be used for logging to DataDog, Langfuse, etc.
         """
-        from litellm.proxy.proxy_server import premium_user
+        from litellm_proxy.proxy_server import premium_user
 
         if premium_user is not True:
             verbose_logger.warning(

litellm/integrations/custom_logger.py

@@ -15,7 +15,6 @@ from typing import (
 from pydantic import BaseModel
 
 from litellm.caching.caching import DualCache
-from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.integrations.argilla import ArgillaItem
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest
 from litellm.types.utils import (
@@ -26,6 +25,7 @@ from litellm.types.utils import (
     StandardCallbackDynamicParams,
     StandardLoggingPayload,
 )
+from litellm_proxy._types import UserAPIKeyAuth
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span

litellm/integrations/email_alerting.py

@@ -6,7 +6,7 @@ import os
 from typing import List, Optional
 
 from litellm._logging import verbose_logger, verbose_proxy_logger
-from litellm.proxy._types import WebhookEvent
+from litellm_proxy._types import WebhookEvent
 
 # we use this for the email header, please send a test email if you change this. verify it looks good on email
 LITELLM_LOGO_URL = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
@@ -19,7 +19,7 @@ async def get_all_team_member_emails(team_id: Optional[str] = None) -> list:
     )
     if team_id is None:
         return []
-    from litellm.proxy.proxy_server import prisma_client
+    from litellm_proxy.proxy_server import prisma_client
 
     if prisma_client is None:
         raise Exception("Not connected to DB!")
@@ -71,7 +71,7 @@ async def send_team_budget_alert(webhook_event: WebhookEvent) -> bool:
     Send an Email Alert to All Team Members when the Team Budget is crossed
     Returns -> True if sent, False if not.
     """
-    from litellm.proxy.utils import send_email
+    from litellm_proxy.utils import send_email
 
     _team_id = webhook_event.team_id
     team_alias = webhook_event.team_alias

litellm/integrations/gcs_bucket/gcs_bucket.py

@@ -9,10 +9,10 @@ from urllib.parse import quote
 from litellm._logging import verbose_logger
 from litellm.integrations.additional_logging_utils import AdditionalLoggingUtils
 from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
-from litellm.proxy._types import CommonProxyErrors
 from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
 from litellm.types.integrations.gcs_bucket import *
 from litellm.types.utils import StandardLoggingPayload
+from litellm_proxy._types import CommonProxyErrors
 
 if TYPE_CHECKING:
     from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
@@ -22,7 +22,7 @@ else:
 
 class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
     def __init__(self, bucket_name: Optional[str] = None) -> None:
-        from litellm.proxy.proxy_server import premium_user
+        from litellm_proxy.proxy_server import premium_user
 
         super().__init__(bucket_name=bucket_name)
 
@@ -48,7 +48,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
 
     #### ASYNC ####
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
-        from litellm.proxy.proxy_server import premium_user
+        from litellm_proxy.proxy_server import premium_user
 
         if premium_user is not True:
             raise ValueError(

litellm/integrations/gcs_pubsub/pub_sub.py

@@ -15,7 +15,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 from litellm.types.utils import StandardLoggingPayload
 
 if TYPE_CHECKING:
-    from litellm.proxy._types import SpendLogsPayload
+    from litellm_proxy._types import SpendLogsPayload
 else:
     SpendLogsPayload = Any
 
@@ -44,7 +44,7 @@ class GcsPubSubLogger(CustomBatchLogger):
             topic_id (str): Pub/Sub topic ID
             credentials_path (str, optional): Path to Google Cloud credentials JSON file
         """
-        from litellm.proxy.utils import _premium_user_check
+        from litellm_proxy.utils import _premium_user_check
 
         _premium_user_check()
 
@@ -108,10 +108,10 @@ class GcsPubSubLogger(CustomBatchLogger):
         Raises:
             Raises a NON Blocking verbose_logger.exception if an error occurs
         """
-        from litellm.proxy.spend_tracking.spend_tracking_utils import (
+        from litellm_proxy.spend_tracking.spend_tracking_utils import (
             get_logging_payload,
         )
-        from litellm.proxy.utils import _premium_user_check
+        from litellm_proxy.utils import _premium_user_check
 
         _premium_user_check()
 

litellm/integrations/langtrace.py

@@ -1,7 +1,7 @@
 import json
 from typing import TYPE_CHECKING, Any, Union
 
-from litellm.proxy._types import SpanAttributes
+from litellm_proxy._types import SpanAttributes
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span

litellm/integrations/opentelemetry.py

@@ -18,10 +18,10 @@ if TYPE_CHECKING:
     from opentelemetry.sdk.trace.export import SpanExporter as _SpanExporter
     from opentelemetry.trace import Span as _Span
 
-    from litellm.proxy._types import (
+    from litellm_proxy._types import (
         ManagementEndpointLoggingPayload as _ManagementEndpointLoggingPayload,
     )
-    from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
+    from litellm_proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
 
     Span = Union[_Span, Any]
     SpanExporter = Union[_SpanExporter, Any]
@@ -350,9 +350,9 @@ class OpenTelemetry(CustomLogger):
         """
         from opentelemetry import trace
 
-        standard_callback_dynamic_params: Optional[
-            StandardCallbackDynamicParams
-        ] = kwargs.get("standard_callback_dynamic_params")
+        standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
+            kwargs.get("standard_callback_dynamic_params")
+        )
         if not standard_callback_dynamic_params:
             return
 
@@ -406,7 +406,7 @@ class OpenTelemetry(CustomLogger):
     def set_tools_attributes(self, span: Span, tools):
         import json
 
-        from litellm.proxy._types import SpanAttributes
+        from litellm_proxy._types import SpanAttributes
 
         if not tools:
             return
@@ -460,7 +460,7 @@ class OpenTelemetry(CustomLogger):
     def _tool_calls_kv_pair(
         tool_calls: List[ChatCompletionMessageToolCall],
     ) -> Dict[str, Any]:
-        from litellm.proxy._types import SpanAttributes
+        from litellm_proxy._types import SpanAttributes
 
         kv_pairs: Dict[str, Any] = {}
         for idx, tool_call in enumerate(tool_calls):
@@ -496,7 +496,7 @@ class OpenTelemetry(CustomLogger):
                     span, kwargs, response_obj
                 )
                 return
-            from litellm.proxy._types import SpanAttributes
+            from litellm_proxy._types import SpanAttributes
 
             optional_params = kwargs.get("optional_params", {})
             litellm_params = kwargs.get("litellm_params", {}) or {}

litellm/integrations/pagerduty/pagerduty.py

@@ -19,7 +19,6 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
-from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.integrations.pagerduty import (
     AlertingConfig,
     PagerDutyInternalEvent,
@@ -30,6 +29,7 @@ from litellm.types.utils import (
     StandardLoggingPayload,
     StandardLoggingPayloadErrorInformation,
 )
+from litellm_proxy._types import UserAPIKeyAuth
 
 PAGERDUTY_DEFAULT_FAILURE_THRESHOLD = 60
 PAGERDUTY_DEFAULT_FAILURE_THRESHOLD_WINDOW_SECONDS = 60
@@ -46,7 +46,7 @@ class PagerDutyAlerting(SlackAlerting):
     def __init__(
         self, alerting_args: Optional[Union[AlertingConfig, dict]] = None, **kwargs
     ):
-        from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
+        from litellm_proxy.proxy_server import CommonProxyErrors, premium_user
 
         super().__init__()
         _api_key = os.getenv("PAGERDUTY_API_KEY")

litellm/integrations/prometheus.py

@@ -18,10 +18,10 @@ from typing import (
 import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.proxy._types import LiteLLM_TeamTable, UserAPIKeyAuth
 from litellm.types.integrations.prometheus import *
 from litellm.types.utils import StandardLoggingPayload
 from litellm.utils import get_end_user_id_for_cost_tracking
+from litellm_proxy._types import LiteLLM_TeamTable, UserAPIKeyAuth
 
 if TYPE_CHECKING:
     from apscheduler.schedulers.asyncio import AsyncIOScheduler
@@ -38,7 +38,7 @@ class PrometheusLogger(CustomLogger):
         try:
             from prometheus_client import Counter, Gauge, Histogram
 
-            from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
+            from litellm_proxy.proxy_server import CommonProxyErrors, premium_user
 
             if premium_user is not True:
                 verbose_logger.warning(
@@ -456,7 +456,7 @@ class PrometheusLogger(CustomLogger):
             and isinstance(user_api_key, str)
             and user_api_key.startswith("sk-")
         ):
-            from litellm.proxy.utils import hash_token
+            from litellm_proxy.utils import hash_token
 
             user_api_key = hash_token(user_api_key)
 
@@ -661,7 +661,7 @@ class PrometheusLogger(CustomLogger):
         kwargs: dict,
         metadata: dict,
     ):
-        from litellm.proxy.common_utils.callback_utils import (
+        from litellm_proxy.common_utils.callback_utils import (
             get_model_group_from_litellm_kwargs,
         )
 
@@ -1363,7 +1363,7 @@ class PrometheusLogger(CustomLogger):
             set_metrics_function: Function to set metrics for the fetched data.
             data_type: String representing the type of data ("teams" or "keys") for logging purposes.
         """
-        from litellm.proxy.proxy_server import prisma_client
+        from litellm_proxy.proxy_server import prisma_client
 
         if prisma_client is None:
             return
@@ -1398,10 +1398,10 @@ class PrometheusLogger(CustomLogger):
         """
         Initialize team budget metrics by reusing the generic pagination logic.
         """
-        from litellm.proxy.management_endpoints.team_endpoints import (
+        from litellm_proxy.management_endpoints.team_endpoints import (
             get_paginated_teams,
         )
-        from litellm.proxy.proxy_server import prisma_client
+        from litellm_proxy.proxy_server import prisma_client
 
         if prisma_client is None:
             verbose_logger.debug(
@@ -1432,10 +1432,10 @@ class PrometheusLogger(CustomLogger):
         from typing import Union
 
         from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
-        from litellm.proxy.management_endpoints.key_management_endpoints import (
+        from litellm_proxy.management_endpoints.key_management_endpoints import (
             _list_key_helper,
         )
-        from litellm.proxy.proxy_server import prisma_client
+        from litellm_proxy.proxy_server import prisma_client
 
         if prisma_client is None:
             verbose_logger.debug(
@@ -1480,7 +1480,7 @@ class PrometheusLogger(CustomLogger):
         - If redis cache is not available, we initialize the metrics directly.
         """
         from litellm.constants import PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME
-        from litellm.proxy.proxy_server import proxy_logging_obj
+        from litellm_proxy.proxy_server import proxy_logging_obj
 
         pod_lock_manager = proxy_logging_obj.db_spend_update_writer.pod_lock_manager
 
@@ -1561,8 +1561,8 @@ class PrometheusLogger(CustomLogger):
         Fields not available in metadata:
         - `budget_reset_at`
         """
-        from litellm.proxy.auth.auth_checks import get_team_object
-        from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
+        from litellm_proxy.auth.auth_checks import get_team_object
+        from litellm_proxy.proxy_server import prisma_client, user_api_key_cache
 
         _total_team_spend = (spend or 0) + response_cost
         team_object = LiteLLM_TeamTable(
@@ -1711,8 +1711,8 @@ class PrometheusLogger(CustomLogger):
         """
         Assemble a UserAPIKeyAuth object
         """
-        from litellm.proxy.auth.auth_checks import get_key_object
-        from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
+        from litellm_proxy.auth.auth_checks import get_key_object
+        from litellm_proxy.proxy_server import prisma_client, user_api_key_cache
 
         _total_key_spend = (key_spend or 0) + response_cost
         user_api_key_dict = UserAPIKeyAuth(
@@ -1803,8 +1803,8 @@ class PrometheusLogger(CustomLogger):
         from prometheus_client import make_asgi_app
 
         from litellm._logging import verbose_proxy_logger
-        from litellm.proxy._types import CommonProxyErrors
-        from litellm.proxy.proxy_server import app
+        from litellm_proxy._types import CommonProxyErrors
+        from litellm_proxy.proxy_server import app
 
         if premium_user is not True:
             verbose_proxy_logger.warning(

litellm/litellm_core_utils/litellm_logging.py

@@ -28,7 +28,6 @@ from litellm._logging import _is_debugging_on, verbose_logger
 from litellm.batches.batch_utils import _handle_completed_batch
 from litellm.caching.caching import DualCache, InMemoryCache
 from litellm.caching.caching_handler import LLMCachingHandler
-
 from litellm.constants import (
     DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
     DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
@@ -2886,7 +2885,7 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
             _in_memory_loggers.append(_otel_logger)
             return _otel_logger  # type: ignore
         elif logging_integration == "dynamic_rate_limiter":
-            from litellm.proxy.hooks.dynamic_rate_limiter import (
+            from litellm_proxy.hooks.dynamic_rate_limiter import (
                 _PROXY_DynamicRateLimitHandler,
             )
 
@@ -3074,7 +3073,7 @@ def get_custom_logger_compatible_class(  # noqa: PLR0915
                     return callback  # type: ignore
 
         elif logging_integration == "dynamic_rate_limiter":
-            from litellm.proxy.hooks.dynamic_rate_limiter import (
+            from litellm_proxy.hooks.dynamic_rate_limiter import (
                 _PROXY_DynamicRateLimitHandler,
             )
 
@@ -3130,7 +3129,7 @@ def _get_custom_logger_settings_from_proxy_server(callback_name: str) -> Dict:
         otel:
             message_logging: False
     """
-    from litellm.proxy.proxy_server import callback_settings
+    from litellm_proxy.proxy_server import callback_settings
 
     if callback_settings:
         return dict(callback_settings.get(callback_name, {}))

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -342,7 +342,7 @@ def get_format_from_file_id(file_id: Optional[str]) -> Optional[str]:
     unified_file_id = litellm_proxy:{};unified_id,{}
     If not a unified file id, returns 'file' as default format
     """
-    from litellm.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
+    from litellm_proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
 
     if not file_id:
         return None

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -37,15 +37,15 @@ class AnthropicMessagesHandler:
         """Helper function to handle Anthropic streaming responses using the existing logging handlers"""
         from datetime import datetime
 
-        from litellm.proxy.pass_through_endpoints.streaming_handler import (
-            PassThroughStreamingHandler,
-        )
-        from litellm.proxy.pass_through_endpoints.success_handler import (
-            PassThroughEndpointLogging,
-        )
         from litellm.types.passthrough_endpoints.pass_through_endpoints import (
             EndpointType,
         )
+        from litellm_proxy.pass_through_endpoints.streaming_handler import (
+            PassThroughStreamingHandler,
+        )
+        from litellm_proxy.pass_through_endpoints.success_handler import (
+            PassThroughEndpointLogging,
+        )
 
         # Create success handler object
         passthrough_success_handler_obj = PassThroughEndpointLogging()

litellm/proxy/.gitignore

@@ -1,2 +0,0 @@
-.env
-secrets.toml

litellm/router.py

@@ -4550,7 +4550,7 @@ class Router:
         Each provider uses diff .env vars for pass-through endpoints, this helper uses the deployment credentials to set the .env vars for pass-through endpoints
         """
         if deployment.litellm_params.use_in_pass_through is True:
-            from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
+            from litellm_proxy.pass_through_endpoints.llm_passthrough_endpoints import (
                 passthrough_endpoint_router,
             )
 

litellm/router_strategy/budget_limiter.py

@@ -10,11 +10,11 @@ This means you can use this with weighted-pick, lowest-latency, simple-shuffle,
 Example:
 ```
 openai:
-	budget_limit: 0.000000000001
-	time_period: 1d
+        budget_limit: 0.000000000001
+        time_period: 1d
 anthropic:
-	budget_limit: 100
-	time_period: 7d
+        budget_limit: 100
+        time_period: 7d
 ```
 """
 
@@ -53,9 +53,9 @@ class RouterBudgetLimiting(CustomLogger):
         self.dual_cache = dual_cache
         self.redis_increment_operation_queue: List[RedisPipelineIncrementOperation] = []
         asyncio.create_task(self.periodic_sync_in_memory_spend_with_redis())
-        self.provider_budget_config: Optional[
-            GenericBudgetConfigType
-        ] = provider_budget_config
+        self.provider_budget_config: Optional[GenericBudgetConfigType] = (
+            provider_budget_config
+        )
         self.deployment_budget_config: Optional[GenericBudgetConfigType] = None
         self.tag_budget_config: Optional[GenericBudgetConfigType] = None
         self._init_provider_budgets()
@@ -797,7 +797,7 @@ class RouterBudgetLimiting(CustomLogger):
     def _init_tag_budgets(self):
         if litellm.tag_budget_config is None:
             return
-        from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
+        from litellm_proxy.proxy_server import CommonProxyErrors, premium_user
 
         if premium_user is not True:
             raise ValueError(

litellm/router_strategy/lowest_cost.py

@@ -187,7 +187,7 @@ class LowestCostLoggingHandler(CustomLogger):
                     self.logged_success += 1
         except Exception as e:
             verbose_logger.exception(
-                "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format(
+                "litellm_proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format(
                     str(e)
                 )
             )

litellm/router_strategy/lowest_latency.py

@@ -170,7 +170,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
                     self.logged_success += 1
         except Exception as e:
             verbose_logger.exception(
-                "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format(
+                "litellm_proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format(
                     str(e)
                 )
             )
@@ -238,7 +238,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
                 return
         except Exception as e:
             verbose_logger.exception(
-                "litellm.proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format(
+                "litellm_proxy.hooks.prompt_injection_detection.py::async_pre_call_hook(): Exception occured - {}".format(
                     str(e)
                 )
             )

litellm/router_strategy/lowest_tpm_rpm_v2.py

@@ -270,7 +270,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
                 self.logged_success += 1
         except Exception as e:
             verbose_logger.exception(
-                "litellm.proxy.hooks.lowest_tpm_rpm_v2.py::log_success_event(): Exception occured - {}".format(
+                "litellm_proxy.hooks.lowest_tpm_rpm_v2.py::log_success_event(): Exception occured - {}".format(
                     str(e)
                 )
             )
@@ -321,7 +321,7 @@ class LowestTPMLoggingHandler_v2(BaseRoutingStrategy, CustomLogger):
                 self.logged_success += 1
         except Exception as e:
             verbose_logger.exception(
-                "litellm.proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - {}".format(
+                "litellm_proxy.hooks.lowest_tpm_rpm_v2.py::async_log_success_event(): Exception occured - {}".format(
                     str(e)
                 )
             )

litellm/secret_managers/aws_secret_manager.py

@@ -4,7 +4,7 @@ This is a file for the AWS Secret Manager Integration
 Relevant issue: https://github.com/BerriAI/litellm/issues/1883
 
 Requires:
-* `os.environ["AWS_REGION_NAME"], 
+* `os.environ["AWS_REGION_NAME"],
 * `pip install boto3>=1.28.57`
 """
 
@@ -15,7 +15,7 @@ import re
 from typing import Any, Dict, Optional
 
 import litellm
-from litellm.proxy._types import KeyManagementSystem
+from litellm_proxy._types import KeyManagementSystem
 
 
 def validate_environment():

litellm/secret_managers/aws_secret_manager_v2.py

@@ -9,7 +9,7 @@ Handles Async Operations for:
 Relevant issue: https://github.com/BerriAI/litellm/issues/1883
 
 Requires:
-* `os.environ["AWS_REGION_NAME"], 
+* `os.environ["AWS_REGION_NAME"],
 * `pip install boto3>=1.28.57`
 """
 
@@ -26,8 +26,8 @@ from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,
     get_async_httpx_client,
 )
-from litellm.proxy._types import KeyManagementSystem
 from litellm.types.llms.custom_http import httpxSpecialProvider
+from litellm_proxy._types import KeyManagementSystem
 
 from .base_secret_manager import BaseSecretManager
 

litellm/secret_managers/google_kms.py

@@ -12,7 +12,7 @@ import os
 from typing import Optional
 
 import litellm
-from litellm.proxy._types import KeyManagementSystem
+from litellm_proxy._types import KeyManagementSystem
 
 
 def validate_environment():

litellm/secret_managers/google_secret_manager.py

@@ -8,7 +8,7 @@ from litellm.caching.caching import InMemoryCache
 from litellm.constants import SECRET_MANAGER_REFRESH_INTERVAL
 from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
 from litellm.llms.custom_httpx.http_handler import _get_httpx_client
-from litellm.proxy._types import CommonProxyErrors, KeyManagementSystem
+from litellm_proxy._types import CommonProxyErrors, KeyManagementSystem
 
 
 class GoogleSecretManager(GCSBucketBase):
@@ -22,7 +22,7 @@ class GoogleSecretManager(GCSBucketBase):
             refresh_interval (int, optional): The refresh interval in seconds. Defaults to 86400. (24 hours)
             always_read_secret_manager (bool, optional): Whether to always read from the secret manager. Defaults to False. Since we do want to cache values
         """
-        from litellm.proxy.proxy_server import premium_user
+        from litellm_proxy.proxy_server import premium_user
 
         if premium_user is not True:
             raise ValueError(

litellm/secret_managers/hashicorp_secret_manager.py

@@ -12,14 +12,14 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
-from litellm.proxy._types import KeyManagementSystem
+from litellm_proxy._types import KeyManagementSystem
 
 from .base_secret_manager import BaseSecretManager
 
 
 class HashicorpSecretManager(BaseSecretManager):
     def __init__(self):
-        from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
+        from litellm_proxy.proxy_server import CommonProxyErrors, premium_user
 
         # Vault-specific config
         self.vault_addr = os.getenv("HCP_VAULT_ADDR", "http://127.0.0.1:8200")

litellm/secret_managers/main.py

@@ -11,7 +11,7 @@ import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.caching.caching import DualCache
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
-from litellm.proxy._types import KeyManagementSystem
+from litellm_proxy._types import KeyManagementSystem
 
 oidc_cache = DualCache()
 

litellm/types/proxy/management_endpoints/internal_user_endpoints.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Literal, Optional, Union
 from fastapi import HTTPException
 from pydantic import BaseModel, EmailStr
 
-from litellm.proxy._types import LiteLLM_UserTableWithKeyCount
+from litellm_proxy._types import LiteLLM_UserTableWithKeyCount
 
 
 class UserListResponse(BaseModel):

litellm/types/proxy/management_endpoints/ui_sso.py

@@ -2,7 +2,7 @@ from typing import List, Literal, Optional, TypedDict
 
 from pydantic import Field
 
-from litellm.proxy._types import LiteLLMPydanticObjectBase, LitellmUserRoles
+from litellm_proxy._types import LiteLLMPydanticObjectBase, LitellmUserRoles
 
 
 class MicrosoftGraphAPIUserGroupDirectoryObject(TypedDict, total=False):

litellm/types/utils.py

@@ -1,3 +1,4 @@
+import enum
 import json
 import time
 import uuid
@@ -2262,3 +2263,16 @@ class SpecialEnums(Enum):
 LLMResponseTypes = Union[
     ModelResponse, EmbeddingResponse, ImageResponse, OpenAIFileObject
 ]
+
+
+AllowedModelRegion = Literal["eu", "us"]
+
+
+class KeyManagementSystem(enum.Enum):
+    GOOGLE_KMS = "google_kms"
+    AZURE_KEY_VAULT = "azure_key_vault"
+    AWS_SECRET_MANAGER = "aws_secret_manager"
+    GOOGLE_SECRET_MANAGER = "google_secret_manager"
+    HASHICORP_VAULT = "hashicorp_vault"
+    LOCAL = "local"
+    AWS_KMS = "aws_kms"

litellm/utils.py

@@ -259,13 +259,13 @@ from .exceptions import (
     UnprocessableEntityError,
     UnsupportedParamsError,
 )
-from .proxy._types import AllowedModelRegion, KeyManagementSystem
 from .types.llms.openai import (
     ChatCompletionDeltaToolCallChunk,
     ChatCompletionToolCallChunk,
     ChatCompletionToolCallFunctionChunk,
 )
 from .types.router import LiteLLM_Params
+from .types.utils import AllowedModelRegion, KeyManagementSystem
 
 ####### ENVIRONMENT VARIABLES ####################
 # Adjust to your specific application needs / system capabilities.

litellm_proxy/_experimental/mcp_server/server.py

@@ -13,8 +13,8 @@ from pydantic import ConfigDict, ValidationError
 from litellm._logging import verbose_logger
 from litellm.constants import MCP_TOOL_NAME_PREFIX
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.proxy._types import UserAPIKeyAuth
-from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm_proxy._types import UserAPIKeyAuth
+from litellm_proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.types.mcp_server.mcp_server_manager import MCPInfo
 from litellm.types.utils import StandardLoggingMCPToolCall
 from litellm.utils import client
@@ -288,7 +288,7 @@ if MCP_AVAILABLE:
         """
         REST API to call a specific MCP tool with the provided arguments
         """
-        from litellm.proxy.proxy_server import add_litellm_data_to_request, proxy_config
+        from litellm_proxy.proxy_server import add_litellm_data_to_request, proxy_config
 
         data = await request.json()
         data = await add_litellm_data_to_request(

litellm_proxy/_experimental/mcp_server/tool_registry.py

@@ -2,7 +2,7 @@ import json
 from typing import Any, Callable, Dict, List, Optional
 
 from litellm._logging import verbose_logger
-from litellm.proxy.types_utils.utils import get_instance_fn
+from litellm_proxy.types_utils.utils import get_instance_fn
 from litellm.types.mcp_server.tool_registry import MCPTool