From 8e42eb624b3427de051a4589c2d26e9d5c520ae7 Mon Sep 17 00:00:00 2001
From: nick-rackauckas <nick@usablemachines.com>
Date: Thu, 13 Jun 2024 15:00:16 -0700
Subject: [PATCH 01/18] Fix file type handling of uppercase extensions

---
 litellm/types/files.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/litellm/types/files.py b/litellm/types/files.py
index 2da08e76c4..577b9b55cc 100644
--- a/litellm/types/files.py
+++ b/litellm/types/files.py
@@ -151,23 +151,16 @@ Util Functions
 """
 
 
-def get_file_mime_type_from_extension(extension: str) -> str:
-    for file_type, extensions in FILE_EXTENSIONS.items():
-        if extension in extensions:
-            return FILE_MIME_TYPES[file_type]
-    raise ValueError(f"Unknown mime type for extension: {extension}")
-
-
 def get_file_extension_from_mime_type(mime_type: str) -> str:
     for file_type, mime in FILE_MIME_TYPES.items():
-        if mime == mime_type:
+        if mime.lower() == mime_type.lower():
             return FILE_EXTENSIONS[file_type][0]
     raise ValueError(f"Unknown extension for mime type: {mime_type}")
 
 
 def get_file_type_from_extension(extension: str) -> FileType:
     for file_type, extensions in FILE_EXTENSIONS.items():
-        if extension in extensions:
+        if extension.lower() in extensions:
             return file_type
 
     raise ValueError(f"Unknown file type for extension: {extension}")
@@ -181,6 +174,11 @@ def get_file_mime_type_for_file_type(file_type: FileType) -> str:
     return FILE_MIME_TYPES[file_type]
 
 
+def get_file_mime_type_from_extension(extension: str) -> str:
+    file_type = get_file_type_from_extension(extension)
+    return get_file_mime_type_for_file_type(file_type)
+
+
 """
 FileType Type Groupings (Videos, Images, etc)
 """

From 6cffd0b62de2c0c263972c7050813461e8e1f2c1 Mon Sep 17 00:00:00 2001
From: = <=>
Date: Fri, 14 Jun 2024 15:19:11 +0200
Subject: [PATCH 02/18] Add mock_tool_calls

---
 litellm/main.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/litellm/main.py b/litellm/main.py
index 31cb8e364f..90feb9aa85 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -41,6 +41,7 @@ from litellm.utils import (
     get_optional_params_embeddings,
     get_optional_params_image_gen,
     supports_httpx_timeout,
+    ChatCompletionMessageToolCall,
 )
 from .llms import (
     anthropic_text,
@@ -398,6 +399,7 @@ def mock_completion(
     messages: List,
     stream: Optional[bool] = False,
     mock_response: Union[str, Exception] = "This is a mock request",
+    mock_tool_calls: Optional[List] = None,
     logging=None,
     **kwargs,
 ):
@@ -465,6 +467,11 @@ def mock_completion(
         model_response["created"] = int(time.time())
         model_response["model"] = model
 
+        if mock_tool_calls:
+            model_response["choices"][0]["message"]["tool_calls"] = [
+                ChatCompletionMessageToolCall(**tool_call) for tool_call in mock_tool_calls
+            ]
+
         setattr(
             model_response,
             "usage",
@@ -578,6 +585,7 @@ def completion(
     args = locals()
     api_base = kwargs.get("api_base", None)
     mock_response = kwargs.get("mock_response", None)
+    mock_tool_calls = kwargs.get("mock_tool_calls", None)
     force_timeout = kwargs.get("force_timeout", 600)  ## deprecated
     logger_fn = kwargs.get("logger_fn", None)
     verbose = kwargs.get("verbose", False)
@@ -896,12 +904,13 @@ def completion(
             litellm_params=litellm_params,
             custom_llm_provider=custom_llm_provider,
         )
-        if mock_response:
+        if mock_response or mock_tool_calls:
             return mock_completion(
                 model,
                 messages,
                 stream=stream,
                 mock_response=mock_response,
+                mock_tool_calls=mock_tool_calls,
                 logging=logging,
                 acompletion=acompletion,
                 mock_delay=kwargs.get("mock_delay", None),

From 464a6e6256f9f532585471dc826dcb2c9468dd13 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 17:21:43 -0700
Subject: [PATCH 03/18] fix - refactor logfire

---
 litellm/litellm_core_utils/litellm_logging.py | 54 +++----------------
 litellm/proxy/proxy_config.yaml               |  2 +-
 2 files changed, 9 insertions(+), 47 deletions(-)

diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index f54bf1d2ff..a66d978e62 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -64,6 +64,7 @@ from ..integrations.litedebugger import LiteDebugger
 from ..integrations.logfire_logger import LogfireLevel, LogfireLogger
 from ..integrations.lunary import LunaryLogger
 from ..integrations.openmeter import OpenMeterLogger
+from ..integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
 from ..integrations.prometheus import PrometheusLogger
 from ..integrations.prometheus_services import PrometheusServicesLogger
 from ..integrations.prompt_layer import PromptLayerLogger
@@ -664,32 +665,6 @@ class Logging:
                             end_time=end_time,
                             print_verbose=print_verbose,
                         )
-                    if callback == "logfire":
-                        global logfireLogger
-                        verbose_logger.debug("reaches logfire for success logging!")
-                        kwargs = {}
-                        for k, v in self.model_call_details.items():
-                            if (
-                                k != "original_response"
-                            ):  # copy.deepcopy raises errors as this could be a coroutine
-                                kwargs[k] = v
-
-                        # this only logs streaming once, complete_streaming_response exists i.e when stream ends
-                        if self.stream:
-                            if "complete_streaming_response" not in kwargs:
-                                continue
-                            else:
-                                print_verbose("reaches logfire for streaming logging!")
-                                result = kwargs["complete_streaming_response"]
-
-                        logfireLogger.log_event(
-                            kwargs=self.model_call_details,
-                            response_obj=result,
-                            start_time=start_time,
-                            end_time=end_time,
-                            print_verbose=print_verbose,
-                            level=LogfireLevel.INFO.value,
-                        )
 
                     if callback == "lunary":
                         print_verbose("reaches lunary for logging!")
@@ -1594,24 +1569,6 @@ class Logging:
                             print_verbose=print_verbose,
                         )
 
-                    if callback == "logfire":
-                        verbose_logger.debug("reaches logfire for failure logging!")
-                        kwargs = {}
-                        for k, v in self.model_call_details.items():
-                            if (
-                                k != "original_response"
-                            ):  # copy.deepcopy raises errors as this could be a coroutine
-                                kwargs[k] = v
-                        kwargs["exception"] = exception
-
-                        logfireLogger.log_event(
-                            kwargs=kwargs,
-                            response_obj=result,
-                            start_time=start_time,
-                            end_time=end_time,
-                            level=LogfireLevel.ERROR.value,
-                            print_verbose=print_verbose,
-                        )
                 except Exception as e:
                     print_verbose(
                         f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging with integrations {str(e)}"
@@ -1754,8 +1711,6 @@ def set_callbacks(callback_list, function_id=None):
                 weightsBiasesLogger = WeightsBiasesLogger()
             elif callback == "langsmith":
                 langsmithLogger = LangsmithLogger()
-            elif callback == "logfire":
-                logfireLogger = LogfireLogger()
             elif callback == "aispend":
                 aispendLogger = AISpendLogger()
             elif callback == "berrispend":
@@ -1789,3 +1744,10 @@ def _init_custom_logger_compatible_class(
         return LagoLogger()  # type: ignore
     elif logging_integration == "openmeter":
         return OpenMeterLogger()  # type: ignore
+    elif logging_integration == "logfire":
+        otel_config = OpenTelemetryConfig(
+            exporter="otlp_http",
+            endpoint="https://logfire-api.pydantic.dev/v1/traces",
+            headers=f"Authorization={os.getenv('LOGFIRE_TOKEN')}",
+        )
+        return OpenTelemetry(config=otel_config)  # type: ignore
diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml
index 27d49a33dd..d5190455f1 100644
--- a/litellm/proxy/proxy_config.yaml
+++ b/litellm/proxy/proxy_config.yaml
@@ -24,9 +24,9 @@ general_settings:
 
 litellm_settings:
   success_callback: ["prometheus"]
+  callbacks: ["otel"]
   failure_callback: ["prometheus"]
   store_audit_logs: true
-  turn_off_message_logging: true
   redact_messages_in_exceptions: True
   enforced_params:  
     - user

From a4b1b9a3aca416789853d9f1b161027baa546830 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 17:31:58 -0700
Subject: [PATCH 04/18] update logging docs

---
 .../docs/observability/athina_integration.md           |  9 +++++++++
 .../docs/observability/greenscale_integration.md       |  9 +++++++++
 .../docs/observability/helicone_integration.md         |  9 +++++++++
 .../docs/observability/langfuse_integration.md         |  2 +-
 .../docs/observability/langsmith_integration.md        | 10 ++++++++++
 .../docs/observability/logfire_integration.md          |  2 +-
 .../docs/observability/lunary_integration.md           |  8 ++++++++
 .../docs/observability/promptlayer_integration.md      |  9 +++++++++
 docs/my-website/docs/observability/sentry.md           |  9 +++++++++
 .../docs/observability/supabase_integration.md         |  8 ++++++++
 .../my-website/docs/observability/wandb_integration.md | 10 ++++++++++
 docs/my-website/sidebars.js                            |  8 ++++----
 12 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/docs/my-website/docs/observability/athina_integration.md b/docs/my-website/docs/observability/athina_integration.md
index 62c8897518..cd1442f35a 100644
--- a/docs/my-website/docs/observability/athina_integration.md
+++ b/docs/my-website/docs/observability/athina_integration.md
@@ -2,6 +2,15 @@ import Image from '@theme/IdealImage';
 
 # Athina
 
+
+:::tip
+
+This is community maintained, Please make an issue if you run into a bug
+https://github.com/BerriAI/litellm
+
+:::
+
+
 [Athina](https://athina.ai/) is an evaluation framework and production monitoring platform for your LLM-powered app. Athina is designed to enhance the performance and reliability of AI applications through real-time monitoring, granular analytics, and plug-and-play evaluations.
 
 <Image img={require('../../img/athina_dashboard.png')} />
diff --git a/docs/my-website/docs/observability/greenscale_integration.md b/docs/my-website/docs/observability/greenscale_integration.md
index 0dd673226e..49eadc6453 100644
--- a/docs/my-website/docs/observability/greenscale_integration.md
+++ b/docs/my-website/docs/observability/greenscale_integration.md
@@ -1,5 +1,14 @@
 # Greenscale - Track LLM Spend and Responsible Usage
 
+
+:::tip
+
+This is community maintained, Please make an issue if you run into a bug
+https://github.com/BerriAI/litellm
+
+:::
+
+
 [Greenscale](https://greenscale.ai/) is a production monitoring platform for your LLM-powered app that provides you granular key insights into your GenAI spending and responsible usage. Greenscale only captures metadata to minimize the exposure risk of personally identifiable information (PII).
 
 ## Getting Started
diff --git a/docs/my-website/docs/observability/helicone_integration.md b/docs/my-website/docs/observability/helicone_integration.md
index de89ba8da8..f7fd330c30 100644
--- a/docs/my-website/docs/observability/helicone_integration.md
+++ b/docs/my-website/docs/observability/helicone_integration.md
@@ -1,4 +1,13 @@
 # Helicone Tutorial 
+
+:::tip
+
+This is community maintained, Please make an issue if you run into a bug
+https://github.com/BerriAI/litellm
+
+:::
+
+
 [Helicone](https://helicone.ai/) is an open source observability platform that proxies your OpenAI traffic and provides you key insights into your spend, latency and usage.
 
 ## Use Helicone to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM)
diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md
index 0fb411dd3f..dbd03620dc 100644
--- a/docs/my-website/docs/observability/langfuse_integration.md
+++ b/docs/my-website/docs/observability/langfuse_integration.md
@@ -1,6 +1,6 @@
 import Image from '@theme/IdealImage';
 
-# Langfuse - Logging LLM Input/Output
+# 🔥 Langfuse - Logging LLM Input/Output
 
 LangFuse is open Source Observability & Analytics for LLM Apps
 Detailed production traces and a granular view on quality, cost and latency
diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md
index b115866d54..c038abd821 100644
--- a/docs/my-website/docs/observability/langsmith_integration.md
+++ b/docs/my-website/docs/observability/langsmith_integration.md
@@ -1,6 +1,16 @@
 import Image from '@theme/IdealImage';
 
 # Langsmith - Logging LLM Input/Output
+
+
+:::tip
+
+This is community maintained, Please make an issue if you run into a bug
+https://github.com/BerriAI/litellm
+
+:::
+
+
 An all-in-one developer platform for every step of the application lifecycle
 https://smith.langchain.com/
 
diff --git a/docs/my-website/docs/observability/logfire_integration.md b/docs/my-website/docs/observability/logfire_integration.md
index c1f425f425..55c15295aa 100644
--- a/docs/my-website/docs/observability/logfire_integration.md
+++ b/docs/my-website/docs/observability/logfire_integration.md
@@ -1,6 +1,6 @@
 import Image from '@theme/IdealImage';
 
-# Logfire - Logging LLM Input/Output
+# 🔥 Logfire - Logging LLM Input/Output
 
 Logfire is open Source Observability & Analytics for LLM Apps
 Detailed production traces and a granular view on quality, cost and latency
diff --git a/docs/my-website/docs/observability/lunary_integration.md b/docs/my-website/docs/observability/lunary_integration.md
index 9b8e90df7b..56e74132f7 100644
--- a/docs/my-website/docs/observability/lunary_integration.md
+++ b/docs/my-website/docs/observability/lunary_integration.md
@@ -1,5 +1,13 @@
 # Lunary - Logging and tracing LLM input/output
 
+:::tip
+
+This is community maintained, Please make an issue if you run into a bug
+https://github.com/BerriAI/litellm
+
+:::
+
+
 [Lunary](https://lunary.ai/) is an open-source AI developer platform providing observability, prompt management, and evaluation tools for AI developers.
 
 <video controls width='900' >
diff --git a/docs/my-website/docs/observability/promptlayer_integration.md b/docs/my-website/docs/observability/promptlayer_integration.md
index ffc0a32ec0..7f62a31697 100644
--- a/docs/my-website/docs/observability/promptlayer_integration.md
+++ b/docs/my-website/docs/observability/promptlayer_integration.md
@@ -2,6 +2,15 @@ import Image from '@theme/IdealImage';
 
 # Promptlayer Tutorial
 
+
+:::tip
+
+This is community maintained, Please make an issue if you run into a bug
+https://github.com/BerriAI/litellm
+
+:::
+
+
 Promptlayer is a platform for prompt engineers. Log OpenAI requests. Search usage history. Track performance. Visually manage prompt templates.
 
 <Image img={require('../../img/promptlayer.png')} />
diff --git a/docs/my-website/docs/observability/sentry.md b/docs/my-website/docs/observability/sentry.md
index 5877db6610..59e43b8f76 100644
--- a/docs/my-website/docs/observability/sentry.md
+++ b/docs/my-website/docs/observability/sentry.md
@@ -1,5 +1,14 @@
 import Image from '@theme/IdealImage';
 
+
+:::tip
+
+This is community maintained, Please make an issue if you run into a bug
+https://github.com/BerriAI/litellm
+
+:::
+
+
 # Sentry - Log LLM Exceptions
 [Sentry](https://sentry.io/) provides error monitoring for production. LiteLLM can add breadcrumbs and send exceptions to Sentry with this integration
 
diff --git a/docs/my-website/docs/observability/supabase_integration.md b/docs/my-website/docs/observability/supabase_integration.md
index a0897a2730..fd3f1c3d5a 100644
--- a/docs/my-website/docs/observability/supabase_integration.md
+++ b/docs/my-website/docs/observability/supabase_integration.md
@@ -1,4 +1,12 @@
 # Supabase Tutorial 
+
+:::tip
+
+This is community maintained, Please make an issue if you run into a bug
+https://github.com/BerriAI/litellm
+
+:::
+
 [Supabase](https://supabase.com/) is an open source Firebase alternative.
 Start your project with a Postgres database, Authentication, instant APIs, Edge Functions, Realtime subscriptions, Storage, and Vector embeddings.
 
diff --git a/docs/my-website/docs/observability/wandb_integration.md b/docs/my-website/docs/observability/wandb_integration.md
index dfa7b46dd3..37057f43db 100644
--- a/docs/my-website/docs/observability/wandb_integration.md
+++ b/docs/my-website/docs/observability/wandb_integration.md
@@ -1,6 +1,16 @@
 import Image from '@theme/IdealImage';
 
 # Weights & Biases - Logging LLM Input/Output
+
+
+:::tip
+
+This is community maintained, Please make an issue if you run into a bug
+https://github.com/BerriAI/litellm
+
+:::
+
+
 Weights & Biases helps AI developers build better models faster https://wandb.ai
 
 <Image img={require('../../img/wandb.png')} />
diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js
index b31ff9a2d4..b718818713 100644
--- a/docs/my-website/sidebars.js
+++ b/docs/my-website/sidebars.js
@@ -172,10 +172,8 @@ const sidebars = {
     "proxy/custom_pricing",
     "routing",
     "scheduler",
-    "rules",
     "set_keys",
     "budget_manager",
-    "contributing",
     "secret",
     "completion/token_usage",
     "load_test",
@@ -183,11 +181,11 @@ const sidebars = {
       type: "category",
       label: "Logging & Observability",
       items: [
+        "observability/langfuse_integration",
+        "observability/logfire_integration",
         "debugging/local_debugging",
         "observability/raw_request_response",
-        "observability/callbacks",
         "observability/custom_callback",
-        "observability/langfuse_integration",
         "observability/sentry",
         "observability/lago",
         "observability/openmeter",
@@ -233,6 +231,8 @@ const sidebars = {
       label: "Extras",
       items: [
         "extras/contributing",
+        "contributing",
+        "rules",
         "proxy_server",
         {
           type: "category",

From 09053ef8b193c5c5cb485d2715d46dab6d574969 Mon Sep 17 00:00:00 2001
From: nick-rackauckas <nick@usablemachines.com>
Date: Mon, 17 Jun 2024 17:50:05 -0700
Subject: [PATCH 05/18] File type unit tests

---
 litellm/tests/test_file_types.py | 54 ++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 litellm/tests/test_file_types.py

diff --git a/litellm/tests/test_file_types.py b/litellm/tests/test_file_types.py
new file mode 100644
index 0000000000..db83ba0e74
--- /dev/null
+++ b/litellm/tests/test_file_types.py
@@ -0,0 +1,54 @@
+from litellm.types.files import (
+    FILE_EXTENSIONS,
+    FILE_MIME_TYPES,
+    FileType,
+    get_file_extension_from_mime_type,
+    get_file_type_from_extension,
+    get_file_extension_for_file_type,
+    get_file_mime_type_for_file_type,
+    get_file_mime_type_from_extension,
+)
+import pytest
+
+
+class TestFileConsts:
+    def test_all_file_types_have_extensions(self):
+        for file_type in FileType:
+            assert file_type in FILE_EXTENSIONS.keys()
+
+    def test_all_file_types_have_mime_types(self):
+        for file_type in FileType:
+            assert file_type in FILE_MIME_TYPES.keys()
+
+    def test_get_file_extension_from_mime_type(self):
+        assert get_file_extension_from_mime_type("audio/aac") == "aac"
+        assert get_file_extension_from_mime_type("application/pdf") == "pdf"
+        with pytest.raises(ValueError):
+            get_file_extension_from_mime_type("application/unknown")
+
+    def test_get_file_type_from_extension(self):
+        assert get_file_type_from_extension("aac") == FileType.AAC
+        assert get_file_type_from_extension("pdf") == FileType.PDF
+        with pytest.raises(ValueError):
+            get_file_type_from_extension("unknown")
+
+    def test_get_file_extension_for_file_type(self):
+        assert get_file_extension_for_file_type(FileType.AAC) == "aac"
+        assert get_file_extension_for_file_type(FileType.PDF) == "pdf"
+
+    def test_get_file_mime_type_for_file_type(self):
+        assert get_file_mime_type_for_file_type(FileType.AAC) == "audio/aac"
+        assert get_file_mime_type_for_file_type(FileType.PDF) == "application/pdf"
+
+    def test_get_file_mime_type_from_extension(self):
+        assert get_file_mime_type_from_extension("aac") == "audio/aac"
+        assert get_file_mime_type_from_extension("pdf") == "application/pdf"
+
+    def test_uppercase_extensions(self):
+        # Test that uppercase extensions return the correct file type
+        assert get_file_type_from_extension("AAC") == FileType.AAC
+        assert get_file_type_from_extension("PDF") == FileType.PDF
+
+        # Test that uppercase extensions return the correct MIME type
+        assert get_file_mime_type_from_extension("AAC") == "audio/aac"
+        assert get_file_mime_type_from_extension("PDF") == "application/pdf"

From ec56ae7c9ac569f83153a068cc98e9dae392b1a3 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 17 Jun 2024 18:34:20 -0700
Subject: [PATCH 06/18] fix(proxy/_types.py): allow internal user to access sso
 routes

---
 litellm/proxy/_types.py                 | 32 +++++----
 litellm/proxy/auth/user_api_key_auth.py | 89 ++++++++++++-------------
 2 files changed, 63 insertions(+), 58 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 9a28478391..8c8059d704 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -1,13 +1,17 @@
-from pydantic import BaseModel, Extra, Field, model_validator, Json, ConfigDict
-from dataclasses import fields
 import enum
-from typing import Optional, List, Union, Dict, Literal, Any, TypedDict, TYPE_CHECKING
+import json
+import os
+import sys
+import uuid
+from dataclasses import fields
 from datetime import datetime
-import uuid, json, sys, os
-from litellm.types.router import UpdateRouterConfig
-from litellm.types.utils import ProviderField
+from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, TypedDict, Union
+
+from pydantic import BaseModel, ConfigDict, Extra, Field, Json, model_validator
 from typing_extensions import Annotated
 
+from litellm.types.router import UpdateRouterConfig
+from litellm.types.utils import ProviderField
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
@@ -283,12 +287,16 @@ class LiteLLMRoutes(enum.Enum):
         "/metrics",
     ]
 
-    internal_user_routes: List = [
-        "/key/generate",
-        "/key/update",
-        "/key/delete",
-        "/key/info",
-    ] + spend_tracking_routes
+    internal_user_routes: List = (
+        [
+            "/key/generate",
+            "/key/update",
+            "/key/delete",
+            "/key/info",
+        ]
+        + spend_tracking_routes
+        + sso_only_routes
+    )
 
 
 # class LiteLLMAllowedRoutes(LiteLLMBase):
diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index c79b0a8f95..be3d801e72 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -7,59 +7,56 @@ Returns a UserAPIKeyAuth object if the API key is valid
 
 """
 
+import asyncio
 import json
+import secrets
+import traceback
 from datetime import datetime, timedelta, timezone
 from typing import Optional
-import secrets
 from uuid import uuid4
 
 import fastapi
-from fastapi import Request
-from pydantic import BaseModel
-import litellm
-import traceback
-import asyncio
 from fastapi import (
-    FastAPI,
-    Request,
-    HTTPException,
-    status,
-    Path,
     Depends,
-    Header,
-    Response,
-    Form,
-    UploadFile,
+    FastAPI,
     File,
+    Form,
+    Header,
+    HTTPException,
+    Path,
+    Request,
+    Response,
+    UploadFile,
+    status,
 )
-from fastapi.responses import (
-    StreamingResponse,
-    FileResponse,
-    ORJSONResponse,
-    JSONResponse,
-)
-from fastapi.openapi.utils import get_openapi
-from fastapi.responses import RedirectResponse
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.staticfiles import StaticFiles
+from fastapi.openapi.utils import get_openapi
+from fastapi.responses import (
+    FileResponse,
+    JSONResponse,
+    ORJSONResponse,
+    RedirectResponse,
+    StreamingResponse,
+)
 from fastapi.security.api_key import APIKeyHeader
-from litellm.proxy._types import *
-from litellm._logging import verbose_logger, verbose_proxy_logger
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel
 
+import litellm
+from litellm._logging import verbose_logger, verbose_proxy_logger
+from litellm.proxy._types import *
 from litellm.proxy.auth.auth_checks import (
+    allowed_routes_check,
     common_checks,
+    get_actual_routes,
     get_end_user_object,
     get_org_object,
     get_team_object,
     get_user_object,
-    allowed_routes_check,
-    get_actual_routes,
     log_to_opentelemetry,
 )
-
-from litellm.proxy.utils import _to_ns
 from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
-
+from litellm.proxy.utils import _to_ns
 
 api_key_header = APIKeyHeader(
     name="Authorization", auto_error=False, description="Bearer token"
@@ -88,20 +85,20 @@ async def user_api_key_auth(
 ) -> UserAPIKeyAuth:
 
     from litellm.proxy.proxy_server import (
-        litellm_proxy_admin_name,
+        allowed_routes_check,
         common_checks,
-        master_key,
-        prisma_client,
-        llm_model_list,
-        user_custom_auth,
         custom_db_client,
         general_settings,
-        proxy_logging_obj,
-        open_telemetry_logger,
-        user_api_key_cache,
-        jwt_handler,
-        allowed_routes_check,
         get_actual_routes,
+        jwt_handler,
+        litellm_proxy_admin_name,
+        llm_model_list,
+        master_key,
+        open_telemetry_logger,
+        prisma_client,
+        proxy_logging_obj,
+        user_api_key_cache,
+        user_custom_auth,
     )
 
     try:
@@ -1004,7 +1001,7 @@ async def user_api_key_auth(
                 ):
 
                     pass
-                elif _user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY:
+                elif _user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value:
                     if route in LiteLLMRoutes.openai_routes.value:
                         raise HTTPException(
                             status_code=status.HTTP_403_FORBIDDEN,
@@ -1031,7 +1028,7 @@ async def user_api_key_auth(
                                 detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}",
                             )
                 elif (
-                    _user_role == LitellmUserRoles.INTERNAL_USER
+                    _user_role == LitellmUserRoles.INTERNAL_USER.value
                     and route in LiteLLMRoutes.internal_user_routes.value
                 ):
                     pass
@@ -1059,6 +1056,7 @@ async def user_api_key_auth(
             # this token is only used for managing the ui
             allowed_routes = [
                 "/sso",
+                "/sso/get/logout_url",
                 "/login",
                 "/key/generate",
                 "/key/update",
@@ -1144,8 +1142,8 @@ async def user_api_key_auth(
             raise Exception()
     except Exception as e:
         verbose_proxy_logger.error(
-            "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {}".format(
-                str(e)
+            "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - {}\n{}".format(
+                str(e), traceback.format_exc()
             )
         )
 
@@ -1156,7 +1154,6 @@ async def user_api_key_auth(
                 user_api_key_dict=UserAPIKeyAuth(parent_otel_span=parent_otel_span),
             )
 
-        verbose_proxy_logger.debug(traceback.format_exc())
         if isinstance(e, litellm.BudgetExceededError):
             raise ProxyException(
                 message=e.message, type="auth_error", param=None, code=400

From 7981ca507430badff5fe338de4281f046d970ccf Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 17 Jun 2024 18:50:16 -0700
Subject: [PATCH 07/18] fix(main.py): fix import

---
 litellm/main.py | 132 ++++++++++++++++++++++++++++--------------------
 1 file changed, 78 insertions(+), 54 deletions(-)

diff --git a/litellm/main.py b/litellm/main.py
index 01ea5dc3ca..8c71b9a979 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -7,109 +7,132 @@
 #
 #  Thank you ! We ❤️ you! - Krrish & Ishaan
 
-import os, openai, sys, json, inspect, uuid, datetime, threading
-from typing import Any, Literal, Union, BinaryIO
-from typing_extensions import overload
-from functools import partial
-
-import dotenv, traceback, random, asyncio, time, contextvars
+import asyncio
+import contextvars
+import datetime
+import inspect
+import json
+import os
+import random
+import sys
+import threading
+import time
+import traceback
+import uuid
+from concurrent.futures import ThreadPoolExecutor
 from copy import deepcopy
+from functools import partial
+from typing import (
+    Any,
+    BinaryIO,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    Union,
+)
+
+import dotenv
 import httpx
+import openai
+import tiktoken
+from typing_extensions import overload
+
 import litellm
-from ._logging import verbose_logger
 from litellm import (  # type: ignore
+    Logging,
     client,
     exception_type,
-    get_optional_params,
     get_litellm_params,
-    Logging,
+    get_optional_params,
 )
 from litellm.utils import (
-    get_secret,
     CustomStreamWrapper,
-    read_config_args,
-    completion_with_fallbacks,
-    get_llm_provider,
-    get_api_key,
-    mock_completion_streaming_obj,
+    Usage,
     async_mock_completion_streaming_obj,
+    completion_with_fallbacks,
     convert_to_model_response_object,
-    token_counter,
     create_pretrained_tokenizer,
     create_tokenizer,
-    Usage,
+    get_api_key,
+    get_llm_provider,
     get_optional_params_embeddings,
     get_optional_params_image_gen,
+    get_secret,
+    mock_completion_streaming_obj,
+    read_config_args,
     supports_httpx_timeout,
-    ChatCompletionMessageToolCall,
+    token_counter,
 )
+
+from ._logging import verbose_logger
+from .caching import disable_cache, enable_cache, update_cache
 from .llms import (
-    anthropic_text,
-    together_ai,
     ai21,
-    sagemaker,
-    bedrock,
-    triton,
-    huggingface_restapi,
-    replicate,
     aleph_alpha,
-    nlp_cloud,
+    anthropic_text,
     baseten,
-    vllm,
-    ollama,
-    ollama_chat,
-    cloudflare,
+    bedrock,
     clarifai,
+    cloudflare,
     cohere,
     cohere_chat,
-    petals,
+    gemini,
+    huggingface_restapi,
+    maritalk,
+    nlp_cloud,
+    ollama,
+    ollama_chat,
     oobabooga,
     openrouter,
     palm,
-    gemini,
+    petals,
+    replicate,
+    sagemaker,
+    together_ai,
+    triton,
     vertex_ai,
     vertex_ai_anthropic,
-    maritalk,
+    vllm,
     watsonx,
 )
-from .llms.openai import OpenAIChatCompletion, OpenAITextCompletion
-from .llms.azure import AzureChatCompletion
-from .llms.databricks import DatabricksChatCompletion
-from .llms.azure_text import AzureTextCompletion
 from .llms.anthropic import AnthropicChatCompletion
 from .llms.anthropic_text import AnthropicTextCompletion
+from .llms.azure import AzureChatCompletion
+from .llms.azure_text import AzureTextCompletion
+from .llms.bedrock_httpx import BedrockConverseLLM, BedrockLLM
+from .llms.databricks import DatabricksChatCompletion
 from .llms.huggingface_restapi import Huggingface
+from .llms.openai import OpenAIChatCompletion, OpenAITextCompletion
 from .llms.predibase import PredibaseChatCompletion
-from .llms.bedrock_httpx import BedrockLLM, BedrockConverseLLM
-from .llms.vertex_httpx import VertexLLM
-from .llms.triton import TritonChatCompletion
-from .llms.text_completion_codestral import CodestralTextCompletion
 from .llms.prompt_templates.factory import (
-    prompt_factory,
     custom_prompt,
     function_call_prompt,
     map_system_message_pt,
+    prompt_factory,
 )
-import tiktoken
-from concurrent.futures import ThreadPoolExecutor
-from typing import Callable, List, Optional, Dict, Union, Mapping
-from .caching import enable_cache, disable_cache, update_cache
+from .llms.text_completion_codestral import CodestralTextCompletion
+from .llms.triton import TritonChatCompletion
+from .llms.vertex_httpx import VertexLLM
 from .types.llms.openai import HttpxBinaryResponseContent
+from .types.utils import ChatCompletionMessageToolCall
 
 encoding = tiktoken.get_encoding("cl100k_base")
 from litellm.utils import (
-    get_secret,
+    Choices,
     CustomStreamWrapper,
-    TextCompletionStreamWrapper,
-    ModelResponse,
-    TextCompletionResponse,
-    TextChoices,
     EmbeddingResponse,
     ImageResponse,
-    read_config_args,
-    Choices,
     Message,
+    ModelResponse,
+    TextChoices,
+    TextCompletionResponse,
+    TextCompletionStreamWrapper,
     TranscriptionResponse,
+    get_secret,
+    read_config_args,
 )
 
 ####### ENVIRONMENT VARIABLES ###################
@@ -477,7 +500,8 @@ def mock_completion(
 
         if mock_tool_calls:
             model_response["choices"][0]["message"]["tool_calls"] = [
-                ChatCompletionMessageToolCall(**tool_call) for tool_call in mock_tool_calls
+                ChatCompletionMessageToolCall(**tool_call)
+                for tool_call in mock_tool_calls
             ]
 
         setattr(

From 986b5ed9c82eded84132d72119f4b3ddad5d1755 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 17 Jun 2024 18:57:20 -0700
Subject: [PATCH 08/18] fix(_logging.py): add loglevel and timestamp to json
 logs

Fixes https://github.com/BerriAI/litellm/issues/4248
---
 litellm/_logging.py                     | 14 ++++++++++----
 litellm/proxy/_logging.py               | 10 +++++++---
 litellm/proxy/_super_secret_config.yaml |  1 +
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/litellm/_logging.py b/litellm/_logging.py
index 52a445b49e..e3844bec22 100644
--- a/litellm/_logging.py
+++ b/litellm/_logging.py
@@ -1,6 +1,8 @@
-import logging, os, json
-from logging import Formatter
+import json
+import logging
+import os
 import traceback
+from logging import Formatter
 
 set_verbose = False
 
@@ -21,8 +23,12 @@ class JsonFormatter(Formatter):
         super(JsonFormatter, self).__init__()
 
     def format(self, record):
-        json_record = {}
-        json_record["message"] = record.getMessage()
+        json_record = {
+            "message": record.getMessage(),
+            "level": record.levelname,
+            "timestamp": self.formatTime(record, self.datefmt),
+        }
+
         return json.dumps(json_record)
 
 
diff --git a/litellm/proxy/_logging.py b/litellm/proxy/_logging.py
index 655da7b294..a3eb35180f 100644
--- a/litellm/proxy/_logging.py
+++ b/litellm/proxy/_logging.py
@@ -1,7 +1,8 @@
 import json
 import logging
-from logging import Formatter
 import os
+from logging import Formatter
+
 from litellm import json_logs
 
 # Set default log level to INFO
@@ -14,8 +15,11 @@ class JsonFormatter(Formatter):
         super(JsonFormatter, self).__init__()
 
     def format(self, record):
-        json_record = {}
-        json_record["message"] = record.getMessage()
+        json_record = {
+            "message": record.getMessage(),
+            "level": record.levelname,
+            "timestamp": self.formatTime(record, self.datefmt),
+        }
         return json.dumps(json_record)
 
 
diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml
index 9f1709b5fe..b5946afd6d 100644
--- a/litellm/proxy/_super_secret_config.yaml
+++ b/litellm/proxy/_super_secret_config.yaml
@@ -79,6 +79,7 @@ litellm_settings:
   success_callback: ["langfuse"]
   failure_callback: ["langfuse"]
   cache: true
+  json_logs: true
 
 general_settings:
   alerting: ["slack"]

From cf07d3abaefccd49461910dce4af481e235c5155 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 17 Jun 2024 19:00:58 -0700
Subject: [PATCH 09/18] ci(config.yml): update config

---
 .circleci/config.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 736bb8e8a1..6052fcba9c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -180,6 +180,7 @@ jobs:
             pip install numpydoc
             pip install prisma            
             pip install fastapi            
+            pip install ijson            
             pip install "httpx==0.24.1"
             pip install "gunicorn==21.2.0"
             pip install "anyio==3.7.1"

From b343911d59be8d75de3524c10b55da470de5a58d Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 19:15:14 -0700
Subject: [PATCH 10/18] fix - otel fixes

---
 litellm/integrations/opentelemetry.py | 54 +++++++++++++++------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py
index e41272c243..e4daff1218 100644
--- a/litellm/integrations/opentelemetry.py
+++ b/litellm/integrations/opentelemetry.py
@@ -1,20 +1,21 @@
 import os
 from dataclasses import dataclass
 from datetime import datetime
-import litellm
-
-from litellm.integrations.custom_logger import CustomLogger
-from litellm._logging import verbose_logger
-from litellm.types.services import ServiceLoggerPayload
 from functools import wraps
-from typing import Union, Optional, TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.services import ServiceLoggerPayload
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
-    from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
+
     from litellm.proxy._types import (
         ManagementEndpointLoggingPayload as _ManagementEndpointLoggingPayload,
     )
+    from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
 
     Span = _Span
     UserAPIKeyAuth = _UserAPIKeyAuth
@@ -107,8 +108,9 @@ class OpenTelemetry(CustomLogger):
         start_time: Optional[datetime] = None,
         end_time: Optional[datetime] = None,
     ):
-        from opentelemetry import trace
         from datetime import datetime
+
+        from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
         _start_time_ns = start_time
@@ -145,8 +147,9 @@ class OpenTelemetry(CustomLogger):
         start_time: Optional[datetime] = None,
         end_time: Optional[datetime] = None,
     ):
-        from opentelemetry import trace
         from datetime import datetime
+
+        from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
         _start_time_ns = start_time
@@ -179,8 +182,8 @@ class OpenTelemetry(CustomLogger):
     async def async_post_call_failure_hook(
         self, original_exception: Exception, user_api_key_dict: UserAPIKeyAuth
     ):
-        from opentelemetry.trace import Status, StatusCode
         from opentelemetry import trace
+        from opentelemetry.trace import Status, StatusCode
 
         parent_otel_span = user_api_key_dict.parent_otel_span
         if parent_otel_span is not None:
@@ -202,8 +205,8 @@ class OpenTelemetry(CustomLogger):
             parent_otel_span.end(end_time=self._to_ns(datetime.now()))
 
     def _handle_sucess(self, kwargs, response_obj, start_time, end_time):
-        from opentelemetry.trace import Status, StatusCode
         from opentelemetry import trace
+        from opentelemetry.trace import Status, StatusCode
 
         verbose_logger.debug(
             "OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s",
@@ -253,9 +256,10 @@ class OpenTelemetry(CustomLogger):
         span.end(end_time=self._to_ns(end_time))
 
     def set_tools_attributes(self, span: Span, tools):
-        from litellm.proxy._types import SpanAttributes
         import json
 
+        from litellm.proxy._types import SpanAttributes
+
         if not tools:
             return
 
@@ -320,7 +324,7 @@ class OpenTelemetry(CustomLogger):
             )
 
         span.set_attribute(
-            SpanAttributes.LLM_IS_STREAMING, optional_params.get("stream", False)
+            SpanAttributes.LLM_IS_STREAMING, str(optional_params.get("stream", False))
         )
 
         if optional_params.get("tools"):
@@ -439,7 +443,7 @@ class OpenTelemetry(CustomLogger):
         #############################################
         ########## LLM Response Attributes ##########
         #############################################
-        if _raw_response:
+        if _raw_response and isinstance(_raw_response, str):
             # cast sr -> dict
             import json
 
@@ -478,10 +482,10 @@ class OpenTelemetry(CustomLogger):
         return _parent_context
 
     def _get_span_context(self, kwargs):
+        from opentelemetry import trace
         from opentelemetry.trace.propagation.tracecontext import (
             TraceContextTextMapPropagator,
         )
-        from opentelemetry import trace
 
         litellm_params = kwargs.get("litellm_params", {}) or {}
         proxy_server_request = litellm_params.get("proxy_server_request", {}) or {}
@@ -505,17 +509,17 @@ class OpenTelemetry(CustomLogger):
             return TraceContextTextMapPropagator().extract(carrier=carrier), None
 
     def _get_span_processor(self):
-        from opentelemetry.sdk.trace.export import (
-            SpanExporter,
-            SimpleSpanProcessor,
-            BatchSpanProcessor,
-            ConsoleSpanExporter,
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+            OTLPSpanExporter as OTLPSpanExporterGRPC,
         )
         from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
             OTLPSpanExporter as OTLPSpanExporterHTTP,
         )
-        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
-            OTLPSpanExporter as OTLPSpanExporterGRPC,
+        from opentelemetry.sdk.trace.export import (
+            BatchSpanProcessor,
+            ConsoleSpanExporter,
+            SimpleSpanProcessor,
+            SpanExporter,
         )
 
         verbose_logger.debug(
@@ -574,8 +578,9 @@ class OpenTelemetry(CustomLogger):
         logging_payload: ManagementEndpointLoggingPayload,
         parent_otel_span: Optional[Span] = None,
     ):
-        from opentelemetry import trace
         from datetime import datetime
+
+        from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
         _start_time_ns = logging_payload.start_time
@@ -619,8 +624,9 @@ class OpenTelemetry(CustomLogger):
         logging_payload: ManagementEndpointLoggingPayload,
         parent_otel_span: Optional[Span] = None,
     ):
-        from opentelemetry import trace
         from datetime import datetime
+
+        from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
         _start_time_ns = logging_payload.start_time

From d89de2fd724b7a242c405a36aa2e6908f4337c44 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 19:21:35 -0700
Subject: [PATCH 11/18] fix logfire

---
 litellm/__init__.py                           |  2 +-
 litellm/litellm_core_utils/litellm_logging.py | 30 ++++++++++++++++---
 litellm/utils.py                              | 11 ++++++-
 3 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/litellm/__init__.py b/litellm/__init__.py
index b85d582606..dbdb2222d7 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -34,7 +34,7 @@ input_callback: List[Union[str, Callable]] = []
 success_callback: List[Union[str, Callable]] = []
 failure_callback: List[Union[str, Callable]] = []
 service_callback: List[Union[str, Callable]] = []
-_custom_logger_compatible_callbacks_literal = Literal["lago", "openmeter"]
+_custom_logger_compatible_callbacks_literal = Literal["lago", "openmeter", "logfire"]
 callbacks: List[Union[Callable, _custom_logger_compatible_callbacks_literal]] = []
 _langfuse_default_tags: Optional[
     List[
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index a66d978e62..b72098fbe7 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -10,7 +10,7 @@ import sys
 import time
 import traceback
 import uuid
-from typing import Callable, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 import litellm
 from litellm import (
@@ -73,6 +73,8 @@ from ..integrations.supabase import Supabase
 from ..integrations.traceloop import TraceloopLogger
 from ..integrations.weights_biases import WeightsBiasesLogger
 
+_in_memory_loggers: List[Any] = []
+
 
 class Logging:
     global supabaseClient, liteDebuggerClient, promptLayerLogger, weightsBiasesLogger, langsmithLogger, logfireLogger, capture_exception, add_breadcrumb, lunaryLogger, logfireLogger, prometheusLogger, slack_app
@@ -1741,13 +1743,33 @@ def _init_custom_logger_compatible_class(
     logging_integration: litellm._custom_logger_compatible_callbacks_literal,
 ) -> Callable:
     if logging_integration == "lago":
-        return LagoLogger()  # type: ignore
+        for callback in _in_memory_loggers:
+            if isinstance(callback, LagoLogger):
+                return callback  # type: ignore
+
+        lago_logger = LagoLogger()
+        _in_memory_loggers.append(lago_logger)
+        return lago_logger  # type: ignore
     elif logging_integration == "openmeter":
-        return OpenMeterLogger()  # type: ignore
+        for callback in _in_memory_loggers:
+            if isinstance(callback, OpenMeterLogger):
+                return callback  # type: ignore
+
+        _openmeter_logger = OpenMeterLogger()
+        _in_memory_loggers.append(_openmeter_logger)
+        return _openmeter_logger  # type: ignore
     elif logging_integration == "logfire":
+        if "LOGFIRE_TOKEN" not in os.environ:
+            raise ValueError("LOGFIRE_TOKEN not found in environment variables")
+
         otel_config = OpenTelemetryConfig(
             exporter="otlp_http",
             endpoint="https://logfire-api.pydantic.dev/v1/traces",
             headers=f"Authorization={os.getenv('LOGFIRE_TOKEN')}",
         )
-        return OpenTelemetry(config=otel_config)  # type: ignore
+        for callback in _in_memory_loggers:
+            if isinstance(callback, OpenTelemetry):
+                return callback  # type: ignore
+        _otel_logger = OpenTelemetry(config=otel_config)
+        _in_memory_loggers.append(_otel_logger)
+        return _otel_logger  # type: ignore
diff --git a/litellm/utils.py b/litellm/utils.py
index 7fbb337d60..e84a262afa 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -340,6 +340,15 @@ def function_setup(
     try:
         global callback_list, add_breadcrumb, user_logger_fn, Logging
         function_id = kwargs["id"] if "id" in kwargs else None
+
+        # Note: maintain backwards compatibility with logfire. some users use litellm.success_callback=["logfire"]
+        if (
+            litellm.success_callback is not None
+            and "logfire" in litellm.success_callback
+        ):
+            litellm.success_callback.remove("logfire")
+            litellm.callbacks.append("logfire")
+
         if len(litellm.callbacks) > 0:
             for callback in litellm.callbacks:
                 # check if callback is a string - e.g. "lago", "openmeter"
@@ -7243,7 +7252,7 @@ def get_secret(
                     else:
                         raise ValueError(
                             f"Google KMS requires the encrypted secret to be encoded in base64"
-                        )#fix for this vulnerability https://huntr.com/bounties/ae623c2f-b64b-4245-9ed4-f13a0a5824ce
+                        )  # fix for this vulnerability https://huntr.com/bounties/ae623c2f-b64b-4245-9ed4-f13a0a5824ce
                     response = client.decrypt(
                         request={
                             "name": litellm._google_kms_resource_name,

From 4aa6a2350a16e17d3da95b0dada5cad0ce5da91c Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 17 Jun 2024 19:22:33 -0700
Subject: [PATCH 12/18] test: skip watsonx tests - account closed

---
 litellm/tests/test_completion.py | 25 ++++++++++++++++++-------
 litellm/tests/test_streaming.py  | 24 +++++++++++++++---------
 2 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py
index 0e1b4a9f41..0cb1f7929f 100644
--- a/litellm/tests/test_completion.py
+++ b/litellm/tests/test_completion.py
@@ -1,20 +1,25 @@
-import sys, os, json
+import json
+import os
+import sys
 import traceback
+
 from dotenv import load_dotenv
 
 load_dotenv()
-import os, io
+import io
+import os
 
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
+from unittest.mock import MagicMock, patch
+
 import pytest
+
 import litellm
-from litellm import embedding, completion, completion_cost, Timeout
-from litellm import RateLimitError
+from litellm import RateLimitError, Timeout, completion, completion_cost, embedding
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.prompt_templates.factory import anthropic_messages_pt
-from unittest.mock import patch, MagicMock
-from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler
 
 # litellm.num_retries =3
 litellm.cache = None
@@ -1500,7 +1505,9 @@ def test_ollama_image():
     data is untouched.
     """
 
-    import io, base64
+    import base64
+    import io
+
     from PIL import Image
 
     def mock_post(url, **kwargs):
@@ -3478,6 +3485,7 @@ def test_completion_palm_stream():
         pytest.fail(f"Error occurred: {e}")
 
 
+@pytest.mark.skip(reason="IBM closed account.")
 def test_completion_watsonx():
     litellm.set_verbose = True
     model_name = "watsonx/ibm/granite-13b-chat-v2"
@@ -3498,6 +3506,7 @@ def test_completion_watsonx():
         pytest.fail(f"Error occurred: {e}")
 
 
+@pytest.mark.skip(reason="IBM closed account.")
 def test_completion_stream_watsonx():
     litellm.set_verbose = True
     model_name = "watsonx/ibm/granite-13b-chat-v2"
@@ -3565,6 +3574,7 @@ def test_unified_auth_params(provider, model, project, region_name, token):
         assert value in translated_optional_params
 
 
+@pytest.mark.skip(reason="IBM closed account.")
 @pytest.mark.asyncio
 async def test_acompletion_watsonx():
     litellm.set_verbose = True
@@ -3585,6 +3595,7 @@ async def test_acompletion_watsonx():
         pytest.fail(f"Error occurred: {e}")
 
 
+@pytest.mark.skip(reason="IBM closed account.")
 @pytest.mark.asyncio
 async def test_acompletion_stream_watsonx():
     litellm.set_verbose = True
diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py
index 622b2efc84..1c402faa52 100644
--- a/litellm/tests/test_streaming.py
+++ b/litellm/tests/test_streaming.py
@@ -1,12 +1,17 @@
 #### What this tests ####
 #    This tests streaming for the completion endpoint
 
-import sys, os, asyncio
+import asyncio
+import os
+import sys
+import time
 import traceback
-import time, pytest, uuid
-from pydantic import BaseModel
+import uuid
 from typing import Tuple
 
+import pytest
+from pydantic import BaseModel
+
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
@@ -15,12 +20,12 @@ from dotenv import load_dotenv
 load_dotenv()
 import litellm
 from litellm import (
-    completion,
-    acompletion,
     AuthenticationError,
     BadRequestError,
-    RateLimitError,
     ModelResponse,
+    RateLimitError,
+    acompletion,
+    completion,
 )
 
 litellm.logging = False
@@ -1644,9 +1649,8 @@ def test_sagemaker_weird_response():
     When the stream ends, flush any remaining holding chunks.
     """
     try:
-        from litellm.llms.sagemaker import TokenIterator
-        import json
         import json
+
         from litellm.llms.sagemaker import TokenIterator
 
         chunk = """<s>[INST] Hey, how's it going? [/INST],
@@ -1772,6 +1776,7 @@ def test_completion_sagemaker_stream():
         pytest.fail(f"Error occurred: {e}")
 
 
+@pytest.mark.skip(reason="IBM closed account.")
 def test_completion_watsonx_stream():
     litellm.set_verbose = True
     try:
@@ -2631,9 +2636,10 @@ def test_success_callback_streaming():
 
 # test_success_callback_streaming()
 
+from typing import List, Optional
+
 #### STREAMING + FUNCTION CALLING ###
 from pydantic import BaseModel
-from typing import List, Optional
 
 
 class Function(BaseModel):

From 246a417e9b0c27c3ca8f5995d318264d77dbe475 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 19:23:24 -0700
Subject: [PATCH 13/18] test logfire

---
 litellm/tests/test_logfire.py | 82 ++++++++---------------------------
 1 file changed, 18 insertions(+), 64 deletions(-)

diff --git a/litellm/tests/test_logfire.py b/litellm/tests/test_logfire.py
index e3078f5954..5ce06ea69c 100644
--- a/litellm/tests/test_logfire.py
+++ b/litellm/tests/test_logfire.py
@@ -1,12 +1,16 @@
-import sys
-import os
+import asyncio
 import json
+import logging
+import os
+import sys
 import time
 
-import logfire
-import litellm
 import pytest
-from logfire.testing import TestExporter, SimpleSpanProcessor
+
+import litellm
+from litellm._logging import verbose_logger, verbose_proxy_logger
+
+verbose_logger.setLevel(logging.DEBUG)
 
 sys.path.insert(0, os.path.abspath("../.."))
 
@@ -17,19 +21,13 @@ sys.path.insert(0, os.path.abspath("../.."))
 # 4. Test logfire logging for completion while streaming is enabled
 
 
-@pytest.mark.skip(reason="Breaks on ci/cd")
+@pytest.mark.skip(reason="Breaks on ci/cd but works locally")
 @pytest.mark.parametrize("stream", [False, True])
 def test_completion_logfire_logging(stream):
+    from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
+
     litellm.success_callback = ["logfire"]
     litellm.set_verbose = True
-
-    exporter = TestExporter()
-    logfire.configure(
-        send_to_logfire=False,
-        console=False,
-        processors=[SimpleSpanProcessor(exporter)],
-        collect_system_metrics=False,
-    )
     messages = [{"role": "user", "content": "what llm are u"}]
     temperature = 0.3
     max_tokens = 10
@@ -47,41 +45,16 @@ def test_completion_logfire_logging(stream):
             print(chunk)
 
     time.sleep(5)
-    exported_spans = exporter.exported_spans_as_dict()
-
-    assert len(exported_spans) == 1
-    assert (
-        exported_spans[0]["attributes"]["logfire.msg"]
-        == "Chat Completion with 'gpt-3.5-turbo'"
-    )
-
-    request_data = json.loads(exported_spans[0]["attributes"]["request_data"])
-
-    assert request_data["model"] == "gpt-3.5-turbo"
-    assert request_data["messages"] == messages
-
-    assert "completion_tokens" in request_data["usage"]
-    assert "prompt_tokens" in request_data["usage"]
-    assert "total_tokens" in request_data["usage"]
-    assert request_data["response"]["choices"][0]["message"]["content"]
-    assert request_data["modelParameters"]["max_tokens"] == max_tokens
-    assert request_data["modelParameters"]["temperature"] == temperature
 
 
-@pytest.mark.skip(reason="Breaks on ci/cd")
+@pytest.mark.skip(reason="Breaks on ci/cd but works locally")
 @pytest.mark.asyncio
 @pytest.mark.parametrize("stream", [False, True])
 async def test_acompletion_logfire_logging(stream):
+    from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
+
     litellm.success_callback = ["logfire"]
     litellm.set_verbose = True
-
-    exporter = TestExporter()
-    logfire.configure(
-        send_to_logfire=False,
-        console=False,
-        processors=[SimpleSpanProcessor(exporter)],
-        collect_system_metrics=False,
-    )
     messages = [{"role": "user", "content": "what llm are u"}]
     temperature = 0.3
     max_tokens = 10
@@ -90,30 +63,11 @@ async def test_acompletion_logfire_logging(stream):
         messages=messages,
         max_tokens=max_tokens,
         temperature=temperature,
+        stream=stream,
     )
     print(response)
     if stream:
-        for chunk in response:
+        async for chunk in response:
             print(chunk)
 
-    time.sleep(5)
-    exported_spans = exporter.exported_spans_as_dict()
-    print("exported_spans", exported_spans)
-
-    assert len(exported_spans) == 1
-    assert (
-        exported_spans[0]["attributes"]["logfire.msg"]
-        == "Chat Completion with 'gpt-3.5-turbo'"
-    )
-
-    request_data = json.loads(exported_spans[0]["attributes"]["request_data"])
-
-    assert request_data["model"] == "gpt-3.5-turbo"
-    assert request_data["messages"] == messages
-
-    assert "completion_tokens" in request_data["usage"]
-    assert "prompt_tokens" in request_data["usage"]
-    assert "total_tokens" in request_data["usage"]
-    assert request_data["response"]["choices"][0]["message"]["content"]
-    assert request_data["modelParameters"]["max_tokens"] == max_tokens
-    assert request_data["modelParameters"]["temperature"] == temperature
+    await asyncio.sleep(5)

From 94709e8309c7865669dc3fc5deb545d59f42fda2 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 19:33:10 -0700
Subject: [PATCH 14/18] fix deps

---
 docs/my-website/docs/observability/logfire_integration.md | 8 ++++++--
 litellm/litellm_core_utils/litellm_logging.py             | 5 ++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/docs/my-website/docs/observability/logfire_integration.md b/docs/my-website/docs/observability/logfire_integration.md
index 55c15295aa..7b826e0f9f 100644
--- a/docs/my-website/docs/observability/logfire_integration.md
+++ b/docs/my-website/docs/observability/logfire_integration.md
@@ -14,10 +14,14 @@ join our [discord](https://discord.gg/wuPM9dRgDw)
 
 ## Pre-Requisites
 
-Ensure you have run `pip install logfire` for this integration
+Ensure you have installed the following packages to use this integration
 
 ```shell
-pip install logfire litellm
+pip install litellm
+
+pip install opentelemetry-api==1.25.0
+pip install opentelemetry-sdk==1.25.0
+pip install opentelemetry-exporter-otlp==1.25.0
 ```
 
 ## Quick Start
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index b72098fbe7..aceb834f8f 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -64,7 +64,6 @@ from ..integrations.litedebugger import LiteDebugger
 from ..integrations.logfire_logger import LogfireLevel, LogfireLogger
 from ..integrations.lunary import LunaryLogger
 from ..integrations.openmeter import OpenMeterLogger
-from ..integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
 from ..integrations.prometheus import PrometheusLogger
 from ..integrations.prometheus_services import PrometheusServicesLogger
 from ..integrations.prompt_layer import PromptLayerLogger
@@ -1761,6 +1760,10 @@ def _init_custom_logger_compatible_class(
     elif logging_integration == "logfire":
         if "LOGFIRE_TOKEN" not in os.environ:
             raise ValueError("LOGFIRE_TOKEN not found in environment variables")
+        from litellm.integrations.opentelemetry import (
+            OpenTelemetry,
+            OpenTelemetryConfig,
+        )
 
         otel_config = OpenTelemetryConfig(
             exporter="otlp_http",

From 0c7665c71f0abb51bc535a73cdd18399a0e30379 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 19:42:39 -0700
Subject: [PATCH 15/18] fix maintain older usage of logfire

---
 litellm/litellm_core_utils/litellm_logging.py | 48 +++++++++++++++++++
 litellm/utils.py                              |  8 ----
 2 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index aceb834f8f..ee3e4bfbb3 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -667,6 +667,33 @@ class Logging:
                             print_verbose=print_verbose,
                         )
 
+                    if callback == "logfire":
+                        global logfireLogger
+                        verbose_logger.debug("reaches logfire for success logging!")
+                        kwargs = {}
+                        for k, v in self.model_call_details.items():
+                            if (
+                                k != "original_response"
+                            ):  # copy.deepcopy raises errors as this could be a coroutine
+                                kwargs[k] = v
+
+                        # this only logs streaming once, complete_streaming_response exists i.e when stream ends
+                        if self.stream:
+                            if "complete_streaming_response" not in kwargs:
+                                continue
+                            else:
+                                print_verbose("reaches logfire for streaming logging!")
+                                result = kwargs["complete_streaming_response"]
+
+                        logfireLogger.log_event(
+                            kwargs=self.model_call_details,
+                            response_obj=result,
+                            start_time=start_time,
+                            end_time=end_time,
+                            print_verbose=print_verbose,
+                            level=LogfireLevel.INFO.value,
+                        )
+
                     if callback == "lunary":
                         print_verbose("reaches lunary for logging!")
                         model = self.model
@@ -1446,6 +1473,25 @@ class Logging:
                             end_time=end_time,
                             print_verbose=print_verbose,
                         )
+
+                    if callback == "logfire":
+                        verbose_logger.debug("reaches logfire for failure logging!")
+                        kwargs = {}
+                        for k, v in self.model_call_details.items():
+                            if (
+                                k != "original_response"
+                            ):  # copy.deepcopy raises errors as this could be a coroutine
+                                kwargs[k] = v
+                        kwargs["exception"] = exception
+
+                        logfireLogger.log_event(
+                            kwargs=kwargs,
+                            response_obj=result,
+                            start_time=start_time,
+                            end_time=end_time,
+                            level=LogfireLevel.ERROR.value,
+                            print_verbose=print_verbose,
+                        )
                     if callback == "sentry":
                         print_verbose("sending exception to sentry")
                         if capture_exception:
@@ -1712,6 +1758,8 @@ def set_callbacks(callback_list, function_id=None):
                 weightsBiasesLogger = WeightsBiasesLogger()
             elif callback == "langsmith":
                 langsmithLogger = LangsmithLogger()
+            elif callback == "logfire":
+                logfireLogger = LogfireLogger()
             elif callback == "aispend":
                 aispendLogger = AISpendLogger()
             elif callback == "berrispend":
diff --git a/litellm/utils.py b/litellm/utils.py
index e84a262afa..2c953b93eb 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -341,14 +341,6 @@ def function_setup(
         global callback_list, add_breadcrumb, user_logger_fn, Logging
         function_id = kwargs["id"] if "id" in kwargs else None
 
-        # Note: maintain backwards compatibility with logfire. some users use litellm.success_callback=["logfire"]
-        if (
-            litellm.success_callback is not None
-            and "logfire" in litellm.success_callback
-        ):
-            litellm.success_callback.remove("logfire")
-            litellm.callbacks.append("logfire")
-
         if len(litellm.callbacks) > 0:
             for callback in litellm.callbacks:
                 # check if callback is a string - e.g. "lago", "openmeter"

From ef20499bd996c539ec953f24c9d0a507ca3501c5 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 19:45:22 -0700
Subject: [PATCH 16/18] use older logfire logging

---
 litellm/litellm_core_utils/litellm_logging.py | 39 +++++++++----------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index ee3e4bfbb3..71cc8e6ab9 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -666,7 +666,6 @@ class Logging:
                             end_time=end_time,
                             print_verbose=print_verbose,
                         )
-
                     if callback == "logfire":
                         global logfireLogger
                         verbose_logger.debug("reaches logfire for success logging!")
@@ -1473,25 +1472,6 @@ class Logging:
                             end_time=end_time,
                             print_verbose=print_verbose,
                         )
-
-                    if callback == "logfire":
-                        verbose_logger.debug("reaches logfire for failure logging!")
-                        kwargs = {}
-                        for k, v in self.model_call_details.items():
-                            if (
-                                k != "original_response"
-                            ):  # copy.deepcopy raises errors as this could be a coroutine
-                                kwargs[k] = v
-                        kwargs["exception"] = exception
-
-                        logfireLogger.log_event(
-                            kwargs=kwargs,
-                            response_obj=result,
-                            start_time=start_time,
-                            end_time=end_time,
-                            level=LogfireLevel.ERROR.value,
-                            print_verbose=print_verbose,
-                        )
                     if callback == "sentry":
                         print_verbose("sending exception to sentry")
                         if capture_exception:
@@ -1616,6 +1596,25 @@ class Logging:
                             print_verbose=print_verbose,
                         )
 
+                    if callback == "logfire":
+                        verbose_logger.debug("reaches logfire for failure logging!")
+                        kwargs = {}
+                        for k, v in self.model_call_details.items():
+                            if (
+                                k != "original_response"
+                            ):  # copy.deepcopy raises errors as this could be a coroutine
+                                kwargs[k] = v
+                        kwargs["exception"] = exception
+
+                        logfireLogger.log_event(
+                            kwargs=kwargs,
+                            response_obj=result,
+                            start_time=start_time,
+                            end_time=end_time,
+                            level=LogfireLevel.ERROR.value,
+                            print_verbose=print_verbose,
+                        )
+
                 except Exception as e:
                     print_verbose(
                         f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while failure logging with integrations {str(e)}"

From 51d5985bca312a9e5907aa3c26b98ed18a730fb6 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 19:47:31 -0700
Subject: [PATCH 17/18] docs - logfire

---
 docs/my-website/docs/observability/logfire_integration.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/my-website/docs/observability/logfire_integration.md b/docs/my-website/docs/observability/logfire_integration.md
index 7b826e0f9f..a2d406f9c6 100644
--- a/docs/my-website/docs/observability/logfire_integration.md
+++ b/docs/my-website/docs/observability/logfire_integration.md
@@ -29,8 +29,7 @@ pip install opentelemetry-exporter-otlp==1.25.0
 Get your Logfire token from [Logfire](https://logfire.pydantic.dev/)
 
 ```python
-litellm.success_callback = ["logfire"]
-litellm.failure_callback = ["logfire"] # logs errors to logfire
+litellm.callbacks = ["logfire"]
 ```
 
 ```python

From 094fbdf1518704e04b0072daf6c632c6243c19d7 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 17 Jun 2024 19:47:48 -0700
Subject: [PATCH 18/18] test - logfire

---
 litellm/tests/test_logfire.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/litellm/tests/test_logfire.py b/litellm/tests/test_logfire.py
index 5ce06ea69c..34bd75ccae 100644
--- a/litellm/tests/test_logfire.py
+++ b/litellm/tests/test_logfire.py
@@ -26,7 +26,7 @@ sys.path.insert(0, os.path.abspath("../.."))
 def test_completion_logfire_logging(stream):
     from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
 
-    litellm.success_callback = ["logfire"]
+    litellm.callbacks = ["logfire"]
     litellm.set_verbose = True
     messages = [{"role": "user", "content": "what llm are u"}]
     temperature = 0.3
@@ -53,7 +53,7 @@ def test_completion_logfire_logging(stream):
 async def test_acompletion_logfire_logging(stream):
     from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
 
-    litellm.success_callback = ["logfire"]
+    litellm.callbacks = ["logfire"]
     litellm.set_verbose = True
     messages = [{"role": "user", "content": "what llm are u"}]
     temperature = 0.3