Litellm dev 11 08 2024 (#6658)

* fix(deepseek/chat): convert content list to str

Fixes https://github.com/BerriAI/litellm/issues/6642

* test(test_deepseek_completion.py): implement base llm unit tests

increase robustness across providers

* fix(router.py): support content policy violation fallbacks with default fallbacks

* fix(opentelemetry.py): refactor to move otel imports behing flag

Fixes https://github.com/BerriAI/litellm/issues/6636

* fix(opentelemtry.py): close span on success completion

* fix(user_api_key_auth.py): allow user_role to default to none

* fix: mark flaky test

* fix(opentelemetry.py): move otelconfig.from_env to inside the init

prevent otel errors raised just by importing the litellm class

* fix(user_api_key_auth.py): fix auth error

tests/llm_translation/base_llm_unit_tests.py

@@ -0,0 +1,46 @@
+import asyncio
+import httpx
+import json
+import pytest
+import sys
+from typing import Any, Dict, List
+from unittest.mock import MagicMock, Mock, patch
+import os
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import litellm
+from litellm.exceptions import BadRequestError
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.utils import CustomStreamWrapper
+
+
+# test_example.py
+from abc import ABC, abstractmethod
+
+
+class BaseLLMChatTest(ABC):
+    """
+    Abstract base test class that enforces a common test across all test classes.
+    """
+
+    @abstractmethod
+    def get_base_completion_call_args(self) -> dict:
+        """Must return the base completion call args"""
+        pass
+
+    def test_content_list_handling(self):
+        """Check if content list is supported by LLM API"""
+        base_completion_call_args = self.get_base_completion_call_args()
+        messages = [
+            {
+                "role": "user",
+                "content": [{"type": "text", "text": "Hello, how are you?"}],
+            }
+        ]
+        response = litellm.completion(
+            **base_completion_call_args,
+            messages=messages,
+        )
+        assert response is not None

tests/llm_translation/test_deepseek_completion.py

@@ -0,0 +1,9 @@
+from base_llm_unit_tests import BaseLLMChatTest
+
+
+# Test implementation
+class TestDeepSeekChatCompletion(BaseLLMChatTest):
+    def get_base_completion_call_args(self) -> dict:
+        return {
+            "model": "deepseek/deepseek-chat",
+        }

tests/local_testing/test_completion.py

@@ -4526,6 +4526,7 @@ async def test_completion_ai21_chat():
     "stream",
     [False, True],
 )
+@pytest.mark.flaky(retries=3, delay=1)
 def test_completion_response_ratelimit_headers(model, stream):
     response = completion(
         model=model,

tests/local_testing/test_opentelemetry_unit_tests.py

@@ -0,0 +1,41 @@
+# What is this?
+## Unit tests for opentelemetry integration
+
+# What is this?
+## Unit test for presidio pii masking
+import sys, os, asyncio, time, random
+from datetime import datetime
+import traceback
+from dotenv import load_dotenv
+
+load_dotenv()
+import os
+import asyncio
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import pytest
+import litellm
+from unittest.mock import patch, MagicMock, AsyncMock
+
+
+@pytest.mark.asyncio
+async def test_opentelemetry_integration():
+    """
+    Unit test to confirm the parent otel span is ended
+    """
+
+    parent_otel_span = MagicMock()
+    litellm.callbacks = ["otel"]
+
+    await litellm.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "Hello, world!"}],
+        mock_response="Hey!",
+        metadata={"litellm_parent_otel_span": parent_otel_span},
+    )
+
+    await asyncio.sleep(1)
+
+    parent_otel_span.end.assert_called_once()

tests/local_testing/test_python_38.py

@@ -72,6 +72,19 @@ def test_litellm_proxy_server_config_no_general_settings():
         # Check if the response is successful
         assert response.status_code == 200
         assert response.json() == "I'm alive!"
+
+        # Test /chat/completions
+        response = requests.post(
+            "http://localhost:4000/chat/completions",
+            headers={"Authorization": "Bearer 1234567890"},
+            json={
+                "model": "test_openai_models",
+                "messages": [{"role": "user", "content": "Hello, how are you?"}],
+            },
+        )
+
+        assert response.status_code == 200
+
     except ImportError:
         pytest.fail("Failed to import litellm.proxy_server")
     except requests.ConnectionError:

tests/local_testing/test_router_fallbacks.py

@@ -1120,9 +1120,10 @@ async def test_client_side_fallbacks_list(sync_mode):
 
 @pytest.mark.parametrize("sync_mode", [True, False])
 @pytest.mark.parametrize("content_filter_response_exception", [True, False])
+@pytest.mark.parametrize("fallback_type", ["model-specific", "default"])
 @pytest.mark.asyncio
 async def test_router_content_policy_fallbacks(
-    sync_mode, content_filter_response_exception
+    sync_mode, content_filter_response_exception, fallback_type
 ):
     os.environ["LITELLM_LOG"] = "DEBUG"
 
@@ -1152,6 +1153,14 @@ async def test_router_content_policy_fallbacks(
                     "mock_response": "This works!",
                 },
             },
+            {
+                "model_name": "my-default-fallback-model",
+                "litellm_params": {
+                    "model": "openai/my-fake-model",
+                    "api_key": "",
+                    "mock_response": "This works 2!",
+                },
+            },
             {
                 "model_name": "my-general-model",
                 "litellm_params": {
@@ -1169,9 +1178,14 @@ async def test_router_content_policy_fallbacks(
                 },
             },
         ],
-        content_policy_fallbacks=[{"claude-2": ["my-fallback-model"]}],
-        fallbacks=[{"claude-2": ["my-general-model"]}],
-        context_window_fallbacks=[{"claude-2": ["my-context-window-model"]}],
+        content_policy_fallbacks=(
+            [{"claude-2": ["my-fallback-model"]}]
+            if fallback_type == "model-specific"
+            else None
+        ),
+        default_fallbacks=(
+            ["my-default-fallback-model"] if fallback_type == "default" else None
+        ),
     )
 
     if sync_mode is True:

tests/router_unit_tests/test_router_helper_utils.py

@@ -452,11 +452,17 @@ def test_update_usage(model_list):
 
 
 @pytest.mark.parametrize(
-    "finish_reason, expected_error", [("content_filter", True), ("stop", False)]
+    "finish_reason, expected_fallback", [("content_filter", True), ("stop", False)]
 )
-def test_should_raise_content_policy_error(model_list, finish_reason, expected_error):
+@pytest.mark.parametrize("fallback_type", ["model-specific", "default"])
+def test_should_raise_content_policy_error(
+    model_list, finish_reason, expected_fallback, fallback_type
+):
     """Test if the '_should_raise_content_policy_error' function is working correctly"""
-    router = Router(model_list=model_list)
+    router = Router(
+        model_list=model_list,
+        default_fallbacks=["gpt-4o"] if fallback_type == "default" else None,
+    )
 
     assert (
         router._should_raise_content_policy_error(
@@ -472,10 +478,14 @@ def test_should_raise_content_policy_error(model_list, finish_reason, expected_e
                 usage={"total_tokens": 100},
             ),
             kwargs={
-                "content_policy_fallbacks": [{"gpt-3.5-turbo": "gpt-4o"}],
+                "content_policy_fallbacks": (
+                    [{"gpt-3.5-turbo": "gpt-4o"}]
+                    if fallback_type == "model-specific"
+                    else None
+                )
             },
         )
-        is expected_error
+        is expected_fallback
     )
 
 
@@ -1019,3 +1029,17 @@ async def test_pass_through_moderation_endpoint_factory(model_list):
     response = await router._pass_through_moderation_endpoint_factory(
         original_function=litellm.amoderation, input="this is valid good text"
     )
+
+
+@pytest.mark.parametrize(
+    "has_default_fallbacks, expected_result",
+    [(True, True), (False, False)],
+)
+def test_has_default_fallbacks(model_list, has_default_fallbacks, expected_result):
+    router = Router(
+        model_list=model_list,
+        default_fallbacks=(
+            ["my-default-fallback-model"] if has_default_fallbacks else None
+        ),
+    )
+    assert router._has_default_fallbacks() is expected_result

tests/test_team.py

@@ -362,7 +362,7 @@ async def test_team_info():
 
         try:
             await get_team_info(session=session, get_team=team_id, call_key=key)
-            pytest.fail(f"Expected call to fail")
+            pytest.fail("Expected call to fail")
         except Exception as e:
             pass