Litellm dev 10 26 2024 (#6472)

* docs(exception_mapping.md): add missing exception types

Fixes https://github.com/Aider-AI/aider/issues/2120#issuecomment-2438971183

* fix(main.py): register custom model pricing with specific key

Ensure custom model pricing is registered to the specific model+provider key combination

* test: make testing more robust for custom pricing

* fix(redis_cache.py): instrument otel logging for sync redis calls

ensures complete coverage for all redis cache calls

tests/documentation_tests/test_exception_types.py

@@ -0,0 +1,81 @@
+import os
+import sys
+import traceback
+
+from dotenv import load_dotenv
+
+load_dotenv()
+import io
+import re
+
+# Backup the original sys.path
+original_sys_path = sys.path.copy()
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import litellm
+
+public_exceptions = litellm.LITELLM_EXCEPTION_TYPES
+# Regular expression to extract the error name
+error_name_pattern = re.compile(r"\.exceptions\.([A-Za-z]+Error)")
+
+# Extract error names from each item
+error_names = {
+    error_name_pattern.search(str(item)).group(1)
+    for item in public_exceptions
+    if error_name_pattern.search(str(item))
+}
+
+
+# sys.path = original_sys_path
+
+
+# Parse the documentation to extract documented keys
+# repo_base = "./"
+repo_base = "../../"
+print(os.listdir(repo_base))
+docs_path = f"{repo_base}/docs/my-website/docs/exception_mapping.md"  # Path to the documentation
+documented_keys = set()
+try:
+    with open(docs_path, "r", encoding="utf-8") as docs_file:
+        content = docs_file.read()
+
+        exceptions_section = re.search(
+            r"## LiteLLM Exceptions(.*?)\n##", content, re.DOTALL
+        )
+        if exceptions_section:
+            # Step 2: Extract the table content
+            table_content = exceptions_section.group(1)
+
+            # Step 3: Create a pattern to capture the Error Types from each row
+            error_type_pattern = re.compile(r"\|\s*[^|]+\s*\|\s*([^\|]+?)\s*\|")
+
+            # Extract the error types
+            exceptions = error_type_pattern.findall(table_content)
+            print(f"exceptions: {exceptions}")
+
+            # Remove extra spaces if any
+            exceptions = [exception.strip() for exception in exceptions]
+
+            print(exceptions)
+            documented_keys.update(exceptions)
+
+except Exception as e:
+    raise Exception(
+        f"Error reading documentation: {e}, \n repo base - {os.listdir(repo_base)}"
+    )
+
+print(documented_keys)
+print(public_exceptions)
+print(error_names)
+
+# Compare and find undocumented keys
+undocumented_keys = error_names - documented_keys
+
+if undocumented_keys:
+    raise Exception(
+        f"\nKeys not documented in 'LiteLLM Exceptions': {undocumented_keys}"
+    )
+else:
+    print("\nAll keys are documented in 'LiteLLM Exceptions'. - {}".format(error_names))

tests/local_testing/test_router_fallbacks.py

@@ -1337,3 +1337,64 @@ async def test_anthropic_streaming_fallbacks(sync_mode):
         mock_client.assert_called_once()
         print(chunks)
         assert len(chunks) > 0
+
+
+def test_router_fallbacks_with_custom_model_costs():
+    """
+    Tests prod use-case where a custom model is registered with a different provider + custom costs.
+
+    Goal: make sure custom model doesn't override default model costs.
+    """
+    model_list = [
+        {
+            "model_name": "claude-3-5-sonnet-20240620",
+            "litellm_params": {
+                "model": "claude-3-5-sonnet-20240620",
+                "api_key": os.environ["ANTHROPIC_API_KEY"],
+                "input_cost_per_token": 30,
+                "output_cost_per_token": 60,
+            },
+        },
+        {
+            "model_name": "claude-3-5-sonnet-aihubmix",
+            "litellm_params": {
+                "model": "openai/claude-3-5-sonnet-20240620",
+                "input_cost_per_token": 0.000003,  # 3$/M
+                "output_cost_per_token": 0.000015,  # 15$/M
+                "api_base": "https://exampleopenaiendpoint-production.up.railway.app",
+                "api_key": "my-fake-key",
+            },
+        },
+    ]
+
+    router = Router(
+        model_list=model_list,
+        fallbacks=[{"claude-3-5-sonnet-20240620": ["claude-3-5-sonnet-aihubmix"]}],
+    )
+
+    router.completion(
+        model="claude-3-5-sonnet-aihubmix",
+        messages=[{"role": "user", "content": "Hey, how's it going?"}],
+    )
+
+    model_info = litellm.get_model_info(model="claude-3-5-sonnet-20240620")
+
+    print(f"key: {model_info['key']}")
+
+    assert model_info["litellm_provider"] == "anthropic"
+
+    response = router.completion(
+        model="claude-3-5-sonnet-20240620",
+        messages=[{"role": "user", "content": "Hey, how's it going?"}],
+    )
+
+    print(f"response_cost: {response._hidden_params['response_cost']}")
+
+    assert response._hidden_params["response_cost"] > 10
+
+    model_info = litellm.get_model_info(model="claude-3-5-sonnet-20240620")
+
+    print(f"key: {model_info['key']}")
+
+    assert model_info["input_cost_per_token"] == 30
+    assert model_info["output_cost_per_token"] == 60