LiteLLM Minor Fixes & Improvements (11/26/2024) (#6913)

* docs(config_settings.md): document all router_settings

* ci(config.yml): add router_settings doc test to ci/cd

* test: debug test on ci/cd

* test: debug ci/cd test

* test: fix test

* fix(team_endpoints.py): skip invalid team object. don't fail `/team/list` call

Causes downstream errors if ui just fails to load team list

* test(base_llm_unit_tests.py): add 'response_format={"type": "text"}' test to base_llm_unit_tests

adds complete coverage for all 'response_format' values to ci/cd

* feat(router.py): support wildcard routes in `get_router_model_info()`

Addresses https://github.com/BerriAI/litellm/issues/6914

* build(model_prices_and_context_window.json): add tpm/rpm limits for all gemini models

Allows for ratelimit tracking for gemini models even with wildcard routing enabled

Addresses https://github.com/BerriAI/litellm/issues/6914

* feat(router.py): add tpm/rpm tracking on success/failure to global_router

Addresses https://github.com/BerriAI/litellm/issues/6914

* feat(router.py): support wildcard routes on router.get_model_group_usage()

* fix(router.py): fix linting error

* fix(router.py): implement get_remaining_tokens_and_requests

Addresses https://github.com/BerriAI/litellm/issues/6914

* fix(router.py): fix linting errors

* test: fix test

* test: fix tests

* docs(config_settings.md): add missing dd env vars to docs

* fix(router.py): check if hidden params is dict

tests/documentation_tests/test_router_settings.py

@@ -0,0 +1,87 @@
+import os
+import re
+import inspect
+from typing import Type
+import sys
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)  # Adds the parent directory to the system path
+import litellm
+
+
+def get_init_params(cls: Type) -> list[str]:
+    """
+    Retrieve all parameters supported by the `__init__` method of a given class.
+
+    Args:
+        cls: The class to inspect.
+
+    Returns:
+        A list of parameter names.
+    """
+    if not hasattr(cls, "__init__"):
+        raise ValueError(
+            f"The provided class {cls.__name__} does not have an __init__ method."
+        )
+
+    init_method = cls.__init__
+    argspec = inspect.getfullargspec(init_method)
+
+    # The first argument is usually 'self', so we exclude it
+    return argspec.args[1:]  # Exclude 'self'
+
+
+router_init_params = set(get_init_params(litellm.router.Router))
+print(router_init_params)
+router_init_params.remove("model_list")
+
+# Parse the documentation to extract documented keys
+repo_base = "./"
+print(os.listdir(repo_base))
+docs_path = (
+    "./docs/my-website/docs/proxy/config_settings.md"  # Path to the documentation
+)
+# docs_path = (
+#     "../../docs/my-website/docs/proxy/config_settings.md"  # Path to the documentation
+# )
+documented_keys = set()
+try:
+    with open(docs_path, "r", encoding="utf-8") as docs_file:
+        content = docs_file.read()
+
+        # Find the section titled "general_settings - Reference"
+        general_settings_section = re.search(
+            r"### router_settings - Reference(.*?)###", content, re.DOTALL
+        )
+        if general_settings_section:
+            # Extract the table rows, which contain the documented keys
+            table_content = general_settings_section.group(1)
+            doc_key_pattern = re.compile(
+                r"\|\s*([^\|]+?)\s*\|"
+            )  # Capture the key from each row of the table
+            documented_keys.update(doc_key_pattern.findall(table_content))
+except Exception as e:
+    raise Exception(
+        f"Error reading documentation: {e}, \n repo base - {os.listdir(repo_base)}"
+    )
+
+
+# Compare and find undocumented keys
+undocumented_keys = router_init_params - documented_keys
+
+# Print results
+print("Keys expected in 'router settings' (found in code):")
+for key in sorted(router_init_params):
+    print(key)
+
+if undocumented_keys:
+    raise Exception(
+        f"\nKeys not documented in 'router settings - Reference': {undocumented_keys}"
+    )
+else:
+    print(
+        "\nAll keys are documented in 'router settings - Reference'. - {}".format(
+            router_init_params
+        )
+    )