add enforce_llms_folder_style (#7175)

.circleci/config.yml

@@ -816,6 +816,7 @@ jobs:
       - run: python ./tests/documentation_tests/test_router_settings.py
       - run: python ./tests/documentation_tests/test_api_docs.py
       - run: python ./tests/code_coverage_tests/ensure_async_clients_test.py
+      - run: python ./tests/code_coverage_tests/enforce_llms_folder_style.py
       - run: helm lint ./deploy/charts/litellm-helm
 
   db_migration_disable_update_check:

tests/code_coverage_tests/enforce_llms_folder_style.py

@@ -2,31 +2,37 @@ import ast
 import os
 import sys
 
-sys.path.insert(
-    0, os.path.abspath("../..")
-) 
+sys.path.insert(0, os.path.abspath("../.."))
 
 import litellm
 
-ALLOWED_FILES_IN_LLMS_FOLDER = ["__init__", "base", "base_llm", "custom_httpx", "custom_llm", "deprecated_providers"]
+ALLOWED_FILES_IN_LLMS_FOLDER = [
+    "__init__",
+    "base",
+    "base_llm",
+    "custom_httpx",
+    "custom_llm",
+    "deprecated_providers",
+]
 
-def get_unique_names_from_llms_dir(base_dir="./litellm/llms/"):
+
+def get_unique_names_from_llms_dir(base_dir: str):
     """
     Returns a set of unique file and folder names from the root level of litellm/llms directory,
     excluding file extensions and __init__.py
     """
     unique_names = set()
-    
+
     if not os.path.exists(base_dir):
         print(f"Warning: Directory {base_dir} does not exist.")
         return unique_names
 
     # Get only root level items
     items = os.listdir(base_dir)
-    
+
     for item in items:
         item_path = os.path.join(base_dir, item)
-        
+
         if os.path.isdir(item_path):
             if item != "__pycache__":
                 unique_names.add(item)
@@ -41,21 +47,26 @@ def run_lint_check(unique_names):
     _all_litellm_providers = [str(provider.value) for provider in litellm.LlmProviders]
     violations = []
     for name in unique_names:
-        if name.lower() not in _all_litellm_providers and name not in ALLOWED_FILES_IN_LLMS_FOLDER:
+        if (
+            name.lower() not in _all_litellm_providers
+            and name not in ALLOWED_FILES_IN_LLMS_FOLDER
+        ):
             violations.append(name)
 
     if len(violations) > 0:
-        raise ValueError(f"There are {len(violations)} violations in the llms folder. \n\n {violations}. \n\n Valid providers: {_all_litellm_providers}")
+        raise ValueError(
+            f"There are {len(violations)} violations in the llms folder. \n\n {violations}. \n\n Valid providers: {_all_litellm_providers}"
+        )
+
 
 def main():
     llms_dir = "./litellm/llms/"  # Update this path if needed
-    llms_dir = "../../litellm/llms/"  # LOCAL TESTING
-    
+    # llms_dir = "../../litellm/llms/"  # LOCAL TESTING
+
     unique_names = get_unique_names_from_llms_dir(llms_dir)
     print("Unique names in llms directory:", sorted(list(unique_names)))
     run_lint_check(unique_names)
 
 
-
 if __name__ == "__main__":
-    main()
+    main()