test(router_code_coverage.py): check if all router functions are dire… (#6186)

* test(router_code_coverage.py): check if all router functions are directly tested prevent regressions * docs(configs.md): document all environment variables (#6185) * docs: make it easier to find anthropic/openai prompt caching doc * aded codecov yml (#6207) * fix codecov.yaml * run ci/cd again * (refactor) caching use LLMCachingHandler for async_get_cache and set_cache (#6208) * use folder for caching * fix importing caching * fix clickhouse pyright * fix linting * fix correctly pass kwargs and args * fix test case for embedding * fix linting * fix embedding caching logic * fix refactor handle utils.py * fix test_embedding_caching_azure_individual_items_reordered * (feat) prometheus have well defined latency buckets (#6211) * fix prometheus have well defined latency buckets * use a well define latency bucket * use types file for prometheus logging * add test for LATENCY_BUCKETS * fix prom testing * fix config.yml * (refactor caching) use LLMCachingHandler for caching streaming responses (#6210) * use folder for caching * fix importing caching * fix clickhouse pyright * fix linting * fix correctly pass kwargs and args * fix test case for embedding * fix linting * fix embedding caching logic * fix refactor handle utils.py * refactor async set stream cache * fix linting * bump (#6187) * update code cov yaml * fix config.yml * add caching component to code cov * fix config.yml ci/cd * add coverage for proxy auth * (refactor caching) use common `_retrieve_from_cache` helper (#6212) * use folder for caching * fix importing caching * fix clickhouse pyright * fix linting * fix correctly pass kwargs and args * fix test case for embedding * fix linting * fix embedding caching logic * fix refactor handle utils.py * refactor async set stream cache * fix linting * refactor - use _retrieve_from_cache * refactor use _convert_cached_result_to_model_response * fix linting errors * bump: version 1.49.2 → 1.49.3 * fix code cov components * test(test_router_helpers.py): add router component unit tests * test: add additional router tests * test: add more router testing * test: add more router testing + more mock functions * ci(router_code_coverage.py): fix check --------- Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com> Co-authored-by: yujonglee <yujonglee.dev@gmail.com>
2025-04-25 10:44:24 +00:00 · 2024-10-14 22:44:00 -07:00 · 2024-10-14 22:44:00 -07:00 · 1eb435e50a
commit 1eb435e50a
parent 39486e2003
15 changed files with 768 additions and 164 deletions
--- a/tests/code_coverage_tests/router_code_coverage.py
+++ b/tests/code_coverage_tests/router_code_coverage.py
@ -0,0 +1,117 @@
+import ast
+import os
+
+
+def get_function_names_from_file(file_path):
+    """
+    Extracts all function names from a given Python file.
+    """
+    with open(file_path, "r") as file:
+        tree = ast.parse(file.read())
+
+    function_names = []
+
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            function_names.append(node.name)
+
+    return function_names
+
+
+def get_all_functions_called_in_tests(base_dir):
+    """
+    Returns a set of function names that are called in test functions
+    inside 'local_testing' and 'router_unit_test' directories,
+    specifically in files containing the word 'router'.
+    """
+    called_functions = set()
+    test_dirs = ["local_testing", "router_unit_tests"]
+
+    for test_dir in test_dirs:
+        dir_path = os.path.join(base_dir, test_dir)
+        if not os.path.exists(dir_path):
+            print(f"Warning: Directory {dir_path} does not exist.")
+            continue
+
+        print("dir_path: ", dir_path)
+        for root, _, files in os.walk(dir_path):
+            for file in files:
+                if file.endswith(".py") and "router" in file.lower():
+                    print("file: ", file)
+                    file_path = os.path.join(root, file)
+                    with open(file_path, "r") as f:
+                        try:
+                            tree = ast.parse(f.read())
+                        except SyntaxError:
+                            print(f"Warning: Syntax error in file {file_path}")
+                            continue
+                    if file == "test_router_validate_fallbacks.py":
+                        print(f"tree: {tree}")
+                    for node in ast.walk(tree):
+                        if isinstance(node, ast.Call) and isinstance(
+                            node.func, ast.Name
+                        ):
+                            called_functions.add(node.func.id)
+                        elif isinstance(node, ast.Call) and isinstance(
+                            node.func, ast.Attribute
+                        ):
+                            called_functions.add(node.func.attr)
+
+    return called_functions
+
+
+def get_functions_from_router(file_path):
+    """
+    Extracts all functions defined in router.py.
+    """
+    return get_function_names_from_file(file_path)
+
+
+ignored_function_names = [
+    "__init__",
+    "_acreate_file",
+    "_acreate_batch",
+    "acreate_assistants",
+    "adelete_assistant",
+    "aget_assistants",
+    "acreate_thread",
+    "aget_thread",
+    "a_add_message",
+    "aget_messages",
+    "arun_thread",
+]
+
+
+def main():
+    router_file = "./litellm/router.py"  # Update this path if it's located elsewhere
+    # router_file = "../../litellm/router.py"  ## LOCAL TESTING
+    tests_dir = (
+        "./tests/"  # Update this path if your tests directory is located elsewhere
+    )
+    # tests_dir = "../../tests/"  # LOCAL TESTING
+
+    router_functions = get_functions_from_router(router_file)
+    print("router_functions: ", router_functions)
+    called_functions_in_tests = get_all_functions_called_in_tests(tests_dir)
+    untested_functions = [
+        fn for fn in router_functions if fn not in called_functions_in_tests
+    ]
+
+    if untested_functions:
+        all_untested_functions = []
+        for func in untested_functions:
+            if func not in ignored_function_names:
+                all_untested_functions.append(func)
+        untested_perc = (len(all_untested_functions)) / len(router_functions)
+        print("perc_covered: ", untested_perc)
+        if untested_perc < 0.3:
+            print("The following functions in router.py are not tested:")
+            raise Exception(
+                f"{untested_perc * 100:.2f}% of functions in router.py are not tested: {all_untested_functions}"
+            )
+    else:
+        print("All functions in router.py are covered by tests.")
+
+
+if __name__ == "__main__":
+    main()