test: initial commit enforcing testing on all anthropic pass through … (#7794)

* test: initial commit enforcing testing on all anthropic pass through functions

prevents future regressions

* test(test_unit_test_anthropic_pass_through.py): add unit test for '_get_user_from_metadata' function

* test(test_unit_test_anthropic_passthrough.py): add unit test for handle_logging_anthropic_collected_chunks

* test(test_unit_test_anthropic_pass_through): add coverage for all anthropic pass through functions

litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py

@@ -162,6 +162,7 @@ class AnthropicPassthroughLoggingHandler:
         - Creates standard logging object
         - Logs in litellm callbacks
         """
+
         model = request_body.get("model", "")
         complete_streaming_response = (
             AnthropicPassthroughLoggingHandler._build_complete_streaming_response(

tests/code_coverage_tests/pass_through_code_coverage.py

@@ -0,0 +1,125 @@
+import ast
+import os
+
+pass_through_classes = [
+    "AnthropicPassthroughLoggingHandler",
+]
+
+
+def get_function_names_from_file(file_path):
+    """
+    Extracts all function names from a given Python file.
+    """
+    with open(file_path, "r") as file:
+        tree = ast.parse(file.read())
+
+    function_names = []
+
+    for node in tree.body:
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            function_names.append(node.name)
+        elif isinstance(node, ast.ClassDef):
+            if node.name in pass_through_classes:
+                for class_node in node.body:
+                    if isinstance(class_node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                        function_names.append(class_node.name)
+
+    return function_names
+
+
+def get_all_functions_called_in_tests(base_dir):
+    """
+    Returns a set of function names that are called in test functions
+    inside 'local_testing' and 'router_unit_test' directories,
+    specifically in files containing the word 'router'.
+    """
+    called_functions = set()
+    test_dirs = ["pass_through_unit_tests"]
+
+    for test_dir in test_dirs:
+        dir_path = os.path.join(base_dir, test_dir)
+        if not os.path.exists(dir_path):
+            print(f"Warning: Directory {dir_path} does not exist.")
+            continue
+
+        print("dir_path: ", dir_path)
+        for root, _, files in os.walk(dir_path):
+            for file in files:
+                if file.endswith(".py"):
+                    print("file: ", file)
+                    file_path = os.path.join(root, file)
+                    with open(file_path, "r") as f:
+                        try:
+                            tree = ast.parse(f.read())
+                        except SyntaxError:
+                            print(f"Warning: Syntax error in file {file_path}")
+                            continue
+
+                    for node in ast.walk(tree):
+                        if isinstance(node, ast.Call):
+                            if isinstance(node.func, ast.Attribute) and isinstance(
+                                node.func.value, ast.Name
+                            ):
+                                if node.func.value.id in pass_through_classes:
+                                    # If it's called via the class, add both the original and non-underscore versions
+                                    method_name = node.func.attr
+                                    called_functions.add(method_name)
+                                    if method_name.startswith("_"):
+                                        called_functions.add(method_name.lstrip("_"))
+
+    return called_functions
+
+
+def get_functions_from_router(file_path):
+    """
+    Extracts all functions defined in router.py.
+    """
+    return get_function_names_from_file(file_path)
+
+
+ignored_function_names = [
+    "__init__",
+]
+
+
+def main():
+    router_file = [
+        "../../litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py",
+    ]
+    tests_dir = "../../tests/"
+
+    router_functions = []
+    for file in router_file:
+        router_functions.extend(get_functions_from_router(file))
+    print("router_functions: ", router_functions)
+    called_functions_in_tests = get_all_functions_called_in_tests(tests_dir)
+    print("called_functions_in_tests: ", called_functions_in_tests)
+
+    untested_functions = []
+    for fn in router_functions:
+        # Check if the function is called either with or without leading underscore
+        clean_name = fn.lstrip("_")
+        if (
+            fn not in called_functions_in_tests
+            and clean_name not in called_functions_in_tests
+        ):
+            untested_functions.append(fn)
+
+    if untested_functions:
+        all_untested_functions = []
+        for func in untested_functions:
+            if func not in ignored_function_names:
+                all_untested_functions.append(func)
+        untested_perc = (len(all_untested_functions)) / len(router_functions)
+        print("untested_perc: ", untested_perc)
+        if untested_perc > 0:
+            print("The following functions in pass_through/ are not tested:")
+            raise Exception(
+                f"{untested_perc * 100:.2f}% of functions in pass_through/ are not tested: {all_untested_functions}"
+            )
+    else:
+        print("All functions in router.py are covered by tests.")
+
+
+if __name__ == "__main__":
+    main()

tests/pass_through_tests/test_anthropic_passthrough.py

@@ -1,6 +1,5 @@
 """
 This test ensures that the proxy can passthrough anthropic requests
-
 """
 
 import pytest

tests/pass_through_unit_tests/test_unit_test_anthropic_pass_through.py

@@ -201,3 +201,177 @@ def test_create_anthropic_response_logging_payload(mock_logging_obj, metadata_pa
     assert "model" in result
     assert "response_cost" in result
     assert "standard_logging_object" in result
+    if metadata_params:
+        assert "test" == result["standard_logging_object"]["end_user"]
+    else:
+        assert "" == result["standard_logging_object"]["end_user"]
+
+
+@pytest.mark.parametrize(
+    "end_user_id",
+    [{"litellm_metadata": {"user": "test"}}, {"metadata": {"user_id": "test"}}],
+)
+def test_get_user_from_metadata(end_user_id):
+    from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
+        AnthropicPassthroughLoggingHandler,
+        PassthroughStandardLoggingPayload,
+    )
+
+    passthrough_logging_payload = PassthroughStandardLoggingPayload(
+        url="https://api.anthropic.com/v1/messages",
+        request_body={**end_user_id},
+        response_body={
+            "id": "msg_015uSaCZBvu9gUSkAmZtMfxC",
+            "type": "message",
+            "role": "assistant",
+            "model": "claude-3-5-sonnet-20241022",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "Now I'll click on the Firefox icon to launch it.",
+                },
+                {
+                    "type": "tool_use",
+                    "id": "toolu_01TQsF5p7Pf4LGKyLUDDySVr",
+                    "name": "computer",
+                    "input": {"action": "mouse_move", "coordinate": [24, 36]},
+                },
+            ],
+            "stop_reason": "tool_use",
+            "stop_sequence": None,
+            "usage": {"input_tokens": 2202, "output_tokens": 89},
+        },
+    )
+
+    response = AnthropicPassthroughLoggingHandler._get_user_from_metadata(
+        passthrough_logging_payload=passthrough_logging_payload
+    )
+
+    assert response == "test"
+
+
+@pytest.fixture
+def all_chunks():
+    return [
+        "event: message_start",
+        'data: {"type":"message_start","message":{"id":"msg_01G7T4YSBzHjmgTyizv1UfkB","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":17,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}}}',
+        "event: content_block_start",
+        'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
+        "event: ping",
+        'data: {"type": "ping"}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Here are 5 "}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"important events from the 19th century ("}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1801-1900):\\n\\n1. The Industrial"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Revolution (ongoing throughout the century)\\nMajor technological"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" advancements and societal changes as manufacturing shifted from han"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d production to machines and factories.\\n\\n2. American Civil War (1861"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"-1865)\\nA conflict between the Union and the"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Confederacy over issues including slavery, resulting in the preservation of the"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" United States and the abolition of slavery.\\n\\n3. Publication"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" of Charles Darwin\'s \\"On the Origin of Species\\" ("}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1859)\\nDarwin\'s groundbreaking work"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" on evolution by natural selection revolutionized biology an"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d scientific thought.\\n\\n4. Unification of Germany"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" (1871)\\nThe consolidation of numerous"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" German states into a single nation-state under Prussian"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" leadership, led by Otto von Bismarck"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":".\\n\\n5. Abolition of Slavery in Various"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Countries\\nIncluding the British Empire (1833),"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" French colonies (1848), and the United States ("}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1865), marking significant progress in human rights."}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\\n\\nThese events had far-reaching consequences that shape"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d the modern world in various ways, from politics and economics to science an"}}',
+        "event: content_block_delta",
+        'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d social structures."}}',
+        "event: content_block_stop",
+        'data: {"type":"content_block_stop","index":0}',
+        "event: message_delta",
+        'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":249}}',
+        "event: message_stop",
+        'data: {"type":"message_stop"}',
+    ]
+
+
+def test_handle_logging_anthropic_collected_chunks(all_chunks):
+    from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
+        AnthropicPassthroughLoggingHandler,
+        PassthroughStandardLoggingPayload,
+        EndpointType,
+    )
+    from litellm.types.utils import ModelResponse
+
+    litellm_logging_obj = Mock()
+    pass_through_logging_obj = Mock()
+
+    sent_args = {
+        "litellm_logging_obj": litellm_logging_obj,
+        "passthrough_success_handler_obj": pass_through_logging_obj,
+        "url_route": "https://api.anthropic.com/v1/messages",
+        "request_body": {
+            "model": "claude-3-5-sonnet-20240620",
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": "List 5 important events in the XIX century",
+                        }
+                    ],
+                }
+            ],
+            "max_tokens": 4096,
+            "stream": True,
+        },
+        "endpoint_type": "anthropic",
+        "start_time": "2025-01-15T16:04:46.155054",
+        "end_time": "2025-01-15T16:04:49.603348",
+        "all_chunks": all_chunks,
+    }
+
+    result = (
+        AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
+            **sent_args
+        )
+    )
+
+    assert isinstance(result["result"], ModelResponse)
+
+
+def test_build_complete_streaming_response(all_chunks):
+    from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
+        AnthropicPassthroughLoggingHandler,
+    )
+    from litellm.types.utils import ModelResponse
+
+    litellm_logging_obj = Mock()
+
+    result = AnthropicPassthroughLoggingHandler._build_complete_streaming_response(
+        all_chunks=all_chunks,
+        model="claude-3-5-sonnet-20240620",
+        litellm_logging_obj=litellm_logging_obj,
+    )
+
+    assert isinstance(result, ModelResponse)