mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-25 18:54:30 +00:00
test: initial commit enforcing testing on all anthropic pass through … (#7794)
* test: initial commit enforcing testing on all anthropic pass through functions prevents future regressions * test(test_unit_test_anthropic_pass_through.py): add unit test for '_get_user_from_metadata' function * test(test_unit_test_anthropic_passthrough.py): add unit test for handle_logging_anthropic_collected_chunks * test(test_unit_test_anthropic_pass_through): add coverage for all anthropic pass through functions
This commit is contained in:
Krish Dholakia
GitHub
parent
843cd3b7c6
commit
c57266c9dc
4 changed files with 300 additions and 1 deletions
|
|
@ -162,6 +162,7 @@ class AnthropicPassthroughLoggingHandler:
|
|||
- Creates standard logging object
|
||||
- Logs in litellm callbacks
|
||||
"""
|
||||
|
||||
model = request_body.get("model", "")
|
||||
complete_streaming_response = (
|
||||
AnthropicPassthroughLoggingHandler._build_complete_streaming_response(
|
||||
|
|
|
|||
125
tests/code_coverage_tests/pass_through_code_coverage.py
Normal file
125
tests/code_coverage_tests/pass_through_code_coverage.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
import ast
|
||||
import os
|
||||
|
||||
pass_through_classes = [
|
||||
"AnthropicPassthroughLoggingHandler",
|
||||
]
|
||||
|
||||
|
||||
def get_function_names_from_file(file_path):
|
||||
"""
|
||||
Extracts all function names from a given Python file.
|
||||
"""
|
||||
with open(file_path, "r") as file:
|
||||
tree = ast.parse(file.read())
|
||||
|
||||
function_names = []
|
||||
|
||||
for node in tree.body:
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
function_names.append(node.name)
|
||||
elif isinstance(node, ast.ClassDef):
|
||||
if node.name in pass_through_classes:
|
||||
for class_node in node.body:
|
||||
if isinstance(class_node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
function_names.append(class_node.name)
|
||||
|
||||
return function_names
|
||||
|
||||
|
||||
def get_all_functions_called_in_tests(base_dir):
|
||||
"""
|
||||
Returns a set of function names that are called in test functions
|
||||
inside 'local_testing' and 'router_unit_test' directories,
|
||||
specifically in files containing the word 'router'.
|
||||
"""
|
||||
called_functions = set()
|
||||
test_dirs = ["pass_through_unit_tests"]
|
||||
|
||||
for test_dir in test_dirs:
|
||||
dir_path = os.path.join(base_dir, test_dir)
|
||||
if not os.path.exists(dir_path):
|
||||
print(f"Warning: Directory {dir_path} does not exist.")
|
||||
continue
|
||||
|
||||
print("dir_path: ", dir_path)
|
||||
for root, _, files in os.walk(dir_path):
|
||||
for file in files:
|
||||
if file.endswith(".py"):
|
||||
print("file: ", file)
|
||||
file_path = os.path.join(root, file)
|
||||
with open(file_path, "r") as f:
|
||||
try:
|
||||
tree = ast.parse(f.read())
|
||||
except SyntaxError:
|
||||
print(f"Warning: Syntax error in file {file_path}")
|
||||
continue
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Call):
|
||||
if isinstance(node.func, ast.Attribute) and isinstance(
|
||||
node.func.value, ast.Name
|
||||
):
|
||||
if node.func.value.id in pass_through_classes:
|
||||
# If it's called via the class, add both the original and non-underscore versions
|
||||
method_name = node.func.attr
|
||||
called_functions.add(method_name)
|
||||
if method_name.startswith("_"):
|
||||
called_functions.add(method_name.lstrip("_"))
|
||||
|
||||
return called_functions
|
||||
|
||||
|
||||
def get_functions_from_router(file_path):
|
||||
"""
|
||||
Extracts all functions defined in router.py.
|
||||
"""
|
||||
return get_function_names_from_file(file_path)
|
||||
|
||||
|
||||
ignored_function_names = [
|
||||
"__init__",
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
router_file = [
|
||||
"../../litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py",
|
||||
]
|
||||
tests_dir = "../../tests/"
|
||||
|
||||
router_functions = []
|
||||
for file in router_file:
|
||||
router_functions.extend(get_functions_from_router(file))
|
||||
print("router_functions: ", router_functions)
|
||||
called_functions_in_tests = get_all_functions_called_in_tests(tests_dir)
|
||||
print("called_functions_in_tests: ", called_functions_in_tests)
|
||||
|
||||
untested_functions = []
|
||||
for fn in router_functions:
|
||||
# Check if the function is called either with or without leading underscore
|
||||
clean_name = fn.lstrip("_")
|
||||
if (
|
||||
fn not in called_functions_in_tests
|
||||
and clean_name not in called_functions_in_tests
|
||||
):
|
||||
untested_functions.append(fn)
|
||||
|
||||
if untested_functions:
|
||||
all_untested_functions = []
|
||||
for func in untested_functions:
|
||||
if func not in ignored_function_names:
|
||||
all_untested_functions.append(func)
|
||||
untested_perc = (len(all_untested_functions)) / len(router_functions)
|
||||
print("untested_perc: ", untested_perc)
|
||||
if untested_perc > 0:
|
||||
print("The following functions in pass_through/ are not tested:")
|
||||
raise Exception(
|
||||
f"{untested_perc * 100:.2f}% of functions in pass_through/ are not tested: {all_untested_functions}"
|
||||
)
|
||||
else:
|
||||
print("All functions in router.py are covered by tests.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
"""
|
||||
This test ensures that the proxy can passthrough anthropic requests
|
||||
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
|
|
|||
|
|
@ -201,3 +201,177 @@ def test_create_anthropic_response_logging_payload(mock_logging_obj, metadata_pa
|
|||
assert "model" in result
|
||||
assert "response_cost" in result
|
||||
assert "standard_logging_object" in result
|
||||
if metadata_params:
|
||||
assert "test" == result["standard_logging_object"]["end_user"]
|
||||
else:
|
||||
assert "" == result["standard_logging_object"]["end_user"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"end_user_id",
|
||||
[{"litellm_metadata": {"user": "test"}}, {"metadata": {"user_id": "test"}}],
|
||||
)
|
||||
def test_get_user_from_metadata(end_user_id):
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
|
||||
AnthropicPassthroughLoggingHandler,
|
||||
PassthroughStandardLoggingPayload,
|
||||
)
|
||||
|
||||
passthrough_logging_payload = PassthroughStandardLoggingPayload(
|
||||
url="https://api.anthropic.com/v1/messages",
|
||||
request_body={**end_user_id},
|
||||
response_body={
|
||||
"id": "msg_015uSaCZBvu9gUSkAmZtMfxC",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-3-5-sonnet-20241022",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Now I'll click on the Firefox icon to launch it.",
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01TQsF5p7Pf4LGKyLUDDySVr",
|
||||
"name": "computer",
|
||||
"input": {"action": "mouse_move", "coordinate": [24, 36]},
|
||||
},
|
||||
],
|
||||
"stop_reason": "tool_use",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 2202, "output_tokens": 89},
|
||||
},
|
||||
)
|
||||
|
||||
response = AnthropicPassthroughLoggingHandler._get_user_from_metadata(
|
||||
passthrough_logging_payload=passthrough_logging_payload
|
||||
)
|
||||
|
||||
assert response == "test"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def all_chunks():
|
||||
return [
|
||||
"event: message_start",
|
||||
'data: {"type":"message_start","message":{"id":"msg_01G7T4YSBzHjmgTyizv1UfkB","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":17,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}}}',
|
||||
"event: content_block_start",
|
||||
'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
|
||||
"event: ping",
|
||||
'data: {"type": "ping"}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Here are 5 "}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"important events from the 19th century ("}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1801-1900):\\n\\n1. The Industrial"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Revolution (ongoing throughout the century)\\nMajor technological"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" advancements and societal changes as manufacturing shifted from han"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d production to machines and factories.\\n\\n2. American Civil War (1861"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"-1865)\\nA conflict between the Union and the"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Confederacy over issues including slavery, resulting in the preservation of the"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" United States and the abolition of slavery.\\n\\n3. Publication"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" of Charles Darwin\'s \\"On the Origin of Species\\" ("}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1859)\\nDarwin\'s groundbreaking work"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" on evolution by natural selection revolutionized biology an"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d scientific thought.\\n\\n4. Unification of Germany"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" (1871)\\nThe consolidation of numerous"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" German states into a single nation-state under Prussian"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" leadership, led by Otto von Bismarck"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":".\\n\\n5. Abolition of Slavery in Various"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Countries\\nIncluding the British Empire (1833),"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" French colonies (1848), and the United States ("}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1865), marking significant progress in human rights."}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\\n\\nThese events had far-reaching consequences that shape"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d the modern world in various ways, from politics and economics to science an"}}',
|
||||
"event: content_block_delta",
|
||||
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d social structures."}}',
|
||||
"event: content_block_stop",
|
||||
'data: {"type":"content_block_stop","index":0}',
|
||||
"event: message_delta",
|
||||
'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":249}}',
|
||||
"event: message_stop",
|
||||
'data: {"type":"message_stop"}',
|
||||
]
|
||||
|
||||
|
||||
def test_handle_logging_anthropic_collected_chunks(all_chunks):
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
|
||||
AnthropicPassthroughLoggingHandler,
|
||||
PassthroughStandardLoggingPayload,
|
||||
EndpointType,
|
||||
)
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
litellm_logging_obj = Mock()
|
||||
pass_through_logging_obj = Mock()
|
||||
|
||||
sent_args = {
|
||||
"litellm_logging_obj": litellm_logging_obj,
|
||||
"passthrough_success_handler_obj": pass_through_logging_obj,
|
||||
"url_route": "https://api.anthropic.com/v1/messages",
|
||||
"request_body": {
|
||||
"model": "claude-3-5-sonnet-20240620",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "List 5 important events in the XIX century",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"max_tokens": 4096,
|
||||
"stream": True,
|
||||
},
|
||||
"endpoint_type": "anthropic",
|
||||
"start_time": "2025-01-15T16:04:46.155054",
|
||||
"end_time": "2025-01-15T16:04:49.603348",
|
||||
"all_chunks": all_chunks,
|
||||
}
|
||||
|
||||
result = (
|
||||
AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
|
||||
**sent_args
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(result["result"], ModelResponse)
|
||||
|
||||
|
||||
def test_build_complete_streaming_response(all_chunks):
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
|
||||
AnthropicPassthroughLoggingHandler,
|
||||
)
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
litellm_logging_obj = Mock()
|
||||
|
||||
result = AnthropicPassthroughLoggingHandler._build_complete_streaming_response(
|
||||
all_chunks=all_chunks,
|
||||
model="claude-3-5-sonnet-20240620",
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
assert isinstance(result, ModelResponse)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue