mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-08-21 17:33:12 +00:00
refactor(tests): make the responses tests nicer (#3161)
# What does this PR do? A _bunch_ on cleanup for the Responses tests. - Got rid of YAML test cases, moved them to just use simple pydantic models - Splitting the large monolithic test file into multiple focused test files: - `test_basic_responses.py` for basic and image response tests - `test_tool_responses.py` for tool-related tests - `test_file_search.py` for file search specific tests - Adding a `StreamingValidator` helper class to standardize streaming response validation ## Test Plan Run the tests: ``` pytest -s -v tests/integration/non_ci/responses/ \ --stack-config=starter \ --text-model openai/gpt-4o \ --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \ -k "client_with_models" ```
This commit is contained in:
parent
ba664474de
commit
8ed69978f9
11 changed files with 1312 additions and 1736 deletions
145
tests/integration/non_ci/responses/streaming_assertions.py
Normal file
145
tests/integration/non_ci/responses/streaming_assertions.py
Normal file
|
@ -0,0 +1,145 @@
|
|||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the terms described in the LICENSE file in
|
||||
# the root directory of this source tree.
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
class StreamingValidator:
|
||||
"""Helper class for validating streaming response events."""
|
||||
|
||||
def __init__(self, chunks: list[Any]):
|
||||
self.chunks = chunks
|
||||
self.event_types = [chunk.type for chunk in chunks]
|
||||
|
||||
def assert_basic_event_sequence(self):
|
||||
"""Verify basic created -> completed event sequence."""
|
||||
assert len(self.chunks) >= 2, f"Expected at least 2 chunks (created + completed), got {len(self.chunks)}"
|
||||
assert self.chunks[0].type == "response.created", (
|
||||
f"First chunk should be response.created, got {self.chunks[0].type}"
|
||||
)
|
||||
assert self.chunks[-1].type == "response.completed", (
|
||||
f"Last chunk should be response.completed, got {self.chunks[-1].type}"
|
||||
)
|
||||
|
||||
# Verify event order
|
||||
created_index = self.event_types.index("response.created")
|
||||
completed_index = self.event_types.index("response.completed")
|
||||
assert created_index < completed_index, "response.created should come before response.completed"
|
||||
|
||||
def assert_response_consistency(self):
|
||||
"""Verify response ID consistency across events."""
|
||||
response_ids = set()
|
||||
for chunk in self.chunks:
|
||||
if hasattr(chunk, "response_id"):
|
||||
response_ids.add(chunk.response_id)
|
||||
elif hasattr(chunk, "response") and hasattr(chunk.response, "id"):
|
||||
response_ids.add(chunk.response.id)
|
||||
|
||||
assert len(response_ids) == 1, f"All events should reference the same response_id, found: {response_ids}"
|
||||
|
||||
def assert_has_incremental_content(self):
|
||||
"""Verify that content is delivered incrementally via delta events."""
|
||||
delta_events = [
|
||||
i for i, event_type in enumerate(self.event_types) if event_type == "response.output_text.delta"
|
||||
]
|
||||
assert len(delta_events) > 0, "Expected delta events for true incremental streaming, but found none"
|
||||
|
||||
# Verify delta events have content
|
||||
non_empty_deltas = 0
|
||||
delta_content_total = ""
|
||||
|
||||
for delta_idx in delta_events:
|
||||
chunk = self.chunks[delta_idx]
|
||||
if hasattr(chunk, "delta") and chunk.delta:
|
||||
delta_content_total += chunk.delta
|
||||
non_empty_deltas += 1
|
||||
|
||||
assert non_empty_deltas > 0, "Delta events found but none contain content"
|
||||
assert len(delta_content_total) > 0, "Delta events found but total delta content is empty"
|
||||
|
||||
return delta_content_total
|
||||
|
||||
def assert_content_quality(self, expected_content: str):
|
||||
"""Verify the final response contains expected content."""
|
||||
final_chunk = self.chunks[-1]
|
||||
if hasattr(final_chunk, "response"):
|
||||
output_text = final_chunk.response.output_text.lower().strip()
|
||||
assert len(output_text) > 0, "Response should have content"
|
||||
assert expected_content.lower() in output_text, f"Expected '{expected_content}' in response"
|
||||
|
||||
def assert_has_tool_calls(self):
|
||||
"""Verify tool call streaming events are present."""
|
||||
# Check for tool call events
|
||||
delta_events = [
|
||||
chunk
|
||||
for chunk in self.chunks
|
||||
if chunk.type in ["response.function_call_arguments.delta", "response.mcp_call.arguments.delta"]
|
||||
]
|
||||
done_events = [
|
||||
chunk
|
||||
for chunk in self.chunks
|
||||
if chunk.type in ["response.function_call_arguments.done", "response.mcp_call.arguments.done"]
|
||||
]
|
||||
|
||||
assert len(delta_events) > 0, f"Expected tool call delta events, got chunk types: {self.event_types}"
|
||||
assert len(done_events) > 0, f"Expected tool call done events, got chunk types: {self.event_types}"
|
||||
|
||||
# Verify output item events
|
||||
item_added_events = [chunk for chunk in self.chunks if chunk.type == "response.output_item.added"]
|
||||
item_done_events = [chunk for chunk in self.chunks if chunk.type == "response.output_item.done"]
|
||||
|
||||
assert len(item_added_events) > 0, (
|
||||
f"Expected response.output_item.added events, got chunk types: {self.event_types}"
|
||||
)
|
||||
assert len(item_done_events) > 0, (
|
||||
f"Expected response.output_item.done events, got chunk types: {self.event_types}"
|
||||
)
|
||||
|
||||
def assert_has_mcp_events(self):
|
||||
"""Verify MCP-specific streaming events are present."""
|
||||
# Tool execution progress events
|
||||
mcp_in_progress_events = [chunk for chunk in self.chunks if chunk.type == "response.mcp_call.in_progress"]
|
||||
mcp_completed_events = [chunk for chunk in self.chunks if chunk.type == "response.mcp_call.completed"]
|
||||
|
||||
assert len(mcp_in_progress_events) > 0, (
|
||||
f"Expected response.mcp_call.in_progress events, got chunk types: {self.event_types}"
|
||||
)
|
||||
assert len(mcp_completed_events) > 0, (
|
||||
f"Expected response.mcp_call.completed events, got chunk types: {self.event_types}"
|
||||
)
|
||||
|
||||
# MCP list tools events
|
||||
mcp_list_tools_in_progress_events = [
|
||||
chunk for chunk in self.chunks if chunk.type == "response.mcp_list_tools.in_progress"
|
||||
]
|
||||
mcp_list_tools_completed_events = [
|
||||
chunk for chunk in self.chunks if chunk.type == "response.mcp_list_tools.completed"
|
||||
]
|
||||
|
||||
assert len(mcp_list_tools_in_progress_events) > 0, (
|
||||
f"Expected response.mcp_list_tools.in_progress events, got chunk types: {self.event_types}"
|
||||
)
|
||||
assert len(mcp_list_tools_completed_events) > 0, (
|
||||
f"Expected response.mcp_list_tools.completed events, got chunk types: {self.event_types}"
|
||||
)
|
||||
|
||||
def assert_rich_streaming(self, min_chunks: int = 10):
|
||||
"""Verify we have substantial streaming activity."""
|
||||
assert len(self.chunks) > min_chunks, (
|
||||
f"Expected rich streaming with many events, got only {len(self.chunks)} chunks"
|
||||
)
|
||||
|
||||
def validate_event_structure(self):
|
||||
"""Validate the structure of various event types."""
|
||||
for chunk in self.chunks:
|
||||
if chunk.type == "response.created":
|
||||
assert chunk.response.status == "in_progress"
|
||||
elif chunk.type == "response.completed":
|
||||
assert chunk.response.status == "completed"
|
||||
elif hasattr(chunk, "item_id"):
|
||||
assert chunk.item_id, "Events with item_id should have non-empty item_id"
|
||||
elif hasattr(chunk, "sequence_number"):
|
||||
assert isinstance(chunk.sequence_number, int), "sequence_number should be an integer"
|
Loading…
Add table
Add a link
Reference in a new issue