mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-06-27 18:50:41 +00:00
feat: add responses input items api (#2239)
# What does this PR do? TSIA ## Test Plan added integration and unit tests
This commit is contained in:
parent
055f48b6a2
commit
15b0a67555
9 changed files with 546 additions and 12 deletions
|
@ -3,11 +3,7 @@
|
|||
#
|
||||
# This source code is licensed under the terms described in the LICENSE file in
|
||||
# the root directory of this source tree.
|
||||
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from openai import OpenAI
|
||||
|
||||
from llama_stack.distribution.library_client import LlamaStackAsLibraryClient
|
||||
|
@ -80,11 +76,10 @@ def test_responses_store(openai_client, client_with_models, text_model_id, strea
|
|||
if not tools:
|
||||
content = response.output[0].content[0].text
|
||||
|
||||
# list responses is not available in the SDK
|
||||
url = urljoin(str(client.base_url), "responses")
|
||||
response = requests.get(url, headers={"Authorization": f"Bearer {client.api_key}"})
|
||||
assert response.status_code == 200
|
||||
data = response.json()["data"]
|
||||
# list responses - use the underlying HTTP client for endpoints not in SDK
|
||||
list_response = client._client.get("/responses")
|
||||
assert list_response.status_code == 200
|
||||
data = list_response.json()["data"]
|
||||
assert response_id in [r["id"] for r in data]
|
||||
|
||||
# test retrieve response
|
||||
|
@ -95,3 +90,133 @@ def test_responses_store(openai_client, client_with_models, text_model_id, strea
|
|||
assert retrieved_response.output[0].type == "function_call"
|
||||
else:
|
||||
assert retrieved_response.output[0].content[0].text == content
|
||||
|
||||
|
||||
def test_list_response_input_items(openai_client, client_with_models, text_model_id):
|
||||
"""Test the new list_openai_response_input_items endpoint."""
|
||||
if isinstance(client_with_models, LlamaStackAsLibraryClient):
|
||||
pytest.skip("OpenAI responses are not supported when testing with library client yet.")
|
||||
|
||||
client = openai_client
|
||||
message = "What is the capital of France?"
|
||||
|
||||
# Create a response first
|
||||
response = client.responses.create(
|
||||
model=text_model_id,
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": message,
|
||||
}
|
||||
],
|
||||
stream=False,
|
||||
)
|
||||
|
||||
response_id = response.id
|
||||
|
||||
# Test the new list input items endpoint
|
||||
input_items_response = client.responses.input_items.list(response_id=response_id)
|
||||
|
||||
# Verify the structure follows OpenAI API spec
|
||||
assert input_items_response.object == "list"
|
||||
assert hasattr(input_items_response, "data")
|
||||
assert isinstance(input_items_response.data, list)
|
||||
assert len(input_items_response.data) > 0
|
||||
|
||||
# Verify the input item contains our message
|
||||
input_item = input_items_response.data[0]
|
||||
assert input_item.type == "message"
|
||||
assert input_item.role == "user"
|
||||
assert message in str(input_item.content)
|
||||
|
||||
|
||||
def test_list_response_input_items_with_limit_and_order(openai_client, client_with_models, text_model_id):
|
||||
"""Test the list input items endpoint with limit and order parameters."""
|
||||
if isinstance(client_with_models, LlamaStackAsLibraryClient):
|
||||
pytest.skip("OpenAI responses are not supported when testing with library client yet.")
|
||||
|
||||
client = openai_client
|
||||
|
||||
# Create a response with multiple input messages to test limit and order
|
||||
# Use distinctive content to make order verification more reliable
|
||||
messages = [
|
||||
{"role": "user", "content": "Message A: What is the capital of France?"},
|
||||
{"role": "assistant", "content": "The capital of France is Paris."},
|
||||
{"role": "user", "content": "Message B: What about Spain?"},
|
||||
{"role": "assistant", "content": "The capital of Spain is Madrid."},
|
||||
{"role": "user", "content": "Message C: And Italy?"},
|
||||
]
|
||||
|
||||
response = client.responses.create(
|
||||
model=text_model_id,
|
||||
input=messages,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
response_id = response.id
|
||||
|
||||
# First get all items to establish baseline
|
||||
all_items_response = client.responses.input_items.list(response_id=response_id)
|
||||
assert all_items_response.object == "list"
|
||||
total_items = len(all_items_response.data)
|
||||
assert total_items == 5 # Should have all 5 input messages
|
||||
|
||||
# Test 1: Limit parameter - request only 2 items
|
||||
limited_response = client.responses.input_items.list(response_id=response_id, limit=2)
|
||||
assert limited_response.object == "list"
|
||||
assert len(limited_response.data) == min(2, total_items) # Should be exactly 2 or total if less
|
||||
|
||||
# Test 2: Edge case - limit larger than available items
|
||||
large_limit_response = client.responses.input_items.list(response_id=response_id, limit=10)
|
||||
assert large_limit_response.object == "list"
|
||||
assert len(large_limit_response.data) == total_items # Should return all available items
|
||||
|
||||
# Test 3: Edge case - limit of 1
|
||||
single_item_response = client.responses.input_items.list(response_id=response_id, limit=1)
|
||||
assert single_item_response.object == "list"
|
||||
assert len(single_item_response.data) == 1
|
||||
|
||||
# Test 4: Order parameter - ascending vs descending
|
||||
asc_response = client.responses.input_items.list(response_id=response_id, order="asc")
|
||||
desc_response = client.responses.input_items.list(response_id=response_id, order="desc")
|
||||
|
||||
assert asc_response.object == "list"
|
||||
assert desc_response.object == "list"
|
||||
assert len(asc_response.data) == len(desc_response.data) == total_items
|
||||
|
||||
# Verify order is actually different (if we have multiple items)
|
||||
if total_items > 1:
|
||||
# First item in asc should be last item in desc (reversed order)
|
||||
first_asc_content = str(asc_response.data[0].content)
|
||||
first_desc_content = str(desc_response.data[0].content)
|
||||
last_asc_content = str(asc_response.data[-1].content)
|
||||
last_desc_content = str(desc_response.data[-1].content)
|
||||
|
||||
# The first item in asc should be the last item in desc (and vice versa)
|
||||
assert first_asc_content == last_desc_content, (
|
||||
f"Expected first asc ({first_asc_content}) to equal last desc ({last_desc_content})"
|
||||
)
|
||||
assert last_asc_content == first_desc_content, (
|
||||
f"Expected last asc ({last_asc_content}) to equal first desc ({first_desc_content})"
|
||||
)
|
||||
|
||||
# Verify the distinctive content markers are in the right positions
|
||||
assert "Message A" in first_asc_content, "First item in ascending order should contain 'Message A'"
|
||||
assert "Message C" in first_desc_content, "First item in descending order should contain 'Message C'"
|
||||
|
||||
# Test 5: Combined limit and order
|
||||
combined_response = client.responses.input_items.list(response_id=response_id, limit=3, order="desc")
|
||||
assert combined_response.object == "list"
|
||||
assert len(combined_response.data) == min(3, total_items)
|
||||
|
||||
# Test 6: Verify combined response has correct order for first few items
|
||||
if total_items >= 3:
|
||||
# Should get the last 3 items in descending order (most recent first)
|
||||
assert "Message C" in str(combined_response.data[0].content), "First item should be most recent (Message C)"
|
||||
# The exact second and third items depend on the implementation, but let's verify structure
|
||||
for item in combined_response.data:
|
||||
assert hasattr(item, "content")
|
||||
assert hasattr(item, "role")
|
||||
assert hasattr(item, "type")
|
||||
assert item.type == "message"
|
||||
assert item.role in ["user", "assistant"]
|
||||
|
|
|
@ -15,7 +15,9 @@ from openai.types.chat.chat_completion_chunk import (
|
|||
ChoiceDeltaToolCallFunction,
|
||||
)
|
||||
|
||||
from llama_stack.apis.agents import Order
|
||||
from llama_stack.apis.agents.openai_responses import (
|
||||
ListOpenAIResponseInputItem,
|
||||
OpenAIResponseInputMessageContentText,
|
||||
OpenAIResponseInputToolFunction,
|
||||
OpenAIResponseInputToolWebSearch,
|
||||
|
@ -504,3 +506,117 @@ async def test_create_openai_response_with_instructions_and_previous_response(
|
|||
assert sent_messages[2].content == "Galway, Longford, Sligo"
|
||||
assert sent_messages[3].role == "user"
|
||||
assert sent_messages[3].content == "Which is the largest?"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_openai_response_input_items_delegation(openai_responses_impl, mock_responses_store):
|
||||
"""Test that list_openai_response_input_items properly delegates to responses_store with correct parameters."""
|
||||
# Setup
|
||||
response_id = "resp_123"
|
||||
after = "msg_after"
|
||||
before = "msg_before"
|
||||
include = ["metadata"]
|
||||
limit = 5
|
||||
order = Order.asc
|
||||
|
||||
input_message = OpenAIResponseMessage(
|
||||
id="msg_123",
|
||||
content="Test message",
|
||||
role="user",
|
||||
)
|
||||
|
||||
expected_result = ListOpenAIResponseInputItem(data=[input_message])
|
||||
mock_responses_store.list_response_input_items.return_value = expected_result
|
||||
|
||||
# Execute with all parameters to test delegation
|
||||
result = await openai_responses_impl.list_openai_response_input_items(
|
||||
response_id, after=after, before=before, include=include, limit=limit, order=order
|
||||
)
|
||||
|
||||
# Verify all parameters are passed through correctly to the store
|
||||
mock_responses_store.list_response_input_items.assert_called_once_with(
|
||||
response_id, after, before, include, limit, order
|
||||
)
|
||||
|
||||
# Verify the result is returned as-is from the store
|
||||
assert result.object == "list"
|
||||
assert len(result.data) == 1
|
||||
assert result.data[0].id == "msg_123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_store_list_input_items_logic():
|
||||
"""Test ResponsesStore list_response_input_items logic - mocks get_response_object to test actual ordering/limiting."""
|
||||
|
||||
# Create mock store and response store
|
||||
mock_sql_store = AsyncMock()
|
||||
responses_store = ResponsesStore(sql_store_config=None)
|
||||
responses_store.sql_store = mock_sql_store
|
||||
|
||||
# Setup test data - multiple input items
|
||||
input_items = [
|
||||
OpenAIResponseMessage(id="msg_1", content="First message", role="user"),
|
||||
OpenAIResponseMessage(id="msg_2", content="Second message", role="user"),
|
||||
OpenAIResponseMessage(id="msg_3", content="Third message", role="user"),
|
||||
OpenAIResponseMessage(id="msg_4", content="Fourth message", role="user"),
|
||||
]
|
||||
|
||||
response_with_input = OpenAIResponseObjectWithInput(
|
||||
id="resp_123",
|
||||
model="test_model",
|
||||
created_at=1234567890,
|
||||
object="response",
|
||||
status="completed",
|
||||
output=[],
|
||||
input=input_items,
|
||||
)
|
||||
|
||||
# Mock the get_response_object method to return our test data
|
||||
mock_sql_store.fetch_one.return_value = {"response_object": response_with_input.model_dump()}
|
||||
|
||||
# Test 1: Default behavior (no limit, desc order)
|
||||
result = await responses_store.list_response_input_items("resp_123")
|
||||
assert result.object == "list"
|
||||
assert len(result.data) == 4
|
||||
# Should be reversed for desc order
|
||||
assert result.data[0].id == "msg_4"
|
||||
assert result.data[1].id == "msg_3"
|
||||
assert result.data[2].id == "msg_2"
|
||||
assert result.data[3].id == "msg_1"
|
||||
|
||||
# Test 2: With limit=2, desc order
|
||||
result = await responses_store.list_response_input_items("resp_123", limit=2, order=Order.desc)
|
||||
assert result.object == "list"
|
||||
assert len(result.data) == 2
|
||||
# Should be first 2 items in desc order
|
||||
assert result.data[0].id == "msg_4"
|
||||
assert result.data[1].id == "msg_3"
|
||||
|
||||
# Test 3: With limit=2, asc order
|
||||
result = await responses_store.list_response_input_items("resp_123", limit=2, order=Order.asc)
|
||||
assert result.object == "list"
|
||||
assert len(result.data) == 2
|
||||
# Should be first 2 items in original order (asc)
|
||||
assert result.data[0].id == "msg_1"
|
||||
assert result.data[1].id == "msg_2"
|
||||
|
||||
# Test 4: Asc order without limit
|
||||
result = await responses_store.list_response_input_items("resp_123", order=Order.asc)
|
||||
assert result.object == "list"
|
||||
assert len(result.data) == 4
|
||||
# Should be in original order (asc)
|
||||
assert result.data[0].id == "msg_1"
|
||||
assert result.data[1].id == "msg_2"
|
||||
assert result.data[2].id == "msg_3"
|
||||
assert result.data[3].id == "msg_4"
|
||||
|
||||
# Test 5: Large limit (larger than available items)
|
||||
result = await responses_store.list_response_input_items("resp_123", limit=10, order=Order.desc)
|
||||
assert result.object == "list"
|
||||
assert len(result.data) == 4 # Should return all available items
|
||||
assert result.data[0].id == "msg_4"
|
||||
|
||||
# Test 6: Zero limit edge case
|
||||
result = await responses_store.list_response_input_items("resp_123", limit=0, order=Order.asc)
|
||||
assert result.object == "list"
|
||||
assert len(result.data) == 0 # Should return no items
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue