phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-06-27 18:50:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: add responses input items api (#2239)

Browse source 
# What does this PR do?
TSIA

## Test Plan
added integration and unit tests
This commit is contained in:
ehhuang 2025-05-24 07:05:53 -07:00 • committed by GitHub
parent 055f48b6a2
commit 15b0a67555
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 546 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

113
docs/_static/llama-stack-spec.html vendored
View file

@ -2994,6 +2994,97 @@
}
}
},
"/v1/openai/v1/responses/{response_id}/input_items": {
"get": {
"responses": {
"200": {
"description": "An ListOpenAIResponseInputItem.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ListOpenAIResponseInputItem"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Agents"
],
"description": "List input items for a given OpenAI response.",
"parameters": [
{
"name": "response_id",
"in": "path",
"description": "The ID of the response to retrieve input items for.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "after",
"in": "query",
"description": "An item ID to list items after, used for pagination.",
"required": false,
"schema": {
"type": "string"
}
},
{
"name": "before",
"in": "query",
"description": "An item ID to list items before, used for pagination.",
"required": false,
"schema": {
"type": "string"
}
},
{
"name": "include",
"in": "query",
"description": "Additional fields to include in the response.",
"required": false,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"name": "limit",
"in": "query",
"description": "A limit on the number of objects to be returned. Limit can range between 1 and 100, and the default is 20.",
"required": false,
"schema": {
"type": "integer"
}
},
{
"name": "order",
"in": "query",
"description": "The order to return the input items in. Default is desc.",
"required": false,
"schema": {
"$ref": "#/components/schemas/Order"
}
}
]
}
},
"/v1/providers": {
"get": {
"responses": {
@ -10247,6 +10338,28 @@
],
"title": "ListModelsResponse"
},
"ListOpenAIResponseInputItem": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"$ref": "#/components/schemas/OpenAIResponseInput"
}
},
"object": {
"type": "string",
"const": "list",
"default": "list"
}
},
"additionalProperties": false,
"required": [
"data",
"object"
],
"title": "ListOpenAIResponseInputItem"
},
"ListOpenAIResponseObject": {
"type": "object",
"properties": {

85
docs/_static/llama-stack-spec.yaml vendored
View file

@ -2085,6 +2085,75 @@ paths:
schema:
$ref: '#/components/schemas/RegisterModelRequest'
required: true
/v1/openai/v1/responses/{response_id}/input_items:
get:
responses:
'200':
description: An ListOpenAIResponseInputItem.
content:
application/json:
schema:
$ref: '#/components/schemas/ListOpenAIResponseInputItem'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Agents
description: >-
List input items for a given OpenAI response.
parameters:
- name: response_id
in: path
description: >-
The ID of the response to retrieve input items for.
required: true
schema:
type: string
- name: after
in: query
description: >-
An item ID to list items after, used for pagination.
required: false
schema:
type: string
- name: before
in: query
description: >-
An item ID to list items before, used for pagination.
required: false
schema:
type: string
- name: include
in: query
description: >-
Additional fields to include in the response.
required: false
schema:
type: array
items:
type: string
- name: limit
in: query
description: >-
A limit on the number of objects to be returned. Limit can range between
1 and 100, and the default is 20.
required: false
schema:
type: integer
- name: order
in: query
description: >-
The order to return the input items in. Default is desc.
required: false
schema:
$ref: '#/components/schemas/Order'
/v1/providers:
get:
responses:
@ -7153,6 +7222,22 @@ components:
required:
- data
title: ListModelsResponse
ListOpenAIResponseInputItem:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/OpenAIResponseInput'
object:
type: string
const: list
default: list
additionalProperties: false
required:
- data
- object
title: ListOpenAIResponseInputItem
ListOpenAIResponseObject:
type: object
properties:

23
llama_stack/apis/agents/agents.py
View file

@ -31,6 +31,7 @@ from llama_stack.apis.tools import ToolDef
from llama_stack.schema_utils import json_schema_type, register_schema, webmethod
from .openai_responses import (
ListOpenAIResponseInputItem,
ListOpenAIResponseObject,
OpenAIResponseInput,
OpenAIResponseInputTool,
@ -630,3 +631,25 @@ class Agents(Protocol):
:returns: A ListOpenAIResponseObject.
"""
...
@webmethod(route="/openai/v1/responses/{response_id}/input_items", method="GET")
async def list_openai_response_input_items(
self,
response_id: str,
after: str | None = None,
before: str | None = None,
include: list[str] | None = None,
limit: int | None = 20,
order: Order | None = Order.desc,
) -> ListOpenAIResponseInputItem:
"""List input items for a given OpenAI response.
:param response_id: The ID of the response to retrieve input items for.
:param after: An item ID to list items after, used for pagination.
:param before: An item ID to list items before, used for pagination.
:param include: Additional fields to include in the response.
:param limit: A limit on the number of objects to be returned. Limit can range between 1 and 100, and the default is 20.
:param order: The order to return the input items in. Default is desc.
:returns: An ListOpenAIResponseInputItem.
"""
...

2
llama_stack/apis/agents/openai_responses.py
View file

@ -216,7 +216,7 @@ OpenAIResponseInputTool = Annotated[
register_schema(OpenAIResponseInputTool, name="OpenAIResponseInputTool")
class OpenAIResponseInputItemList(BaseModel):
class ListOpenAIResponseInputItem(BaseModel):
data: list[OpenAIResponseInput]
object: Literal["list"] = "list"

14
llama_stack/providers/inline/agents/meta_reference/agents.py
View file

@ -20,6 +20,7 @@ from llama_stack.apis.agents import (
AgentTurnCreateRequest,
AgentTurnResumeRequest,
Document,
ListOpenAIResponseInputItem,
ListOpenAIResponseObject,
OpenAIResponseInput,
OpenAIResponseInputTool,
@ -337,3 +338,16 @@ class MetaReferenceAgentsImpl(Agents):
order: Order | None = Order.desc,
) -> ListOpenAIResponseObject:
return await self.openai_responses_impl.list_openai_responses(after, limit, model, order)
async def list_openai_response_input_items(
self,
response_id: str,
after: str | None = None,
before: str | None = None,
include: list[str] | None = None,
limit: int | None = 20,
order: Order | None = Order.desc,
) -> ListOpenAIResponseInputItem:
return await self.openai_responses_impl.list_openai_response_input_items(
response_id, after, before, include, limit, order
)

25
llama_stack/providers/inline/agents/meta_reference/openai_responses.py
View file

@ -14,10 +14,10 @@ from pydantic import BaseModel
from llama_stack.apis.agents import Order
from llama_stack.apis.agents.openai_responses import (
ListOpenAIResponseInputItem,
ListOpenAIResponseObject,
OpenAIResponseInput,
OpenAIResponseInputFunctionToolCallOutput,
OpenAIResponseInputItemList,
OpenAIResponseInputMessageContent,
OpenAIResponseInputMessageContentImage,
OpenAIResponseInputMessageContentText,
@ -164,7 +164,7 @@ async def _get_message_type_by_role(role: str):
class OpenAIResponsePreviousResponseWithInputItems(BaseModel):
input_items: OpenAIResponseInputItemList
input_items: ListOpenAIResponseInputItem
response: OpenAIResponseObject
@ -223,6 +223,27 @@ class OpenAIResponsesImpl:
) -> ListOpenAIResponseObject:
return await self.responses_store.list_responses(after, limit, model, order)
async def list_openai_response_input_items(
self,
response_id: str,
after: str | None = None,
before: str | None = None,
include: list[str] | None = None,
limit: int | None = 20,
order: Order | None = Order.desc,
) -> ListOpenAIResponseInputItem:
"""List input items for a given OpenAI response.
:param response_id: The ID of the response to retrieve input items for.
:param after: An item ID to list items after, used for pagination.
:param before: An item ID to list items before, used for pagination.
:param include: Additional fields to include in the response.
:param limit: A limit on the number of objects to be returned.
:param order: The order to return the input items in.
:returns: An ListOpenAIResponseInputItem.
"""
return await self.responses_store.list_response_input_items(response_id, after, before, include, limit, order)
async def create_openai_response(
self,
input: str | list[OpenAIResponseInput],

37
llama_stack/providers/utils/responses/responses_store.py
View file

@ -7,6 +7,7 @@ from llama_stack.apis.agents import (
Order,
)
from llama_stack.apis.agents.openai_responses import (
ListOpenAIResponseInputItem,
ListOpenAIResponseObject,
OpenAIResponseInput,
OpenAIResponseObject,
@ -96,3 +97,39 @@ class ResponsesStore:
if not row:
raise ValueError(f"Response with id {response_id} not found") from None
return OpenAIResponseObjectWithInput(**row["response_object"])
async def list_response_input_items(
self,
response_id: str,
after: str | None = None,
before: str | None = None,
include: list[str] | None = None,
limit: int | None = 20,
order: Order | None = Order.desc,
) -> ListOpenAIResponseInputItem:
"""
List input items for a given response.
:param response_id: The ID of the response to retrieve input items for.
:param after: An item ID to list items after, used for pagination.
:param before: An item ID to list items before, used for pagination.
:param include: Additional fields to include in the response.
:param limit: A limit on the number of objects to be returned.
:param order: The order to return the input items in.
"""
# TODO: support after/before pagination
if after or before:
raise NotImplementedError("After/before pagination is not supported yet")
if include:
raise NotImplementedError("Include is not supported yet")
response_with_input = await self.get_response_object(response_id)
input_items = response_with_input.input
if order == Order.desc:
input_items = list(reversed(input_items))
if limit is not None and len(input_items) > limit:
input_items = input_items[:limit]
return ListOpenAIResponseInputItem(data=input_items)

143
tests/integration/agents/test_openai_responses.py
View file

@ -3,11 +3,7 @@
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from urllib.parse import urljoin
import pytest
import requests
from openai import OpenAI
from llama_stack.distribution.library_client import LlamaStackAsLibraryClient
@ -80,11 +76,10 @@ def test_responses_store(openai_client, client_with_models, text_model_id, strea
if not tools:
content = response.output[0].content[0].text
# list responses is not available in the SDK
url = urljoin(str(client.base_url), "responses")
response = requests.get(url, headers={"Authorization": f"Bearer {client.api_key}"})
assert response.status_code == 200
data = response.json()["data"]
# list responses - use the underlying HTTP client for endpoints not in SDK
list_response = client._client.get("/responses")
assert list_response.status_code == 200
data = list_response.json()["data"]
assert response_id in [r["id"] for r in data]
# test retrieve response
@ -95,3 +90,133 @@ def test_responses_store(openai_client, client_with_models, text_model_id, strea
assert retrieved_response.output[0].type == "function_call"
else:
assert retrieved_response.output[0].content[0].text == content
def test_list_response_input_items(openai_client, client_with_models, text_model_id):
"""Test the new list_openai_response_input_items endpoint."""
if isinstance(client_with_models, LlamaStackAsLibraryClient):
pytest.skip("OpenAI responses are not supported when testing with library client yet.")
client = openai_client
message = "What is the capital of France?"
# Create a response first
response = client.responses.create(
model=text_model_id,
input=[
{
"role": "user",
"content": message,
}
],
stream=False,
)
response_id = response.id
# Test the new list input items endpoint
input_items_response = client.responses.input_items.list(response_id=response_id)
# Verify the structure follows OpenAI API spec
assert input_items_response.object == "list"
assert hasattr(input_items_response, "data")
assert isinstance(input_items_response.data, list)
assert len(input_items_response.data) > 0
# Verify the input item contains our message
input_item = input_items_response.data[0]
assert input_item.type == "message"
assert input_item.role == "user"
assert message in str(input_item.content)
def test_list_response_input_items_with_limit_and_order(openai_client, client_with_models, text_model_id):
"""Test the list input items endpoint with limit and order parameters."""
if isinstance(client_with_models, LlamaStackAsLibraryClient):
pytest.skip("OpenAI responses are not supported when testing with library client yet.")
client = openai_client
# Create a response with multiple input messages to test limit and order
# Use distinctive content to make order verification more reliable
messages = [
{"role": "user", "content": "Message A: What is the capital of France?"},
{"role": "assistant", "content": "The capital of France is Paris."},
{"role": "user", "content": "Message B: What about Spain?"},
{"role": "assistant", "content": "The capital of Spain is Madrid."},
{"role": "user", "content": "Message C: And Italy?"},
]
response = client.responses.create(
model=text_model_id,
input=messages,
stream=False,
)
response_id = response.id
# First get all items to establish baseline
all_items_response = client.responses.input_items.list(response_id=response_id)
assert all_items_response.object == "list"
total_items = len(all_items_response.data)
assert total_items == 5 # Should have all 5 input messages
# Test 1: Limit parameter - request only 2 items
limited_response = client.responses.input_items.list(response_id=response_id, limit=2)
assert limited_response.object == "list"
assert len(limited_response.data) == min(2, total_items) # Should be exactly 2 or total if less
# Test 2: Edge case - limit larger than available items
large_limit_response = client.responses.input_items.list(response_id=response_id, limit=10)
assert large_limit_response.object == "list"
assert len(large_limit_response.data) == total_items # Should return all available items
# Test 3: Edge case - limit of 1
single_item_response = client.responses.input_items.list(response_id=response_id, limit=1)
assert single_item_response.object == "list"
assert len(single_item_response.data) == 1
# Test 4: Order parameter - ascending vs descending
asc_response = client.responses.input_items.list(response_id=response_id, order="asc")
desc_response = client.responses.input_items.list(response_id=response_id, order="desc")
assert asc_response.object == "list"
assert desc_response.object == "list"
assert len(asc_response.data) == len(desc_response.data) == total_items
# Verify order is actually different (if we have multiple items)
if total_items > 1:
# First item in asc should be last item in desc (reversed order)
first_asc_content = str(asc_response.data[0].content)
first_desc_content = str(desc_response.data[0].content)
last_asc_content = str(asc_response.data[-1].content)
last_desc_content = str(desc_response.data[-1].content)
# The first item in asc should be the last item in desc (and vice versa)
assert first_asc_content == last_desc_content, (
f"Expected first asc ({first_asc_content}) to equal last desc ({last_desc_content})"
)
assert last_asc_content == first_desc_content, (
f"Expected last asc ({last_asc_content}) to equal first desc ({first_desc_content})"
)
# Verify the distinctive content markers are in the right positions
assert "Message A" in first_asc_content, "First item in ascending order should contain 'Message A'"
assert "Message C" in first_desc_content, "First item in descending order should contain 'Message C'"
# Test 5: Combined limit and order
combined_response = client.responses.input_items.list(response_id=response_id, limit=3, order="desc")
assert combined_response.object == "list"
assert len(combined_response.data) == min(3, total_items)
# Test 6: Verify combined response has correct order for first few items
if total_items >= 3:
# Should get the last 3 items in descending order (most recent first)
assert "Message C" in str(combined_response.data[0].content), "First item should be most recent (Message C)"
# The exact second and third items depend on the implementation, but let's verify structure
for item in combined_response.data:
assert hasattr(item, "content")
assert hasattr(item, "role")
assert hasattr(item, "type")
assert item.type == "message"
assert item.role in ["user", "assistant"]

116
tests/unit/providers/agents/meta_reference/test_openai_responses.py
View file

@ -15,7 +15,9 @@ from openai.types.chat.chat_completion_chunk import (
ChoiceDeltaToolCallFunction,
)
from llama_stack.apis.agents import Order
from llama_stack.apis.agents.openai_responses import (
ListOpenAIResponseInputItem,
OpenAIResponseInputMessageContentText,
OpenAIResponseInputToolFunction,
OpenAIResponseInputToolWebSearch,
@ -504,3 +506,117 @@ async def test_create_openai_response_with_instructions_and_previous_response(
assert sent_messages[2].content == "Galway, Longford, Sligo"
assert sent_messages[3].role == "user"
assert sent_messages[3].content == "Which is the largest?"
@pytest.mark.asyncio
async def test_list_openai_response_input_items_delegation(openai_responses_impl, mock_responses_store):
"""Test that list_openai_response_input_items properly delegates to responses_store with correct parameters."""
# Setup
response_id = "resp_123"
after = "msg_after"
before = "msg_before"
include = ["metadata"]
limit = 5
order = Order.asc
input_message = OpenAIResponseMessage(
id="msg_123",
content="Test message",
role="user",
)
expected_result = ListOpenAIResponseInputItem(data=[input_message])
mock_responses_store.list_response_input_items.return_value = expected_result
# Execute with all parameters to test delegation
result = await openai_responses_impl.list_openai_response_input_items(
response_id, after=after, before=before, include=include, limit=limit, order=order
)
# Verify all parameters are passed through correctly to the store
mock_responses_store.list_response_input_items.assert_called_once_with(
response_id, after, before, include, limit, order
)
# Verify the result is returned as-is from the store
assert result.object == "list"
assert len(result.data) == 1
assert result.data[0].id == "msg_123"
@pytest.mark.asyncio
async def test_responses_store_list_input_items_logic():
"""Test ResponsesStore list_response_input_items logic - mocks get_response_object to test actual ordering/limiting."""
# Create mock store and response store
mock_sql_store = AsyncMock()
responses_store = ResponsesStore(sql_store_config=None)
responses_store.sql_store = mock_sql_store
# Setup test data - multiple input items
input_items = [
OpenAIResponseMessage(id="msg_1", content="First message", role="user"),
OpenAIResponseMessage(id="msg_2", content="Second message", role="user"),
OpenAIResponseMessage(id="msg_3", content="Third message", role="user"),
OpenAIResponseMessage(id="msg_4", content="Fourth message", role="user"),
]
response_with_input = OpenAIResponseObjectWithInput(
id="resp_123",
model="test_model",
created_at=1234567890,
object="response",
status="completed",
output=[],
input=input_items,
)
# Mock the get_response_object method to return our test data
mock_sql_store.fetch_one.return_value = {"response_object": response_with_input.model_dump()}
# Test 1: Default behavior (no limit, desc order)
result = await responses_store.list_response_input_items("resp_123")
assert result.object == "list"
assert len(result.data) == 4
# Should be reversed for desc order
assert result.data[0].id == "msg_4"
assert result.data[1].id == "msg_3"
assert result.data[2].id == "msg_2"
assert result.data[3].id == "msg_1"
# Test 2: With limit=2, desc order
result = await responses_store.list_response_input_items("resp_123", limit=2, order=Order.desc)
assert result.object == "list"
assert len(result.data) == 2
# Should be first 2 items in desc order
assert result.data[0].id == "msg_4"
assert result.data[1].id == "msg_3"
# Test 3: With limit=2, asc order
result = await responses_store.list_response_input_items("resp_123", limit=2, order=Order.asc)
assert result.object == "list"
assert len(result.data) == 2
# Should be first 2 items in original order (asc)
assert result.data[0].id == "msg_1"
assert result.data[1].id == "msg_2"
# Test 4: Asc order without limit
result = await responses_store.list_response_input_items("resp_123", order=Order.asc)
assert result.object == "list"
assert len(result.data) == 4
# Should be in original order (asc)
assert result.data[0].id == "msg_1"
assert result.data[1].id == "msg_2"
assert result.data[2].id == "msg_3"
assert result.data[3].id == "msg_4"
# Test 5: Large limit (larger than available items)
result = await responses_store.list_response_input_items("resp_123", limit=10, order=Order.desc)
assert result.object == "list"
assert len(result.data) == 4 # Should return all available items
assert result.data[0].id == "msg_4"
# Test 6: Zero limit edge case
result = await responses_store.list_response_input_items("resp_123", limit=0, order=Order.asc)
assert result.object == "list"
assert len(result.data) == 0 # Should return no items