diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html index d21ff81fd..7a6e82001 100644 --- a/docs/_static/llama-stack-spec.html +++ b/docs/_static/llama-stack-spec.html @@ -2994,6 +2994,97 @@ } } }, + "/v1/openai/v1/responses/{response_id}/input_items": { + "get": { + "responses": { + "200": { + "description": "An ListOpenAIResponseInputItem.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListOpenAIResponseInputItem" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest400" + }, + "429": { + "$ref": "#/components/responses/TooManyRequests429" + }, + "500": { + "$ref": "#/components/responses/InternalServerError500" + }, + "default": { + "$ref": "#/components/responses/DefaultError" + } + }, + "tags": [ + "Agents" + ], + "description": "List input items for a given OpenAI response.", + "parameters": [ + { + "name": "response_id", + "in": "path", + "description": "The ID of the response to retrieve input items for.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "after", + "in": "query", + "description": "An item ID to list items after, used for pagination.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "before", + "in": "query", + "description": "An item ID to list items before, used for pagination.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "include", + "in": "query", + "description": "Additional fields to include in the response.", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "limit", + "in": "query", + "description": "A limit on the number of objects to be returned. Limit can range between 1 and 100, and the default is 20.", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "order", + "in": "query", + "description": "The order to return the input items in. Default is desc.", + "required": false, + "schema": { + "$ref": "#/components/schemas/Order" + } + } + ] + } + }, "/v1/providers": { "get": { "responses": { @@ -10247,6 +10338,28 @@ ], "title": "ListModelsResponse" }, + "ListOpenAIResponseInputItem": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/OpenAIResponseInput" + } + }, + "object": { + "type": "string", + "const": "list", + "default": "list" + } + }, + "additionalProperties": false, + "required": [ + "data", + "object" + ], + "title": "ListOpenAIResponseInputItem" + }, "ListOpenAIResponseObject": { "type": "object", "properties": { diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml index 8a936fcee..74c4852d4 100644 --- a/docs/_static/llama-stack-spec.yaml +++ b/docs/_static/llama-stack-spec.yaml @@ -2085,6 +2085,75 @@ paths: schema: $ref: '#/components/schemas/RegisterModelRequest' required: true + /v1/openai/v1/responses/{response_id}/input_items: + get: + responses: + '200': + description: An ListOpenAIResponseInputItem. + content: + application/json: + schema: + $ref: '#/components/schemas/ListOpenAIResponseInputItem' + '400': + $ref: '#/components/responses/BadRequest400' + '429': + $ref: >- + #/components/responses/TooManyRequests429 + '500': + $ref: >- + #/components/responses/InternalServerError500 + default: + $ref: '#/components/responses/DefaultError' + tags: + - Agents + description: >- + List input items for a given OpenAI response. + parameters: + - name: response_id + in: path + description: >- + The ID of the response to retrieve input items for. + required: true + schema: + type: string + - name: after + in: query + description: >- + An item ID to list items after, used for pagination. + required: false + schema: + type: string + - name: before + in: query + description: >- + An item ID to list items before, used for pagination. + required: false + schema: + type: string + - name: include + in: query + description: >- + Additional fields to include in the response. + required: false + schema: + type: array + items: + type: string + - name: limit + in: query + description: >- + A limit on the number of objects to be returned. Limit can range between + 1 and 100, and the default is 20. + required: false + schema: + type: integer + - name: order + in: query + description: >- + The order to return the input items in. Default is desc. + required: false + schema: + $ref: '#/components/schemas/Order' /v1/providers: get: responses: @@ -7153,6 +7222,22 @@ components: required: - data title: ListModelsResponse + ListOpenAIResponseInputItem: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/OpenAIResponseInput' + object: + type: string + const: list + default: list + additionalProperties: false + required: + - data + - object + title: ListOpenAIResponseInputItem ListOpenAIResponseObject: type: object properties: diff --git a/llama_stack/apis/agents/agents.py b/llama_stack/apis/agents/agents.py index bb185b8a3..b79c512b8 100644 --- a/llama_stack/apis/agents/agents.py +++ b/llama_stack/apis/agents/agents.py @@ -31,6 +31,7 @@ from llama_stack.apis.tools import ToolDef from llama_stack.schema_utils import json_schema_type, register_schema, webmethod from .openai_responses import ( + ListOpenAIResponseInputItem, ListOpenAIResponseObject, OpenAIResponseInput, OpenAIResponseInputTool, @@ -630,3 +631,25 @@ class Agents(Protocol): :returns: A ListOpenAIResponseObject. """ ... + + @webmethod(route="/openai/v1/responses/{response_id}/input_items", method="GET") + async def list_openai_response_input_items( + self, + response_id: str, + after: str | None = None, + before: str | None = None, + include: list[str] | None = None, + limit: int | None = 20, + order: Order | None = Order.desc, + ) -> ListOpenAIResponseInputItem: + """List input items for a given OpenAI response. + + :param response_id: The ID of the response to retrieve input items for. + :param after: An item ID to list items after, used for pagination. + :param before: An item ID to list items before, used for pagination. + :param include: Additional fields to include in the response. + :param limit: A limit on the number of objects to be returned. Limit can range between 1 and 100, and the default is 20. + :param order: The order to return the input items in. Default is desc. + :returns: An ListOpenAIResponseInputItem. + """ + ... diff --git a/llama_stack/apis/agents/openai_responses.py b/llama_stack/apis/agents/openai_responses.py index 5d8f2b80b..7740b5643 100644 --- a/llama_stack/apis/agents/openai_responses.py +++ b/llama_stack/apis/agents/openai_responses.py @@ -216,7 +216,7 @@ OpenAIResponseInputTool = Annotated[ register_schema(OpenAIResponseInputTool, name="OpenAIResponseInputTool") -class OpenAIResponseInputItemList(BaseModel): +class ListOpenAIResponseInputItem(BaseModel): data: list[OpenAIResponseInput] object: Literal["list"] = "list" diff --git a/llama_stack/providers/inline/agents/meta_reference/agents.py b/llama_stack/providers/inline/agents/meta_reference/agents.py index 787135488..bcbfcbe31 100644 --- a/llama_stack/providers/inline/agents/meta_reference/agents.py +++ b/llama_stack/providers/inline/agents/meta_reference/agents.py @@ -20,6 +20,7 @@ from llama_stack.apis.agents import ( AgentTurnCreateRequest, AgentTurnResumeRequest, Document, + ListOpenAIResponseInputItem, ListOpenAIResponseObject, OpenAIResponseInput, OpenAIResponseInputTool, @@ -337,3 +338,16 @@ class MetaReferenceAgentsImpl(Agents): order: Order | None = Order.desc, ) -> ListOpenAIResponseObject: return await self.openai_responses_impl.list_openai_responses(after, limit, model, order) + + async def list_openai_response_input_items( + self, + response_id: str, + after: str | None = None, + before: str | None = None, + include: list[str] | None = None, + limit: int | None = 20, + order: Order | None = Order.desc, + ) -> ListOpenAIResponseInputItem: + return await self.openai_responses_impl.list_openai_response_input_items( + response_id, after, before, include, limit, order + ) diff --git a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py index 939282005..9acda3b8c 100644 --- a/llama_stack/providers/inline/agents/meta_reference/openai_responses.py +++ b/llama_stack/providers/inline/agents/meta_reference/openai_responses.py @@ -14,10 +14,10 @@ from pydantic import BaseModel from llama_stack.apis.agents import Order from llama_stack.apis.agents.openai_responses import ( + ListOpenAIResponseInputItem, ListOpenAIResponseObject, OpenAIResponseInput, OpenAIResponseInputFunctionToolCallOutput, - OpenAIResponseInputItemList, OpenAIResponseInputMessageContent, OpenAIResponseInputMessageContentImage, OpenAIResponseInputMessageContentText, @@ -164,7 +164,7 @@ async def _get_message_type_by_role(role: str): class OpenAIResponsePreviousResponseWithInputItems(BaseModel): - input_items: OpenAIResponseInputItemList + input_items: ListOpenAIResponseInputItem response: OpenAIResponseObject @@ -223,6 +223,27 @@ class OpenAIResponsesImpl: ) -> ListOpenAIResponseObject: return await self.responses_store.list_responses(after, limit, model, order) + async def list_openai_response_input_items( + self, + response_id: str, + after: str | None = None, + before: str | None = None, + include: list[str] | None = None, + limit: int | None = 20, + order: Order | None = Order.desc, + ) -> ListOpenAIResponseInputItem: + """List input items for a given OpenAI response. + + :param response_id: The ID of the response to retrieve input items for. + :param after: An item ID to list items after, used for pagination. + :param before: An item ID to list items before, used for pagination. + :param include: Additional fields to include in the response. + :param limit: A limit on the number of objects to be returned. + :param order: The order to return the input items in. + :returns: An ListOpenAIResponseInputItem. + """ + return await self.responses_store.list_response_input_items(response_id, after, before, include, limit, order) + async def create_openai_response( self, input: str | list[OpenAIResponseInput], diff --git a/llama_stack/providers/utils/responses/responses_store.py b/llama_stack/providers/utils/responses/responses_store.py index 19da6785a..15354e3e2 100644 --- a/llama_stack/providers/utils/responses/responses_store.py +++ b/llama_stack/providers/utils/responses/responses_store.py @@ -7,6 +7,7 @@ from llama_stack.apis.agents import ( Order, ) from llama_stack.apis.agents.openai_responses import ( + ListOpenAIResponseInputItem, ListOpenAIResponseObject, OpenAIResponseInput, OpenAIResponseObject, @@ -96,3 +97,39 @@ class ResponsesStore: if not row: raise ValueError(f"Response with id {response_id} not found") from None return OpenAIResponseObjectWithInput(**row["response_object"]) + + async def list_response_input_items( + self, + response_id: str, + after: str | None = None, + before: str | None = None, + include: list[str] | None = None, + limit: int | None = 20, + order: Order | None = Order.desc, + ) -> ListOpenAIResponseInputItem: + """ + List input items for a given response. + + :param response_id: The ID of the response to retrieve input items for. + :param after: An item ID to list items after, used for pagination. + :param before: An item ID to list items before, used for pagination. + :param include: Additional fields to include in the response. + :param limit: A limit on the number of objects to be returned. + :param order: The order to return the input items in. + """ + # TODO: support after/before pagination + if after or before: + raise NotImplementedError("After/before pagination is not supported yet") + if include: + raise NotImplementedError("Include is not supported yet") + + response_with_input = await self.get_response_object(response_id) + input_items = response_with_input.input + + if order == Order.desc: + input_items = list(reversed(input_items)) + + if limit is not None and len(input_items) > limit: + input_items = input_items[:limit] + + return ListOpenAIResponseInputItem(data=input_items) diff --git a/tests/integration/agents/test_openai_responses.py b/tests/integration/agents/test_openai_responses.py index 8af1c1870..c9c1d4fa8 100644 --- a/tests/integration/agents/test_openai_responses.py +++ b/tests/integration/agents/test_openai_responses.py @@ -3,11 +3,7 @@ # # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. - -from urllib.parse import urljoin - import pytest -import requests from openai import OpenAI from llama_stack.distribution.library_client import LlamaStackAsLibraryClient @@ -80,11 +76,10 @@ def test_responses_store(openai_client, client_with_models, text_model_id, strea if not tools: content = response.output[0].content[0].text - # list responses is not available in the SDK - url = urljoin(str(client.base_url), "responses") - response = requests.get(url, headers={"Authorization": f"Bearer {client.api_key}"}) - assert response.status_code == 200 - data = response.json()["data"] + # list responses - use the underlying HTTP client for endpoints not in SDK + list_response = client._client.get("/responses") + assert list_response.status_code == 200 + data = list_response.json()["data"] assert response_id in [r["id"] for r in data] # test retrieve response @@ -95,3 +90,133 @@ def test_responses_store(openai_client, client_with_models, text_model_id, strea assert retrieved_response.output[0].type == "function_call" else: assert retrieved_response.output[0].content[0].text == content + + +def test_list_response_input_items(openai_client, client_with_models, text_model_id): + """Test the new list_openai_response_input_items endpoint.""" + if isinstance(client_with_models, LlamaStackAsLibraryClient): + pytest.skip("OpenAI responses are not supported when testing with library client yet.") + + client = openai_client + message = "What is the capital of France?" + + # Create a response first + response = client.responses.create( + model=text_model_id, + input=[ + { + "role": "user", + "content": message, + } + ], + stream=False, + ) + + response_id = response.id + + # Test the new list input items endpoint + input_items_response = client.responses.input_items.list(response_id=response_id) + + # Verify the structure follows OpenAI API spec + assert input_items_response.object == "list" + assert hasattr(input_items_response, "data") + assert isinstance(input_items_response.data, list) + assert len(input_items_response.data) > 0 + + # Verify the input item contains our message + input_item = input_items_response.data[0] + assert input_item.type == "message" + assert input_item.role == "user" + assert message in str(input_item.content) + + +def test_list_response_input_items_with_limit_and_order(openai_client, client_with_models, text_model_id): + """Test the list input items endpoint with limit and order parameters.""" + if isinstance(client_with_models, LlamaStackAsLibraryClient): + pytest.skip("OpenAI responses are not supported when testing with library client yet.") + + client = openai_client + + # Create a response with multiple input messages to test limit and order + # Use distinctive content to make order verification more reliable + messages = [ + {"role": "user", "content": "Message A: What is the capital of France?"}, + {"role": "assistant", "content": "The capital of France is Paris."}, + {"role": "user", "content": "Message B: What about Spain?"}, + {"role": "assistant", "content": "The capital of Spain is Madrid."}, + {"role": "user", "content": "Message C: And Italy?"}, + ] + + response = client.responses.create( + model=text_model_id, + input=messages, + stream=False, + ) + + response_id = response.id + + # First get all items to establish baseline + all_items_response = client.responses.input_items.list(response_id=response_id) + assert all_items_response.object == "list" + total_items = len(all_items_response.data) + assert total_items == 5 # Should have all 5 input messages + + # Test 1: Limit parameter - request only 2 items + limited_response = client.responses.input_items.list(response_id=response_id, limit=2) + assert limited_response.object == "list" + assert len(limited_response.data) == min(2, total_items) # Should be exactly 2 or total if less + + # Test 2: Edge case - limit larger than available items + large_limit_response = client.responses.input_items.list(response_id=response_id, limit=10) + assert large_limit_response.object == "list" + assert len(large_limit_response.data) == total_items # Should return all available items + + # Test 3: Edge case - limit of 1 + single_item_response = client.responses.input_items.list(response_id=response_id, limit=1) + assert single_item_response.object == "list" + assert len(single_item_response.data) == 1 + + # Test 4: Order parameter - ascending vs descending + asc_response = client.responses.input_items.list(response_id=response_id, order="asc") + desc_response = client.responses.input_items.list(response_id=response_id, order="desc") + + assert asc_response.object == "list" + assert desc_response.object == "list" + assert len(asc_response.data) == len(desc_response.data) == total_items + + # Verify order is actually different (if we have multiple items) + if total_items > 1: + # First item in asc should be last item in desc (reversed order) + first_asc_content = str(asc_response.data[0].content) + first_desc_content = str(desc_response.data[0].content) + last_asc_content = str(asc_response.data[-1].content) + last_desc_content = str(desc_response.data[-1].content) + + # The first item in asc should be the last item in desc (and vice versa) + assert first_asc_content == last_desc_content, ( + f"Expected first asc ({first_asc_content}) to equal last desc ({last_desc_content})" + ) + assert last_asc_content == first_desc_content, ( + f"Expected last asc ({last_asc_content}) to equal first desc ({first_desc_content})" + ) + + # Verify the distinctive content markers are in the right positions + assert "Message A" in first_asc_content, "First item in ascending order should contain 'Message A'" + assert "Message C" in first_desc_content, "First item in descending order should contain 'Message C'" + + # Test 5: Combined limit and order + combined_response = client.responses.input_items.list(response_id=response_id, limit=3, order="desc") + assert combined_response.object == "list" + assert len(combined_response.data) == min(3, total_items) + + # Test 6: Verify combined response has correct order for first few items + if total_items >= 3: + # Should get the last 3 items in descending order (most recent first) + assert "Message C" in str(combined_response.data[0].content), "First item should be most recent (Message C)" + # The exact second and third items depend on the implementation, but let's verify structure + for item in combined_response.data: + assert hasattr(item, "content") + assert hasattr(item, "role") + assert hasattr(item, "type") + assert item.type == "message" + assert item.role in ["user", "assistant"] diff --git a/tests/unit/providers/agents/meta_reference/test_openai_responses.py b/tests/unit/providers/agents/meta_reference/test_openai_responses.py index bf36d7b64..d046057eb 100644 --- a/tests/unit/providers/agents/meta_reference/test_openai_responses.py +++ b/tests/unit/providers/agents/meta_reference/test_openai_responses.py @@ -15,7 +15,9 @@ from openai.types.chat.chat_completion_chunk import ( ChoiceDeltaToolCallFunction, ) +from llama_stack.apis.agents import Order from llama_stack.apis.agents.openai_responses import ( + ListOpenAIResponseInputItem, OpenAIResponseInputMessageContentText, OpenAIResponseInputToolFunction, OpenAIResponseInputToolWebSearch, @@ -504,3 +506,117 @@ async def test_create_openai_response_with_instructions_and_previous_response( assert sent_messages[2].content == "Galway, Longford, Sligo" assert sent_messages[3].role == "user" assert sent_messages[3].content == "Which is the largest?" + + +@pytest.mark.asyncio +async def test_list_openai_response_input_items_delegation(openai_responses_impl, mock_responses_store): + """Test that list_openai_response_input_items properly delegates to responses_store with correct parameters.""" + # Setup + response_id = "resp_123" + after = "msg_after" + before = "msg_before" + include = ["metadata"] + limit = 5 + order = Order.asc + + input_message = OpenAIResponseMessage( + id="msg_123", + content="Test message", + role="user", + ) + + expected_result = ListOpenAIResponseInputItem(data=[input_message]) + mock_responses_store.list_response_input_items.return_value = expected_result + + # Execute with all parameters to test delegation + result = await openai_responses_impl.list_openai_response_input_items( + response_id, after=after, before=before, include=include, limit=limit, order=order + ) + + # Verify all parameters are passed through correctly to the store + mock_responses_store.list_response_input_items.assert_called_once_with( + response_id, after, before, include, limit, order + ) + + # Verify the result is returned as-is from the store + assert result.object == "list" + assert len(result.data) == 1 + assert result.data[0].id == "msg_123" + + +@pytest.mark.asyncio +async def test_responses_store_list_input_items_logic(): + """Test ResponsesStore list_response_input_items logic - mocks get_response_object to test actual ordering/limiting.""" + + # Create mock store and response store + mock_sql_store = AsyncMock() + responses_store = ResponsesStore(sql_store_config=None) + responses_store.sql_store = mock_sql_store + + # Setup test data - multiple input items + input_items = [ + OpenAIResponseMessage(id="msg_1", content="First message", role="user"), + OpenAIResponseMessage(id="msg_2", content="Second message", role="user"), + OpenAIResponseMessage(id="msg_3", content="Third message", role="user"), + OpenAIResponseMessage(id="msg_4", content="Fourth message", role="user"), + ] + + response_with_input = OpenAIResponseObjectWithInput( + id="resp_123", + model="test_model", + created_at=1234567890, + object="response", + status="completed", + output=[], + input=input_items, + ) + + # Mock the get_response_object method to return our test data + mock_sql_store.fetch_one.return_value = {"response_object": response_with_input.model_dump()} + + # Test 1: Default behavior (no limit, desc order) + result = await responses_store.list_response_input_items("resp_123") + assert result.object == "list" + assert len(result.data) == 4 + # Should be reversed for desc order + assert result.data[0].id == "msg_4" + assert result.data[1].id == "msg_3" + assert result.data[2].id == "msg_2" + assert result.data[3].id == "msg_1" + + # Test 2: With limit=2, desc order + result = await responses_store.list_response_input_items("resp_123", limit=2, order=Order.desc) + assert result.object == "list" + assert len(result.data) == 2 + # Should be first 2 items in desc order + assert result.data[0].id == "msg_4" + assert result.data[1].id == "msg_3" + + # Test 3: With limit=2, asc order + result = await responses_store.list_response_input_items("resp_123", limit=2, order=Order.asc) + assert result.object == "list" + assert len(result.data) == 2 + # Should be first 2 items in original order (asc) + assert result.data[0].id == "msg_1" + assert result.data[1].id == "msg_2" + + # Test 4: Asc order without limit + result = await responses_store.list_response_input_items("resp_123", order=Order.asc) + assert result.object == "list" + assert len(result.data) == 4 + # Should be in original order (asc) + assert result.data[0].id == "msg_1" + assert result.data[1].id == "msg_2" + assert result.data[2].id == "msg_3" + assert result.data[3].id == "msg_4" + + # Test 5: Large limit (larger than available items) + result = await responses_store.list_response_input_items("resp_123", limit=10, order=Order.desc) + assert result.object == "list" + assert len(result.data) == 4 # Should return all available items + assert result.data[0].id == "msg_4" + + # Test 6: Zero limit edge case + result = await responses_store.list_response_input_items("resp_123", limit=0, order=Order.asc) + assert result.object == "list" + assert len(result.data) == 0 # Should return no items