phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-29 05:38:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

OpenAPI Responses - move tests under tests/verifications

Browse source 
This moves the OpenAI Responses API tests under
tests/verifications/openai_api/test_response.py and starts to wire
them up to our verification suite, so that we can test multiple
providers as well as OpenAI directly for the Responses API.

Signed-off-by: Ben Browning <bbrownin@redhat.com>
This commit is contained in:
Ben Browning 2025-04-18 15:26:34 -04:00 • committed by Ashwin Bharambe
parent 591e6a3972
commit 207224a811
14 changed files with 353 additions and 273 deletions
Download patch file Download diff file Expand all files Collapse all files

5
tests/integration/openai_responses/__init__.py
View file

@ -1,5 +0,0 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.

83
tests/integration/openai_responses/test_basic.py
View file

@ -1,83 +0,0 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
import pytest
from ..test_cases.test_case import TestCase
@pytest.mark.parametrize(
"test_case",
[
"openai:responses:non_streaming_01",
"openai:responses:non_streaming_02",
],
)
def test_basic_non_streaming(openai_client, client_with_models, text_model_id, test_case):
tc = TestCase(test_case)
question = tc["question"]
expected = tc["expected"]
response = openai_client.responses.create(
model=text_model_id,
input=question,
stream=False,
)
output_text = response.output_text.lower().strip()
assert len(output_text) > 0
assert expected.lower() in output_text
retrieved_response = openai_client.responses.retrieve(response_id=response.id)
assert retrieved_response.output_text == response.output_text
next_response = openai_client.responses.create(
model=text_model_id, input="Repeat your previous response in all caps.", previous_response_id=response.id
)
next_output_text = next_response.output_text.strip()
assert expected.upper() in next_output_text
@pytest.mark.parametrize(
"test_case",
[
"openai:responses:streaming_01",
"openai:responses:streaming_02",
],
)
def test_basic_streaming(openai_client, client_with_models, text_model_id, test_case):
tc = TestCase(test_case)
question = tc["question"]
expected = tc["expected"]
response = openai_client.responses.create(
model=text_model_id,
input=question,
stream=True,
timeout=120, # Increase timeout to 2 minutes for large conversation history
)
streamed_content = []
response_id = ""
for chunk in response:
response_id = chunk.response.id
streamed_content.append(chunk.response.output_text.strip())
assert len(streamed_content) > 0
assert expected.lower() in "".join(streamed_content).lower()
retrieved_response = openai_client.responses.retrieve(response_id=response_id)
assert retrieved_response.output_text == "".join(streamed_content)
next_response = openai_client.responses.create(
model=text_model_id,
input="Repeat your previous response in all caps.",
previous_response_id=response_id,
stream=True,
)
next_streamed_content = []
for chunk in next_response:
next_streamed_content.append(chunk.response.output_text.strip())
assert expected.upper() in "".join(next_streamed_content)

101
tests/integration/openai_responses/test_web_search_builtin.py
View file

@ -1,101 +0,0 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
import pytest
from ..test_cases.test_case import TestCase
@pytest.mark.parametrize(
"test_case",
[
"openai:responses:tools_web_search_01",
],
)
def test_web_search_non_streaming(openai_client, client_with_models, text_model_id, test_case):
tc = TestCase(test_case)
input = tc["input"]
expected = tc["expected"]
tools = tc["tools"]
response = openai_client.responses.create(
model=text_model_id,
input=input,
tools=tools,
stream=False,
)
assert len(response.output) > 1
assert response.output[0].type == "web_search_call"
assert response.output[0].status == "completed"
assert response.output[1].type == "message"
assert response.output[1].status == "completed"
assert response.output[1].role == "assistant"
assert len(response.output[1].content) > 0
assert expected.lower() in response.output_text.lower().strip()
def test_input_image_non_streaming(openai_client, vision_model_id):
supported_models = ["llama-4", "gpt-4o", "llama4"]
if not any(model in vision_model_id.lower() for model in supported_models):
pytest.skip(f"Skip for non-supported model: {vision_model_id}")
response = openai_client.with_options(max_retries=0).responses.create(
model=vision_model_id,
input=[
{
"role": "user",
"content": [
{
"type": "input_text",
"text": "Identify the type of animal in this image.",
},
{
"type": "input_image",
"image_url": "https://upload.wikimedia.org/wikipedia/commons/f/f7/Llamas%2C_Vernagt-Stausee%2C_Italy.jpg",
},
],
}
],
)
output_text = response.output_text.lower()
assert "llama" in output_text
def test_multi_turn_web_search_from_image_non_streaming(openai_client, vision_model_id):
supported_models = ["llama-4", "gpt-4o", "llama4"]
if not any(model in vision_model_id.lower() for model in supported_models):
pytest.skip(f"Skip for non-supported model: {vision_model_id}")
response = openai_client.with_options(max_retries=0).responses.create(
model=vision_model_id,
input=[
{
"role": "user",
"content": [
{
"type": "input_text",
"text": "Extract a single search keyword that represents the type of animal in this image.",
},
{
"type": "input_image",
"image_url": "https://upload.wikimedia.org/wikipedia/commons/f/f7/Llamas%2C_Vernagt-Stausee%2C_Italy.jpg",
},
],
}
],
)
output_text = response.output_text.lower()
assert "llama" in output_text
search_response = openai_client.with_options(max_retries=0).responses.create(
model=vision_model_id,
input="Search the web using the search tool for those keywords plus the words 'maverick' and 'scout' and summarize the results.",
previous_response_id=response.id,
tools=[{"type": "web_search"}],
)
output_text = search_response.output_text.lower()
assert "model" in output_text