diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ff13a4cb0..cfc26000b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,7 +54,7 @@ jobs: echo "REPORT_FILE=${REPORT_OUTPUT}" >> "$GITHUB_ENV" export INFERENCE_MODEL=meta-llama/Llama-3.1-8B-Instruct - LLAMA_STACK_CONFIG=./llama_stack/templates/${{ matrix.provider }}/run.yaml pytest --md-report --md-report-verbose=1 ./tests/client-sdk/inference/test_inference.py --md-report-output "$REPORT_OUTPUT" + LLAMA_STACK_CONFIG=./llama_stack/templates/${{ matrix.provider }}/run.yaml pytest --md-report --md-report-verbose=1 ./tests/client-sdk/inference/ --md-report-output "$REPORT_OUTPUT" - name: Output reports to the job summary if: always() diff --git a/tests/client-sdk/README.md b/tests/client-sdk/README.md index 13142d46f..d4d439d96 100644 --- a/tests/client-sdk/README.md +++ b/tests/client-sdk/README.md @@ -4,18 +4,18 @@ You can run llama stack integration tests on either a Llama Stack Library or a L To test on a Llama Stack library with certain configuration, run ```bash LLAMA_STACK_CONFIG=./llama_stack/templates/cerebras/run.yaml -pytest -s -v tests/client-sdk/inference/test_inference.py +pytest -s -v tests/client-sdk/inference/ ``` or just the template name ```bash LLAMA_STACK_CONFIG=together -pytest -s -v tests/client-sdk/inference/test_inference.py +pytest -s -v tests/client-sdk/inference/ ``` To test on a Llama Stack endpoint, run ```bash LLAMA_STACK_BASE_URL=http//localhost:8089 -pytest -s -v tests/client-sdk/inference/test_inference.py +pytest -s -v tests/client-sdk/inference ``` ## Report Generation diff --git a/tests/client-sdk/inference/test_inference.py b/tests/client-sdk/inference/test_text_inference.py similarity index 73% rename from tests/client-sdk/inference/test_inference.py rename to tests/client-sdk/inference/test_text_inference.py index 9bbd1061a..4b24f1d38 100644 --- a/tests/client-sdk/inference/test_inference.py +++ b/tests/client-sdk/inference/test_text_inference.py @@ -4,9 +4,6 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -import base64 -import pathlib - import pytest from pydantic import BaseModel @@ -56,23 +53,6 @@ def get_weather_tool_definition(): } -@pytest.fixture -def image_path(): - return pathlib.Path(__file__).parent / "dog.png" - - -@pytest.fixture -def base64_image_data(image_path): - # Convert the image to base64 - return base64.b64encode(image_path.read_bytes()).decode("utf-8") - - -@pytest.fixture -def base64_image_url(base64_image_data, image_path): - # suffix includes the ., so we remove it - return f"data:image/{image_path.suffix[1:]};base64,{base64_image_data}" - - def test_text_completion_non_streaming(llama_stack_client, text_model_id): response = llama_stack_client.inference.completion( content="Complete the sentence using one word: Roses are red, violets are ", @@ -299,101 +279,3 @@ def test_text_chat_completion_structured_output(llama_stack_client, text_model_i assert answer.last_name == "Jordan" assert answer.year_of_birth == 1963 assert answer.num_seasons_in_nba == 15 - - -def test_image_chat_completion_non_streaming(llama_stack_client, vision_model_id): - message = { - "role": "user", - "content": [ - { - "type": "image", - "image": { - "url": { - # TODO: Replace with Github based URI to resources/sample1.jpg - "uri": "https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" - }, - }, - }, - { - "type": "text", - "text": "Describe what is in this image.", - }, - ], - } - response = llama_stack_client.inference.chat_completion( - model_id=vision_model_id, - messages=[message], - stream=False, - ) - message_content = response.completion_message.content.lower().strip() - assert len(message_content) > 0 - assert any(expected in message_content for expected in {"dog", "puppy", "pup"}) - - -def test_image_chat_completion_streaming(llama_stack_client, vision_model_id): - message = { - "role": "user", - "content": [ - { - "type": "image", - "image": { - "url": { - # TODO: Replace with Github based URI to resources/sample1.jpg - "uri": "https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" - }, - }, - }, - { - "type": "text", - "text": "Describe what is in this image.", - }, - ], - } - response = llama_stack_client.inference.chat_completion( - model_id=vision_model_id, - messages=[message], - stream=True, - ) - streamed_content = "" - for chunk in response: - streamed_content += chunk.event.delta.text.lower() - assert len(streamed_content) > 0 - assert any(expected in streamed_content for expected in {"dog", "puppy", "pup"}) - - -@pytest.mark.parametrize("type_", ["url", "data"]) -def test_image_chat_completion_base64(llama_stack_client, vision_model_id, base64_image_data, base64_image_url, type_): - image_spec = { - "url": { - "type": "image", - "image": { - "url": { - "uri": base64_image_url, - }, - }, - }, - "data": { - "type": "image", - "image": { - "data": base64_image_data, - }, - }, - }[type_] - - message = { - "role": "user", - "content": [ - image_spec, - { - "type": "text", - "text": "Describe what is in this image.", - }, - ], - } - response = llama_stack_client.inference.chat_completion( - model_id=vision_model_id, - messages=[message], - stream=False, - ) - message_content = response.completion_message.content.lower().strip() - assert len(message_content) > 0 diff --git a/tests/client-sdk/inference/test_vision_inference.py b/tests/client-sdk/inference/test_vision_inference.py new file mode 100644 index 000000000..df4b9d933 --- /dev/null +++ b/tests/client-sdk/inference/test_vision_inference.py @@ -0,0 +1,133 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import base64 +import pathlib + +import pytest + + +@pytest.fixture(scope="session") +def inference_provider_type(llama_stack_client): + providers = llama_stack_client.providers.list() + inference_providers = [p for p in providers if p.api == "inference"] + assert len(inference_providers) > 0, "No inference providers found" + return inference_providers[0].provider_type + + +@pytest.fixture +def image_path(): + return pathlib.Path(__file__).parent / "dog.png" + + +@pytest.fixture +def base64_image_data(image_path): + # Convert the image to base64 + return base64.b64encode(image_path.read_bytes()).decode("utf-8") + + +@pytest.fixture +def base64_image_url(base64_image_data, image_path): + # suffix includes the ., so we remove it + return f"data:image/{image_path.suffix[1:]};base64,{base64_image_data}" + + +def test_image_chat_completion_non_streaming(llama_stack_client, vision_model_id): + message = { + "role": "user", + "content": [ + { + "type": "image", + "image": { + "url": { + # TODO: Replace with Github based URI to resources/sample1.jpg + "uri": "https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + }, + }, + }, + { + "type": "text", + "text": "Describe what is in this image.", + }, + ], + } + response = llama_stack_client.inference.chat_completion( + model_id=vision_model_id, + messages=[message], + stream=False, + ) + message_content = response.completion_message.content.lower().strip() + assert len(message_content) > 0 + assert any(expected in message_content for expected in {"dog", "puppy", "pup"}) + + +def test_image_chat_completion_streaming(llama_stack_client, vision_model_id): + message = { + "role": "user", + "content": [ + { + "type": "image", + "image": { + "url": { + # TODO: Replace with Github based URI to resources/sample1.jpg + "uri": "https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + }, + }, + }, + { + "type": "text", + "text": "Describe what is in this image.", + }, + ], + } + response = llama_stack_client.inference.chat_completion( + model_id=vision_model_id, + messages=[message], + stream=True, + ) + streamed_content = "" + for chunk in response: + streamed_content += chunk.event.delta.text.lower() + assert len(streamed_content) > 0 + assert any(expected in streamed_content for expected in {"dog", "puppy", "pup"}) + + +@pytest.mark.parametrize("type_", ["url", "data"]) +def test_image_chat_completion_base64(llama_stack_client, vision_model_id, base64_image_data, base64_image_url, type_): + image_spec = { + "url": { + "type": "image", + "image": { + "url": { + "uri": base64_image_url, + }, + }, + }, + "data": { + "type": "image", + "image": { + "data": base64_image_data, + }, + }, + }[type_] + + message = { + "role": "user", + "content": [ + image_spec, + { + "type": "text", + "text": "Describe what is in this image.", + }, + ], + } + response = llama_stack_client.inference.chat_completion( + model_id=vision_model_id, + messages=[message], + stream=False, + ) + message_content = response.completion_message.content.lower().strip() + assert len(message_content) > 0