From 5736c7d6826569329f163c4bfa6146e6e0368f29 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Mon, 3 Mar 2025 17:28:12 -0800 Subject: [PATCH] refactor: move tests/client-sdk to tests/api (#1376) This PR moves the client-sdk tests to the api directory to better reflect their purpose and improve code organization. --- CONTRIBUTING.md | 2 +- docs/source/contributing/new_api_provider.md | 2 +- .../tests/inference/test_vision_inference.py | 4 ++-- llama_stack/scripts/run_client_sdk_tests.py | 6 +++--- tests/{client-sdk => api}/README.md | 8 ++++---- tests/{client-sdk => api}/__init__.py | 0 tests/{client-sdk => api}/agents/__init__.py | 0 tests/{client-sdk => api}/agents/test_agents.py | 0 tests/{client-sdk => api}/conftest.py | 0 .../{client-sdk => api}/fixtures/recordable_mock.py | 0 .../recorded_responses/chat_completion.json | 0 .../recorded_responses/chat_completion.pickle | Bin .../fixtures/recorded_responses/invoke_tool.json | 0 .../fixtures/recorded_responses/invoke_tool.pickle | Bin tests/{client-sdk => api}/inference/__init__.py | 0 tests/{client-sdk => api}/inference/dog.png | Bin .../{client-sdk => api}/inference/test_embedding.py | 0 .../inference/test_text_inference.py | 0 .../inference/test_vision_inference.py | 4 ++-- tests/{client-sdk => api}/metadata.py | 0 tests/{client-sdk => api}/report.py | 0 tests/{client-sdk => api}/safety/__init__.py | 0 tests/{client-sdk => api}/safety/conftest.py | 0 .../safety/resources/example_safe.jpg | Bin .../safety/resources/example_unsafe.jpg | Bin tests/{client-sdk => api}/safety/test_safety.py | 0 .../tool_runtime/test_rag_tool.py | 0 tests/{client-sdk => api}/vector_io/__init__.py | 0 .../{client-sdk => api}/vector_io/test_vector_io.py | 0 29 files changed, 13 insertions(+), 13 deletions(-) rename tests/{client-sdk => api}/README.md (79%) rename tests/{client-sdk => api}/__init__.py (100%) rename tests/{client-sdk => api}/agents/__init__.py (100%) rename tests/{client-sdk => api}/agents/test_agents.py (100%) rename tests/{client-sdk => api}/conftest.py (100%) rename tests/{client-sdk => api}/fixtures/recordable_mock.py (100%) rename tests/{client-sdk => api}/fixtures/recorded_responses/chat_completion.json (100%) rename tests/{client-sdk => api}/fixtures/recorded_responses/chat_completion.pickle (100%) rename tests/{client-sdk => api}/fixtures/recorded_responses/invoke_tool.json (100%) rename tests/{client-sdk => api}/fixtures/recorded_responses/invoke_tool.pickle (100%) rename tests/{client-sdk => api}/inference/__init__.py (100%) rename tests/{client-sdk => api}/inference/dog.png (100%) rename tests/{client-sdk => api}/inference/test_embedding.py (100%) rename tests/{client-sdk => api}/inference/test_text_inference.py (100%) rename tests/{client-sdk => api}/inference/test_vision_inference.py (96%) rename tests/{client-sdk => api}/metadata.py (100%) rename tests/{client-sdk => api}/report.py (100%) rename tests/{client-sdk => api}/safety/__init__.py (100%) rename tests/{client-sdk => api}/safety/conftest.py (100%) rename tests/{client-sdk => api}/safety/resources/example_safe.jpg (100%) rename tests/{client-sdk => api}/safety/resources/example_unsafe.jpg (100%) rename tests/{client-sdk => api}/safety/test_safety.py (100%) rename tests/{client-sdk => api}/tool_runtime/test_rag_tool.py (100%) rename tests/{client-sdk => api}/vector_io/__init__.py (100%) rename tests/{client-sdk => api}/vector_io/test_vector_io.py (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0da84d9b0..224dc4d14 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,7 +80,7 @@ LLAMA_STACK_CONFIG= And then use this dotenv file when running client SDK tests via the following: ```bash -$ uv run --env-file .env -- pytest -v tests/client-sdk/inference/test_text_inference.py +$ uv run --env-file .env -- pytest -v tests/api/inference/test_text_inference.py ``` ## Pre-commit Hooks diff --git a/docs/source/contributing/new_api_provider.md b/docs/source/contributing/new_api_provider.md index dbbdb9646..78f49df82 100644 --- a/docs/source/contributing/new_api_provider.md +++ b/docs/source/contributing/new_api_provider.md @@ -21,7 +21,7 @@ Here are some example PRs to help you get started: - Create integration tests that use real provider instances and configurations - For remote services, test actual API interactions - Avoid mocking at the provider level since adapter layers tend to be thin -- Reference examples in {repopath}`tests/client-sdk` +- Reference examples in {repopath}`tests/api` ### 2. Unit Testing (Optional) - Add unit tests for provider-specific functionality diff --git a/llama_stack/providers/tests/inference/test_vision_inference.py b/llama_stack/providers/tests/inference/test_vision_inference.py index 4d7183c49..b3e490f0e 100644 --- a/llama_stack/providers/tests/inference/test_vision_inference.py +++ b/llama_stack/providers/tests/inference/test_vision_inference.py @@ -39,7 +39,7 @@ class TestVisionModelInference: ImageContentItem( image=dict( url=URL( - uri="https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/client-sdk/inference/dog.png" + uri="https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/api/inference/dog.png" ) ) ), @@ -80,7 +80,7 @@ class TestVisionModelInference: ImageContentItem( image=dict( url=URL( - uri="https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/client-sdk/inference/dog.png" + uri="https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/api/inference/dog.png" ) ) ), diff --git a/llama_stack/scripts/run_client_sdk_tests.py b/llama_stack/scripts/run_client_sdk_tests.py index 6aaeb3273..e70d187aa 100644 --- a/llama_stack/scripts/run_client_sdk_tests.py +++ b/llama_stack/scripts/run_client_sdk_tests.py @@ -11,14 +11,14 @@ from pathlib import Path import pytest """ -Script for running client-sdk on AsyncLlamaStackAsLibraryClient with templates +Script for running api on AsyncLlamaStackAsLibraryClient with templates Assuming directory structure: - llama-stack - llama_stack - scripts - tests - - client-sdk + - api Example command: @@ -29,7 +29,7 @@ python llama_stack/scripts/run_client_sdk_tests.py --templates together firework """ REPO_ROOT = Path(__file__).parent.parent.parent -CLIENT_SDK_TESTS_RELATIVE_PATH = "tests/client-sdk/" +CLIENT_SDK_TESTS_RELATIVE_PATH = "tests/api/" def main(parser: argparse.ArgumentParser): diff --git a/tests/client-sdk/README.md b/tests/api/README.md similarity index 79% rename from tests/client-sdk/README.md rename to tests/api/README.md index 703d06a39..cd2b07b8c 100644 --- a/tests/client-sdk/README.md +++ b/tests/api/README.md @@ -3,23 +3,23 @@ You can run llama stack integration tests on either a Llama Stack Library or a L To test on a Llama Stack library with certain configuration, run ```bash -LLAMA_STACK_CONFIG=./llama_stack/templates/cerebras/run.yaml pytest -s -v tests/client-sdk/inference/ +LLAMA_STACK_CONFIG=./llama_stack/templates/cerebras/run.yaml pytest -s -v tests/api/inference/ ``` or just the template name ```bash -LLAMA_STACK_CONFIG=together pytest -s -v tests/client-sdk/inference/ +LLAMA_STACK_CONFIG=together pytest -s -v tests/api/inference/ ``` To test on a Llama Stack endpoint, run ```bash -LLAMA_STACK_BASE_URL=http://localhost:8089 pytest -s -v tests/client-sdk/inference +LLAMA_STACK_BASE_URL=http://localhost:8089 pytest -s -v tests/api/inference ``` ## Report Generation To generate a report, run with `--report` option ```bash -LLAMA_STACK_CONFIG=together pytest -s -v report.md tests/client-sdk/ --report +LLAMA_STACK_CONFIG=together pytest -s -v report.md tests/api/ --report ``` ## Common options diff --git a/tests/client-sdk/__init__.py b/tests/api/__init__.py similarity index 100% rename from tests/client-sdk/__init__.py rename to tests/api/__init__.py diff --git a/tests/client-sdk/agents/__init__.py b/tests/api/agents/__init__.py similarity index 100% rename from tests/client-sdk/agents/__init__.py rename to tests/api/agents/__init__.py diff --git a/tests/client-sdk/agents/test_agents.py b/tests/api/agents/test_agents.py similarity index 100% rename from tests/client-sdk/agents/test_agents.py rename to tests/api/agents/test_agents.py diff --git a/tests/client-sdk/conftest.py b/tests/api/conftest.py similarity index 100% rename from tests/client-sdk/conftest.py rename to tests/api/conftest.py diff --git a/tests/client-sdk/fixtures/recordable_mock.py b/tests/api/fixtures/recordable_mock.py similarity index 100% rename from tests/client-sdk/fixtures/recordable_mock.py rename to tests/api/fixtures/recordable_mock.py diff --git a/tests/client-sdk/fixtures/recorded_responses/chat_completion.json b/tests/api/fixtures/recorded_responses/chat_completion.json similarity index 100% rename from tests/client-sdk/fixtures/recorded_responses/chat_completion.json rename to tests/api/fixtures/recorded_responses/chat_completion.json diff --git a/tests/client-sdk/fixtures/recorded_responses/chat_completion.pickle b/tests/api/fixtures/recorded_responses/chat_completion.pickle similarity index 100% rename from tests/client-sdk/fixtures/recorded_responses/chat_completion.pickle rename to tests/api/fixtures/recorded_responses/chat_completion.pickle diff --git a/tests/client-sdk/fixtures/recorded_responses/invoke_tool.json b/tests/api/fixtures/recorded_responses/invoke_tool.json similarity index 100% rename from tests/client-sdk/fixtures/recorded_responses/invoke_tool.json rename to tests/api/fixtures/recorded_responses/invoke_tool.json diff --git a/tests/client-sdk/fixtures/recorded_responses/invoke_tool.pickle b/tests/api/fixtures/recorded_responses/invoke_tool.pickle similarity index 100% rename from tests/client-sdk/fixtures/recorded_responses/invoke_tool.pickle rename to tests/api/fixtures/recorded_responses/invoke_tool.pickle diff --git a/tests/client-sdk/inference/__init__.py b/tests/api/inference/__init__.py similarity index 100% rename from tests/client-sdk/inference/__init__.py rename to tests/api/inference/__init__.py diff --git a/tests/client-sdk/inference/dog.png b/tests/api/inference/dog.png similarity index 100% rename from tests/client-sdk/inference/dog.png rename to tests/api/inference/dog.png diff --git a/tests/client-sdk/inference/test_embedding.py b/tests/api/inference/test_embedding.py similarity index 100% rename from tests/client-sdk/inference/test_embedding.py rename to tests/api/inference/test_embedding.py diff --git a/tests/client-sdk/inference/test_text_inference.py b/tests/api/inference/test_text_inference.py similarity index 100% rename from tests/client-sdk/inference/test_text_inference.py rename to tests/api/inference/test_text_inference.py diff --git a/tests/client-sdk/inference/test_vision_inference.py b/tests/api/inference/test_vision_inference.py similarity index 96% rename from tests/client-sdk/inference/test_vision_inference.py rename to tests/api/inference/test_vision_inference.py index 8fa0d8023..6029a8c72 100644 --- a/tests/client-sdk/inference/test_vision_inference.py +++ b/tests/api/inference/test_vision_inference.py @@ -35,7 +35,7 @@ def test_image_chat_completion_non_streaming(client_with_models, vision_model_id "type": "image", "image": { "url": { - "uri": "https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/client-sdk/inference/dog.png" + "uri": "https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/api/inference/dog.png" }, }, }, @@ -63,7 +63,7 @@ def test_image_chat_completion_streaming(client_with_models, vision_model_id): "type": "image", "image": { "url": { - "uri": "https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/client-sdk/inference/dog.png" + "uri": "https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/api/inference/dog.png" }, }, }, diff --git a/tests/client-sdk/metadata.py b/tests/api/metadata.py similarity index 100% rename from tests/client-sdk/metadata.py rename to tests/api/metadata.py diff --git a/tests/client-sdk/report.py b/tests/api/report.py similarity index 100% rename from tests/client-sdk/report.py rename to tests/api/report.py diff --git a/tests/client-sdk/safety/__init__.py b/tests/api/safety/__init__.py similarity index 100% rename from tests/client-sdk/safety/__init__.py rename to tests/api/safety/__init__.py diff --git a/tests/client-sdk/safety/conftest.py b/tests/api/safety/conftest.py similarity index 100% rename from tests/client-sdk/safety/conftest.py rename to tests/api/safety/conftest.py diff --git a/tests/client-sdk/safety/resources/example_safe.jpg b/tests/api/safety/resources/example_safe.jpg similarity index 100% rename from tests/client-sdk/safety/resources/example_safe.jpg rename to tests/api/safety/resources/example_safe.jpg diff --git a/tests/client-sdk/safety/resources/example_unsafe.jpg b/tests/api/safety/resources/example_unsafe.jpg similarity index 100% rename from tests/client-sdk/safety/resources/example_unsafe.jpg rename to tests/api/safety/resources/example_unsafe.jpg diff --git a/tests/client-sdk/safety/test_safety.py b/tests/api/safety/test_safety.py similarity index 100% rename from tests/client-sdk/safety/test_safety.py rename to tests/api/safety/test_safety.py diff --git a/tests/client-sdk/tool_runtime/test_rag_tool.py b/tests/api/tool_runtime/test_rag_tool.py similarity index 100% rename from tests/client-sdk/tool_runtime/test_rag_tool.py rename to tests/api/tool_runtime/test_rag_tool.py diff --git a/tests/client-sdk/vector_io/__init__.py b/tests/api/vector_io/__init__.py similarity index 100% rename from tests/client-sdk/vector_io/__init__.py rename to tests/api/vector_io/__init__.py diff --git a/tests/client-sdk/vector_io/test_vector_io.py b/tests/api/vector_io/test_vector_io.py similarity index 100% rename from tests/client-sdk/vector_io/test_vector_io.py rename to tests/api/vector_io/test_vector_io.py