From f4f47970e536f007a53ba0dbe23026a5a1e13821 Mon Sep 17 00:00:00 2001 From: Sixian Yi Date: Wed, 22 Jan 2025 15:35:19 -0800 Subject: [PATCH] [client sdk test] add options for inference_model, safety_shield, embedding_model (#843) # What does this PR do? Default inference_model for testing: "meta-llama/Llama-3.1-8B-Instruct" Default vision inference_model for testing: "meta-llama/Llama-3.2-11B-Vision-Instruct" ## Test Plan `/opt/miniconda3/envs/stack/bin/pytest -s -v --inference-model=meta-llama/Llama-3.2-3B-Instruct tests/client-sdk/agents` `/opt/miniconda3/envs/stack/bin/pytest -s -v --embedding-model=all-MiniLM-L6-v2 tests/client-sdk/vector_io` `/opt/miniconda3/envs/stack/bin/pytest -s -v --safety-shield=meta-llama/Llama-Guard-3-1B tests/client-sdk/safety` ## Sources Please link relevant resources if necessary. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Ran pre-commit to handle lint / formatting issues. - [ ] Read the [contributor guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md), Pull Request section? - [ ] Updated relevant documentation. - [ ] Wrote necessary unit or integration tests. --- tests/client-sdk/agents/test_agents.py | 12 ------ tests/client-sdk/conftest.py | 35 +++++++++++++++-- tests/client-sdk/inference/test_inference.py | 26 +------------ tests/client-sdk/safety/conftest.py | 22 +++++++++++ tests/client-sdk/safety/test_safety.py | 10 ----- tests/client-sdk/vector_io/conftest.py | 22 +++++++++++ tests/client-sdk/vector_io/test_vector_io.py | 40 ++++---------------- 7 files changed, 84 insertions(+), 83 deletions(-) create mode 100644 tests/client-sdk/safety/conftest.py create mode 100644 tests/client-sdk/vector_io/conftest.py diff --git a/tests/client-sdk/agents/test_agents.py b/tests/client-sdk/agents/test_agents.py index f1da4e124..7c13f5768 100644 --- a/tests/client-sdk/agents/test_agents.py +++ b/tests/client-sdk/agents/test_agents.py @@ -79,18 +79,6 @@ class TestClientTool(ClientTool): return -1 -@pytest.fixture(scope="session") -def text_model_id(llama_stack_client): - available_models = [ - model.identifier - for model in llama_stack_client.models.list() - if model.identifier.startswith("meta-llama") and "405" not in model.identifier - ] - model_id = available_models[0] - print(f"Using model: {model_id}") - return model_id - - @pytest.fixture(scope="session") def agent_config(llama_stack_client, text_model_id): available_shields = [ diff --git a/tests/client-sdk/conftest.py b/tests/client-sdk/conftest.py index c19546887..0f0733010 100644 --- a/tests/client-sdk/conftest.py +++ b/tests/client-sdk/conftest.py @@ -20,6 +20,10 @@ def pytest_configure(config): config.pluginmanager.register(Report()) +TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" +VISION_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct" + + def pytest_addoption(parser): parser.addoption( "--report", @@ -27,10 +31,18 @@ def pytest_addoption(parser): action="store_true", help="Knob to determine if we should generate report, e.g. --output=True", ) - - -TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" -INFERENCE_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct" + parser.addoption( + "--inference-model", + action="store", + default=TEXT_MODEL, + help="Specify the inference model to use for testing", + ) + parser.addoption( + "--vision-inference-model", + action="store", + default=VISION_MODEL, + help="Specify the vision inference model to use for testing", + ) @pytest.fixture(scope="session") @@ -61,3 +73,18 @@ def llama_stack_client(provider_data): else: raise ValueError("LLAMA_STACK_CONFIG or LLAMA_STACK_BASE_URL must be set") return client + + +def pytest_generate_tests(metafunc): + if "text_model_id" in metafunc.fixturenames: + metafunc.parametrize( + "text_model_id", + [metafunc.config.getoption("--inference-model")], + scope="session", + ) + if "vision_model_id" in metafunc.fixturenames: + metafunc.parametrize( + "vision_model_id", + [metafunc.config.getoption("--vision-inference-model")], + scope="session", + ) diff --git a/tests/client-sdk/inference/test_inference.py b/tests/client-sdk/inference/test_inference.py index b1f1dd139..01bbd7dc0 100644 --- a/tests/client-sdk/inference/test_inference.py +++ b/tests/client-sdk/inference/test_inference.py @@ -34,30 +34,6 @@ def inference_provider_type(llama_stack_client): return inference_providers[0].provider_type -@pytest.fixture(scope="session") -def text_model_id(llama_stack_client): - available_models = [ - model.identifier - for model in llama_stack_client.models.list() - if model.identifier.startswith("meta-llama") and "405" not in model.identifier - ] - assert len(available_models) > 0 - return available_models[0] - - -@pytest.fixture(scope="session") -def vision_model_id(llama_stack_client): - available_models = [ - model.identifier - for model in llama_stack_client.models.list() - if "vision" in model.identifier.lower() - ] - if len(available_models) == 0: - pytest.skip("No vision models available") - - return available_models[0] - - @pytest.fixture def get_weather_tool_definition(): return { @@ -107,6 +83,7 @@ def test_text_completion_streaming(llama_stack_client, text_model_id): assert "blue" in "".join(streamed_content).lower().strip() +@pytest.mark.skip("Most inference providers don't support log probs yet") def test_completion_log_probs_non_streaming(llama_stack_client, text_model_id): response = llama_stack_client.inference.completion( content="Complete the sentence: Micheael Jordan is born in ", @@ -124,6 +101,7 @@ def test_completion_log_probs_non_streaming(llama_stack_client, text_model_id): assert all(len(logprob.logprobs_by_token) == 3 for logprob in response.logprobs) +@pytest.mark.skip("Most inference providers don't support log probs yet") def test_completion_log_probs_streaming(llama_stack_client, text_model_id): response = llama_stack_client.inference.completion( content="Complete the sentence: Micheael Jordan is born in ", diff --git a/tests/client-sdk/safety/conftest.py b/tests/client-sdk/safety/conftest.py new file mode 100644 index 000000000..9c5ff7352 --- /dev/null +++ b/tests/client-sdk/safety/conftest.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +def pytest_addoption(parser): + parser.addoption( + "--safety_shield", + action="store", + default="meta-llama/Llama-Guard-3-1B", + help="Specify the safety shield model to use for testing", + ) + + +def pytest_generate_tests(metafunc): + if "llama_guard_text_shield_id" in metafunc.fixturenames: + metafunc.parametrize( + "llama_guard_text_shield_id", + [metafunc.config.getoption("--safety_shield")], + ) diff --git a/tests/client-sdk/safety/test_safety.py b/tests/client-sdk/safety/test_safety.py index ac3221364..7456fb88f 100644 --- a/tests/client-sdk/safety/test_safety.py +++ b/tests/client-sdk/safety/test_safety.py @@ -32,16 +32,6 @@ def available_shields(llama_stack_client): return [shield.identifier for shield in llama_stack_client.shields.list()] -@pytest.fixture(scope="session") -def llama_guard_text_shield_id(available_shields): - if "meta-llama/Llama-Guard-3-1B" in available_shields: - return "meta-llama/Llama-Guard-3-1B" - elif "meta-llama/Llama-Guard-3-8B" in available_shields: - return "meta-llama/Llama-Guard-3-8B" - else: - pytest.skip("Llama-Guard shield is not available. Skipping.") - - @pytest.fixture(scope="session") def code_scanner_shield_id(available_shields): if "CodeScanner" in available_shields: diff --git a/tests/client-sdk/vector_io/conftest.py b/tests/client-sdk/vector_io/conftest.py new file mode 100644 index 000000000..64cac27d2 --- /dev/null +++ b/tests/client-sdk/vector_io/conftest.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +def pytest_addoption(parser): + parser.addoption( + "--embedding-model", + action="store", + default="all-MiniLM-L6-v2", + help="Specify the embedding model to use for testing", + ) + + +def pytest_generate_tests(metafunc): + if "embedding_model" in metafunc.fixturenames: + metafunc.parametrize( + "embedding_model", + [metafunc.config.getoption("--embedding-model")], + ) diff --git a/tests/client-sdk/vector_io/test_vector_io.py b/tests/client-sdk/vector_io/test_vector_io.py index 04b639667..20e49d805 100644 --- a/tests/client-sdk/vector_io/test_vector_io.py +++ b/tests/client-sdk/vector_io/test_vector_io.py @@ -6,39 +6,13 @@ import random -import pytest - -@pytest.fixture(scope="function") -def empty_vector_db_registry(llama_stack_client): - vector_dbs = [ - vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() - ] - for vector_db_id in vector_dbs: - llama_stack_client.vector_dbs.unregister(vector_db_id=vector_db_id) - - -@pytest.fixture(scope="function") -def single_entry_vector_db_registry(llama_stack_client, empty_vector_db_registry): - vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" - llama_stack_client.vector_dbs.register( - vector_db_id=vector_db_id, - embedding_model="all-MiniLM-L6-v2", - embedding_dimension=384, - provider_id="faiss", - ) - vector_dbs = [ - vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() - ] - return vector_dbs - - -def test_vector_db_retrieve(llama_stack_client, empty_vector_db_registry): +def test_vector_db_retrieve(llama_stack_client, embedding_model): # Register a memory bank first vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" llama_stack_client.vector_dbs.register( vector_db_id=vector_db_id, - embedding_model="all-MiniLM-L6-v2", + embedding_model=embedding_model, embedding_dimension=384, provider_id="faiss", ) @@ -47,23 +21,23 @@ def test_vector_db_retrieve(llama_stack_client, empty_vector_db_registry): response = llama_stack_client.vector_dbs.retrieve(vector_db_id=vector_db_id) assert response is not None assert response.identifier == vector_db_id - assert response.embedding_model == "all-MiniLM-L6-v2" + assert response.embedding_model == embedding_model assert response.provider_id == "faiss" assert response.provider_resource_id == vector_db_id -def test_vector_db_list(llama_stack_client, empty_vector_db_registry): +def test_vector_db_list(llama_stack_client): vector_dbs_after_register = [ vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() ] assert len(vector_dbs_after_register) == 0 -def test_vector_db_register(llama_stack_client, empty_vector_db_registry): +def test_vector_db_register(llama_stack_client, embedding_model): vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" llama_stack_client.vector_dbs.register( vector_db_id=vector_db_id, - embedding_model="all-MiniLM-L6-v2", + embedding_model=embedding_model, embedding_dimension=384, provider_id="faiss", ) @@ -74,7 +48,7 @@ def test_vector_db_register(llama_stack_client, empty_vector_db_registry): assert vector_dbs_after_register == [vector_db_id] -def test_vector_db_unregister(llama_stack_client, single_entry_vector_db_registry): +def test_vector_db_unregister(llama_stack_client): vector_dbs = [ vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() ]