diff --git a/tests/client-sdk/agents/test_agents.py b/tests/client-sdk/agents/test_agents.py index f1da4e124..7c13f5768 100644 --- a/tests/client-sdk/agents/test_agents.py +++ b/tests/client-sdk/agents/test_agents.py @@ -79,18 +79,6 @@ class TestClientTool(ClientTool): return -1 -@pytest.fixture(scope="session") -def text_model_id(llama_stack_client): - available_models = [ - model.identifier - for model in llama_stack_client.models.list() - if model.identifier.startswith("meta-llama") and "405" not in model.identifier - ] - model_id = available_models[0] - print(f"Using model: {model_id}") - return model_id - - @pytest.fixture(scope="session") def agent_config(llama_stack_client, text_model_id): available_shields = [ diff --git a/tests/client-sdk/conftest.py b/tests/client-sdk/conftest.py index c19546887..0f0733010 100644 --- a/tests/client-sdk/conftest.py +++ b/tests/client-sdk/conftest.py @@ -20,6 +20,10 @@ def pytest_configure(config): config.pluginmanager.register(Report()) +TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" +VISION_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct" + + def pytest_addoption(parser): parser.addoption( "--report", @@ -27,10 +31,18 @@ def pytest_addoption(parser): action="store_true", help="Knob to determine if we should generate report, e.g. --output=True", ) - - -TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" -INFERENCE_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct" + parser.addoption( + "--inference-model", + action="store", + default=TEXT_MODEL, + help="Specify the inference model to use for testing", + ) + parser.addoption( + "--vision-inference-model", + action="store", + default=VISION_MODEL, + help="Specify the vision inference model to use for testing", + ) @pytest.fixture(scope="session") @@ -61,3 +73,18 @@ def llama_stack_client(provider_data): else: raise ValueError("LLAMA_STACK_CONFIG or LLAMA_STACK_BASE_URL must be set") return client + + +def pytest_generate_tests(metafunc): + if "text_model_id" in metafunc.fixturenames: + metafunc.parametrize( + "text_model_id", + [metafunc.config.getoption("--inference-model")], + scope="session", + ) + if "vision_model_id" in metafunc.fixturenames: + metafunc.parametrize( + "vision_model_id", + [metafunc.config.getoption("--vision-inference-model")], + scope="session", + ) diff --git a/tests/client-sdk/inference/test_inference.py b/tests/client-sdk/inference/test_inference.py index b1f1dd139..01bbd7dc0 100644 --- a/tests/client-sdk/inference/test_inference.py +++ b/tests/client-sdk/inference/test_inference.py @@ -34,30 +34,6 @@ def inference_provider_type(llama_stack_client): return inference_providers[0].provider_type -@pytest.fixture(scope="session") -def text_model_id(llama_stack_client): - available_models = [ - model.identifier - for model in llama_stack_client.models.list() - if model.identifier.startswith("meta-llama") and "405" not in model.identifier - ] - assert len(available_models) > 0 - return available_models[0] - - -@pytest.fixture(scope="session") -def vision_model_id(llama_stack_client): - available_models = [ - model.identifier - for model in llama_stack_client.models.list() - if "vision" in model.identifier.lower() - ] - if len(available_models) == 0: - pytest.skip("No vision models available") - - return available_models[0] - - @pytest.fixture def get_weather_tool_definition(): return { @@ -107,6 +83,7 @@ def test_text_completion_streaming(llama_stack_client, text_model_id): assert "blue" in "".join(streamed_content).lower().strip() +@pytest.mark.skip("Most inference providers don't support log probs yet") def test_completion_log_probs_non_streaming(llama_stack_client, text_model_id): response = llama_stack_client.inference.completion( content="Complete the sentence: Micheael Jordan is born in ", @@ -124,6 +101,7 @@ def test_completion_log_probs_non_streaming(llama_stack_client, text_model_id): assert all(len(logprob.logprobs_by_token) == 3 for logprob in response.logprobs) +@pytest.mark.skip("Most inference providers don't support log probs yet") def test_completion_log_probs_streaming(llama_stack_client, text_model_id): response = llama_stack_client.inference.completion( content="Complete the sentence: Micheael Jordan is born in ", diff --git a/tests/client-sdk/safety/conftest.py b/tests/client-sdk/safety/conftest.py new file mode 100644 index 000000000..9c5ff7352 --- /dev/null +++ b/tests/client-sdk/safety/conftest.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +def pytest_addoption(parser): + parser.addoption( + "--safety_shield", + action="store", + default="meta-llama/Llama-Guard-3-1B", + help="Specify the safety shield model to use for testing", + ) + + +def pytest_generate_tests(metafunc): + if "llama_guard_text_shield_id" in metafunc.fixturenames: + metafunc.parametrize( + "llama_guard_text_shield_id", + [metafunc.config.getoption("--safety_shield")], + ) diff --git a/tests/client-sdk/safety/test_safety.py b/tests/client-sdk/safety/test_safety.py index ac3221364..7456fb88f 100644 --- a/tests/client-sdk/safety/test_safety.py +++ b/tests/client-sdk/safety/test_safety.py @@ -32,16 +32,6 @@ def available_shields(llama_stack_client): return [shield.identifier for shield in llama_stack_client.shields.list()] -@pytest.fixture(scope="session") -def llama_guard_text_shield_id(available_shields): - if "meta-llama/Llama-Guard-3-1B" in available_shields: - return "meta-llama/Llama-Guard-3-1B" - elif "meta-llama/Llama-Guard-3-8B" in available_shields: - return "meta-llama/Llama-Guard-3-8B" - else: - pytest.skip("Llama-Guard shield is not available. Skipping.") - - @pytest.fixture(scope="session") def code_scanner_shield_id(available_shields): if "CodeScanner" in available_shields: diff --git a/tests/client-sdk/vector_io/conftest.py b/tests/client-sdk/vector_io/conftest.py new file mode 100644 index 000000000..64cac27d2 --- /dev/null +++ b/tests/client-sdk/vector_io/conftest.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +def pytest_addoption(parser): + parser.addoption( + "--embedding-model", + action="store", + default="all-MiniLM-L6-v2", + help="Specify the embedding model to use for testing", + ) + + +def pytest_generate_tests(metafunc): + if "embedding_model" in metafunc.fixturenames: + metafunc.parametrize( + "embedding_model", + [metafunc.config.getoption("--embedding-model")], + ) diff --git a/tests/client-sdk/vector_io/test_vector_io.py b/tests/client-sdk/vector_io/test_vector_io.py index 04b639667..20e49d805 100644 --- a/tests/client-sdk/vector_io/test_vector_io.py +++ b/tests/client-sdk/vector_io/test_vector_io.py @@ -6,39 +6,13 @@ import random -import pytest - -@pytest.fixture(scope="function") -def empty_vector_db_registry(llama_stack_client): - vector_dbs = [ - vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() - ] - for vector_db_id in vector_dbs: - llama_stack_client.vector_dbs.unregister(vector_db_id=vector_db_id) - - -@pytest.fixture(scope="function") -def single_entry_vector_db_registry(llama_stack_client, empty_vector_db_registry): - vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" - llama_stack_client.vector_dbs.register( - vector_db_id=vector_db_id, - embedding_model="all-MiniLM-L6-v2", - embedding_dimension=384, - provider_id="faiss", - ) - vector_dbs = [ - vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() - ] - return vector_dbs - - -def test_vector_db_retrieve(llama_stack_client, empty_vector_db_registry): +def test_vector_db_retrieve(llama_stack_client, embedding_model): # Register a memory bank first vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" llama_stack_client.vector_dbs.register( vector_db_id=vector_db_id, - embedding_model="all-MiniLM-L6-v2", + embedding_model=embedding_model, embedding_dimension=384, provider_id="faiss", ) @@ -47,23 +21,23 @@ def test_vector_db_retrieve(llama_stack_client, empty_vector_db_registry): response = llama_stack_client.vector_dbs.retrieve(vector_db_id=vector_db_id) assert response is not None assert response.identifier == vector_db_id - assert response.embedding_model == "all-MiniLM-L6-v2" + assert response.embedding_model == embedding_model assert response.provider_id == "faiss" assert response.provider_resource_id == vector_db_id -def test_vector_db_list(llama_stack_client, empty_vector_db_registry): +def test_vector_db_list(llama_stack_client): vector_dbs_after_register = [ vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() ] assert len(vector_dbs_after_register) == 0 -def test_vector_db_register(llama_stack_client, empty_vector_db_registry): +def test_vector_db_register(llama_stack_client, embedding_model): vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" llama_stack_client.vector_dbs.register( vector_db_id=vector_db_id, - embedding_model="all-MiniLM-L6-v2", + embedding_model=embedding_model, embedding_dimension=384, provider_id="faiss", ) @@ -74,7 +48,7 @@ def test_vector_db_register(llama_stack_client, empty_vector_db_registry): assert vector_dbs_after_register == [vector_db_id] -def test_vector_db_unregister(llama_stack_client, single_entry_vector_db_registry): +def test_vector_db_unregister(llama_stack_client): vector_dbs = [ vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() ]