[client sdk test] add options for inference_model, safety_shield, embedding_model (#843)

# What does this PR do?
Default inference_model for testing: "meta-llama/Llama-3.1-8B-Instruct"
Default vision inference_model for testing:
"meta-llama/Llama-3.2-11B-Vision-Instruct"


## Test Plan
`/opt/miniconda3/envs/stack/bin/pytest -s -v
--inference-model=meta-llama/Llama-3.2-3B-Instruct
tests/client-sdk/agents`


`/opt/miniconda3/envs/stack/bin/pytest -s -v
--embedding-model=all-MiniLM-L6-v2 tests/client-sdk/vector_io`

`/opt/miniconda3/envs/stack/bin/pytest -s -v
--safety-shield=meta-llama/Llama-Guard-3-1B tests/client-sdk/safety`

## Sources

Please link relevant resources if necessary.


## Before submitting

- [ ] This PR fixes a typo or improves the docs (you can dismiss the
other checks if that's the case).
- [ ] Ran pre-commit to handle lint / formatting issues.
- [ ] Read the [contributor
guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md),
      Pull Request section?
- [ ] Updated relevant documentation.
- [ ] Wrote necessary unit or integration tests.

tests/client-sdk/agents/test_agents.py

@@ -79,18 +79,6 @@ class TestClientTool(ClientTool):
             return -1
 
 
-@pytest.fixture(scope="session")
-def text_model_id(llama_stack_client):
-    available_models = [
-        model.identifier
-        for model in llama_stack_client.models.list()
-        if model.identifier.startswith("meta-llama") and "405" not in model.identifier
-    ]
-    model_id = available_models[0]
-    print(f"Using model: {model_id}")
-    return model_id
-
-
 @pytest.fixture(scope="session")
 def agent_config(llama_stack_client, text_model_id):
     available_shields = [

tests/client-sdk/conftest.py

@@ -20,6 +20,10 @@ def pytest_configure(config):
         config.pluginmanager.register(Report())
 
 
+TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct"
+VISION_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct"
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--report",
@@ -27,10 +31,18 @@ def pytest_addoption(parser):
         action="store_true",
         help="Knob to determine if we should generate report, e.g. --output=True",
     )
-
-
-TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct"
-INFERENCE_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct"
+    parser.addoption(
+        "--inference-model",
+        action="store",
+        default=TEXT_MODEL,
+        help="Specify the inference model to use for testing",
+    )
+    parser.addoption(
+        "--vision-inference-model",
+        action="store",
+        default=VISION_MODEL,
+        help="Specify the vision inference model to use for testing",
+    )
 
 
 @pytest.fixture(scope="session")
@@ -61,3 +73,18 @@ def llama_stack_client(provider_data):
     else:
         raise ValueError("LLAMA_STACK_CONFIG or LLAMA_STACK_BASE_URL must be set")
     return client
+
+
+def pytest_generate_tests(metafunc):
+    if "text_model_id" in metafunc.fixturenames:
+        metafunc.parametrize(
+            "text_model_id",
+            [metafunc.config.getoption("--inference-model")],
+            scope="session",
+        )
+    if "vision_model_id" in metafunc.fixturenames:
+        metafunc.parametrize(
+            "vision_model_id",
+            [metafunc.config.getoption("--vision-inference-model")],
+            scope="session",
+        )

tests/client-sdk/inference/test_inference.py

@@ -34,30 +34,6 @@ def inference_provider_type(llama_stack_client):
     return inference_providers[0].provider_type
 
 
-@pytest.fixture(scope="session")
-def text_model_id(llama_stack_client):
-    available_models = [
-        model.identifier
-        for model in llama_stack_client.models.list()
-        if model.identifier.startswith("meta-llama") and "405" not in model.identifier
-    ]
-    assert len(available_models) > 0
-    return available_models[0]
-
-
-@pytest.fixture(scope="session")
-def vision_model_id(llama_stack_client):
-    available_models = [
-        model.identifier
-        for model in llama_stack_client.models.list()
-        if "vision" in model.identifier.lower()
-    ]
-    if len(available_models) == 0:
-        pytest.skip("No vision models available")
-
-    return available_models[0]
-
-
 @pytest.fixture
 def get_weather_tool_definition():
     return {
@@ -107,6 +83,7 @@ def test_text_completion_streaming(llama_stack_client, text_model_id):
     assert "blue" in "".join(streamed_content).lower().strip()
 
 
+@pytest.mark.skip("Most inference providers don't support log probs yet")
 def test_completion_log_probs_non_streaming(llama_stack_client, text_model_id):
     response = llama_stack_client.inference.completion(
         content="Complete the sentence: Micheael Jordan is born in ",
@@ -124,6 +101,7 @@ def test_completion_log_probs_non_streaming(llama_stack_client, text_model_id):
     assert all(len(logprob.logprobs_by_token) == 3 for logprob in response.logprobs)
 
 
+@pytest.mark.skip("Most inference providers don't support log probs yet")
 def test_completion_log_probs_streaming(llama_stack_client, text_model_id):
     response = llama_stack_client.inference.completion(
         content="Complete the sentence: Micheael Jordan is born in ",

tests/client-sdk/safety/conftest.py

@@ -0,0 +1,22 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--safety_shield",
+        action="store",
+        default="meta-llama/Llama-Guard-3-1B",
+        help="Specify the safety shield model to use for testing",
+    )
+
+
+def pytest_generate_tests(metafunc):
+    if "llama_guard_text_shield_id" in metafunc.fixturenames:
+        metafunc.parametrize(
+            "llama_guard_text_shield_id",
+            [metafunc.config.getoption("--safety_shield")],
+        )

tests/client-sdk/safety/test_safety.py

@@ -32,16 +32,6 @@ def available_shields(llama_stack_client):
     return [shield.identifier for shield in llama_stack_client.shields.list()]
 
 
-@pytest.fixture(scope="session")
-def llama_guard_text_shield_id(available_shields):
-    if "meta-llama/Llama-Guard-3-1B" in available_shields:
-        return "meta-llama/Llama-Guard-3-1B"
-    elif "meta-llama/Llama-Guard-3-8B" in available_shields:
-        return "meta-llama/Llama-Guard-3-8B"
-    else:
-        pytest.skip("Llama-Guard shield is not available. Skipping.")
-
-
 @pytest.fixture(scope="session")
 def code_scanner_shield_id(available_shields):
     if "CodeScanner" in available_shields:

tests/client-sdk/vector_io/conftest.py

@@ -0,0 +1,22 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--embedding-model",
+        action="store",
+        default="all-MiniLM-L6-v2",
+        help="Specify the embedding model to use for testing",
+    )
+
+
+def pytest_generate_tests(metafunc):
+    if "embedding_model" in metafunc.fixturenames:
+        metafunc.parametrize(
+            "embedding_model",
+            [metafunc.config.getoption("--embedding-model")],
+        )

tests/client-sdk/vector_io/test_vector_io.py

@@ -6,39 +6,13 @@
 
 import random
 
-import pytest
 
-
-@pytest.fixture(scope="function")
-def empty_vector_db_registry(llama_stack_client):
-    vector_dbs = [
-        vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list()
-    ]
-    for vector_db_id in vector_dbs:
-        llama_stack_client.vector_dbs.unregister(vector_db_id=vector_db_id)
-
-
-@pytest.fixture(scope="function")
-def single_entry_vector_db_registry(llama_stack_client, empty_vector_db_registry):
-    vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}"
-    llama_stack_client.vector_dbs.register(
-        vector_db_id=vector_db_id,
-        embedding_model="all-MiniLM-L6-v2",
-        embedding_dimension=384,
-        provider_id="faiss",
-    )
-    vector_dbs = [
-        vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list()
-    ]
-    return vector_dbs
-
-
-def test_vector_db_retrieve(llama_stack_client, empty_vector_db_registry):
+def test_vector_db_retrieve(llama_stack_client, embedding_model):
     # Register a memory bank first
     vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}"
     llama_stack_client.vector_dbs.register(
         vector_db_id=vector_db_id,
-        embedding_model="all-MiniLM-L6-v2",
+        embedding_model=embedding_model,
         embedding_dimension=384,
         provider_id="faiss",
     )
@@ -47,23 +21,23 @@ def test_vector_db_retrieve(llama_stack_client, empty_vector_db_registry):
     response = llama_stack_client.vector_dbs.retrieve(vector_db_id=vector_db_id)
     assert response is not None
     assert response.identifier == vector_db_id
-    assert response.embedding_model == "all-MiniLM-L6-v2"
+    assert response.embedding_model == embedding_model
     assert response.provider_id == "faiss"
     assert response.provider_resource_id == vector_db_id
 
 
-def test_vector_db_list(llama_stack_client, empty_vector_db_registry):
+def test_vector_db_list(llama_stack_client):
     vector_dbs_after_register = [
         vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list()
     ]
     assert len(vector_dbs_after_register) == 0
 
 
-def test_vector_db_register(llama_stack_client, empty_vector_db_registry):
+def test_vector_db_register(llama_stack_client, embedding_model):
     vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}"
     llama_stack_client.vector_dbs.register(
         vector_db_id=vector_db_id,
-        embedding_model="all-MiniLM-L6-v2",
+        embedding_model=embedding_model,
         embedding_dimension=384,
         provider_id="faiss",
     )
@@ -74,7 +48,7 @@ def test_vector_db_register(llama_stack_client, empty_vector_db_registry):
     assert vector_dbs_after_register == [vector_db_id]
 
 
-def test_vector_db_unregister(llama_stack_client, single_entry_vector_db_registry):
+def test_vector_db_unregister(llama_stack_client):
     vector_dbs = [
         vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list()
     ]