From 6c5e71b1db7012844a4d419538793568f8dcdaa6 Mon Sep 17 00:00:00 2001 From: Sixian Yi Date: Wed, 22 Jan 2025 15:32:11 -0800 Subject: [PATCH] add options for vectorio and safety --- tests/client-sdk/safety/conftest.py | 22 +++++++++++ tests/client-sdk/safety/test_safety.py | 10 ----- tests/client-sdk/vector_io/conftest.py | 22 +++++++++++ tests/client-sdk/vector_io/test_vector_io.py | 40 ++++---------------- 4 files changed, 51 insertions(+), 43 deletions(-) create mode 100644 tests/client-sdk/safety/conftest.py create mode 100644 tests/client-sdk/vector_io/conftest.py diff --git a/tests/client-sdk/safety/conftest.py b/tests/client-sdk/safety/conftest.py new file mode 100644 index 000000000..9c5ff7352 --- /dev/null +++ b/tests/client-sdk/safety/conftest.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +def pytest_addoption(parser): + parser.addoption( + "--safety_shield", + action="store", + default="meta-llama/Llama-Guard-3-1B", + help="Specify the safety shield model to use for testing", + ) + + +def pytest_generate_tests(metafunc): + if "llama_guard_text_shield_id" in metafunc.fixturenames: + metafunc.parametrize( + "llama_guard_text_shield_id", + [metafunc.config.getoption("--safety_shield")], + ) diff --git a/tests/client-sdk/safety/test_safety.py b/tests/client-sdk/safety/test_safety.py index ac3221364..7456fb88f 100644 --- a/tests/client-sdk/safety/test_safety.py +++ b/tests/client-sdk/safety/test_safety.py @@ -32,16 +32,6 @@ def available_shields(llama_stack_client): return [shield.identifier for shield in llama_stack_client.shields.list()] -@pytest.fixture(scope="session") -def llama_guard_text_shield_id(available_shields): - if "meta-llama/Llama-Guard-3-1B" in available_shields: - return "meta-llama/Llama-Guard-3-1B" - elif "meta-llama/Llama-Guard-3-8B" in available_shields: - return "meta-llama/Llama-Guard-3-8B" - else: - pytest.skip("Llama-Guard shield is not available. Skipping.") - - @pytest.fixture(scope="session") def code_scanner_shield_id(available_shields): if "CodeScanner" in available_shields: diff --git a/tests/client-sdk/vector_io/conftest.py b/tests/client-sdk/vector_io/conftest.py new file mode 100644 index 000000000..64cac27d2 --- /dev/null +++ b/tests/client-sdk/vector_io/conftest.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +def pytest_addoption(parser): + parser.addoption( + "--embedding-model", + action="store", + default="all-MiniLM-L6-v2", + help="Specify the embedding model to use for testing", + ) + + +def pytest_generate_tests(metafunc): + if "embedding_model" in metafunc.fixturenames: + metafunc.parametrize( + "embedding_model", + [metafunc.config.getoption("--embedding-model")], + ) diff --git a/tests/client-sdk/vector_io/test_vector_io.py b/tests/client-sdk/vector_io/test_vector_io.py index 04b639667..20e49d805 100644 --- a/tests/client-sdk/vector_io/test_vector_io.py +++ b/tests/client-sdk/vector_io/test_vector_io.py @@ -6,39 +6,13 @@ import random -import pytest - -@pytest.fixture(scope="function") -def empty_vector_db_registry(llama_stack_client): - vector_dbs = [ - vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() - ] - for vector_db_id in vector_dbs: - llama_stack_client.vector_dbs.unregister(vector_db_id=vector_db_id) - - -@pytest.fixture(scope="function") -def single_entry_vector_db_registry(llama_stack_client, empty_vector_db_registry): - vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" - llama_stack_client.vector_dbs.register( - vector_db_id=vector_db_id, - embedding_model="all-MiniLM-L6-v2", - embedding_dimension=384, - provider_id="faiss", - ) - vector_dbs = [ - vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() - ] - return vector_dbs - - -def test_vector_db_retrieve(llama_stack_client, empty_vector_db_registry): +def test_vector_db_retrieve(llama_stack_client, embedding_model): # Register a memory bank first vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" llama_stack_client.vector_dbs.register( vector_db_id=vector_db_id, - embedding_model="all-MiniLM-L6-v2", + embedding_model=embedding_model, embedding_dimension=384, provider_id="faiss", ) @@ -47,23 +21,23 @@ def test_vector_db_retrieve(llama_stack_client, empty_vector_db_registry): response = llama_stack_client.vector_dbs.retrieve(vector_db_id=vector_db_id) assert response is not None assert response.identifier == vector_db_id - assert response.embedding_model == "all-MiniLM-L6-v2" + assert response.embedding_model == embedding_model assert response.provider_id == "faiss" assert response.provider_resource_id == vector_db_id -def test_vector_db_list(llama_stack_client, empty_vector_db_registry): +def test_vector_db_list(llama_stack_client): vector_dbs_after_register = [ vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() ] assert len(vector_dbs_after_register) == 0 -def test_vector_db_register(llama_stack_client, empty_vector_db_registry): +def test_vector_db_register(llama_stack_client, embedding_model): vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" llama_stack_client.vector_dbs.register( vector_db_id=vector_db_id, - embedding_model="all-MiniLM-L6-v2", + embedding_model=embedding_model, embedding_dimension=384, provider_id="faiss", ) @@ -74,7 +48,7 @@ def test_vector_db_register(llama_stack_client, empty_vector_db_registry): assert vector_dbs_after_register == [vector_db_id] -def test_vector_db_unregister(llama_stack_client, single_entry_vector_db_registry): +def test_vector_db_unregister(llama_stack_client): vector_dbs = [ vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() ]