phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-06-28 10:54:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update tests to ignore with library client

Browse source
This commit is contained in:
Hardik Shah 2025-06-12 15:31:27 -07:00
parent f8b85c2176
commit d54c58c8dd
1 changed files with 56 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

102
tests/integration/vector_io/test_openai_vector_stores.py
View file

@ -34,6 +34,13 @@ def openai_client(client_with_models):
return OpenAI(base_url=base_url, api_key="fake") return OpenAI(base_url=base_url, api_key="fake")
@pytest.fixture(params=["openai_client"]) # , "llama_stack_client"])
def compat_client(request, client_with_models):
if request.param == "openai_client" and isinstance(client_with_models, LlamaStackAsLibraryClient):
pytest.skip("OpenAI client tests not supported with library client")
return request.getfixturevalue(request.param)
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def sample_chunks(): def sample_chunks():
return [ return [
@ -57,29 +64,29 @@ def sample_chunks():
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def openai_client_with_empty_stores(openai_client): def compat_client_with_empty_stores(compat_client):
def clear_vector_stores(): def clear_vector_stores():
# List and delete all existing vector stores # List and delete all existing vector stores
try: try:
response = openai_client.vector_stores.list() response = compat_client.vector_stores.list()
for store in response.data: for store in response.data:
openai_client.vector_stores.delete(vector_store_id=store.id) compat_client.vector_stores.delete(vector_store_id=store.id)
except Exception: except Exception:
# If the API is not available or fails, just continue # If the API is not available or fails, just continue
logger.warning("Failed to clear vector stores") logger.warning("Failed to clear vector stores")
pass pass
clear_vector_stores() clear_vector_stores()
yield openai_client yield compat_client
# Clean up after the test # Clean up after the test
clear_vector_stores() clear_vector_stores()
def test_openai_create_vector_store(openai_client_with_empty_stores, client_with_models): def test_openai_create_vector_store(compat_client_with_empty_stores, client_with_models):
"""Test creating a vector store using OpenAI API.""" """Test creating a vector store using OpenAI API."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
client = openai_client_with_empty_stores client = compat_client_with_empty_stores
# Create a vector store # Create a vector store
vector_store = client.vector_stores.create( vector_store = client.vector_stores.create(
@ -96,11 +103,11 @@ def test_openai_create_vector_store(openai_client_with_empty_stores, client_with
assert hasattr(vector_store, "created_at") assert hasattr(vector_store, "created_at")
def test_openai_list_vector_stores(openai_client_with_empty_stores, client_with_models): def test_openai_list_vector_stores(compat_client_with_empty_stores, client_with_models):
"""Test listing vector stores using OpenAI API.""" """Test listing vector stores using OpenAI API."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
client = openai_client_with_empty_stores client = compat_client_with_empty_stores
# Create a few vector stores # Create a few vector stores
store1 = client.vector_stores.create(name="store1", metadata={"type": "test"}) store1 = client.vector_stores.create(name="store1", metadata={"type": "test"})
@ -123,11 +130,11 @@ def test_openai_list_vector_stores(openai_client_with_empty_stores, client_with_
assert len(limited_response.data) == 1 assert len(limited_response.data) == 1
def test_openai_retrieve_vector_store(openai_client_with_empty_stores, client_with_models): def test_openai_retrieve_vector_store(compat_client_with_empty_stores, client_with_models):
"""Test retrieving a specific vector store using OpenAI API.""" """Test retrieving a specific vector store using OpenAI API."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
client = openai_client_with_empty_stores client = compat_client_with_empty_stores
# Create a vector store # Create a vector store
created_store = client.vector_stores.create(name="retrieve_test_store", metadata={"purpose": "retrieval_test"}) created_store = client.vector_stores.create(name="retrieve_test_store", metadata={"purpose": "retrieval_test"})
@ -142,11 +149,11 @@ def test_openai_retrieve_vector_store(openai_client_with_empty_stores, client_wi
assert retrieved_store.object == "vector_store" assert retrieved_store.object == "vector_store"
def test_openai_update_vector_store(openai_client_with_empty_stores, client_with_models): def test_openai_update_vector_store(compat_client_with_empty_stores, client_with_models):
"""Test modifying a vector store using OpenAI API.""" """Test modifying a vector store using OpenAI API."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
client = openai_client_with_empty_stores client = compat_client_with_empty_stores
# Create a vector store # Create a vector store
created_store = client.vector_stores.create(name="original_name", metadata={"version": "1.0"}) created_store = client.vector_stores.create(name="original_name", metadata={"version": "1.0"})
@ -165,11 +172,11 @@ def test_openai_update_vector_store(openai_client_with_empty_stores, client_with
assert modified_store.last_active_at > created_store.last_active_at assert modified_store.last_active_at > created_store.last_active_at
def test_openai_delete_vector_store(openai_client_with_empty_stores, client_with_models): def test_openai_delete_vector_store(compat_client_with_empty_stores, client_with_models):
"""Test deleting a vector store using OpenAI API.""" """Test deleting a vector store using OpenAI API."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
client = openai_client_with_empty_stores client = compat_client_with_empty_stores
# Create a vector store # Create a vector store
created_store = client.vector_stores.create(name="delete_test_store", metadata={"purpose": "deletion_test"}) created_store = client.vector_stores.create(name="delete_test_store", metadata={"purpose": "deletion_test"})
@ -187,11 +194,11 @@ def test_openai_delete_vector_store(openai_client_with_empty_stores, client_with
client.vector_stores.retrieve(vector_store_id=created_store.id) client.vector_stores.retrieve(vector_store_id=created_store.id)
def test_openai_vector_store_search_empty(openai_client_with_empty_stores, client_with_models): def test_openai_vector_store_search_empty(compat_client_with_empty_stores, client_with_models):
"""Test searching an empty vector store using OpenAI API.""" """Test searching an empty vector store using OpenAI API."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
client = openai_client_with_empty_stores client = compat_client_with_empty_stores
# Create a vector store # Create a vector store
vector_store = client.vector_stores.create(name="search_test_store", metadata={"purpose": "search_testing"}) vector_store = client.vector_stores.create(name="search_test_store", metadata={"purpose": "search_testing"})
@ -208,15 +215,15 @@ def test_openai_vector_store_search_empty(openai_client_with_empty_stores, clien
assert search_response.has_more is False assert search_response.has_more is False
def test_openai_vector_store_with_chunks(openai_client_with_empty_stores, client_with_models, sample_chunks): def test_openai_vector_store_with_chunks(compat_client_with_empty_stores, client_with_models, sample_chunks):
"""Test vector store functionality with actual chunks using both OpenAI and native APIs.""" """Test vector store functionality with actual chunks using both OpenAI and native APIs."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
openai_client = openai_client_with_empty_stores compat_client = compat_client_with_empty_stores
llama_client = client_with_models llama_client = client_with_models
# Create a vector store using OpenAI API # Create a vector store using OpenAI API
vector_store = openai_client.vector_stores.create(name="chunks_test_store", metadata={"purpose": "chunks_testing"}) vector_store = compat_client.vector_stores.create(name="chunks_test_store", metadata={"purpose": "chunks_testing"})
# Insert chunks using the native LlamaStack API (since OpenAI API doesn't have direct chunk insertion) # Insert chunks using the native LlamaStack API (since OpenAI API doesn't have direct chunk insertion)
llama_client.vector_io.insert( llama_client.vector_io.insert(
@ -225,7 +232,7 @@ def test_openai_vector_store_with_chunks(openai_client_with_empty_stores, client
) )
# Search using OpenAI API # Search using OpenAI API
search_response = openai_client.vector_stores.search( search_response = compat_client.vector_stores.search(
vector_store_id=vector_store.id, query="What is Python programming language?", max_num_results=3 vector_store_id=vector_store.id, query="What is Python programming language?", max_num_results=3
) )
assert search_response is not None assert search_response is not None
@ -233,18 +240,19 @@ def test_openai_vector_store_with_chunks(openai_client_with_empty_stores, client
# The top result should be about Python (doc1) # The top result should be about Python (doc1)
top_result = search_response.data[0] top_result = search_response.data[0]
assert "python" in top_result.content.lower() or "programming" in top_result.content.lower() top_content = top_result.content[0].text
assert top_result.metadata["document_id"] == "doc1" assert "python" in top_content.lower() or "programming" in top_content.lower()
assert top_result.attributes["document_id"] == "doc1"
# Test filtering by metadata # Test filtering by metadata
filtered_search = openai_client.vector_stores.search( filtered_search = compat_client.vector_stores.search(
vector_store_id=vector_store.id, query="artificial intelligence", filters={"topic": "ai"}, max_num_results=5 vector_store_id=vector_store.id, query="artificial intelligence", filters={"topic": "ai"}, max_num_results=5
) )
assert filtered_search is not None assert filtered_search is not None
# All results should have topic "ai" # All results should have topic "ai"
for result in filtered_search.data: for result in filtered_search.data:
assert result.metadata["topic"] == "ai" assert result.attributes["topic"] == "ai"
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -257,18 +265,18 @@ def test_openai_vector_store_with_chunks(openai_client_with_empty_stores, client
], ],
) )
def test_openai_vector_store_search_relevance( def test_openai_vector_store_search_relevance(
openai_client_with_empty_stores, client_with_models, sample_chunks, test_case compat_client_with_empty_stores, client_with_models, sample_chunks, test_case
): ):
"""Test that OpenAI vector store search returns relevant results for different queries.""" """Test that OpenAI vector store search returns relevant results for different queries."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
openai_client = openai_client_with_empty_stores compat_client = compat_client_with_empty_stores
llama_client = client_with_models llama_client = client_with_models
query, expected_doc_id, expected_topic = test_case query, expected_doc_id, expected_topic = test_case
# Create a vector store # Create a vector store
vector_store = openai_client.vector_stores.create( vector_store = compat_client.vector_stores.create(
name=f"relevance_test_{expected_doc_id}", metadata={"purpose": "relevance_testing"} name=f"relevance_test_{expected_doc_id}", metadata={"purpose": "relevance_testing"}
) )
@ -279,7 +287,7 @@ def test_openai_vector_store_search_relevance(
) )
# Search using OpenAI API # Search using OpenAI API
search_response = openai_client.vector_stores.search( search_response = compat_client.vector_stores.search(
vector_store_id=vector_store.id, query=query, max_num_results=4 vector_store_id=vector_store.id, query=query, max_num_results=4
) )
@ -288,8 +296,9 @@ def test_openai_vector_store_search_relevance(
# The top result should match the expected document # The top result should match the expected document
top_result = search_response.data[0] top_result = search_response.data[0]
assert top_result.metadata["document_id"] == expected_doc_id
assert top_result.metadata["topic"] == expected_topic assert top_result.attributes["document_id"] == expected_doc_id
assert top_result.attributes["topic"] == expected_topic
# Verify score is included and reasonable # Verify score is included and reasonable
assert isinstance(top_result.score, int | float) assert isinstance(top_result.score, int | float)
@ -297,16 +306,16 @@ def test_openai_vector_store_search_relevance(
def test_openai_vector_store_search_with_ranking_options( def test_openai_vector_store_search_with_ranking_options(
openai_client_with_empty_stores, client_with_models, sample_chunks compat_client_with_empty_stores, client_with_models, sample_chunks
): ):
"""Test OpenAI vector store search with ranking options.""" """Test OpenAI vector store search with ranking options."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
openai_client = openai_client_with_empty_stores compat_client = compat_client_with_empty_stores
llama_client = client_with_models llama_client = client_with_models
# Create a vector store # Create a vector store
vector_store = openai_client.vector_stores.create( vector_store = compat_client.vector_stores.create(
name="ranking_test_store", metadata={"purpose": "ranking_testing"} name="ranking_test_store", metadata={"purpose": "ranking_testing"}
) )
@ -318,7 +327,7 @@ def test_openai_vector_store_search_with_ranking_options(
# Search with ranking options # Search with ranking options
threshold = 0.1 threshold = 0.1
search_response = openai_client.vector_stores.search( search_response = compat_client.vector_stores.search(
vector_store_id=vector_store.id, vector_store_id=vector_store.id,
query="machine learning and artificial intelligence", query="machine learning and artificial intelligence",
max_num_results=3, max_num_results=3,
@ -334,16 +343,16 @@ def test_openai_vector_store_search_with_ranking_options(
def test_openai_vector_store_search_with_high_score_filter( def test_openai_vector_store_search_with_high_score_filter(
openai_client_with_empty_stores, client_with_models, sample_chunks compat_client_with_empty_stores, client_with_models, sample_chunks
): ):
"""Test that searching with text very similar to a document and high score threshold returns only that document.""" """Test that searching with text very similar to a document and high score threshold returns only that document."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
openai_client = openai_client_with_empty_stores compat_client = compat_client_with_empty_stores
llama_client = client_with_models llama_client = client_with_models
# Create a vector store # Create a vector store
vector_store = openai_client.vector_stores.create( vector_store = compat_client.vector_stores.create(
name="high_score_filter_test", metadata={"purpose": "high_score_filtering"} name="high_score_filter_test", metadata={"purpose": "high_score_filtering"}
) )
@ -358,7 +367,7 @@ def test_openai_vector_store_search_with_high_score_filter(
query = "Python is a high-level programming language with code readability and fewer lines than C++ or Java" query = "Python is a high-level programming language with code readability and fewer lines than C++ or Java"
# picking up thrshold to be slightly higher than the second result # picking up thrshold to be slightly higher than the second result
search_response = openai_client.vector_stores.search( search_response = compat_client.vector_stores.search(
vector_store_id=vector_store.id, vector_store_id=vector_store.id,
query=query, query=query,
max_num_results=3, max_num_results=3,
@ -367,7 +376,7 @@ def test_openai_vector_store_search_with_high_score_filter(
threshold = search_response.data[1].score + 0.0001 threshold = search_response.data[1].score + 0.0001
# we expect only one result with the requested threshold # we expect only one result with the requested threshold
search_response = openai_client.vector_stores.search( search_response = compat_client.vector_stores.search(
vector_store_id=vector_store.id, vector_store_id=vector_store.id,
query=query, query=query,
max_num_results=10, # Allow more results but expect filtering max_num_results=10, # Allow more results but expect filtering
@ -379,25 +388,26 @@ def test_openai_vector_store_search_with_high_score_filter(
# The top result should be the Python document (doc1) # The top result should be the Python document (doc1)
top_result = search_response.data[0] top_result = search_response.data[0]
assert top_result.metadata["document_id"] == "doc1" assert top_result.attributes["document_id"] == "doc1"
assert top_result.metadata["topic"] == "programming" assert top_result.attributes["topic"] == "programming"
assert top_result.score >= threshold assert top_result.score >= threshold
# Verify the content contains Python-related terms # Verify the content contains Python-related terms
assert "python" in top_result.content.lower() or "programming" in top_result.content.lower() top_content = top_result.content[0].text
assert "python" in top_content.lower() or "programming" in top_content.lower()
def test_openai_vector_store_search_with_max_num_results( def test_openai_vector_store_search_with_max_num_results(
openai_client_with_empty_stores, client_with_models, sample_chunks compat_client_with_empty_stores, client_with_models, sample_chunks
): ):
"""Test OpenAI vector store search with max_num_results.""" """Test OpenAI vector store search with max_num_results."""
skip_if_provider_doesnt_support_openai_vector_stores(client_with_models) skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
openai_client = openai_client_with_empty_stores compat_client = compat_client_with_empty_stores
llama_client = client_with_models llama_client = client_with_models
# Create a vector store # Create a vector store
vector_store = openai_client.vector_stores.create( vector_store = compat_client.vector_stores.create(
name="max_num_results_test_store", metadata={"purpose": "max_num_results_testing"} name="max_num_results_test_store", metadata={"purpose": "max_num_results_testing"}
) )
@ -408,7 +418,7 @@ def test_openai_vector_store_search_with_max_num_results(
) )
# Search with max_num_results # Search with max_num_results
search_response = openai_client.vector_stores.search( search_response = compat_client.vector_stores.search(
vector_store_id=vector_store.id, vector_store_id=vector_store.id,
query="machine learning and artificial intelligence", query="machine learning and artificial intelligence",
max_num_results=2, max_num_results=2,