mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-04 04:04:14 +00:00
47b640370e
Some checks failed
Integration Auth Tests / test-matrix (oauth2_token) (push) Failing after 1s
SqlStore Integration Tests / test-postgres (3.13) (push) Failing after 0s
Test External Providers Installed via Module / test-external-providers-from-module (venv) (push) Has been skipped
Python Package Build Test / build (3.13) (push) Failing after 1s
SqlStore Integration Tests / test-postgres (3.12) (push) Failing after 4s
Integration Tests (Replay) / Integration Tests (, , , client=, ) (push) Failing after 4s
Vector IO Integration Tests / test-matrix (push) Failing after 4s
Python Package Build Test / build (3.12) (push) Failing after 3s
Test External API and Providers / test-external (venv) (push) Failing after 4s
Unit Tests / unit-tests (3.12) (push) Failing after 4s
Unit Tests / unit-tests (3.13) (push) Failing after 3s
UI Tests / ui-tests (22) (push) Successful in 33s
Pre-commit / pre-commit (push) Successful in 1m15s
Our integration tests need to be 'grouped' because each group often needs a specific set of models it works with. We separated vision tests due to this, and we have a separate set of tests which test "Responses" API. This PR makes this system a bit more official so it is very easy to target these groups and apply all testing infrastructure towards all the groups (for example, record-replay) uniformly. There are three suites declared: - base - vision - responses Note that our CI currently runs the "base" and "vision" suites. You can use the `--suite` option when running pytest (or any of the testing scripts or workflows.) For example: ``` OLLAMA_URL=http://localhost:11434 \ pytest -s -v tests/integration/ --stack-config starter --suite vision ```
64 lines
2.3 KiB
Python
64 lines
2.3 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
import time
|
|
|
|
|
|
def new_vector_store(openai_client, name):
|
|
"""Create a new vector store, cleaning up any existing one with the same name."""
|
|
# Ensure we don't reuse an existing vector store
|
|
vector_stores = openai_client.vector_stores.list()
|
|
for vector_store in vector_stores:
|
|
if vector_store.name == name:
|
|
openai_client.vector_stores.delete(vector_store_id=vector_store.id)
|
|
|
|
# Create a new vector store
|
|
vector_store = openai_client.vector_stores.create(name=name)
|
|
return vector_store
|
|
|
|
|
|
def upload_file(openai_client, name, file_path):
|
|
"""Upload a file, cleaning up any existing file with the same name."""
|
|
# Ensure we don't reuse an existing file
|
|
files = openai_client.files.list()
|
|
for file in files:
|
|
if file.filename == name:
|
|
openai_client.files.delete(file_id=file.id)
|
|
|
|
# Upload a text file with our document content
|
|
return openai_client.files.create(file=open(file_path, "rb"), purpose="assistants")
|
|
|
|
|
|
def wait_for_file_attachment(compat_client, vector_store_id, file_id):
|
|
"""Wait for a file to be attached to a vector store."""
|
|
file_attach_response = compat_client.vector_stores.files.retrieve(
|
|
vector_store_id=vector_store_id,
|
|
file_id=file_id,
|
|
)
|
|
|
|
while file_attach_response.status == "in_progress":
|
|
time.sleep(0.1)
|
|
file_attach_response = compat_client.vector_stores.files.retrieve(
|
|
vector_store_id=vector_store_id,
|
|
file_id=file_id,
|
|
)
|
|
|
|
assert file_attach_response.status == "completed", f"Expected file to be attached, got {file_attach_response}"
|
|
assert not file_attach_response.last_error
|
|
return file_attach_response
|
|
|
|
|
|
def setup_mcp_tools(tools, mcp_server_info):
|
|
"""Replace placeholder MCP server URLs with actual server info."""
|
|
# Create a deep copy to avoid modifying the original test case
|
|
import copy
|
|
|
|
tools_copy = copy.deepcopy(tools)
|
|
|
|
for tool in tools_copy:
|
|
if tool["type"] == "mcp" and tool["server_url"] == "<FILLED_BY_TEST_RUNNER>":
|
|
tool["server_url"] = mcp_server_info["server_url"]
|
|
return tools_copy
|