diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index ec782c331..86adf8a15 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -15,8 +15,14 @@ on: - '.github/workflows/integration-tests.yml' # This workflow jobs: - ollama: + test-matrix: runs-on: ubuntu-latest + strategy: + matrix: + # Listing tests manually since some of them currently fail + # TODO: generate matrix list from tests/integration when fixed + test-type: [inference, datasets, inspect, scoring, post_training, providers] + fail-fast: false # we want to run all tests regardless of failure steps: - name: Checkout repository @@ -43,6 +49,8 @@ jobs: run: | uv sync --extra dev --extra test uv pip install ollama faiss-cpu + # always test against the latest version of the client + uv pip install git+https://github.com/meta-llama/llama-stack-client-python.git@main uv pip install -e . - name: Wait for Ollama to start @@ -72,17 +80,17 @@ jobs: echo "Waiting for Llama Stack server..." for i in {1..30}; do if curl -s http://localhost:8321/v1/health | grep -q "OK"; then - echo " Llama Stack server is up!" + echo "Llama Stack server is up!" exit 0 fi sleep 1 done - echo " Llama Stack server failed to start" + echo "Llama Stack server failed to start" cat server.log exit 1 - - name: Run Inference Integration Tests + - name: Run Integration Tests env: INFERENCE_MODEL: "meta-llama/Llama-3.2-3B-Instruct" run: | - uv run pytest -v tests/integration/inference --stack-config=ollama --text-model="meta-llama/Llama-3.2-3B-Instruct" --embedding-model=all-MiniLM-L6-v2 + uv run pytest -v tests/integration/${{ matrix.test-type }} --stack-config=ollama --text-model="meta-llama/Llama-3.2-3B-Instruct" --embedding-model=all-MiniLM-L6-v2