From f6afb3c26ba453edbf04893a66c71d565d72899f Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Tue, 29 Jul 2025 17:48:04 -0700 Subject: [PATCH] feat(ci): keep only one re-recording job because independent recordings will conflict (#2956) A couple of important updates: - When recording tests, we cannot be generating a matrix because all the independent recordings will conflict. - In fact, we just don't need a matrix on test types any more because things are very fast and the overhead of `llama stack build` and setting up `uv` etc. is much more. - Refactored the running of tests into an independent action --- .../actions/run-integration-tests/action.yml | 73 +++++++ .github/workflows/integration-tests.yml | 188 ++++++++---------- 2 files changed, 158 insertions(+), 103 deletions(-) create mode 100644 .github/actions/run-integration-tests/action.yml diff --git a/.github/actions/run-integration-tests/action.yml b/.github/actions/run-integration-tests/action.yml new file mode 100644 index 000000000..e2db846e4 --- /dev/null +++ b/.github/actions/run-integration-tests/action.yml @@ -0,0 +1,73 @@ +name: 'Run Integration Tests' +description: 'Run integration tests with configurable execution mode and provider settings' + +inputs: + test-types: + description: 'Test types to run (JSON array)' + required: true + stack-config: + description: 'Stack configuration: "ci-tests" or "server:ci-tests"' + required: true + provider: + description: 'Provider to use: "ollama" or "vllm"' + required: true + inference-mode: + description: 'Inference mode: "record" or "replay"' + required: true + +outputs: + logs-path: + description: 'Path to generated log files' + value: '*.log' + +runs: + using: 'composite' + steps: + - name: Run Integration Tests + env: + LLAMA_STACK_CLIENT_TIMEOUT: "300" + LLAMA_STACK_TEST_RECORDING_DIR: "tests/integration/recordings" + LLAMA_STACK_TEST_INFERENCE_MODE: ${{ inputs.inference-mode }} + shell: bash + run: | + stack_config="${{ inputs.stack-config }}" + EXCLUDE_TESTS="builtin_tool or safety_with_image or code_interpreter or test_rag" + + # Configure provider-specific settings + if [ "${{ inputs.provider }}" == "ollama" ]; then + export OLLAMA_URL="http://0.0.0.0:11434" + export TEXT_MODEL="ollama/llama3.2:3b-instruct-fp16" + export SAFETY_MODEL="ollama/llama-guard3:1b" + EXTRA_PARAMS="--safety-shield=llama-guard" + else + export VLLM_URL="http://localhost:8000/v1" + export TEXT_MODEL="vllm/meta-llama/Llama-3.2-1B-Instruct" + EXTRA_PARAMS="" + EXCLUDE_TESTS="${EXCLUDE_TESTS} or test_inference_store_tool_calls" + fi + + TEST_TYPES='${{ inputs.test-types }}' + echo "Test types to run: $TEST_TYPES" + + for test_type in $(echo "$TEST_TYPES" | jq -r '.[]'); do + # if provider is vllm, exclude the following tests: (safety, post_training, tool_runtime) + if [ "${{ inputs.provider }}" == "vllm" ]; then + if [ "$test_type" == "safety" ] || [ "$test_type" == "post_training" ] || [ "$test_type" == "tool_runtime" ]; then + continue + fi + fi + + echo "=== Running tests for: $test_type ===" + + if uv run pytest -s -v tests/integration/$test_type --stack-config=${stack_config} \ + -k "not( ${EXCLUDE_TESTS} )" \ + --text-model=$TEXT_MODEL \ + --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \ + --color=yes ${EXTRA_PARAMS} \ + --capture=tee-sys | tee pytest-${{ inputs.inference-mode }}-$test_type.log; then + echo "✅ Tests completed for $test_type" + else + echo "❌ Tests failed for $test_type" + exit 1 + fi + done diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 70f0722e4..a6ba00b6d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,6 +1,6 @@ name: Integration Tests -run-name: Run the integration test suite with Ollama +run-name: Run the integration test suite from tests/integration on: push: @@ -16,6 +16,7 @@ on: - 'requirements.txt' - '.github/workflows/integration-tests.yml' # This workflow - '.github/actions/setup-ollama/action.yml' + - '.github/actions/run-integration-tests/action.yml' schedule: # If changing the cron schedule, update the provider in the test-matrix job - cron: '0 0 * * *' # (test latest client) Daily at 12 AM UTC @@ -39,21 +40,21 @@ jobs: discover-tests: runs-on: ubuntu-latest outputs: - test-type: ${{ steps.generate-matrix.outputs.test-type }} + test-types: ${{ steps.generate-test-types.outputs.test-types }} rerecord-tests: ${{ steps.check-rerecord-tests.outputs.rerecord-tests }} steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Generate test matrix - id: generate-matrix + - name: Generate test types + id: generate-test-types run: | # Get test directories dynamically, excluding non-test directories TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" | grep -Ev "^(__pycache__|fixtures|test_cases|recordings)$" | sort | jq -R -s -c 'split("\n")[:-1]') - echo "test-type=$TEST_TYPES" >> $GITHUB_OUTPUT + echo "test-types=$TEST_TYPES" >> $GITHUB_OUTPUT - name: Check if re-record-tests label exists id: check-rerecord-tests @@ -64,49 +65,93 @@ jobs: echo "rerecord-tests=false" >> $GITHUB_OUTPUT fi - test-matrix: + record-tests: + # Sequential job for recording to avoid SQLite conflicts + if: ${{ needs.discover-tests.outputs.rerecord-tests == 'true' }} needs: discover-tests runs-on: ubuntu-latest permissions: - # Set write permissions since we might need to commit recordings contents: write pull-requests: write - env: - # Create reusable variable for the re-record tests condition - SHOULD_RECORD: ${{ needs.discover-tests.outputs.rerecord-tests == 'true' }} - # TODO: set up another var to track whether we need ollama or not - # not every matrix type needs ollama + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install dependencies + uses: ./.github/actions/setup-runner + with: + python-version: "3.12" # Use single Python version for recording + client-version: "latest" + + - name: Setup ollama + if: ${{ inputs.test-provider == 'ollama' }} + uses: ./.github/actions/setup-ollama + + - name: Setup vllm + if: ${{ inputs.test-provider == 'vllm' }} + uses: ./.github/actions/setup-vllm + + - name: Build Llama Stack + run: | + uv run llama stack build --template ci-tests --image-type venv + + - name: Configure git for commits + run: | + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + + - name: Run Integration Tests for All Types (Recording Mode) + uses: ./.github/actions/run-integration-tests + with: + test-types: ${{ needs.discover-tests.outputs.test-types }} + stack-config: 'ci-tests' + provider: ${{ inputs.test-provider }} + inference-mode: 'record' + + - name: Commit and push recordings + run: | + if ! git diff --quiet tests/integration/recordings/; then + echo "Committing recordings" + git add tests/integration/recordings/ + git commit -m "Update recordings" + echo "Pushing all recording commits to PR" + git push origin HEAD:${{ github.head_ref }} + else + echo "No recording changes" + fi + + - name: Write inference logs to file + if: ${{ always() }} + run: | + sudo docker logs ollama > ollama-recording.log || true + + - name: Upload recording logs + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: recording-logs-${{ github.run_id }} + path: | + *.log + retention-days: 1 + + run-tests: + # Skip this job if we're in recording mode (handled by record-tests job) + if: ${{ needs.discover-tests.outputs.rerecord-tests != 'true' }} + needs: discover-tests + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - test-type: ${{ fromJson(needs.discover-tests.outputs.test-type) }} client-type: [library, server] # Use vllm on weekly schedule, otherwise use test-provider input (defaults to ollama) provider: ${{ (github.event.schedule == '1 0 * * 0') && fromJSON('["vllm"]') || fromJSON(format('["{0}"]', github.event.inputs.test-provider || 'ollama')) }} python-version: ["3.12", "3.13"] client-version: ${{ (github.event.schedule == '0 0 * * 0' || github.event.inputs.test-all-client-versions == 'true') && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }} - exclude: # TODO: look into why these tests are failing and fix them - - provider: vllm - test-type: safety - - provider: vllm - test-type: post_training - - provider: vllm - test-type: tool_runtime steps: - - name: Debug - run: | - echo "test-type: ${{ matrix.test-type }}" - echo "client-type: ${{ matrix.client-type }}" - echo "provider: ${{ matrix.provider }}" - echo "python-version: ${{ matrix.python-version }}" - echo "client-version: ${{ matrix.client-version }}" - echo "SHOULD_RECORD: ${{ env.SHOULD_RECORD }}" - echo "rerecord-tests: ${{ needs.discover-tests.outputs.rerecord-tests }}" - - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -116,14 +161,6 @@ jobs: python-version: ${{ matrix.python-version }} client-version: ${{ matrix.client-version }} - - name: Setup ollama - if: ${{ matrix.provider == 'ollama' && env.SHOULD_RECORD == 'true' }} - uses: ./.github/actions/setup-ollama - - - name: Setup vllm - if: ${{ matrix.provider == 'vllm' }} - uses: ./.github/actions/setup-vllm - - name: Build Llama Stack run: | uv run llama stack build --template ci-tests --image-type venv @@ -134,62 +171,13 @@ jobs: free -h df -h - - name: Run Integration Tests - env: - LLAMA_STACK_CLIENT_TIMEOUT: "300" # Increased timeout for eval operations - # Use 'shell' to get pipefail behavior - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference - # TODO: write a precommit hook to detect if a test contains a pipe but does not use 'shell: bash' - shell: bash - run: | - if [ "${{ matrix.client-type }}" == "library" ]; then - stack_config="ci-tests" - else - stack_config="server:ci-tests" - fi - - EXCLUDE_TESTS="builtin_tool or safety_with_image or code_interpreter or test_rag" - export LLAMA_STACK_TEST_RECORDING_DIR="tests/integration/recordings" - - if [ "$SHOULD_RECORD" == "true" ]; then - export LLAMA_STACK_TEST_INFERENCE_MODE="record" - else - export LLAMA_STACK_TEST_INFERENCE_MODE="replay" - fi - - if [ "${{ matrix.provider }}" == "ollama" ]; then - export OLLAMA_URL="http://0.0.0.0:11434" - export TEXT_MODEL=ollama/llama3.2:3b-instruct-fp16 - export SAFETY_MODEL="ollama/llama-guard3:1b" - EXTRA_PARAMS="--safety-shield=llama-guard" - else - export VLLM_URL="http://localhost:8000/v1" - export TEXT_MODEL=vllm/meta-llama/Llama-3.2-1B-Instruct - # TODO: remove the not(test_inference_store_tool_calls) once we can get the tool called consistently - EXTRA_PARAMS= - EXCLUDE_TESTS="${EXCLUDE_TESTS} or test_inference_store_tool_calls" - fi - - uv run pytest -s -v tests/integration/${{ matrix.test-type }} --stack-config=${stack_config} \ - -k "not( ${EXCLUDE_TESTS} )" \ - --text-model=$TEXT_MODEL \ - --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \ - --color=yes ${EXTRA_PARAMS} \ - --capture=tee-sys | tee pytest-${{ matrix.test-type }}.log - - - name: Update the PR if tests/integration/recordings/ has changed - if: ${{ env.SHOULD_RECORD == 'true' }} - run: | - if ! git diff --quiet tests/integration/recordings/; then - echo "Updating PR with updated recordings" - git config --local user.email "github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git add tests/integration/recordings/ - git commit -m "Update recordings from integration tests" - git push origin HEAD:${{ github.head_ref }} - else - echo "No changes to recordings detected" - fi + - name: Run Integration Tests (Replay Mode) + uses: ./.github/actions/run-integration-tests + with: + test-types: ${{ needs.discover-tests.outputs.test-types }} + stack-config: ${{ matrix.client-type == 'library' && 'ci-tests' || 'server:ci-tests' }} + provider: ${{ matrix.provider }} + inference-mode: 'replay' - name: Check Storage and Memory Available After Tests if: ${{ always() }} @@ -197,17 +185,11 @@ jobs: free -h df -h - - name: Write inference logs to file - if: ${{ env.SHOULD_RECORD == 'true' }} - run: | - sudo docker logs ollama > ollama.log || true - sudo docker logs vllm > vllm.log || true - - - name: Upload all logs to artifacts - if: ${{ env.SHOULD_RECORD == 'true' }} + - name: Upload test logs on failure + if: ${{ failure() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.provider }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }}-${{ matrix.client-version }} + name: test-logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.provider }}-${{ matrix.client-type }}-${{ matrix.python-version }}-${{ matrix.client-version }} path: | *.log retention-days: 1