diff --git a/.github/actions/install-llama-stack-client/action.yml b/.github/actions/install-llama-stack-client/action.yml index 3c1c77d9c..fc359f1fa 100644 --- a/.github/actions/install-llama-stack-client/action.yml +++ b/.github/actions/install-llama-stack-client/action.yml @@ -6,6 +6,10 @@ inputs: description: 'Client version to install on non-release branches (latest or published). Ignored on release branches.' required: false default: "" + sdk_install_url: + description: 'URL to install Python SDK from (for testing preview builds). If provided, overrides client-version.' + required: false + default: "" outputs: uv-extra-index-url: @@ -25,6 +29,14 @@ runs: id: configure shell: bash run: | + # If sdk_install_url is provided (e.g., from Stainless preview), use it directly + if [ -n "${{ inputs.sdk_install_url }}" ]; then + echo "Using provided sdk_install_url: ${{ inputs.sdk_install_url }}" + echo "install-after-sync=true" >> $GITHUB_OUTPUT + echo "install-source=${{ inputs.sdk_install_url }}" >> $GITHUB_OUTPUT + exit 0 + fi + # Determine the branch we're working with BRANCH="${{ github.base_ref || github.ref }}" BRANCH="${BRANCH#refs/heads/}" diff --git a/.github/actions/setup-runner/action.yml b/.github/actions/setup-runner/action.yml index 6c642b310..b65288764 100644 --- a/.github/actions/setup-runner/action.yml +++ b/.github/actions/setup-runner/action.yml @@ -9,6 +9,10 @@ inputs: description: The llama-stack-client-python version to test against (latest or published) required: false default: "latest" + sdk_install_url: + description: 'URL to install Python SDK from (for testing preview builds). If provided, overrides client-version.' + required: false + default: "" runs: using: "composite" steps: @@ -22,6 +26,7 @@ runs: uses: ./.github/actions/install-llama-stack-client with: client-version: ${{ inputs.client-version }} + sdk_install_url: ${{ inputs.sdk_install_url }} - name: Install dependencies shell: bash diff --git a/.github/actions/setup-test-environment/action.yml b/.github/actions/setup-test-environment/action.yml index 1c9d019cc..55d403406 100644 --- a/.github/actions/setup-test-environment/action.yml +++ b/.github/actions/setup-test-environment/action.yml @@ -8,6 +8,10 @@ inputs: client-version: description: 'Client version (latest or published)' required: true + sdk_install_url: + description: 'URL to install Python SDK from (for testing preview builds). If provided, overrides client-version.' + required: false + default: '' setup: description: 'Setup to configure (ollama, vllm, gpt, etc.)' required: false @@ -28,6 +32,7 @@ runs: with: python-version: ${{ inputs.python-version }} client-version: ${{ inputs.client-version }} + sdk_install_url: ${{ inputs.sdk_install_url }} - name: Setup ollama if: ${{ (inputs.setup == 'ollama' || inputs.setup == 'ollama-vision') && inputs.inference-mode == 'record' }} @@ -65,6 +70,15 @@ runs: sleep 2 done + - name: Verify client installation + shell: bash + run: | + echo "Verifying llama-stack-client installation:" + uv pip show llama-stack-client || echo "llama-stack-client not found" + echo "" + echo "All installed llama packages:" + uv pip list | grep llama || true + - name: Build Llama Stack shell: bash run: | diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b731dfae7..f3b5b95a9 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -37,6 +37,22 @@ on: description: 'Test against a specific setup' type: string default: 'ollama' + workflow_call: + inputs: + sdk_install_url: + required: false + type: string + description: 'URL to install Python SDK from (for testing preview builds)' + matrix_key: + required: false + type: string + default: 'default' + description: 'Matrix configuration key from ci_matrix.json (e.g., "default", "stainless")' + test-all-client-versions: + required: false + type: boolean + default: false + description: 'Test against both the latest and published versions' concurrency: # Skip concurrency for pushes to main - each commit should be tested independently @@ -55,11 +71,12 @@ jobs: - name: Generate test matrix id: set-matrix run: | - # Generate matrix from CI_MATRIX in tests/integration/suites.py - # Supports schedule-based and manual input overrides + # Generate matrix from CI_MATRIX in tests/integration/ci_matrix.json + # Supports schedule-based, manual input, and workflow_call overrides MATRIX=$(PYTHONPATH=. python3 scripts/generate_ci_matrix.py \ --schedule "${{ github.event.schedule }}" \ - --test-setup "${{ github.event.inputs.test-setup }}") + --test-setup "${{ github.event.inputs.test-setup || '' }}" \ + --matrix-key "${{ inputs.matrix_key || 'default' }}") echo "matrix=$MATRIX" >> $GITHUB_OUTPUT echo "Generated matrix: $MATRIX" @@ -75,8 +92,8 @@ jobs: # Use Python 3.13 only on nightly schedule (daily latest client test), otherwise use 3.12 python-version: ${{ github.event.schedule == '0 0 * * *' && fromJSON('["3.12", "3.13"]') || fromJSON('["3.12"]') }} node-version: [22] - client-version: ${{ (github.event.schedule == '0 0 * * *' || github.event.inputs.test-all-client-versions == 'true') && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }} - # Test configurations: Generated from CI_MATRIX in tests/integration/suites.py + client-version: ${{ (github.event.schedule == '0 0 * * *' || github.event.inputs.test-all-client-versions == 'true' || inputs.test-all-client-versions == true) && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }} + # Test configurations: Generated from CI_MATRIX in tests/integration/ci_matrix.json # See scripts/generate_ci_matrix.py for generation logic config: ${{ fromJSON(needs.generate-matrix.outputs.matrix).include }} @@ -90,6 +107,7 @@ jobs: with: python-version: ${{ matrix.python-version }} client-version: ${{ matrix.client-version }} + sdk_install_url: ${{ inputs.sdk_install_url || '' }} setup: ${{ matrix.config.setup }} suite: ${{ matrix.config.suite }} inference-mode: 'replay' diff --git a/.github/workflows/stainless-builds.yml b/.github/workflows/stainless-builds.yml index 5ab0cb4e0..880324756 100644 --- a/.github/workflows/stainless-builds.yml +++ b/.github/workflows/stainless-builds.yml @@ -22,6 +22,10 @@ on: description: 'PR number to run Stainless build for' required: true type: number + sdk_install_url: + description: 'Python SDK install URL (optional, for testing specific builds)' + required: false + type: string concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.pr_number || github.run_id }} @@ -61,6 +65,7 @@ jobs: pr_head_sha: ${{ steps.compute.outputs.pr_head_sha }} pr_base_sha: ${{ steps.compute.outputs.pr_base_sha }} pr_base_ref: ${{ steps.compute.outputs.pr_base_ref }} + pr_title: ${{ steps.compute.outputs.pr_title }} steps: - name: Fetch PR details for workflow_dispatch if: github.event_name == 'workflow_dispatch' @@ -68,7 +73,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - PR_DATA=$(gh pr view ${{ inputs.pr_number }} --repo ${{ github.repository }} --json headRefName,headRepository,headRefOid,baseRefName,baseRefOid,headRepositoryOwner) + PR_DATA=$(gh pr view ${{ inputs.pr_number }} --repo ${{ github.repository }} --json headRefName,headRepository,headRefOid,baseRefName,baseRefOid,headRepositoryOwner,title) echo "pr_data=$PR_DATA" >> $GITHUB_OUTPUT - name: Compute branch names @@ -84,6 +89,7 @@ jobs: HEAD_SHA=$(echo "$PR_DATA" | jq -r '.headRefOid') BASE_SHA=$(echo "$PR_DATA" | jq -r '.baseRefOid') BASE_REF=$(echo "$PR_DATA" | jq -r '.baseRefName') + PR_TITLE=$(echo "$PR_DATA" | jq -r '.title') else # Use pull_request_target event data HEAD_REPO="${{ github.event.pull_request.head.repo.full_name }}" @@ -92,6 +98,7 @@ jobs: HEAD_SHA="${{ github.event.pull_request.head.sha }}" BASE_SHA="${{ github.event.pull_request.base.sha }}" BASE_REF="${{ github.event.pull_request.base.ref }}" + PR_TITLE="${{ github.event.pull_request.title }}" fi BASE_REPO="${{ github.repository }}" @@ -118,14 +125,20 @@ jobs: echo "pr_head_sha=${HEAD_SHA}" >> $GITHUB_OUTPUT echo "pr_base_sha=${BASE_SHA}" >> $GITHUB_OUTPUT echo "pr_base_ref=${BASE_REF}" >> $GITHUB_OUTPUT + echo "pr_title=${PR_TITLE}" >> $GITHUB_OUTPUT preview: needs: compute-branch - if: github.event_name == 'workflow_dispatch' || github.event.action != 'closed' + # Skip preview if workflow_dispatch provides sdk_install_url, or if PR is being closed + if: | + (github.event_name == 'workflow_dispatch' && inputs.sdk_install_url == '') || + (github.event_name == 'pull_request_target' && github.event.action != 'closed') runs-on: ubuntu-latest permissions: contents: read pull-requests: write + outputs: + sdk_install_url: ${{ fromJSON(steps.run-preview.outputs.outcomes || '{}').python.install_url || '' }} steps: # Checkout the PR's code to access the OpenAPI spec and config files. # This is necessary to read the spec/config from the PR (including from forks). @@ -137,7 +150,10 @@ jobs: fetch-depth: 2 - name: Run preview builds + id: run-preview uses: stainless-api/upload-openapi-spec-action/preview@979824f1ea5f44334940f0768d04642b6cdaa0d1 # 1.8.1 + env: + PR_NUMBER: ${{ inputs.pr_number || github.event.pull_request.number }} with: stainless_api_key: ${{ secrets.STAINLESS_API_KEY }} org: ${{ env.STAINLESS_ORG }} @@ -150,6 +166,21 @@ jobs: head_sha: ${{ needs.compute-branch.outputs.pr_head_sha }} branch: ${{ needs.compute-branch.outputs.preview_branch }} base_branch: ${{ needs.compute-branch.outputs.base_branch }} + commit_message: ${{ needs.compute-branch.outputs.pr_title }} + make_comment: true + + run-integration-tests: + needs: [compute-branch, preview] + if: | + always() && + (needs.preview.result == 'success' || needs.preview.result == 'skipped') && + (github.event_name == 'workflow_dispatch' || github.event.action != 'closed') + uses: ./.github/workflows/integration-tests.yml + with: + # Use provided sdk_install_url from workflow_dispatch, or from preview build + sdk_install_url: ${{ inputs.sdk_install_url || needs.preview.outputs.sdk_install_url }} + matrix_key: 'stainless' + test-all-client-versions: false merge: needs: compute-branch diff --git a/scripts/generate_ci_matrix.py b/scripts/generate_ci_matrix.py index 0d4e924b3..f6e68ba8a 100755 --- a/scripts/generate_ci_matrix.py +++ b/scripts/generate_ci_matrix.py @@ -24,24 +24,30 @@ DEFAULT_MATRIX = matrix_config["default"] SCHEDULE_MATRICES: dict[str, list[dict[str, str]]] = matrix_config.get("schedules", {}) -def generate_matrix(schedule="", test_setup=""): +def generate_matrix(schedule="", test_setup="", matrix_key="default"): """ - Generate test matrix based on schedule or manual input. + Generate test matrix based on schedule, manual input, or matrix key. Args: schedule: GitHub cron schedule string (e.g., "1 0 * * 0" for weekly) test_setup: Manual test setup input (e.g., "ollama-vision") + matrix_key: Matrix configuration key from ci_matrix.json (e.g., "default", "stainless") Returns: Matrix configuration as JSON string """ - # Weekly scheduled test matrices + # Weekly scheduled test matrices (highest priority) if schedule and schedule in SCHEDULE_MATRICES: matrix = SCHEDULE_MATRICES[schedule] # Manual input for specific setup elif test_setup == "ollama-vision": matrix = [{"suite": "vision", "setup": "ollama-vision"}] - # Default: use JSON-defined matrix + # Use specified matrix key from ci_matrix.json + elif matrix_key: + if matrix_key not in matrix_config: + raise ValueError(f"Invalid matrix_key '{matrix_key}'. Available keys: {list(matrix_config.keys())}") + matrix = matrix_config[matrix_key] + # Default: use JSON-defined default matrix else: matrix = DEFAULT_MATRIX @@ -55,7 +61,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description="Generate CI test matrix") parser.add_argument("--schedule", default="", help="GitHub schedule cron string") parser.add_argument("--test-setup", default="", help="Manual test setup input") + parser.add_argument("--matrix-key", default="default", help="Matrix configuration key from ci_matrix.json") args = parser.parse_args() - print(generate_matrix(args.schedule, args.test_setup)) + print(generate_matrix(args.schedule, args.test_setup, args.matrix_key)) diff --git a/tests/integration/ci_matrix.json b/tests/integration/ci_matrix.json index 43678e5c7..fcadf95df 100644 --- a/tests/integration/ci_matrix.json +++ b/tests/integration/ci_matrix.json @@ -6,6 +6,9 @@ {"suite": "responses", "setup": "gpt"}, {"suite": "base-vllm-subset", "setup": "vllm"} ], + "stainless": [ + {"suite": "base", "setup": "ollama", "allowed_clients": ["library"]} + ], "schedules": { "1 0 * * 0": [ {"suite": "base", "setup": "vllm"}