test: add tests against published client (#2752)

# What does this PR do?
closes #2751

## Test Plan

---------

Co-authored-by: Nathan Weinberg <31703736+nathan-weinberg@users.noreply.github.com>

.github/actions/setup-runner/action.yml

@@ -5,6 +5,10 @@ inputs:
     description: The Python version to use
     required: false
     default: "3.12"
+  client-version:
+    description: The llama-stack-client-python version to test against (latest or published)
+    required: false
+    default: "latest"
 runs:
   using: "composite"
   steps:
@@ -20,8 +24,17 @@ runs:
       run: |
         uv sync --all-groups
         uv pip install ollama faiss-cpu
-        # always test against the latest version of the client
-        # TODO: this is not necessarily a good idea. we need to test against both published and latest
-        # to find out backwards compatibility issues.
-        uv pip install git+https://github.com/meta-llama/llama-stack-client-python.git@main
+
+        # Install llama-stack-client-python based on the client-version input
+        if [ "${{ inputs.client-version }}" = "latest" ]; then
+          echo "Installing latest llama-stack-client-python from main branch"
+          uv pip install git+https://github.com/meta-llama/llama-stack-client-python.git@main
+        elif [ "${{ inputs.client-version }}" = "published" ]; then
+          echo "Installing published llama-stack-client-python from PyPI"
+          uv pip install llama-stack-client
+        else
+          echo "Invalid client-version: ${{ inputs.client-version }}"
+          exit 1
+        fi
+
         uv pip install -e .

.github/workflows/integration-tests.yml

@@ -12,6 +12,14 @@ on:
       - 'pyproject.toml'
       - 'requirements.txt'
       - '.github/workflows/integration-tests.yml' # This workflow
+  schedule:
+    - cron: '0 0 * * *'  # Daily at 12 AM UTC
+  workflow_dispatch:
+    inputs:
+      test-all-client-versions:
+        description: 'Test against both the latest and published versions'
+        type: boolean
+        default: false
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -45,6 +53,7 @@ jobs:
         test-type: ${{ fromJson(needs.discover-tests.outputs.test-type) }}
         client-type: [library, server]
         python-version: ["3.12", "3.13"]
+        client-version: ${{ (github.event_name == 'schedule' || github.event.inputs.test-all-client-versions == 'true') && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }}
 
     steps:
       - name: Checkout repository
@@ -54,6 +63,7 @@ jobs:
         uses: ./.github/actions/setup-runner
         with:
           python-version: ${{ matrix.python-version }}
+          client-version: ${{ matrix.client-version }}
 
       - name: Setup ollama
         uses: ./.github/actions/setup-ollama
@@ -108,7 +118,7 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }}
+          name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }}-${{ matrix.client-version }}
           path: |
             *.log
           retention-days: 1